diff options
Diffstat (limited to 'pygments')
44 files changed, 2865 insertions, 663 deletions
diff --git a/pygments/__init__.py b/pygments/__init__.py index ce3312b1..a47f686e 100644 --- a/pygments/__init__.py +++ b/pygments/__init__.py @@ -26,7 +26,7 @@ :license: BSD, see LICENSE for details. """ -__version__ = '1.6' +__version__ = '2.0pre' __docformat__ = 'restructuredtext' __all__ = ['lex', 'format', 'highlight'] @@ -43,7 +43,7 @@ def lex(code, lexer): """ try: return lexer.get_tokens(code) - except TypeError, err: + except TypeError as err: if isinstance(err.args[0], str) and \ 'unbound method get_tokens' in err.args[0]: raise TypeError('lex() argument must be a lexer instance, ' @@ -67,7 +67,7 @@ def format(tokens, formatter, outfile=None): return realoutfile.getvalue() else: formatter.format(tokens, outfile) - except TypeError, err: + except TypeError as err: if isinstance(err.args[0], str) and \ 'unbound method format' in err.args[0]: raise TypeError('format() argument must be a formatter instance, ' diff --git a/pygments/cmdline.py b/pygments/cmdline.py index 687cdad0..7c23ebee 100644 --- a/pygments/cmdline.py +++ b/pygments/cmdline.py @@ -8,6 +8,9 @@ :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ + +from __future__ import print_function + import sys import getopt from textwrap import dedent @@ -16,6 +19,7 @@ from pygments import __version__, highlight from pygments.util import ClassNotFound, OptionError, docstring_headline from pygments.lexers import get_all_lexers, get_lexer_by_name, get_lexer_for_filename, \ find_lexer_class, guess_lexer, TextLexer +from pygments.formatters.latex import LatexEmbeddedLexer, LatexFormatter from pygments.formatters import get_all_formatters, get_formatter_by_name, \ get_formatter_for_filename, find_formatter_class, \ TerminalFormatter # pylint:disable-msg=E0611 @@ -119,25 +123,25 @@ def _print_help(what, name): try: if what == 'lexer': cls = find_lexer_class(name) - print "Help on the %s lexer:" % cls.name - print dedent(cls.__doc__) + print("Help on the %s lexer:" % cls.name) + print(dedent(cls.__doc__)) elif what == 'formatter': cls = find_formatter_class(name) - print "Help on the %s formatter:" % cls.name - print dedent(cls.__doc__) + print("Help on the %s formatter:" % cls.name) + print(dedent(cls.__doc__)) elif what == 'filter': cls = find_filter_class(name) - print "Help on the %s filter:" % name - print dedent(cls.__doc__) + print("Help on the %s filter:" % name) + print(dedent(cls.__doc__)) except AttributeError: - print >>sys.stderr, "%s not found!" % what + print("%s not found!" % what, file=sys.stderr) def _print_list(what): if what == 'lexer': - print - print "Lexers:" - print "~~~~~~~" + print() + print("Lexers:") + print("~~~~~~~") info = [] for fullname, names, exts, _ in get_all_lexers(): @@ -146,12 +150,12 @@ def _print_list(what): info.append(tup) info.sort() for i in info: - print ('* %s\n %s %s') % i + print(('* %s\n %s %s') % i) elif what == 'formatter': - print - print "Formatters:" - print "~~~~~~~~~~~" + print() + print("Formatters:") + print("~~~~~~~~~~~") info = [] for cls in get_all_formatters(): @@ -161,27 +165,27 @@ def _print_list(what): info.append(tup) info.sort() for i in info: - print ('* %s\n %s %s') % i + print(('* %s\n %s %s') % i) elif what == 'filter': - print - print "Filters:" - print "~~~~~~~~" + print() + print("Filters:") + print("~~~~~~~~") for name in get_all_filters(): cls = find_filter_class(name) - print "* " + name + ':' - print " %s" % docstring_headline(cls) + print("* " + name + ':') + print(" %s" % docstring_headline(cls)) elif what == 'style': - print - print "Styles:" - print "~~~~~~~" + print() + print("Styles:") + print("~~~~~~~") for name in get_all_styles(): cls = get_style_by_name(name) - print "* " + name + ':' - print " %s" % docstring_headline(cls) + print("* " + name + ':') + print(" %s" % docstring_headline(cls)) def main(args=sys.argv): @@ -202,8 +206,8 @@ def main(args=sys.argv): try: popts, args = getopt.getopt(args[1:], "l:f:F:o:O:P:LS:a:N:hVHg") - except getopt.GetoptError, err: - print >>sys.stderr, usage + except getopt.GetoptError: + print(usage, file=sys.stderr) return 2 opts = {} O_opts = [] @@ -219,22 +223,22 @@ def main(args=sys.argv): opts[opt] = arg if not opts and not args: - print usage + print(usage) return 0 if opts.pop('-h', None) is not None: - print usage + print(usage) return 0 if opts.pop('-V', None) is not None: - print 'Pygments version %s, (c) 2006-2014 by Georg Brandl.' % __version__ + print('Pygments version %s, (c) 2006-2014 by Georg Brandl.' % __version__) return 0 # handle ``pygmentize -L`` L_opt = opts.pop('-L', None) if L_opt is not None: if opts: - print >>sys.stderr, usage + print(usage, file=sys.stderr) return 2 # print version @@ -249,12 +253,12 @@ def main(args=sys.argv): H_opt = opts.pop('-H', None) if H_opt is not None: if opts or len(args) != 2: - print >>sys.stderr, usage + print(usage, file=sys.stderr) return 2 what, name = args if what not in ('lexer', 'formatter', 'filter'): - print >>sys.stderr, usage + print(usage, file=sys.stderr) return 2 _print_help(what, name) @@ -279,13 +283,13 @@ def main(args=sys.argv): if infn is not None: try: lexer = get_lexer_for_filename(infn, **parsed_opts) - except ClassNotFound, err: + except ClassNotFound as err: lexer = TextLexer() - except OptionError, err: - print >>sys.stderr, 'Error:', err + except OptionError as err: + print('Error:', err, file=sys.stderr) return 1 - print lexer.aliases[0] + print(lexer.aliases[0]) return 0 # handle ``pygmentize -S`` @@ -294,30 +298,30 @@ def main(args=sys.argv): if S_opt is not None: f_opt = opts.pop('-f', None) if not f_opt: - print >>sys.stderr, usage + print(usage, file=sys.stderr) return 2 if opts or args: - print >>sys.stderr, usage + print(usage, file=sys.stderr) return 2 try: parsed_opts['style'] = S_opt fmter = get_formatter_by_name(f_opt, **parsed_opts) - except ClassNotFound, err: - print >>sys.stderr, err + except ClassNotFound as err: + print(err, file=sys.stderr) return 1 arg = a_opt or '' try: - print fmter.get_style_defs(arg) - except Exception, err: - print >>sys.stderr, 'Error:', err + print(fmter.get_style_defs(arg)) + except Exception as err: + print('Error:', err, file=sys.stderr) return 1 return 0 # if no -S is given, -a is not allowed if a_opt is not None: - print >>sys.stderr, usage + print(usage, file=sys.stderr) return 2 # parse -F options @@ -330,21 +334,21 @@ def main(args=sys.argv): if fmter: try: fmter = get_formatter_by_name(fmter, **parsed_opts) - except (OptionError, ClassNotFound), err: - print >>sys.stderr, 'Error:', err + except (OptionError, ClassNotFound) as err: + print('Error:', err, file=sys.stderr) return 1 if outfn: if not fmter: try: fmter = get_formatter_for_filename(outfn, **parsed_opts) - except (OptionError, ClassNotFound), err: - print >>sys.stderr, 'Error:', err + except (OptionError, ClassNotFound) as err: + print('Error:', err, file=sys.stderr) return 1 try: outfile = open(outfn, 'wb') - except Exception, err: - print >>sys.stderr, 'Error: cannot open outfile:', err + except Exception as err: + print('Error: cannot open outfile:', err, file=sys.stderr) return 1 else: if not fmter: @@ -356,36 +360,36 @@ def main(args=sys.argv): if lexer: try: lexer = get_lexer_by_name(lexer, **parsed_opts) - except (OptionError, ClassNotFound), err: - print >>sys.stderr, 'Error:', err + except (OptionError, ClassNotFound) as err: + print('Error:', err, file=sys.stderr) return 1 if args: if len(args) > 1: - print >>sys.stderr, usage + print(usage, file=sys.stderr) return 2 infn = args[0] try: code = open(infn, 'rb').read() - except Exception, err: - print >>sys.stderr, 'Error: cannot read infile:', err + except Exception as err: + print('Error: cannot read infile:', err, file=sys.stderr) return 1 if not lexer: try: lexer = get_lexer_for_filename(infn, code, **parsed_opts) - except ClassNotFound, err: + except ClassNotFound as err: if '-g' in opts: try: lexer = guess_lexer(code, **parsed_opts) except ClassNotFound: lexer = TextLexer(**parsed_opts) else: - print >>sys.stderr, 'Error:', err + print('Error:', err, file=sys.stderr) return 1 - except OptionError, err: - print >>sys.stderr, 'Error:', err + except OptionError as err: + print('Error:', err, file=sys.stderr) return 1 else: @@ -396,12 +400,21 @@ def main(args=sys.argv): except ClassNotFound: lexer = TextLexer(**parsed_opts) elif not lexer: - print >>sys.stderr, 'Error: no lexer name given and reading ' + \ - 'from stdin (try using -g or -l <lexer>)' + print('Error: no lexer name given and reading ' + \ + 'from stdin (try using -g or -l <lexer>)', file=sys.stderr) return 2 else: code = sys.stdin.read() + # When using the LaTeX formatter and the option `escapeinside` is + # specified, we need a special lexer which collects escaped text + # before running the chosen language lexer. + escapeinside = parsed_opts.get('escapeinside', '') + if len(escapeinside) == 2 and isinstance(fmter, LatexFormatter): + left = escapeinside[0] + right = escapeinside[1] + lexer = LatexEmbeddedLexer(left, right, lexer) + # No encoding given? Use latin1 if output file given, # stdin/stdout encoding otherwise. # (This is a compromise, I'm not too happy with it...) @@ -426,16 +439,16 @@ def main(args=sys.argv): for fname, fopts in F_opts: lexer.add_filter(fname, **fopts) highlight(code, lexer, fmter, outfile) - except Exception, err: + except Exception: import traceback info = traceback.format_exception(*sys.exc_info()) msg = info[-1].strip() if len(info) >= 3: # extract relevant file and position info msg += '\n (f%s)' % info[-2].split('\n')[0].strip()[1:] - print >>sys.stderr - print >>sys.stderr, '*** Error while highlighting:' - print >>sys.stderr, msg + print(file=sys.stderr) + print('*** Error while highlighting:', file=sys.stderr) + print(msg, file=sys.stderr) return 1 return 0 diff --git a/pygments/filters/__init__.py b/pygments/filters/__init__.py index c33dac7e..2de661c7 100644 --- a/pygments/filters/__init__.py +++ b/pygments/filters/__init__.py @@ -16,7 +16,7 @@ from pygments.token import String, Comment, Keyword, Name, Error, Whitespace, \ string_to_tokentype from pygments.filter import Filter from pygments.util import get_list_opt, get_int_opt, get_bool_opt, \ - get_choice_opt, ClassNotFound, OptionError + get_choice_opt, ClassNotFound, OptionError, text_type, string_types from pygments.plugin import find_plugin_filters @@ -117,7 +117,7 @@ class KeywordCaseFilter(Filter): def __init__(self, **options): Filter.__init__(self, **options) case = get_choice_opt(options, 'case', ['lower', 'upper', 'capitalize'], 'lower') - self.convert = getattr(unicode, case) + self.convert = getattr(text_type, case) def filter(self, lexer, stream): for ttype, value in stream: @@ -182,7 +182,7 @@ class RaiseOnErrorTokenFilter(Filter): The exception class to raise. The default is `pygments.filters.ErrorToken`. - *New in Pygments 0.8.* + .. versionadded:: 0.8 """ def __init__(self, **options): @@ -230,14 +230,16 @@ class VisibleWhitespaceFilter(Filter): styling the visible whitespace differently (e.g. greyed out), but it can disrupt background colors. The default is ``True``. - *New in Pygments 0.8.* + .. versionadded:: 0.8 """ def __init__(self, **options): Filter.__init__(self, **options) - for name, default in {'spaces': u'·', 'tabs': u'»', 'newlines': u'¶'}.items(): + for name, default in [('spaces', u'·'), + ('tabs', u'»'), + ('newlines', u'¶')]: opt = options.get(name, False) - if isinstance(opt, basestring) and len(opt) == 1: + if isinstance(opt, string_types) and len(opt) == 1: setattr(self, name, opt) else: setattr(self, name, (opt and default or '')) @@ -293,7 +295,7 @@ class GobbleFilter(Filter): `n` : int The number of characters to gobble. - *New in Pygments 1.2.* + .. versionadded:: 1.2 """ def __init__(self, **options): Filter.__init__(self, **options) @@ -325,7 +327,7 @@ class TokenMergeFilter(Filter): Merges consecutive tokens with the same token type in the output stream of a lexer. - *New in Pygments 1.2.* + .. versionadded:: 1.2 """ def __init__(self, **options): Filter.__init__(self, **options) diff --git a/pygments/formatter.py b/pygments/formatter.py index e5ad0d25..b16ffee8 100644 --- a/pygments/formatter.py +++ b/pygments/formatter.py @@ -11,14 +11,14 @@ import codecs -from pygments.util import get_bool_opt +from pygments.util import get_bool_opt, string_types from pygments.styles import get_style_by_name __all__ = ['Formatter'] def _lookup_style(style): - if isinstance(style, basestring): + if isinstance(style, string_types): return get_style_by_name(style) return style diff --git a/pygments/formatters/_mapping.py b/pygments/formatters/_mapping.py index d2aabeca..79f592b3 100755 --- a/pygments/formatters/_mapping.py +++ b/pygments/formatters/_mapping.py @@ -13,6 +13,8 @@ :license: BSD, see LICENSE for details. """ +from __future__ import print_function + # start from pygments.formatters.bbcode import BBCodeFormatter from pygments.formatters.html import HtmlFormatter @@ -57,7 +59,7 @@ if __name__ == '__main__': for filename in os.listdir('.'): if filename.endswith('.py') and not filename.startswith('_'): module_name = 'pygments.formatters.%s' % filename[:-3] - print module_name + print(module_name) module = __import__(module_name, None, None, ['']) for formatter_name in module.__all__: imports.append((module_name, formatter_name)) diff --git a/pygments/formatters/html.py b/pygments/formatters/html.py index 5e5f1e40..3bc60e8a 100644 --- a/pygments/formatters/html.py +++ b/pygments/formatters/html.py @@ -9,14 +9,16 @@ :license: BSD, see LICENSE for details. """ +from __future__ import print_function + import os import sys import os.path -import StringIO from pygments.formatter import Formatter from pygments.token import Token, Text, STANDARD_TYPES -from pygments.util import get_bool_opt, get_int_opt, get_list_opt, bytes +from pygments.util import get_bool_opt, get_int_opt, get_list_opt, \ + StringIO, string_types, iteritems try: import ctags @@ -218,29 +220,34 @@ class HtmlFormatter(Formatter): If you set this option, the default selector for `get_style_defs()` will be this class. - *New in Pygments 0.9:* If you select the ``'table'`` line numbers, the - wrapping table will have a CSS class of this string plus ``'table'``, - the default is accordingly ``'highlighttable'``. + .. versionadded:: 0.9 + If you select the ``'table'`` line numbers, the wrapping table will + have a CSS class of this string plus ``'table'``, the default is + accordingly ``'highlighttable'``. `cssstyles` Inline CSS styles for the wrapping ``<div>`` tag (default: ``''``). `prestyles` - Inline CSS styles for the ``<pre>`` tag (default: ``''``). *New in - Pygments 0.11.* + Inline CSS styles for the ``<pre>`` tag (default: ``''``). + + .. versionadded:: 0.11 `cssfile` If the `full` option is true and this option is given, it must be the name of an external file. If the filename does not include an absolute path, the file's path will be assumed to be relative to the main output file's path, if the latter can be found. The stylesheet is then written - to this file instead of the HTML file. *New in Pygments 0.6.* + to this file instead of the HTML file. + + .. versionadded:: 0.6 `noclobber_cssfile` If `cssfile` is given and the specified file exists, the css file will not be overwritten. This allows the use of the `full` option in combination with a user specified css file. Default is ``False``. - *New in Pygments 1.1.* + + .. versionadded:: 1.1 `linenos` If set to ``'table'``, output line numbers as a table with two cells, @@ -263,7 +270,9 @@ class HtmlFormatter(Formatter): 125%``). `hl_lines` - Specify a list of lines to be highlighted. *New in Pygments 0.11.* + Specify a list of lines to be highlighted. + + .. versionadded:: 0.11 `linenostart` The line number for the first line (default: ``1``). @@ -279,24 +288,30 @@ class HtmlFormatter(Formatter): If set to ``True``, the formatter won't output the background color for the wrapping element (this automatically defaults to ``False`` when there is no wrapping element [eg: no argument for the - `get_syntax_defs` method given]) (default: ``False``). *New in - Pygments 0.6.* + `get_syntax_defs` method given]) (default: ``False``). + + .. versionadded:: 0.6 `lineseparator` This string is output between lines of code. It defaults to ``"\n"``, which is enough to break a line inside ``<pre>`` tags, but you can - e.g. set it to ``"<br>"`` to get HTML line breaks. *New in Pygments - 0.7.* + e.g. set it to ``"<br>"`` to get HTML line breaks. + + .. versionadded:: 0.7 `lineanchors` If set to a nonempty string, e.g. ``foo``, the formatter will wrap each output line in an anchor tag with a ``name`` of ``foo-linenumber``. - This allows easy linking to certain lines. *New in Pygments 0.9.* + This allows easy linking to certain lines. + + .. versionadded:: 0.9 `linespans` If set to a nonempty string, e.g. ``foo``, the formatter will wrap each output line in a span tag with an ``id`` of ``foo-linenumber``. - This allows easy access to lines via javascript. *New in Pygments 1.6.* + This allows easy access to lines via javascript. + + .. versionadded:: 1.6 `anchorlinenos` If set to `True`, will wrap line numbers in <a> tags. Used in @@ -306,18 +321,20 @@ class HtmlFormatter(Formatter): If set to the path of a ctags file, wrap names in anchor tags that link to their definitions. `lineanchors` should be used, and the tags file should specify line numbers (see the `-n` option to ctags). - *New in Pygments 1.6.* + + .. versionadded:: 1.6 `tagurlformat` A string formatting pattern used to generate links to ctags definitions. Available variables are `%(path)s`, `%(fname)s` and `%(fext)s`. Defaults to an empty string, resulting in just `#prefix-number` links. - *New in Pygments 1.6.* + + .. versionadded:: 1.6 **Subclassing the HTML formatter** - *New in Pygments 0.7.* + .. versionadded:: 0.7 The HTML formatter is now built in a way that allows easy subclassing, thus customizing the output HTML code. The `format()` method calls @@ -453,7 +470,7 @@ class HtmlFormatter(Formatter): """ if arg is None: arg = ('cssclass' in self.options and '.'+self.cssclass or '') - if isinstance(arg, basestring): + if isinstance(arg, string_types): args = [arg] else: args = list(arg) @@ -467,7 +484,7 @@ class HtmlFormatter(Formatter): return ', '.join(tmp) styles = [(level, ttype, cls, style) - for cls, (style, ttype, level) in self.class2style.iteritems() + for cls, (style, ttype, level) in iteritems(self.class2style) if cls and style] styles.sort() lines = ['%s { %s } /* %s */' % (prefix(cls), style, repr(ttype)[6:]) @@ -505,8 +522,8 @@ class HtmlFormatter(Formatter): cssfilename = os.path.join(os.path.dirname(filename), self.cssfile) except AttributeError: - print >>sys.stderr, 'Note: Cannot determine output file name, ' \ - 'using current directory as base for the CSS file name' + print('Note: Cannot determine output file name, ' \ + 'using current directory as base for the CSS file name', file=sys.stderr) cssfilename = self.cssfile # write CSS file only if noclobber_cssfile isn't given as an option. try: @@ -515,7 +532,7 @@ class HtmlFormatter(Formatter): cf.write(CSSFILE_TEMPLATE % {'styledefs': self.get_style_defs('body')}) cf.close() - except IOError, err: + except IOError as err: err.strerror = 'Error writing CSS file: ' + err.strerror raise @@ -534,7 +551,7 @@ class HtmlFormatter(Formatter): yield 0, DOC_FOOTER def _wrap_tablelinenos(self, inner): - dummyoutfile = StringIO.StringIO() + dummyoutfile = StringIO() lncount = 0 for t, line in inner: if t: diff --git a/pygments/formatters/img.py b/pygments/formatters/img.py index 615c722d..8e2b5f9e 100644 --- a/pygments/formatters/img.py +++ b/pygments/formatters/img.py @@ -12,8 +12,8 @@ import sys from pygments.formatter import Formatter -from pygments.util import get_bool_opt, get_int_opt, \ - get_list_opt, get_choice_opt +from pygments.util import get_bool_opt, get_int_opt, get_list_opt, \ + get_choice_opt, xrange # Import this carefully try: @@ -25,7 +25,10 @@ except ImportError: try: import _winreg except ImportError: - _winreg = None + try: + import winreg as _winreg + except ImportError: + _winreg = None __all__ = ['ImageFormatter', 'GifImageFormatter', 'JpgImageFormatter', 'BmpImageFormatter'] @@ -72,7 +75,10 @@ class FontManager(object): self._create_nix() def _get_nix_font_path(self, name, style): - from commands import getstatusoutput + try: + from commands import getstatusoutput + except ImportError: + from subprocess import getstatusoutput exit, out = getstatusoutput('fc-list "%s:style=%s" file' % (name, style)) if not exit: @@ -169,7 +175,7 @@ class ImageFormatter(Formatter): Create a PNG image from source code. This uses the Python Imaging Library to generate a pixmap from the source code. - *New in Pygments 0.10.* + .. versionadded:: 0.10 Additional options accepted: @@ -258,12 +264,16 @@ class ImageFormatter(Formatter): Default: 6 `hl_lines` - Specify a list of lines to be highlighted. *New in Pygments 1.2.* + Specify a list of lines to be highlighted. + + .. versionadded:: 1.2 Default: empty list `hl_color` - Specify the color for highlighting lines. *New in Pygments 1.2.* + Specify the color for highlighting lines. + + .. versionadded:: 1.2 Default: highlight color of the selected style """ @@ -513,8 +523,7 @@ class GifImageFormatter(ImageFormatter): Create a GIF image from source code. This uses the Python Imaging Library to generate a pixmap from the source code. - *New in Pygments 1.0.* (You could create GIF images before by passing a - suitable `image_format` option to the `ImageFormatter`.) + .. versionadded:: 1.0 """ name = 'img_gif' @@ -528,8 +537,7 @@ class JpgImageFormatter(ImageFormatter): Create a JPEG image from source code. This uses the Python Imaging Library to generate a pixmap from the source code. - *New in Pygments 1.0.* (You could create JPEG images before by passing a - suitable `image_format` option to the `ImageFormatter`.) + .. versionadded:: 1.0 """ name = 'img_jpg' @@ -543,8 +551,7 @@ class BmpImageFormatter(ImageFormatter): Create a bitmap image from source code. This uses the Python Imaging Library to generate a pixmap from the source code. - *New in Pygments 1.0.* (You could create bitmap images before by passing a - suitable `image_format` option to the `ImageFormatter`.) + .. versionadded:: 1.0 """ name = 'img_bmp' diff --git a/pygments/formatters/latex.py b/pygments/formatters/latex.py index 2cacca86..fee177c5 100644 --- a/pygments/formatters/latex.py +++ b/pygments/formatters/latex.py @@ -9,9 +9,13 @@ :license: BSD, see LICENSE for details. """ +from __future__ import division + from pygments.formatter import Formatter +from pygments.lexer import Lexer from pygments.token import Token, STANDARD_TYPES -from pygments.util import get_bool_opt, get_int_opt, StringIO +from pygments.util import get_bool_opt, get_int_opt, StringIO, xrange, \ + iteritems __all__ = ['LatexFormatter'] @@ -205,19 +209,33 @@ class LatexFormatter(Formatter): `commandprefix` The LaTeX commands used to produce colored output are constructed using this prefix and some letters (default: ``'PY'``). - *New in Pygments 0.7.* - *New in Pygments 0.10:* the default is now ``'PY'`` instead of ``'C'``. + .. versionadded:: 0.7 + .. versionchanged:: 0.10 + The default is now ``'PY'`` instead of ``'C'``. `texcomments` If set to ``True``, enables LaTeX comment lines. That is, LaTex markup in comment tokens is not escaped so that LaTeX can render it (default: - ``False``). *New in Pygments 1.2.* + ``False``). + + .. versionadded:: 1.2 `mathescape` If set to ``True``, enables LaTeX math mode escape in comments. That is, ``'$...$'`` inside a comment will trigger math mode (default: - ``False``). *New in Pygments 1.2.* + ``False``). + + .. versionadded:: 1.2 + + `escapeinside` + If set to a string of length 2, enables escaping to LaTeX. Text + delimited by these 2 characters is read as LaTeX code and + typeset accordingly. It has no effect in string literals. It has + no effect in comments if `texcomments` or `mathescape` is + set. (default: ``''``). + + .. versionadded:: 2.0 """ name = 'LaTeX' aliases = ['latex', 'tex'] @@ -235,6 +253,13 @@ class LatexFormatter(Formatter): self.commandprefix = options.get('commandprefix', 'PY') self.texcomments = get_bool_opt(options, 'texcomments', False) self.mathescape = get_bool_opt(options, 'mathescape', False) + self.escapeinside = options.get('escapeinside', '') + + if len(self.escapeinside) == 2: + self.left = self.escapeinside[0] + self.right = self.escapeinside[1] + else: + self.escapeinside = '' self._create_stylesheet() @@ -291,7 +316,7 @@ class LatexFormatter(Formatter): """ cp = self.commandprefix styles = [] - for name, definition in self.cmd2def.iteritems(): + for name, definition in iteritems(self.cmd2def): styles.append(r'\expandafter\def\csname %s@tok@%s\endcsname{%s}' % (cp, name, definition)) return STYLE_TEMPLATE % {'cp': self.commandprefix, @@ -306,14 +331,14 @@ class LatexFormatter(Formatter): realoutfile = outfile outfile = StringIO() - outfile.write(ur'\begin{Verbatim}[commandchars=\\\{\}') + outfile.write(u'\\begin{Verbatim}[commandchars=\\\\\\{\\}') if self.linenos: start, step = self.linenostart, self.linenostep outfile.write(u',numbers=left' + (start and u',firstnumber=%d' % start or u'') + (step and u',stepnumber=%d' % step or u'')) - if self.mathescape or self.texcomments: - outfile.write(ur',codes={\catcode`\$=3\catcode`\^=7\catcode`\_=8}') + if self.mathescape or self.texcomments or self.escapeinside: + outfile.write(u',codes={\\catcode`\\$=3\\catcode`\\^=7\\catcode`\\_=8}') if self.verboptions: outfile.write(u',' + self.verboptions) outfile.write(u']\n') @@ -342,9 +367,22 @@ class LatexFormatter(Formatter): parts[i] = escape_tex(part, self.commandprefix) in_math = not in_math value = '$'.join(parts) + elif self.escapeinside: + text = value + value = '' + while len(text) > 0: + a,sep1,text = text.partition(self.left) + if len(sep1) > 0: + b,sep2,text = text.partition(self.right) + if len(sep2) > 0: + value = value + escape_tex(a, self.commandprefix) + b + else: + value = value + escape_tex(a + sep1 + b, self.commandprefix) + else: + value = value + escape_tex(a, self.commandprefix) else: value = escape_tex(value, self.commandprefix) - else: + elif not (ttype in Token.Escape): value = escape_tex(value, self.commandprefix) styles = [] while ttype is not Token: @@ -376,3 +414,57 @@ class LatexFormatter(Formatter): encoding = self.encoding or 'latin1', styledefs = self.get_style_defs(), code = outfile.getvalue())) + + +class LatexEmbeddedLexer(Lexer): + r""" + + This lexer takes one lexer as argument, the lexer for the language + being formatted, and the left and right delimiters for escaped text. + + First everything is scanned using the language lexer to obtain + strings and comments. All other consecutive tokens are merged and + the resulting text is scanned for escaped segments, which are given + the Token.Escape type. Finally text that is not escaped is scanned + again with the language lexer. + """ + def __init__(self, left, right, lang, **options): + self.left = left + self.right = right + self.lang = lang + Lexer.__init__(self, **options) + + def get_tokens_unprocessed(self, text): + buf = '' + for i, t, v in self.lang.get_tokens_unprocessed(text): + if t in Token.Comment or t in Token.String: + if buf: + for x in self.get_tokens_aux(idx, buf): + yield x + buf = '' + yield i, t, v + else: + if not buf: + idx = i; + buf += v + if buf: + for x in self.get_tokens_aux(idx, buf): + yield x + + def get_tokens_aux(self, index, text): + while text: + a, sep1, text = text.partition(self.left) + if a: + for i, t, v in self.lang.get_tokens_unprocessed(a): + yield index + i, t, v + index += len(a) + if sep1: + b, sep2, text = text.partition(self.right) + if sep2: + yield index + len(sep1), Token.Escape, b + index += len(sep1) + len(b) + len(sep2) + else: + yield index, Token.Error, sep1 + index += len(sep1) + text = b + diff --git a/pygments/formatters/other.py b/pygments/formatters/other.py index 00fe8ba6..7368a642 100644 --- a/pygments/formatters/other.py +++ b/pygments/formatters/other.py @@ -10,7 +10,7 @@ """ from pygments.formatter import Formatter -from pygments.util import OptionError, get_choice_opt, b +from pygments.util import OptionError, get_choice_opt from pygments.token import Token from pygments.console import colorize @@ -40,7 +40,7 @@ class RawTokenFormatter(Formatter): The format is ``tokentype<TAB>repr(tokenstring)\n``. The output can later be converted to a token stream with the `RawTokenLexer`, described in the - `lexer list <lexers.txt>`_. + :doc:`lexer list <lexers>`. Only two options are accepted: @@ -50,7 +50,8 @@ class RawTokenFormatter(Formatter): `error_color` If set to a color name, highlight error tokens using that color. If set but with no value, defaults to ``'red'``. - *New in Pygments 0.11.* + + .. versionadded:: 0.11 """ name = 'Raw tokens' @@ -79,7 +80,7 @@ class RawTokenFormatter(Formatter): def format(self, tokensource, outfile): try: - outfile.write(b('')) + outfile.write(b'') except TypeError: raise TypeError('The raw tokens formatter needs a binary ' 'output file') diff --git a/pygments/formatters/rtf.py b/pygments/formatters/rtf.py index 45cb2d88..9d87e8f1 100644 --- a/pygments/formatters/rtf.py +++ b/pygments/formatters/rtf.py @@ -10,6 +10,7 @@ """ from pygments.formatter import Formatter +from pygments.util import get_int_opt __all__ = ['RtfFormatter'] @@ -21,7 +22,7 @@ class RtfFormatter(Formatter): documents with color information and other useful stuff. Perfect for Copy and Paste into Microsoft® Word® documents. - *New in Pygments 0.6.* + .. versionadded:: 0.6 Additional options accepted: @@ -32,6 +33,12 @@ class RtfFormatter(Formatter): `fontface` The used font famliy, for example ``Bitstream Vera Sans``. Defaults to some generic font which is supposed to have fixed width. + + `fontsize` + Size of the font used. Size is specified in half points. The + default is 24 half-points, giving a size 12 font. + + .. versionadded:: 2.0 """ name = 'RTF' aliases = ['rtf'] @@ -49,9 +56,11 @@ class RtfFormatter(Formatter): specification claims that ``\fmodern`` are "Fixed-pitch serif and sans serif fonts". Hope every RTF implementation thinks the same about modern... + """ Formatter.__init__(self, **options) self.fontface = options.get('fontface') or '' + self.fontsize = get_int_opt(options, 'fontsize', 0) def _escape(self, text): return text.replace('\\', '\\\\') \ @@ -106,6 +115,8 @@ class RtfFormatter(Formatter): )) offset += 1 outfile.write(r'}\f0') + if self.fontsize: + outfile.write(r'\fs%d' % (self.fontsize)) # highlight stream for ttype, value in tokensource: diff --git a/pygments/formatters/svg.py b/pygments/formatters/svg.py index b67b54a2..07636943 100644 --- a/pygments/formatters/svg.py +++ b/pygments/formatters/svg.py @@ -35,7 +35,7 @@ class SvgFormatter(Formatter): By default, this formatter outputs a full SVG document including doctype declaration and the ``<svg>`` root element. - *New in Pygments 0.9.* + .. versionadded:: 0.9 Additional options accepted: diff --git a/pygments/formatters/terminal256.py b/pygments/formatters/terminal256.py index b40bdd0d..60b698c9 100644 --- a/pygments/formatters/terminal256.py +++ b/pygments/formatters/terminal256.py @@ -76,7 +76,7 @@ class Terminal256Formatter(Formatter): and converts them to nearest ANSI 256-color escape sequences. Bold and underline attributes from the style are preserved (and displayed). - *New in Pygments 0.9.* + .. versionadded:: 0.9 Options accepted: diff --git a/pygments/lexer.py b/pygments/lexer.py index 2f191619..567e85f8 100644 --- a/pygments/lexer.py +++ b/pygments/lexer.py @@ -14,18 +14,18 @@ from pygments.filter import apply_filters, Filter from pygments.filters import get_filter_by_name from pygments.token import Error, Text, Other, _TokenType from pygments.util import get_bool_opt, get_int_opt, get_list_opt, \ - make_analysator + make_analysator, text_type, add_metaclass, iteritems __all__ = ['Lexer', 'RegexLexer', 'ExtendedRegexLexer', 'DelegatingLexer', 'LexerContext', 'include', 'inherit', 'bygroups', 'using', 'this'] -_encoding_map = [('\xef\xbb\xbf', 'utf-8'), - ('\xff\xfe\0\0', 'utf-32'), - ('\0\0\xfe\xff', 'utf-32be'), - ('\xff\xfe', 'utf-16'), - ('\xfe\xff', 'utf-16be')] +_encoding_map = [(b'\xef\xbb\xbf', 'utf-8'), + (b'\xff\xfe\0\0', 'utf-32'), + (b'\0\0\xfe\xff', 'utf-32be'), + (b'\xff\xfe', 'utf-16'), + (b'\xfe\xff', 'utf-16be')] _default_analyse = staticmethod(lambda x: 0.0) @@ -42,6 +42,7 @@ class LexerMeta(type): return type.__new__(cls, name, bases, d) +@add_metaclass(LexerMeta) class Lexer(object): """ Lexer for a specific language. @@ -55,7 +56,9 @@ class Lexer(object): ``ensurenl`` Make sure that the input ends with a newline (default: True). This is required for some lexers that consume input linewise. - *New in Pygments 1.3.* + + .. versionadded:: 1.3 + ``tabsize`` If given and greater than 0, expand tabs in the input (default: 0). ``encoding`` @@ -84,8 +87,6 @@ class Lexer(object): #: Priority, should multiple lexers match and no content is provided priority = 0 - __metaclass__ = LexerMeta - def __init__(self, **options): self.options = options self.stripnl = get_bool_opt(options, 'stripnl', True) @@ -136,7 +137,7 @@ class Lexer(object): Also preprocess the text, i.e. expand tabs and strip it if wanted and applies registered filters. """ - if not isinstance(text, unicode): + if not isinstance(text, text_type): if self.encoding == 'guess': try: text = text.decode('utf-8') @@ -155,14 +156,13 @@ class Lexer(object): decoded = None for bom, encoding in _encoding_map: if text.startswith(bom): - decoded = unicode(text[len(bom):], encoding, - errors='replace') + decoded = text[len(bom):].decode(encoding, 'replace') break # no BOM found, so use chardet if decoded is None: enc = chardet.detect(text[:1024]) # Guess using first 1KB - decoded = unicode(text, enc.get('encoding') or 'utf-8', - errors='replace') + decoded = text.decode(enc.get('encoding') or 'utf-8', + 'replace') text = decoded else: text = text.decode(self.encoding) @@ -457,7 +457,7 @@ class RegexLexerMeta(LexerMeta): try: rex = cls._process_regex(tdef[0], rflags) - except Exception, err: + except Exception as err: raise ValueError("uncompilable regex %r in state %r of %r: %s" % (tdef[0], state, cls, err)) @@ -476,7 +476,7 @@ class RegexLexerMeta(LexerMeta): """Preprocess a dictionary of token definitions.""" processed = cls._all_tokens[name] = {} tokendefs = tokendefs or cls.tokens[name] - for state in tokendefs.keys(): + for state in list(tokendefs): cls._process_state(tokendefs, processed, state) return processed @@ -497,7 +497,7 @@ class RegexLexerMeta(LexerMeta): for c in itertools.chain((cls,), cls.__mro__): toks = c.__dict__.get('tokens', {}) - for state, items in toks.iteritems(): + for state, items in iteritems(toks): curitems = tokens.get(state) if curitems is None: tokens[state] = items @@ -537,13 +537,13 @@ class RegexLexerMeta(LexerMeta): return type.__call__(cls, *args, **kwds) +@add_metaclass(RegexLexerMeta) class RegexLexer(Lexer): """ Base for simple stateful regular expression-based lexers. Simplifies the lexing process so that you need only provide a list of states and regular expressions. """ - __metaclass__ = RegexLexerMeta #: Flags for compiling the regular expressions. #: Defaults to MULTILINE. @@ -722,7 +722,7 @@ def do_insertions(insertions, tokens): """ insertions = iter(insertions) try: - index, itokens = insertions.next() + index, itokens = next(insertions) except StopIteration: # no insertions for item in tokens: @@ -748,7 +748,7 @@ def do_insertions(insertions, tokens): realpos += len(it_value) oldi = index - i try: - index, itokens = insertions.next() + index, itokens = next(insertions) except StopIteration: insleft = False break # not strictly necessary @@ -763,7 +763,7 @@ def do_insertions(insertions, tokens): yield realpos, t, v realpos += len(v) try: - index, itokens = insertions.next() + index, itokens = next(insertions) except StopIteration: insleft = False break # not strictly necessary diff --git a/pygments/lexers/__init__.py b/pygments/lexers/__init__.py index 6334b54f..caedd479 100644 --- a/pygments/lexers/__init__.py +++ b/pygments/lexers/__init__.py @@ -18,11 +18,11 @@ from os.path import basename from pygments.lexers._mapping import LEXERS from pygments.modeline import get_filetype_from_buffer from pygments.plugin import find_plugin_lexers -from pygments.util import ClassNotFound, bytes +from pygments.util import ClassNotFound, itervalues __all__ = ['get_lexer_by_name', 'get_lexer_for_filename', 'find_lexer_class', - 'guess_lexer'] + LEXERS.keys() + 'guess_lexer'] + list(LEXERS) _lexer_cache = {} _pattern_cache = {} @@ -55,7 +55,7 @@ def get_all_lexers(): Return a generator of tuples in the form ``(name, aliases, filenames, mimetypes)`` of all know lexers. """ - for item in LEXERS.itervalues(): + for item in itervalues(LEXERS): yield item[1:] for lexer in find_plugin_lexers(): yield lexer.name, lexer.aliases, lexer.filenames, lexer.mimetypes @@ -68,7 +68,7 @@ def find_lexer_class(name): if name in _lexer_cache: return _lexer_cache[name] # lookup builtin lexers - for module_name, lname, aliases, _, _ in LEXERS.itervalues(): + for module_name, lname, aliases, _, _ in itervalues(LEXERS): if name == lname: _load_lexers(module_name) return _lexer_cache[name] @@ -82,8 +82,11 @@ def get_lexer_by_name(_alias, **options): """ Get a lexer by an alias. """ + if not _alias: + raise ClassNotFound('no lexer for alias %r found' % _alias) + # lookup builtin lexers - for module_name, name, aliases, _, _ in LEXERS.itervalues(): + for module_name, name, aliases, _, _ in itervalues(LEXERS): if _alias.lower() in aliases: if name not in _lexer_cache: _load_lexers(module_name) @@ -98,12 +101,12 @@ def get_lexer_by_name(_alias, **options): def get_lexer_for_filename(_fn, code=None, **options): """ Get a lexer for a filename. If multiple lexers match the filename - pattern, use ``analyze_text()`` to figure out which one is more + pattern, use ``analyse_text()`` to figure out which one is more appropriate. """ matches = [] fn = basename(_fn) - for modname, name, _, filenames, _ in LEXERS.itervalues(): + for modname, name, _, filenames, _ in itervalues(LEXERS): for filename in filenames: if _fn_matches(fn, filename): if name not in _lexer_cache: @@ -141,7 +144,7 @@ def get_lexer_for_mimetype(_mime, **options): """ Get a lexer for a mimetype. """ - for modname, name, _, _, mimetypes in LEXERS.itervalues(): + for modname, name, _, _, mimetypes in itervalues(LEXERS): if _mime in mimetypes: if name not in _lexer_cache: _load_lexers(modname) diff --git a/pygments/lexers/_cocoabuiltins.py b/pygments/lexers/_cocoabuiltins.py index 312e28d0..1bfa0cdf 100644 --- a/pygments/lexers/_cocoabuiltins.py +++ b/pygments/lexers/_cocoabuiltins.py @@ -12,6 +12,8 @@ :license: BSD, see LICENSE for details. """ +from __future__ import print_function + COCOA_INTERFACES = set(['UITableViewCell', 'NSURLSessionDataTask', 'NSLinguisticTagger', 'NSStream', 'UIPrintInfo', 'SKPaymentTransaction', 'SKPhysicsWorld', 'NSString', 'CMAttitude', 'SKSpriteNode', 'JSContext', 'UICollectionReusableView', 'AVMutableCompositionTrack', 'GKLeaderboard', 'NSFetchedResultsController', 'MKTileOverlayRenderer', 'MIDINetworkSession', 'UITextSelectionRect', 'MKRoute', 'MPVolumeView', 'UIKeyCommand', 'AVMutableAudioMix', 'GLKEffectPropertyLight', 'UICollectionViewLayout', 'NSMutableCharacterSet', 'UIAccessibilityElement', 'NSShadow', 'NSAtomicStoreCacheNode', 'UIPushBehavior', 'CBCharacteristic', 'CBUUID', 'CMStepCounter', 'NSNetService', 'UICollectionView', 'UIViewPrintFormatter', 'CAShapeLayer', 'MCPeerID', 'NSFileVersion', 'CMGyroData', 'SKPhysicsJointSpring', 'CIFilter', 'UIView', 'MKMapItem', 'PKPass', 'MKPolygonRenderer', 'JSValue', 'CLGeocoder', 'NSByteCountFormatter', 'AVCaptureScreenInput', 'CAAnimation', 'MKOverlayPathView', 'UIActionSheet', 'UIMotionEffectGroup', 'UIBarItem', 'SKProduct', 'AVAssetExportSession', 'NSKeyedUnarchiver', 'NSMutableSet', 'MKMapView', 'CATransition', 'CLCircularRegion', 'MKTileOverlay', 'UICollisionBehavior', 'ACAccountCredential', 'SKPhysicsJointLimit', 'AVMediaSelectionGroup', 'NSIndexSet', 'AVAudioRecorder', 'NSURL', 'CBCentral', 'NSNumber', 'UITableView', 'AVCaptureStillImageOutput', 'GCController', 'NSAssertionHandler', 'AVAudioSessionPortDescription', 'NSHTTPURLResponse', 'NSPropertyListSerialization', 'AVPlayerItemAccessLogEvent', 'UISwipeGestureRecognizer', 'MKOverlayRenderer', 'NSDecimalNumber', 'EKReminder', 'MKPolylineView', 'AVCaptureMovieFileOutput', 'UIImagePickerController', 'GKAchievementDescription', 'EKParticipant', 'NSBlockOperation', 'UIActivityItemProvider', 'CLLocation', 'GKLeaderboardViewController', 'MPMoviePlayerController', 'GKScore', 'NSURLConnection', 'ABUnknownPersonViewController', 'UIMenuController', 'NSEvent', 'SKTextureAtlas', 'NSKeyedArchiver', 'GKLeaderboardSet', 'NSSimpleCString', 'CBATTRequest', 'GKMatchRequest', 'AVMetadataObject', 'UIAlertView', 'NSIncrementalStore', 'MFMailComposeViewController', 'SSReadingList', 'MPMovieAccessLog', 'NSManagedObjectContext', 'AVCaptureAudioDataOutput', 'ACAccount', 'AVMetadataItem', 'AVCaptureDeviceInputSource', 'CLLocationManager', 'UIStepper', 'UIRefreshControl', 'GKTurnBasedParticipant', 'UICollectionViewTransitionLayout', 'CBCentralManager', 'NSPurgeableData', 'SLComposeViewController', 'NSHashTable', 'MKUserTrackingBarButtonItem', 'UITabBarController', 'CMMotionActivity', 'SKAction', 'AVPlayerItemOutput', 'UIDocumentInteractionController', 'UIDynamicItemBehavior', 'NSMutableDictionary', 'UILabel', 'AVCaptureInputPort', 'NSExpression', 'SKMutablePayment', 'UIStoryboardSegue', 'NSOrderedSet', 'UIPopoverBackgroundView', 'UIToolbar', 'NSNotificationCenter', 'NSEntityMigrationPolicy', 'NSLocale', 'NSURLSession', 'NSTimeZone', 'UIManagedDocument', 'AVMutableVideoCompositionLayerInstruction', 'AVAssetTrackGroup', 'NSInvocationOperation', 'ALAssetRepresentation', 'AVQueuePlayer', 'UIPasteboard', 'NSLayoutManager', 'EKCalendarChooser', 'EKObject', 'CATiledLayer', 'GLKReflectionMapEffect', 'NSManagedObjectID', 'NSUserDefaults', 'SLRequest', 'AVPlayerLayer', 'NSPointerArray', 'AVAudioMix', 'MCAdvertiserAssistant', 'MKMapSnapshotOptions', 'GKMatch', 'AVTimedMetadataGroup', 'CBMutableCharacteristic', 'NSFetchRequest', 'UIDevice', 'NSManagedObject', 'NKAssetDownload', 'AVOutputSettingsAssistant', 'SKPhysicsJointPin', 'UITabBar', 'UITextInputMode', 'NSFetchRequestExpression', 'NSPipe', 'AVComposition', 'ADBannerView', 'AVPlayerItem', 'AVSynchronizedLayer', 'MKDirectionsRequest', 'NSMetadataItem', 'UINavigationItem', 'CBPeripheralManager', 'UIStoryboardPopoverSegue', 'SKProductsRequest', 'UIGravityBehavior', 'UIWindow', 'CBMutableDescriptor', 'UIBezierPath', 'UINavigationController', 'ABPeoplePickerNavigationController', 'EKSource', 'AVAssetWriterInput', 'AVPlayerItemTrack', 'GLKEffectPropertyTexture', 'NSURLResponse', 'SKPaymentQueue', 'MKReverseGeocoder', 'GCControllerAxisInput', 'MKMapSnapshotter', 'NSOrthography', 'NSURLSessionUploadTask', 'NSCharacterSet', 'AVAssetReaderOutput', 'EAGLContext', 'UICollectionViewController', 'AVAssetTrack', 'SKEmitterNode', 'AVCaptureDeviceInput', 'AVVideoCompositionCoreAnimationTool', 'NSURLRequest', 'CMAccelerometerData', 'NSNetServiceBrowser', 'AVAsynchronousVideoCompositionRequest', 'CAGradientLayer', 'NSFormatter', 'CATransaction', 'MPMovieAccessLogEvent', 'UIStoryboard', 'MPMediaLibrary', 'UITapGestureRecognizer', 'MPMediaItemArtwork', 'NSURLSessionTask', 'MCBrowserViewController', 'NSRelationshipDescription', 'NSMutableAttributedString', 'MPNowPlayingInfoCenter', 'MKLocalSearch', 'EAAccessory', 'MKETAResponse', 'CATextLayer', 'NSNotificationQueue', 'NSValue', 'NSMutableIndexSet', 'SKPhysicsContact', 'NSProgress', 'CAScrollLayer', 'NSTextCheckingResult', 'NSEntityDescription', 'NSURLCredentialStorage', 'UIApplication', 'SKDownload', 'MKLocalSearchRequest', 'SKScene', 'UISearchDisplayController', 'CAReplicatorLayer', 'UIPrintPageRenderer', 'EKCalendarItem', 'NSUUID', 'EAAccessoryManager', 'AVAssetResourceLoader', 'AVMutableVideoCompositionInstruction', 'MyClass', 'CTCall', 'CIVector', 'UINavigationBar', 'UIPanGestureRecognizer', 'MPMediaQuery', 'ABNewPersonViewController', 'ACAccountType', 'GKSession', 'SKVideoNode', 'GCExtendedGamepadSnapshot', 'GCExtendedGamepad', 'CAValueFunction', 'UIActivityIndicatorView', 'NSNotification', 'SKReceiptRefreshRequest', 'AVCaptureDeviceFormat', 'AVPlayerItemErrorLog', 'NSMapTable', 'NSSet', 'CMMotionManager', 'GKVoiceChatService', 'UIPageControl', 'MKGeodesicPolyline', 'AVMutableComposition', 'NSLayoutConstraint', 'UIWebView', 'NSIncrementalStoreNode', 'EKEventStore', 'UISlider', 'AVAssetResourceLoadingRequest', 'AVCaptureInput', 'SKPhysicsBody', 'NSOperation', 'MKMapCamera', 'SKProductsResponse', 'GLKEffectPropertyMaterial', 'AVCaptureDevice', 'CTCallCenter', 'CBMutableService', 'SKTransition', 'UIDynamicAnimator', 'NSMutableArray', 'MCNearbyServiceBrowser', 'NSOperationQueue', 'MKPolylineRenderer', 'UICollectionViewLayoutAttributes', 'NSValueTransformer', 'UICollectionViewFlowLayout', 'NSEntityMapping', 'SKTexture', 'NSMergePolicy', 'UITextInputStringTokenizer', 'NSRecursiveLock', 'AVAsset', 'NSUndoManager', 'MPMediaPickerController', 'NSFileCoordinator', 'NSFileHandle', 'NSConditionLock', 'UISegmentedControl', 'NSManagedObjectModel', 'UITabBarItem', 'MPMediaItem', 'EKRecurrenceRule', 'UIEvent', 'UITouch', 'UIPrintInteractionController', 'CMDeviceMotion', 'NSCompoundPredicate', 'MKMultiPoint', 'UIPrintFormatter', 'SKView', 'NSConstantString', 'UIPopoverController', 'AVMetadataFaceObject', 'EKEventViewController', 'NSPort', 'MKCircleRenderer', 'AVCompositionTrack', 'UINib', 'NSUbiquitousKeyValueStore', 'NSMetadataQueryResultGroup', 'AVAssetResourceLoadingDataRequest', 'UITableViewHeaderFooterView', 'UISplitViewController', 'AVAudioSession', 'CAEmitterLayer', 'NSNull', 'MKCircleView', 'UIColor', 'UIAttachmentBehavior', 'CLBeacon', 'NSInputStream', 'NSURLCache', 'GKPlayer', 'NSMappingModel', 'NSHTTPCookie', 'AVMutableVideoComposition', 'NSAttributeDescription', 'AVPlayer', 'MKAnnotationView', 'UIFontDescriptor', 'NSTimer', 'CBDescriptor', 'MKOverlayView', 'EKEventEditViewController', 'NSSaveChangesRequest', 'UIReferenceLibraryViewController', 'SKPhysicsJointFixed', 'UILocalizedIndexedCollation', 'UIInterpolatingMotionEffect', 'AVAssetWriter', 'NSBundle', 'SKStoreProductViewController', 'GLKViewController', 'NSMetadataQueryAttributeValueTuple', 'GKTurnBasedMatch', 'UIActivity', 'MKShape', 'NSMergeConflict', 'CIImage', 'UIRotationGestureRecognizer', 'AVPlayerItemLegibleOutput', 'AVAssetImageGenerator', 'GCControllerButtonInput', 'NSSortDescriptor', 'MPTimedMetadata', 'NKIssue', 'UIScreenMode', 'GKTurnBasedEventHandler', 'MKPolyline', 'JSVirtualMachine', 'AVAssetReader', 'NSAttributedString', 'GKMatchmakerViewController', 'NSCountedSet', 'UIButton', 'GKLocalPlayer', 'MPMovieErrorLog', 'AVSpeechUtterance', 'AVURLAsset', 'CBPeripheral', 'AVAssetWriterInputGroup', 'AVAssetReaderAudioMixOutput', 'NSEnumerator', 'UIDocument', 'MKLocalSearchResponse', 'UISimpleTextPrintFormatter', 'CBService', 'MCSession', 'QLPreviewController', 'CAMediaTimingFunction', 'UITextPosition', 'NSNumberFormatter', 'UIPinchGestureRecognizer', 'UIMarkupTextPrintFormatter', 'MKRouteStep', 'NSMetadataQuery', 'AVAssetResourceLoadingContentInformationRequest', 'CTSubscriber', 'CTCarrier', 'NSFileSecurity', 'UIAcceleration', 'UIMotionEffect', 'CLHeading', 'NSFileWrapper', 'MKDirectionsResponse', 'UILocalNotification', 'UICollectionViewCell', 'UITextView', 'CMMagnetometerData', 'UIProgressView', 'GKInvite', 'UISearchBar', 'MKPlacemark', 'AVCaptureConnection', 'ALAssetsFilter', 'AVPlayerItemErrorLogEvent', 'NSJSONSerialization', 'AVAssetReaderVideoCompositionOutput', 'ABPersonViewController', 'CIDetector', 'GKTurnBasedMatchmakerViewController', 'MPMediaItemCollection', 'NSCondition', 'NSURLCredential', 'MIDINetworkConnection', 'NSDecimalNumberHandler', 'NSURLSessionConfiguration', 'EKCalendar', 'NSDictionary', 'CAPropertyAnimation', 'UIPercentDrivenInteractiveTransition', 'MKPolygon', 'AVAssetTrackSegment', 'NSExpressionDescription', 'UIViewController', 'NSURLAuthenticationChallenge', 'NSDirectoryEnumerator', 'MKDistanceFormatter', 'GCControllerElement', 'GKPeerPickerController', 'UITableViewController', 'GKNotificationBanner', 'MKPointAnnotation', 'NSCache', 'SKPhysicsJoint', 'NSXMLParser', 'MFMessageComposeViewController', 'AVCaptureSession', 'NSDataDetector', 'AVCaptureVideoPreviewLayer', 'NSURLComponents', 'UISnapBehavior', 'AVMetadataMachineReadableCodeObject', 'GLKTextureLoader', 'NSTextAttachment', 'NSException', 'UIMenuItem', 'CMMotionActivityManager', 'MKUserLocation', 'CIFeature', 'NSMachPort', 'ALAsset', 'NSURLSessionDownloadTask', 'MPMoviePlayerViewController', 'NSMutableOrderedSet', 'AVCaptureVideoDataOutput', 'NSCachedURLResponse', 'ALAssetsLibrary', 'NSInvocation', 'UILongPressGestureRecognizer', 'NSTextStorage', 'CIFaceFeature', 'MKMapSnapshot', 'GLKEffectPropertyFog', 'NSPersistentStoreRequest', 'AVAudioMixInputParameters', 'CAEmitterBehavior', 'PKPassLibrary', 'NSLock', 'UIDynamicBehavior', 'AVPlayerMediaSelectionCriteria', 'CALayer', 'UIBarButtonItem', 'AVAudioSessionRouteDescription', 'CLBeaconRegion', 'SKEffectNode', 'CABasicAnimation', 'AVVideoCompositionInstruction', 'AVMutableTimedMetadataGroup', 'EKRecurrenceEnd', 'NSTextContainer', 'TWTweetComposeViewController', 'UIScrollView', 'EKRecurrenceDayOfWeek', 'ASIdentifierManager', 'UIScreen', 'CLRegion', 'NSProcessInfo', 'GLKTextureInfo', 'AVCaptureMetadataOutput', 'NSTextTab', 'JSManagedValue', 'NSDate', 'UITextChecker', 'NSData', 'NSParagraphStyle', 'AVMutableMetadataItem', 'EKAlarm', 'NSMutableURLRequest', 'UIVideoEditorController', 'NSAtomicStore', 'UIResponder', 'AVCompositionTrackSegment', 'GCGamepadSnapshot', 'MPMediaEntity', 'GLKSkyboxEffect', 'UISwitch', 'EKStructuredLocation', 'UIGestureRecognizer', 'NSProxy', 'GLKBaseEffect', 'GKScoreChallenge', 'NSCoder', 'MPMediaPlaylist', 'NSDateComponents', 'EKEvent', 'NSDateFormatter', 'AVAssetWriterInputPixelBufferAdaptor', 'UICollectionViewFlowLayoutInvalidationContext', 'UITextField', 'CLPlacemark', 'AVCaptureOutput', 'NSPropertyDescription', 'GCGamepad', 'NSPersistentStoreCoordinator', 'GKMatchmaker', 'CIContext', 'NSThread', 'SKRequest', 'SKPhysicsJointSliding', 'NSPredicate', 'GKVoiceChat', 'SKCropNode', 'AVCaptureAudioPreviewOutput', 'NSStringDrawingContext', 'GKGameCenterViewController', 'UIPrintPaper', 'UICollectionViewLayoutInvalidationContext', 'GLKEffectPropertyTransform', 'UIDatePicker', 'MKDirections', 'ALAssetsGroup', 'CAEmitterCell', 'UIFont', 'MKPinAnnotationView', 'UIPickerView', 'UIImageView', 'SKNode', 'MPMediaQuerySection', 'GKFriendRequestComposeViewController', 'NSError', 'CTSubscriberInfo', 'AVPlayerItemAccessLog', 'MPMediaPropertyPredicate', 'CMLogItem', 'NSAutoreleasePool', 'NSSocketPort', 'AVAssetReaderTrackOutput', 'AVSpeechSynthesisVoice', 'UIImage', 'AVCaptureAudioChannel', 'GKTurnBasedExchangeReply', 'AVVideoCompositionLayerInstruction', 'AVSpeechSynthesizer', 'GKChallengeEventHandler', 'AVCaptureFileOutput', 'UIControl', 'SKPayment', 'ADInterstitialAd', 'AVAudioSessionDataSourceDescription', 'NSArray', 'GCControllerDirectionPad', 'NSFileManager', 'AVMutableAudioMixInputParameters', 'UIScreenEdgePanGestureRecognizer', 'CAKeyframeAnimation', 'EASession', 'UIInputView', 'NSHTTPCookieStorage', 'NSPointerFunctions', 'AVMediaSelectionOption', 'NSRunLoop', 'CAAnimationGroup', 'MKCircle', 'NSMigrationManager', 'UICollectionViewUpdateItem', 'NSMutableData', 'NSMutableParagraphStyle', 'GLKEffectProperty', 'SKShapeNode', 'MPMovieErrorLogEvent', 'MKPolygonView', 'UIAccelerometer', 'NSScanner', 'GKAchievementChallenge', 'AVAudioPlayer', 'AVVideoComposition', 'NKLibrary', 'NSPersistentStore', 'NSPropertyMapping', 'GKChallenge', 'NSURLProtectionSpace', 'ACAccountStore', 'UITextRange', 'NSComparisonPredicate', 'NSOutputStream', 'PKAddPassesViewController', 'CTTelephonyNetworkInfo', 'AVTextStyleRule', 'NSFetchedPropertyDescription', 'UIPageViewController', 'CATransformLayer', 'MCNearbyServiceAdvertiser', 'NSObject', 'MPMusicPlayerController', 'MKOverlayPathRenderer', 'GKAchievement', 'AVCaptureAudioFileOutput', 'TWRequest', 'SKLabelNode', 'MIDINetworkHost', 'MPMediaPredicate', 'AVFrameRateRange', 'NSIndexPath', 'AVVideoCompositionRenderContext', 'CADisplayLink', 'CAEAGLLayer', 'NSMutableString', 'NSMessagePort', 'AVAudioSessionChannelDescription', 'GLKView', 'UIActivityViewController', 'GKAchievementViewController', 'NSURLProtocol', 'NSCalendar', 'SKKeyframeSequence', 'AVMetadataItemFilter', 'NSMethodSignature', 'NSRegularExpression', 'EAGLSharegroup', 'AVPlayerItemVideoOutput', 'CIColor', 'UIDictationPhrase']) COCOA_PROTOCOLS = set(['SKStoreProductViewControllerDelegate', 'AVVideoCompositionInstruction', 'AVAudioSessionDelegate', 'GKMatchDelegate', 'NSFileManagerDelegate', 'UILayoutSupport', 'NSCopying', 'UIPrintInteractionControllerDelegate', 'QLPreviewControllerDataSource', 'SKProductsRequestDelegate', 'NSTextStorageDelegate', 'MCBrowserViewControllerDelegate', 'UIViewControllerTransitionCoordinatorContext', 'NSTextAttachmentContainer', 'NSDecimalNumberBehaviors', 'NSMutableCopying', 'UIViewControllerTransitioningDelegate', 'UIAlertViewDelegate', 'AVAudioPlayerDelegate', 'MKReverseGeocoderDelegate', 'NSCoding', 'UITextInputTokenizer', 'GKFriendRequestComposeViewControllerDelegate', 'UIActivityItemSource', 'NSCacheDelegate', 'UITableViewDelegate', 'GKAchievementViewControllerDelegate', 'EKEventEditViewDelegate', 'NSURLConnectionDelegate', 'GKPeerPickerControllerDelegate', 'UIGuidedAccessRestrictionDelegate', 'AVSpeechSynthesizerDelegate', 'MFMailComposeViewControllerDelegate', 'AVPlayerItemLegibleOutputPushDelegate', 'ADInterstitialAdDelegate', 'AVAssetResourceLoaderDelegate', 'UITabBarControllerDelegate', 'SKPaymentTransactionObserver', 'AVCaptureAudioDataOutputSampleBufferDelegate', 'UIInputViewAudioFeedback', 'GKChallengeListener', 'UIPickerViewDelegate', 'UIWebViewDelegate', 'UIApplicationDelegate', 'GKInviteEventListener', 'MPMediaPlayback', 'MyClassJavaScriptMethods', 'AVAsynchronousKeyValueLoading', 'QLPreviewItem', 'NSPortDelegate', 'SKRequestDelegate', 'SKPhysicsContactDelegate', 'UIPageViewControllerDataSource', 'AVPlayerItemOutputPushDelegate', 'UICollectionViewDelegate', 'UIImagePickerControllerDelegate', 'UIToolbarDelegate', 'UIViewControllerTransitionCoordinator', 'NSURLConnectionDataDelegate', 'MKOverlay', 'CBCentralManagerDelegate', 'JSExport', 'NSTextLayoutOrientationProvider', 'UIPickerViewDataSource', 'UITextInputTraits', 'NSLayoutManagerDelegate', 'NSFetchedResultsControllerDelegate', 'ABPeoplePickerNavigationControllerDelegate', 'NSDiscardableContent', 'UITextFieldDelegate', 'GKGameCenterControllerDelegate', 'MPMediaPickerControllerDelegate', 'UIAppearance', 'UIPickerViewAccessibilityDelegate', 'UIScrollViewAccessibilityDelegate', 'ADBannerViewDelegate', 'NSURLSessionDelegate', 'NSXMLParserDelegate', 'UIViewControllerRestoration', 'UISearchBarDelegate', 'UIBarPositioning', 'CBPeripheralDelegate', 'UISearchDisplayDelegate', 'CAAction', 'PKAddPassesViewControllerDelegate', 'MCNearbyServiceAdvertiserDelegate', 'GKTurnBasedMatchmakerViewControllerDelegate', 'UIActionSheetDelegate', 'AVCaptureVideoDataOutputSampleBufferDelegate', 'UIAppearanceContainer', 'UIStateRestoring', 'NSURLSessionTaskDelegate', 'NSFilePresenter', 'UIViewControllerContextTransitioning', 'UITextInput', 'CBPeripheralManagerDelegate', 'UITextInputDelegate', 'NSFastEnumeration', 'NSURLAuthenticationChallengeSender', 'AVVideoCompositing', 'NSSecureCoding', 'MCAdvertiserAssistantDelegate', 'GKLocalPlayerListener', 'GLKNamedEffect', 'UIPopoverControllerDelegate', 'AVCaptureMetadataOutputObjectsDelegate', 'MFMessageComposeViewControllerDelegate', 'UITextSelecting', 'NSURLProtocolClient', 'UIVideoEditorControllerDelegate', 'UITableViewDataSource', 'UIDynamicAnimatorDelegate', 'NSURLSessionDataDelegate', 'UICollisionBehaviorDelegate', 'NSStreamDelegate', 'MCNearbyServiceBrowserDelegate', 'UINavigationControllerDelegate', 'MCSessionDelegate', 'UIViewControllerInteractiveTransitioning', 'GKTurnBasedEventListener', 'GLKViewDelegate', 'EAAccessoryDelegate', 'NSKeyedUnarchiverDelegate', 'NSMachPortDelegate', 'UIBarPositioningDelegate', 'ABPersonViewControllerDelegate', 'NSNetServiceBrowserDelegate', 'EKEventViewDelegate', 'UIScrollViewDelegate', 'NSURLConnectionDownloadDelegate', 'UIGestureRecognizerDelegate', 'UINavigationBarDelegate', 'GKVoiceChatClient', 'NSFetchedResultsSectionInfo', 'UIDocumentInteractionControllerDelegate', 'QLPreviewControllerDelegate', 'UIAccessibilityReadingContent', 'ABUnknownPersonViewControllerDelegate', 'GLKViewControllerDelegate', 'UICollectionViewDelegateFlowLayout', 'UISplitViewControllerDelegate', 'MKAnnotation', 'UIAccessibilityIdentification', 'ABNewPersonViewControllerDelegate', 'CAMediaTiming', 'AVCaptureFileOutputRecordingDelegate', 'UITextViewDelegate', 'UITabBarDelegate', 'GKLeaderboardViewControllerDelegate', 'MKMapViewDelegate', 'UIKeyInput', 'UICollectionViewDataSource', 'NSLocking', 'AVCaptureFileOutputDelegate', 'GKChallengeEventHandlerDelegate', 'UIObjectRestoration', 'CIFilterConstructor', 'AVPlayerItemOutputPullDelegate', 'EAGLDrawable', 'AVVideoCompositionValidationHandling', 'UIViewControllerAnimatedTransitioning', 'NSURLSessionDownloadDelegate', 'UIAccelerometerDelegate', 'UIPageViewControllerDelegate', 'UIDataSourceModelAssociation', 'AVAudioRecorderDelegate', 'GKSessionDelegate', 'NSKeyedArchiverDelegate', 'UIDynamicItem', 'CLLocationManagerDelegate', 'NSMetadataQueryDelegate', 'NSNetServiceDelegate', 'GKMatchmakerViewControllerDelegate', 'EKCalendarChooserDelegate']) COCOA_PRIMITIVES = set(['ROTAHeader', '__CFBundle', 'MortSubtable', 'AudioFilePacketTableInfo', 'CGPDFOperatorTable', 'KerxStateEntry', 'ExtendedTempoEvent', 'CTParagraphStyleSetting', 'OpaqueMIDIPort', 'CFStreamErrorHTTP', '__CFMachPort', '_GLKMatrix4', 'ExtendedControlEvent', 'CAFAudioDescription', 'KernVersion0Header', 'CGTextDrawingMode', 'EKErrorCode', 'gss_buffer_desc_struct', 'AudioUnitParameterInfo', '__SCPreferences', '__CTFrame', '__CTLine', 'CFStreamSocketSecurityProtocol', 'gss_krb5_lucid_context_v1', 'OpaqueJSValue', 'TrakTableEntry', 'AudioFramePacketTranslation', 'CGImageSource', 'OpaqueJSPropertyNameAccumulator', 'JustPCGlyphRepeatAddAction', 'BslnFormat0Part', 'OpaqueMIDIThruConnection', 'opaqueCMBufferQueue', 'OpaqueMusicSequence', 'MortRearrangementSubtable', 'MixerDistanceParams', 'MorxSubtable', 'MIDIObjectPropertyChangeNotification', '__CFDictionary', 'CGImageMetadataErrors', 'CGPath', 'OpaqueMIDIEndpoint', 'ALMXHeader', 'AudioComponentPlugInInterface', 'gss_ctx_id_t_desc_struct', 'sfntFontFeatureSetting', 'OpaqueJSContextGroup', '__SCNetworkConnection', 'AudioUnitParameterValueTranslation', 'CGImageMetadataType', 'CGPattern', 'AudioFileTypeAndFormatID', 'CGContext', 'AUNodeInteraction', 'SFNTLookupTable', 'JustPCDecompositionAction', 'KerxControlPointHeader', 'PKErrorCode', 'AudioStreamPacketDescription', 'KernSubtableHeader', '__CFNull', 'AUMIDIOutputCallbackStruct', 'MIDIMetaEvent', 'AudioQueueChannelAssignment', '__CFString', 'AnchorPoint', 'JustTable', '__CFNetService', 'gss_krb5_lucid_key', 'CGPDFDictionary', 'MIDIThruConnectionParams', 'CAF_UUID_ChunkHeader', 'gss_krb5_cfx_keydata', '_GLKMatrix3', 'CGGradient', 'OpaqueMIDISetup', '_GLKMatrix2', 'JustPostcompTable', '__CTParagraphStyle', 'AudioUnitParameterHistoryInfo', 'OpaqueJSContext', 'CGShading', '__CFBinaryHeap', 'SFNTLookupSingle', '__CFHost', '__SecRandom', '__CTFontDescriptor', '_NSRange', 'sfntDirectory', 'AudioQueueLevelMeterState', 'CAFPositionPeak', '__CFBoolean', 'PropLookupSegment', '__CVOpenGLESTextureCache', 'sfntInstance', '_GLKQuaternion', 'KernStateEntry', '__SCNetworkProtocol', 'CAFFileHeader', 'KerxOrderedListHeader', 'CGBlendMode', 'STXEntryOne', 'CAFRegion', 'SFNTLookupTrimmedArrayHeader', 'KerxControlPointEntry', '__CFCharacterSet', 'OpaqueMusicTrack', '_GLKVector4', 'gss_OID_set_desc_struct', 'OpaqueMusicPlayer', '_CFHTTPAuthentication', 'CGAffineTransform', 'CAFMarkerChunk', 'AUHostIdentifier', 'ROTAGlyphEntry', 'BslnTable', 'gss_krb5_lucid_context_version', '_GLKMatrixStack', 'CGImage', 'AnkrTable', 'SFNTLookupSingleHeader', 'MortLigatureSubtable', 'AudioFile_SMPTE_Time', 'CAFUMIDChunk', 'SMPTETime', 'CAFDataChunk', 'CGPDFStream', 'AudioFileRegionList', 'STEntryTwo', 'SFNTLookupBinarySearchHeader', 'OpbdTable', '__CTGlyphInfo', 'BslnFormat2Part', 'KerxIndexArrayHeader', 'TrakTable', 'KerxKerningPair', '__CFBitVector', 'KernVersion0SubtableHeader', 'OpaqueAudioComponentInstance', 'AudioChannelLayout', '__CFUUID', 'MIDISysexSendRequest', '__CFNumberFormatter', 'CGImageSourceStatus', '__CFURL', 'AudioFileMarkerList', 'AUSamplerBankPresetData', 'CGDataProvider', 'AudioFormatInfo', '__SecIdentity', 'sfntCMapExtendedSubHeader', 'MIDIChannelMessage', 'KernOffsetTable', 'CGColorSpaceModel', 'MFMailComposeErrorCode', 'CGFunction', '__SecTrust', 'CFHostInfoType', 'KernSimpleArrayHeader', 'CGFontPostScriptFormat', 'KernStateHeader', 'AudioUnitCocoaViewInfo', 'CGDataConsumer', 'OpaqueMIDIDevice', 'OpaqueCMBlockBuffer', 'AnchorPointTable', 'CGImageDestination', 'CAFInstrumentChunk', 'AudioUnitMeterClipping', '__CFNumber', 'MorxChain', '__CTFontCollection', 'STEntryOne', 'STXEntryTwo', 'ExtendedNoteOnEvent', '__CFArray', 'CGColorRenderingIntent', 'KerxSimpleArrayHeader', 'MorxTable', '_GLKVector3', '_GLKVector2', 'MortTable', 'CGPDFBox', 'AudioUnitParameterValueFromString', '__CFSocket', 'ALCdevice_struct', 'MIDINoteMessage', 'sfntFeatureHeader', 'CGRect', '__SCNetworkInterface', '__CFTree', 'MusicEventUserData', 'TrakTableData', 'MortContextualSubtable', '__CTRun', 'AudioUnitFrequencyResponseBin', 'MortChain', 'MorxInsertionSubtable', 'CGImageMetadata', 'gss_auth_identity', 'AudioUnitMIDIControlMapping', 'CAFChunkHeader', 'PropTable', 'CGPDFScanner', 'OpaqueMusicEventIterator', '__CFFileSecurity', 'AudioUnitNodeConnection', 'OpaqueMIDIDeviceList', 'ExtendedAudioFormatInfo', 'CGRectEdge', 'sfntFontDescriptor', '__CFRunLoopObserver', 'CGPatternTiling', 'MIDINotification', 'MorxLigatureSubtable', 'SFNTLookupSegment', 'MessageComposeResult', 'MIDIThruConnectionEndpoint', 'MusicDeviceStdNoteParams', 'opaqueCMSimpleQueue', 'ALCcontext_struct', 'OpaqueAudioQueue', 'PropLookupSingle', 'CGColor', 'AudioOutputUnitStartAtTimeParams', 'gss_name_t_desc_struct', 'CGFunctionCallbacks', 'CAFPacketTableHeader', 'AudioChannelDescription', 'sfntFeatureName', 'MorxContextualSubtable', 'CVSMPTETime', 'AudioValueRange', 'CGTextEncoding', 'AudioStreamBasicDescription', 'AUNodeRenderCallback', 'AudioPanningInfo', '__CFData', '__CFDate', 'KerxOrderedListEntry', '__CFAllocator', 'OpaqueJSPropertyNameArray', '__SCDynamicStore', 'OpaqueMIDIEntity', 'CFHostClientContext', 'CFNetServiceClientContext', 'AudioUnitPresetMAS_SettingData', 'opaqueCMBufferQueueTriggerToken', 'AudioUnitProperty', 'CAFRegionChunk', 'CGPDFString', '__CFWriteStream', '__CFAttributedString', '__CFStringTokenizer', 'JustWidthDeltaEntry', '__CFSet', 'sfntVariationAxis', '__CFNetDiagnostic', 'CAFOverviewSample', 'sfntCMapEncoding', 'CGVector', '__SCNetworkService', 'opaqueCMSampleBuffer', 'AUHostVersionIdentifier', 'AudioBalanceFade', 'sfntFontRunFeature', 'KerxCoordinateAction', 'sfntCMapSubHeader', 'CVPlanarPixelBufferInfo', 'AUNumVersion', '__CFTimeZone', 'AUSamplerInstrumentData', 'AUPreset', '__CTRunDelegate', 'OpaqueAudioQueueProcessingTap', 'KerxTableHeader', '_NSZone', 'OpaqueExtAudioFile', '__CFRunLoopSource', 'KerxAnchorPointAction', 'OpaqueJSString', 'AudioQueueParameterEvent', '__CFHTTPMessage', 'OpaqueCMClock', 'ScheduledAudioFileRegion', 'STEntryZero', 'gss_channel_bindings_struct', 'sfntVariationHeader', 'AUChannelInfo', 'UIOffset', 'GLKEffectPropertyPrv', 'KerxStateHeader', 'CGLineJoin', 'CGPDFDocument', '__CFBag', 'CFStreamErrorHTTPAuthentication', 'KernOrderedListHeader', '__SCNetworkSet', '__SecKey', 'MIDIObjectAddRemoveNotification', 'sfntDescriptorHeader', 'AudioUnitParameter', 'JustPCActionSubrecord', 'AudioComponentDescription', 'AudioUnitParameterValueName', 'AudioUnitParameterEvent', 'KerxControlPointAction', 'AudioTimeStamp', 'KernKerningPair', 'gss_buffer_set_desc_struct', 'MortFeatureEntry', 'FontVariation', 'CAFStringID', 'LcarCaretClassEntry', 'AudioUnitParameterStringFromValue', 'ACErrorCode', 'ALMXGlyphEntry', 'LtagTable', '__CTTypesetter', 'AuthorizationOpaqueRef', 'UIEdgeInsets', 'CGPathElement', 'CAFMarker', 'KernTableHeader', 'NoteParamsControlValue', 'SSLContext', 'gss_cred_id_t_desc_struct', 'AudioUnitParameterNameInfo', '__SecCertificate', 'CGDataConsumerCallbacks', 'CGInterpolationQuality', 'CGLineCap', 'MIDIControlTransform', 'BslnFormat1Part', 'CGPDFArray', '__SecPolicy', 'AudioConverterPrimeInfo', '__CTTextTab', '__CFNetServiceMonitor', 'AUInputSamplesInOutputCallbackStruct', '__CTFramesetter', 'CGPDFDataFormat', 'STHeader', 'CVPlanarPixelBufferInfo_YCbCrPlanar', 'MIDIValueMap', 'JustDirectionTable', '__SCBondStatus', 'SFNTLookupSegmentHeader', 'OpaqueCMMemoryPool', 'CGPathDrawingMode', 'CGFont', '__SCNetworkReachability', 'AudioClassDescription', 'CGPoint', 'CAFStrings', '__CFNetServiceBrowser', 'opaqueMTAudioProcessingTap', 'sfntNameRecord', 'CGPDFPage', 'CGLayer', 'ComponentInstanceRecord', 'CAFInfoStrings', 'HostCallbackInfo', 'MusicDeviceNoteParams', 'KernIndexArrayHeader', 'CVPlanarPixelBufferInfo_YCbCrBiPlanar', 'MusicTrackLoopInfo', 'opaqueCMFormatDescription', 'STClassTable', 'sfntDirectoryEntry', 'OpaqueCMTimebase', 'CGDataProviderDirectCallbacks', 'MIDIPacketList', 'CAFOverviewChunk', 'MIDIPacket', 'ScheduledAudioSlice', 'CGDataProviderSequentialCallbacks', 'AudioBuffer', 'MorxRearrangementSubtable', 'CGPatternCallbacks', 'AUDistanceAttenuationData', 'MIDIIOErrorNotification', 'CGPDFContentStream', 'IUnknownVTbl', 'MIDITransform', 'MortInsertionSubtable', 'CABarBeatTime', 'AudioBufferList', 'KerxSubtableHeader', '__CVBuffer', 'AURenderCallbackStruct', 'STXEntryZero', 'JustPCDuctilityAction', 'OpaqueAudioQueueTimeline', 'OpaqueMIDIClient', '__CFPlugInInstance', 'AudioQueueBuffer', '__CFFileDescriptor', 'AudioUnitConnection', '_GKTurnBasedExchangeStatus', 'LcarCaretTable', 'CVPlanarComponentInfo', 'JustWidthDeltaGroup', 'OpaqueAudioComponent', 'ParameterEvent', '__CVPixelBufferPool', '__CTFont', 'OpaqueJSClass', 'CGColorSpace', 'CGSize', 'AUDependentParameter', 'MIDIDriverInterface', 'gss_krb5_rfc1964_keydata', '__CFDateFormatter', 'LtagStringRange', 'CFNetServiceMonitorType', 'gss_iov_buffer_desc_struct', 'AUPresetEvent', 'CFNetServicesError', 'KernOrderedListEntry', '__CFLocale', 'gss_OID_desc_struct', 'AudioUnitPresetMAS_Settings', 'AudioFileMarker', 'JustPCConditionalAddAction', 'BslnFormat3Part', '__CFNotificationCenter', 'MortSwashSubtable', 'AUParameterMIDIMapping', 'OpaqueAudioConverter', 'MIDIRawData', 'CFNetDiagnosticStatusValues', 'sfntNameHeader', '__CFRunLoop', 'MFMailComposeResult', 'CATransform3D', 'OpbdSideValues', 'CAF_SMPTE_Time', 'JustPCAction', 'CGPathElementType', '__CFRunLoopTimer', '__CFError', 'AudioFormatListItem', '__CFReadStream', 'AudioUnitExternalBuffer', 'AudioFileRegion', 'AudioValueTranslation', 'CGImageMetadataTag', 'CAFPeakChunk', 'AudioBytePacketTranslation', 'CFNetworkErrors', 'sfntCMapHeader', '__CFURLEnumerator', '__CFCalendar', '__CFMessagePort', 'STXHeader', 'CGPDFObjectType', 'SFNTLookupArrayHeader']) @@ -61,11 +63,11 @@ if __name__ == '__main__': all_primitives.add(r) - print "ALL interfaces: \n" - print all_interfaces + print("ALL interfaces: \n") + print(all_interfaces) - print "\nALL protocols: \n" - print all_protocols + print("\nALL protocols: \n") + print(all_protocols) - print "\nALL primitives: \n" - print all_primitives + print("\nALL primitives: \n") + print(all_primitives) diff --git a/pygments/lexers/_luabuiltins.py b/pygments/lexers/_luabuiltins.py index 671dfeaa..40037357 100644 --- a/pygments/lexers/_luabuiltins.py +++ b/pygments/lexers/_luabuiltins.py @@ -13,6 +13,9 @@ :license: BSD, see LICENSE for details. """ +from __future__ import print_function + + MODULES = {'basic': ['_G', '_VERSION', 'assert', @@ -142,7 +145,10 @@ MODULES = {'basic': ['_G', if __name__ == '__main__': import re - import urllib + try: + from urllib import urlopen + except ImportError: + from urllib.request import urlopen import pprint # you can't generally find out what module a function belongs to if you @@ -188,7 +194,7 @@ if __name__ == '__main__': def get_newest_version(): - f = urllib.urlopen('http://www.lua.org/manual/') + f = urlopen('http://www.lua.org/manual/') r = re.compile(r'^<A HREF="(\d\.\d)/">Lua \1</A>') for line in f: m = r.match(line) @@ -196,7 +202,7 @@ if __name__ == '__main__': return m.groups()[0] def get_lua_functions(version): - f = urllib.urlopen('http://www.lua.org/manual/%s/' % version) + f = urlopen('http://www.lua.org/manual/%s/' % version) r = re.compile(r'^<A HREF="manual.html#pdf-(.+)">\1</A>') functions = [] for line in f: @@ -206,7 +212,7 @@ if __name__ == '__main__': return functions def get_function_module(name): - for mod, cb in module_callbacks().iteritems(): + for mod, cb in module_callbacks().items(): if cb(name): return mod if '.' in name: @@ -233,13 +239,13 @@ if __name__ == '__main__': def run(): version = get_newest_version() - print '> Downloading function index for Lua %s' % version + print('> Downloading function index for Lua %s' % version) functions = get_lua_functions(version) - print '> %d functions found:' % len(functions) + print('> %d functions found:' % len(functions)) modules = {} for full_function_name in functions: - print '>> %s' % full_function_name + print('>> %s' % full_function_name) m = get_function_module(full_function_name) modules.setdefault(m, []).append(full_function_name) diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py index e9380914..1a190d1c 100644 --- a/pygments/lexers/_mapping.py +++ b/pygments/lexers/_mapping.py @@ -13,8 +13,11 @@ :license: BSD, see LICENSE for details. """ +from __future__ import print_function + LEXERS = { 'ABAPLexer': ('pygments.lexers.other', 'ABAP', ('abap',), ('*.abap',), ('text/x-abap',)), + 'APLLexer': ('pygments.lexers.other', 'APL', ('apl',), ('*.apl',), ()), 'ActionScript3Lexer': ('pygments.lexers.web', 'ActionScript 3', ('as3', 'actionscript3'), ('*.as',), ('application/x-actionscript', 'text/x-actionscript', 'text/actionscript')), 'ActionScriptLexer': ('pygments.lexers.web', 'ActionScript', ('as', 'actionscript'), ('*.as',), ('application/x-actionscript3', 'text/x-actionscript3', 'text/actionscript3')), 'AdaLexer': ('pygments.lexers.compiled', 'Ada', ('ada', 'ada95ada2005'), ('*.adb', '*.ads', '*.ada'), ('text/x-ada',)), @@ -61,14 +64,17 @@ LEXERS = { 'CheetahJavascriptLexer': ('pygments.lexers.templates', 'JavaScript+Cheetah', ('js+cheetah', 'javascript+cheetah', 'js+spitfire', 'javascript+spitfire'), (), ('application/x-javascript+cheetah', 'text/x-javascript+cheetah', 'text/javascript+cheetah', 'application/x-javascript+spitfire', 'text/x-javascript+spitfire', 'text/javascript+spitfire')), 'CheetahLexer': ('pygments.lexers.templates', 'Cheetah', ('cheetah', 'spitfire'), ('*.tmpl', '*.spt'), ('application/x-cheetah', 'application/x-spitfire')), 'CheetahXmlLexer': ('pygments.lexers.templates', 'XML+Cheetah', ('xml+cheetah', 'xml+spitfire'), (), ('application/xml+cheetah', 'application/xml+spitfire')), + 'CirruLexer': ('pygments.lexers.web', 'Cirru', ('cirru',), ('*.cirru', '*.cr'), ('text/x-cirru',)), 'ClayLexer': ('pygments.lexers.compiled', 'Clay', ('clay',), ('*.clay',), ('text/x-clay',)), 'ClojureLexer': ('pygments.lexers.jvm', 'Clojure', ('clojure', 'clj'), ('*.clj',), ('text/x-clojure', 'application/x-clojure')), + 'ClojureScriptLexer': ('pygments.lexers.jvm', 'ClojureScript', ('clojurescript', 'cljs'), ('*.cljs',), ('text/x-clojurescript', 'application/x-clojurescript')), 'CobolFreeformatLexer': ('pygments.lexers.compiled', 'COBOLFree', ('cobolfree',), ('*.cbl', '*.CBL'), ()), 'CobolLexer': ('pygments.lexers.compiled', 'COBOL', ('cobol',), ('*.cob', '*.COB', '*.cpy', '*.CPY'), ('text/x-cobol',)), 'CoffeeScriptLexer': ('pygments.lexers.web', 'CoffeeScript', ('coffee-script', 'coffeescript', 'coffee'), ('*.coffee',), ('text/coffeescript',)), - 'ColdfusionHtmlLexer': ('pygments.lexers.templates', 'Coldfusion HTML', ('cfm',), ('*.cfm', '*.cfml', '*.cfc'), ('application/x-coldfusion',)), + 'ColdfusionCFCLexer': ('pygments.lexers.templates', 'Coldfusion CFC', ('cfc',), ('*.cfc',), ()), + 'ColdfusionHtmlLexer': ('pygments.lexers.templates', 'Coldfusion HTML', ('cfm',), ('*.cfm', '*.cfml'), ('application/x-coldfusion',)), 'ColdfusionLexer': ('pygments.lexers.templates', 'cfstatement', ('cfs',), (), ()), - 'CommonLispLexer': ('pygments.lexers.functional', 'Common Lisp', ('common-lisp', 'cl', 'lisp'), ('*.cl', '*.lisp', '*.el'), ('text/x-common-lisp',)), + 'CommonLispLexer': ('pygments.lexers.functional', 'Common Lisp', ('common-lisp', 'cl', 'lisp', 'elisp', 'emacs'), ('*.cl', '*.lisp', '*.el'), ('text/x-common-lisp',)), 'CoqLexer': ('pygments.lexers.functional', 'Coq', ('coq',), ('*.v',), ('text/x-coq',)), 'CppLexer': ('pygments.lexers.compiled', 'C++', ('cpp', 'c++'), ('*.cpp', '*.hpp', '*.c++', '*.h++', '*.cc', '*.hh', '*.cxx', '*.hxx', '*.C', '*.H', '*.cp', '*.CPP'), ('text/x-c++hdr', 'text/x-c++src')), 'CppObjdumpLexer': ('pygments.lexers.asm', 'cpp-objdump', ('cpp-objdump', 'c++-objdumb', 'cxx-objdump'), ('*.cpp-objdump', '*.c++-objdump', '*.cxx-objdump'), ('text/x-cpp-objdump',)), @@ -114,6 +120,7 @@ LEXERS = { 'FelixLexer': ('pygments.lexers.compiled', 'Felix', ('felix', 'flx'), ('*.flx', '*.flxh'), ('text/x-felix',)), 'FortranLexer': ('pygments.lexers.compiled', 'Fortran', ('fortran',), ('*.f', '*.f90', '*.F', '*.F90'), ('text/x-fortran',)), 'FoxProLexer': ('pygments.lexers.foxpro', 'FoxPro', ('foxpro', 'vfp', 'clipper', 'xbase'), ('*.PRG', '*.prg'), ()), + 'GAPLexer': ('pygments.lexers.math', 'GAP', ('gap',), ('*.g', '*.gd', '*.gi', '*.gap'), ()), 'GLShaderLexer': ('pygments.lexers.compiled', 'GLSL', ('glsl',), ('*.vert', '*.frag', '*.geo'), ('text/x-glslsrc',)), 'GasLexer': ('pygments.lexers.asm', 'GAS', ('gas', 'asm'), ('*.s', '*.S'), ('text/x-gas',)), 'GenshiLexer': ('pygments.lexers.templates', 'Genshi', ('genshi', 'kid', 'xml+genshi', 'xml+kid'), ('*.kid',), ('application/x-genshi', 'application/x-kid')), @@ -140,7 +147,11 @@ LEXERS = { 'HyLexer': ('pygments.lexers.agile', 'Hy', ('hylang',), ('*.hy',), ('text/x-hy', 'application/x-hy')), 'HybrisLexer': ('pygments.lexers.other', 'Hybris', ('hybris', 'hy'), ('*.hy', '*.hyb'), ('text/x-hybris', 'application/x-hybris')), 'IDLLexer': ('pygments.lexers.math', 'IDL', ('idl',), ('*.pro',), ('text/idl',)), + 'IdrisLexer': ('pygments.lexers.functional', 'Idris', ('idris', 'idr'), ('*.idr',), ('text/x-idris',)), 'IgorLexer': ('pygments.lexers.math', 'Igor', ('igor', 'igorpro'), ('*.ipf',), ('text/ipf',)), + 'Inform6Lexer': ('pygments.lexers.compiled', 'Inform 6', ('inform6', 'i6'), ('*.inf',), ()), + 'Inform6TemplateLexer': ('pygments.lexers.compiled', 'Inform 6 template', ('i6t',), ('*.i6t',), ()), + 'Inform7Lexer': ('pygments.lexers.compiled', 'Inform 7', ('inform7', 'i7'), ('*.ni', '*.i7x'), ()), 'IniLexer': ('pygments.lexers.text', 'INI', ('ini', 'cfg', 'dosini'), ('*.ini', '*.cfg'), ('text/x-ini',)), 'IoLexer': ('pygments.lexers.agile', 'Io', ('io',), ('*.io',), ('text/x-iosrc',)), 'IokeLexer': ('pygments.lexers.jvm', 'Ioke', ('ioke', 'ik'), ('*.ik',), ('text/x-iokesrc',)), @@ -170,6 +181,7 @@ LEXERS = { 'LighttpdConfLexer': ('pygments.lexers.text', 'Lighttpd configuration file', ('lighty', 'lighttpd'), (), ('text/x-lighttpd-conf',)), 'LiterateAgdaLexer': ('pygments.lexers.functional', 'Literate Agda', ('lagda', 'literate-agda'), ('*.lagda',), ('text/x-literate-agda',)), 'LiterateHaskellLexer': ('pygments.lexers.functional', 'Literate Haskell', ('lhs', 'literate-haskell', 'lhaskell'), ('*.lhs',), ('text/x-literate-haskell',)), + 'LiterateIdrisLexer': ('pygments.lexers.functional', 'Literate Idris', ('lidr', 'literate-idris', 'lidris'), ('*.lidr',), ('text/x-literate-idris',)), 'LiveScriptLexer': ('pygments.lexers.web', 'LiveScript', ('live-script', 'livescript'), ('*.ls',), ('text/livescript',)), 'LlvmLexer': ('pygments.lexers.asm', 'LLVM', ('llvm',), ('*.ll',), ('text/x-llvm',)), 'LogosLexer': ('pygments.lexers.compiled', 'Logos', ('logos',), ('*.x', '*.xi', '*.xm', '*.xmi'), ('text/x-logos',)), @@ -183,6 +195,7 @@ LEXERS = { 'MakoLexer': ('pygments.lexers.templates', 'Mako', ('mako',), ('*.mao',), ('application/x-mako',)), 'MakoXmlLexer': ('pygments.lexers.templates', 'XML+Mako', ('xml+mako',), (), ('application/xml+mako',)), 'MaqlLexer': ('pygments.lexers.other', 'MAQL', ('maql',), ('*.maql',), ('text/x-gooddata-maql', 'application/x-gooddata-maql')), + 'MaskLexer': ('pygments.lexers.web', 'Mask', ('mask',), ('*.mask',), ('text/x-mask',)), 'MasonLexer': ('pygments.lexers.templates', 'Mason', ('mason',), ('*.m', '*.mhtml', '*.mc', '*.mi', 'autohandler', 'dhandler'), ('application/x-mason',)), 'MathematicaLexer': ('pygments.lexers.math', 'Mathematica', ('mathematica', 'mma', 'nb'), ('*.nb', '*.cdf', '*.nbp', '*.ma'), ('application/mathematica', 'application/vnd.wolfram.mathematica', 'application/vnd.wolfram.mathematica.package', 'application/vnd.wolfram.cdf')), 'MatlabLexer': ('pygments.lexers.math', 'Matlab', ('matlab',), ('*.m',), ('text/matlab',)), @@ -193,6 +206,7 @@ LEXERS = { 'MoinWikiLexer': ('pygments.lexers.text', 'MoinMoin/Trac Wiki markup', ('trac-wiki', 'moin'), (), ('text/x-trac-wiki',)), 'MonkeyLexer': ('pygments.lexers.compiled', 'Monkey', ('monkey',), ('*.monkey',), ('text/x-monkey',)), 'MoonScriptLexer': ('pygments.lexers.agile', 'MoonScript', ('moon', 'moonscript'), ('*.moon',), ('text/x-moonscript', 'application/x-moonscript')), + 'MqlLexer': ('pygments.lexers.compiled', 'MQL', ('mql', 'mq4', 'mq5', 'mql4', 'mql5'), ('*.mq4', '*.mq5', '*.mqh'), ('text/x-mql',)), 'MscgenLexer': ('pygments.lexers.other', 'Mscgen', ('mscgen', 'msc'), ('*.msc',), ()), 'MuPADLexer': ('pygments.lexers.math', 'MuPAD', ('mupad',), ('*.mu',), ()), 'MxmlLexer': ('pygments.lexers.web', 'MXML', ('mxml',), ('*.mxml',), ()), @@ -204,6 +218,7 @@ LEXERS = { 'MyghtyXmlLexer': ('pygments.lexers.templates', 'XML+Myghty', ('xml+myghty',), (), ('application/xml+myghty',)), 'NSISLexer': ('pygments.lexers.other', 'NSIS', ('nsis', 'nsi', 'nsh'), ('*.nsi', '*.nsh'), ('text/x-nsis',)), 'NasmLexer': ('pygments.lexers.asm', 'NASM', ('nasm',), ('*.asm', '*.ASM'), ('text/x-nasm',)), + 'NasmObjdumpLexer': ('pygments.lexers.asm', 'objdump-nasm', ('objdump-nasm',), ('*.objdump-intel',), ('text/x-nasm-objdump',)), 'NemerleLexer': ('pygments.lexers.dotnet', 'Nemerle', ('nemerle',), ('*.n',), ('text/x-nemerle',)), 'NesCLexer': ('pygments.lexers.compiled', 'nesC', ('nesc',), ('*.nc',), ('text/x-nescsrc',)), 'NewLispLexer': ('pygments.lexers.functional', 'NewLisp', ('newlisp',), ('*.lsp', '*.nl'), ('text/x-newlisp', 'application/x-newlisp')), @@ -224,6 +239,7 @@ LEXERS = { 'Perl6Lexer': ('pygments.lexers.agile', 'Perl6', ('perl6', 'pl6'), ('*.pl', '*.pm', '*.nqp', '*.p6', '*.6pl', '*.p6l', '*.pl6', '*.6pm', '*.p6m', '*.pm6', '*.t'), ('text/x-perl6', 'application/x-perl6')), 'PerlLexer': ('pygments.lexers.agile', 'Perl', ('perl', 'pl'), ('*.pl', '*.pm', '*.t'), ('text/x-perl', 'application/x-perl')), 'PhpLexer': ('pygments.lexers.web', 'PHP', ('php', 'php3', 'php4', 'php5'), ('*.php', '*.php[345]', '*.inc'), ('text/x-php',)), + 'PigLexer': ('pygments.lexers.jvm', 'Pig', ('pig',), ('*.pig',), ('text/x-pig',)), 'PikeLexer': ('pygments.lexers.compiled', 'Pike', ('pike',), ('*.pike', '*.pmod'), ('text/x-pike',)), 'PlPgsqlLexer': ('pygments.lexers.sql', 'PL/pgSQL', ('plpgsql',), (), ('text/x-plpgsql',)), 'PostScriptLexer': ('pygments.lexers.other', 'PostScript', ('postscript', 'postscr'), ('*.ps', '*.eps'), ('application/postscript',)), @@ -264,7 +280,7 @@ LEXERS = { 'RstLexer': ('pygments.lexers.text', 'reStructuredText', ('rst', 'rest', 'restructuredtext'), ('*.rst', '*.rest'), ('text/x-rst', 'text/prs.fallenstein.rst')), 'RubyConsoleLexer': ('pygments.lexers.agile', 'Ruby irb session', ('rbcon', 'irb'), (), ('text/x-ruby-shellsession',)), 'RubyLexer': ('pygments.lexers.agile', 'Ruby', ('rb', 'ruby', 'duby'), ('*.rb', '*.rbw', 'Rakefile', '*.rake', '*.gemspec', '*.rbx', '*.duby'), ('text/x-ruby', 'application/x-ruby')), - 'RustLexer': ('pygments.lexers.compiled', 'Rust', ('rust',), ('*.rs', '*.rc'), ('text/x-rustsrc',)), + 'RustLexer': ('pygments.lexers.compiled', 'Rust', ('rust',), ('*.rs',), ('text/x-rustsrc',)), 'SLexer': ('pygments.lexers.math', 'S', ('splus', 's', 'r'), ('*.S', '*.R', '.Rhistory', '.Rprofile'), ('text/S-plus', 'text/S', 'text/x-r-source', 'text/x-r', 'text/x-R', 'text/x-r-history', 'text/x-r-profile')), 'SMLLexer': ('pygments.lexers.functional', 'Standard ML', ('sml',), ('*.sml', '*.sig', '*.fun'), ('text/x-standardml', 'application/x-standardml')), 'SassLexer': ('pygments.lexers.web', 'Sass', ('sass',), ('*.sass',), ('text/x-sass',)), @@ -315,6 +331,7 @@ LEXERS = { 'XsltLexer': ('pygments.lexers.web', 'XSLT', ('xslt',), ('*.xsl', '*.xslt', '*.xpl'), ('application/xsl+xml', 'application/xslt+xml')), 'XtendLexer': ('pygments.lexers.jvm', 'Xtend', ('xtend',), ('*.xtend',), ('text/x-xtend',)), 'YamlLexer': ('pygments.lexers.text', 'YAML', ('yaml',), ('*.yaml', '*.yml'), ('text/x-yaml',)), + 'ZephirLexer': ('pygments.lexers.web', 'Zephir', ('zephir',), ('*.zep',), ()), } if __name__ == '__main__': @@ -327,7 +344,7 @@ if __name__ == '__main__': for filename in os.listdir('.'): if filename.endswith('.py') and not filename.startswith('_'): module_name = 'pygments.lexers.%s' % filename[:-3] - print module_name + print(module_name) module = __import__(module_name, None, None, ['']) for lexer_name in module.__all__: lexer = getattr(module, lexer_name) diff --git a/pygments/lexers/_phpbuiltins.py b/pygments/lexers/_phpbuiltins.py index 571f564a..2f5ec851 100644 --- a/pygments/lexers/_phpbuiltins.py +++ b/pygments/lexers/_phpbuiltins.py @@ -16,6 +16,7 @@ :license: BSD, see LICENSE for details. """ +from __future__ import print_function MODULES = {'.NET': ['dotnet_load'], 'APC': ['apc_add', @@ -3711,7 +3712,10 @@ if __name__ == '__main__': import re import shutil import tarfile - import urllib + try: + from urllib import urlretrieve + except ImportError: + from urllib.request import urlretrieve PHP_MANUAL_URL = 'http://us3.php.net/distributions/manual/php_manual_en.tar.gz' PHP_MANUAL_DIR = './php-chunked-xhtml/' @@ -3752,7 +3756,7 @@ if __name__ == '__main__': return modules def get_php_references(): - download = urllib.urlretrieve(PHP_MANUAL_URL) + download = urlretrieve(PHP_MANUAL_URL) tar = tarfile.open(download[0]) tar.extractall() tar.close() @@ -3777,10 +3781,10 @@ if __name__ == '__main__': f.close() def run(): - print '>> Downloading Function Index' + print('>> Downloading Function Index') modules = get_php_functions() - total = sum(len(v) for v in modules.itervalues()) - print '%d functions found' % total + total = sum(len(v) for v in modules.values()) + print('%d functions found' % total) regenerate(__file__, modules) shutil.rmtree(PHP_MANUAL_DIR) diff --git a/pygments/lexers/_postgres_builtins.py b/pygments/lexers/_postgres_builtins.py index 32206e9b..11dc6dec 100644 --- a/pygments/lexers/_postgres_builtins.py +++ b/pygments/lexers/_postgres_builtins.py @@ -10,7 +10,10 @@ """ import re -import urllib +try: + from urllib import urlopen +except ImportError: + from urllib.request import urlopen # One man's constant is another man's variable. SOURCE_URL = 'https://github.com/postgres/postgres/raw/master' @@ -18,11 +21,11 @@ KEYWORDS_URL = SOURCE_URL + '/doc/src/sgml/keywords.sgml' DATATYPES_URL = SOURCE_URL + '/doc/src/sgml/datatype.sgml' def update_myself(): - data_file = list(fetch(DATATYPES_URL)) + data_file = list(urlopen(DATATYPES_URL)) datatypes = parse_datatypes(data_file) pseudos = parse_pseudos(data_file) - keywords = parse_keywords(fetch(KEYWORDS_URL)) + keywords = parse_keywords(urlopen(KEYWORDS_URL)) update_consts(__file__, 'DATATYPES', datatypes) update_consts(__file__, 'PSEUDO_TYPES', pseudos) update_consts(__file__, 'KEYWORDS', keywords) @@ -96,9 +99,6 @@ def parse_pseudos(f): return dt -def fetch(url): - return urllib.urlopen(url) - def update_consts(filename, constname, content): f = open(filename) lines = f.readlines() diff --git a/pygments/lexers/_robotframeworklexer.py b/pygments/lexers/_robotframeworklexer.py index e90918da..2889e1b8 100644 --- a/pygments/lexers/_robotframeworklexer.py +++ b/pygments/lexers/_robotframeworklexer.py @@ -27,6 +27,7 @@ import re from pygments.lexer import Lexer from pygments.token import Token +from pygments.util import text_type HEADING = Token.Generic.Heading @@ -57,7 +58,7 @@ class RobotFrameworkLexer(Lexer): Supports both space and pipe separated plain text formats. - *New in Pygments 1.6.* + .. versionadded:: 1.6 """ name = 'RobotFramework' aliases = ['robotframework'] @@ -77,7 +78,7 @@ class RobotFrameworkLexer(Lexer): for value, token in row_tokenizer.tokenize(row): for value, token in var_tokenizer.tokenize(value, token): if value: - yield index, token, unicode(value) + yield index, token, text_type(value) index += len(value) diff --git a/pygments/lexers/_sourcemodbuiltins.py b/pygments/lexers/_sourcemodbuiltins.py index 03967055..eee84d0b 100644 --- a/pygments/lexers/_sourcemodbuiltins.py +++ b/pygments/lexers/_sourcemodbuiltins.py @@ -12,6 +12,8 @@ :license: BSD, see LICENSE for details. """ +from __future__ import print_function + FUNCTIONS = ['TopMenuHandler', 'CreateTopMenu', 'LoadTopMenuConfig', @@ -1012,7 +1014,10 @@ if __name__ == '__main__': import pprint import re import sys - import urllib + try: + from urllib import urlopen + except ImportError: + from urllib.request import urlopen # urllib ends up wanting to import a module called 'math' -- if # pygments/lexers is in the path, this ends badly. @@ -1021,7 +1026,7 @@ if __name__ == '__main__': del sys.path[i] def get_version(): - f = urllib.urlopen('http://docs.sourcemod.net/api/index.php') + f = urlopen('http://docs.sourcemod.net/api/index.php') r = re.compile(r'SourceMod v\.<b>([\d\.]+)</td>') for line in f: m = r.search(line) @@ -1029,7 +1034,7 @@ if __name__ == '__main__': return m.groups()[0] def get_sm_functions(): - f = urllib.urlopen('http://docs.sourcemod.net/api/SMfuncs.js') + f = urlopen('http://docs.sourcemod.net/api/SMfuncs.js') r = re.compile(r'SMfunctions\[\d+\] = Array \("(?:public )?([^,]+)",".+"\);') functions = [] for line in f: @@ -1057,13 +1062,13 @@ if __name__ == '__main__': def run(): version = get_version() - print '> Downloading function index for SourceMod %s' % version + print('> Downloading function index for SourceMod %s' % version) functions = get_sm_functions() - print '> %d functions found:' % len(functions) + print('> %d functions found:' % len(functions)) functionlist = [] for full_function_name in functions: - print '>> %s' % full_function_name + print('>> %s' % full_function_name) functionlist.append(full_function_name) regenerate(__file__, functionlist) diff --git a/pygments/lexers/agile.py b/pygments/lexers/agile.py index 88f0d983..a49289dc 100644 --- a/pygments/lexers/agile.py +++ b/pygments/lexers/agile.py @@ -15,7 +15,7 @@ from pygments.lexer import Lexer, RegexLexer, ExtendedRegexLexer, \ LexerContext, include, combined, do_insertions, bygroups, using, this from pygments.token import Error, Text, Other, \ Comment, Operator, Keyword, Name, String, Number, Generic, Punctuation -from pygments.util import get_bool_opt, get_list_opt, shebang_matches +from pygments.util import get_bool_opt, get_list_opt, shebang_matches, iteritems from pygments import unistring as uni @@ -194,7 +194,7 @@ class Python3Lexer(RegexLexer): """ For `Python <http://www.python.org>`_ source code (version 3.0). - *New in Pygments 0.10.* + .. versionadded:: 0.10 """ name = 'Python 3' @@ -308,7 +308,8 @@ class PythonConsoleLexer(Lexer): `python3` Use Python 3 lexer for code. Default is ``False``. - *New in Pygments 1.0.* + + .. versionadded:: 1.0 """ name = 'Python console session' aliases = ['pycon'] @@ -353,7 +354,7 @@ class PythonConsoleLexer(Lexer): curcode = '' insertions = [] if (line.startswith(u'Traceback (most recent call last):') or - re.match(ur' File "[^"]+", line \d+\n$', line)): + re.match(u' File "[^"]+", line \\d+\\n$', line)): tb = 1 curtb = line tbindex = match.start() @@ -377,7 +378,7 @@ class PythonTracebackLexer(RegexLexer): """ For Python tracebacks. - *New in Pygments 0.7.* + .. versionadded:: 0.7 """ name = 'Python Traceback' @@ -414,7 +415,7 @@ class Python3TracebackLexer(RegexLexer): """ For Python 3.0 tracebacks, with support for chained exceptions. - *New in Pygments 1.0.* + .. versionadded:: 1.0 """ name = 'Python 3.0 Traceback' @@ -1020,7 +1021,7 @@ class PerlLexer(RegexLexer): def analyse_text(text): if shebang_matches(text, r'perl'): return True - if 'my $' in text: + if re.search('(?:my|our)\s+[$@%(]', text): return 0.9 @@ -1126,7 +1127,7 @@ class LuaLexer(RegexLexer): self._functions = set() if self.func_name_highlighting: from pygments.lexers._luabuiltins import MODULES - for mod, func in MODULES.iteritems(): + for mod, func in iteritems(MODULES): if mod not in self.disabled_modules: self._functions.update(func) RegexLexer.__init__(self, **options) @@ -1151,7 +1152,7 @@ class MoonScriptLexer(LuaLexer): """ For `MoonScript <http://moonscript.org.org>`_ source code. - *New in Pygments 1.5.* + .. versionadded:: 1.5 """ name = "MoonScript" @@ -1290,7 +1291,7 @@ class IoLexer(RegexLexer): For `Io <http://iolanguage.com/>`_ (a small, prototype-based programming language) source. - *New in Pygments 0.10.* + .. versionadded:: 0.10 """ name = 'Io' filenames = ['*.io'] @@ -1336,7 +1337,7 @@ class TclLexer(RegexLexer): """ For Tcl source code. - *New in Pygments 0.10.* + .. versionadded:: 0.10 """ keyword_cmds_re = ( @@ -1466,7 +1467,7 @@ class FactorLexer(RegexLexer): """ Lexer for the `Factor <http://factorcode.org>`_ language. - *New in Pygments 1.4.* + .. versionadded:: 1.4 """ name = 'Factor' aliases = ['factor'] @@ -1757,7 +1758,7 @@ class FancyLexer(RegexLexer): class-based, concurrent general-purpose programming language running on Rubinius, the Ruby VM. - *New in Pygments 1.5.* + .. versionadded:: 1.5 """ name = 'Fancy' filenames = ['*.fy', '*.fancypack'] @@ -1839,7 +1840,7 @@ class DgLexer(RegexLexer): a functional and object-oriented programming language running on the CPython 3 VM. - *New in Pygments 1.6.* + .. versionadded:: 1.6 """ name = 'dg' aliases = ['dg'] @@ -1929,7 +1930,7 @@ class Perl6Lexer(ExtendedRegexLexer): """ For `Perl 6 <http://www.perl6.org>`_ source code. - *New in Pygments 1.7.* + .. versionadded:: 2.0 """ name = 'Perl6' @@ -1939,7 +1940,7 @@ class Perl6Lexer(ExtendedRegexLexer): mimetypes = ['text/x-perl6', 'application/x-perl6'] flags = re.MULTILINE | re.DOTALL | re.UNICODE - PERL6_IDENTIFIER_RANGE = "['a-zA-Z0-9_:-]" + PERL6_IDENTIFIER_RANGE = "['a-zA-Z0-9_:-]" # if you alter this, search for a copy made of it below PERL6_KEYWORDS = ( 'BEGIN', 'CATCH', 'CHECK', 'CONTROL', 'END', 'ENTER', 'FIRST', 'INIT', @@ -2124,12 +2125,16 @@ class Perl6Lexer(ExtendedRegexLexer): end_pos = next_close_pos + if end_pos < 0: # if we didn't find a closer, just highlight the + # rest of the text in this class + end_pos = len(text) + if adverbs is not None and re.search(r':to\b', adverbs): heredoc_terminator = text[match.start('delimiter') + n_chars : end_pos] - end_heredoc = re.search(r'^\s*' + re.escape(heredoc_terminator) + r'\s*$', text[ match.end('delimiter') : ], re.MULTILINE) + end_heredoc = re.search(r'^\s*' + re.escape(heredoc_terminator) + r'\s*$', text[ end_pos : ], re.MULTILINE) if end_heredoc: - end_pos = match.end('delimiter') + end_heredoc.end() + end_pos += end_heredoc.end() else: end_pos = len(text) @@ -2177,7 +2182,7 @@ class Perl6Lexer(ExtendedRegexLexer): # process the corresponding one! tokens = { 'common' : [ - (r'#[`|=](?P<delimiter>(?P<first_char>[' + ''.join(PERL6_BRACKETS.keys()) + r'])(?P=first_char)*)', brackets_callback(Comment.Multiline)), + (r'#[`|=](?P<delimiter>(?P<first_char>[' + ''.join(PERL6_BRACKETS) + r'])(?P=first_char)*)', brackets_callback(Comment.Multiline)), (r'#[^\n]*$', Comment.Singleline), (r'^(\s*)=begin\s+(\w+)\b.*?^\1=end\s+\2', Comment.Multiline), (r'^(\s*)=for.*?\n\s*?\n', Comment.Multiline), @@ -2206,7 +2211,7 @@ class Perl6Lexer(ExtendedRegexLexer): (r'(?<=~~)\s*/(?:\\\\|\\/|.)*?/', String.Regex), (r'(?<=[=(,])\s*/(?:\\\\|\\/|.)*?/', String.Regex), (r'm\w+(?=\()', Name), - (r'(?:m|ms|rx)\s*(?P<adverbs>:[\w\s:]+)?\s*(?P<delimiter>(?P<first_char>[^0-9a-zA-Z:\s])(?P=first_char)*)', brackets_callback(String.Regex)), + (r'(?:m|ms|rx)\s*(?P<adverbs>:[\w\s:]+)?\s*(?P<delimiter>(?P<first_char>[^0-9a-zA-Z_:\s])(?P=first_char)*)', brackets_callback(String.Regex)), (r'(?:s|ss|tr)\s*(?::[\w\s:]+)?\s*/(?:\\\\|\\/|.)*?/(?:\\\\|\\/|.)*?/', String.Regex), (r'<[^\s=].*?\S>', String), (_build_word_match(PERL6_OPERATORS), Operator), @@ -2226,7 +2231,7 @@ class Perl6Lexer(ExtendedRegexLexer): (r'.+?', Text), ], 'token-sym-brackets' : [ - (r'(?P<delimiter>(?P<first_char>[' + ''.join(PERL6_BRACKETS.keys()) + '])(?P=first_char)*)', brackets_callback(Name), ('#pop', 'pre-token')), + (r'(?P<delimiter>(?P<first_char>[' + ''.join(PERL6_BRACKETS) + '])(?P=first_char)*)', brackets_callback(Name), ('#pop', 'pre-token')), (r'', Name, ('#pop', 'pre-token')), ], 'token': [ @@ -2244,9 +2249,6 @@ class Perl6Lexer(ExtendedRegexLexer): } def analyse_text(text): - # disabled for now; the lexer is not bug-free and will loop sometimes, - # so let's be sure to use it only for "real" Perl 6 code. - return False def strip_pod(lines): in_pod = False stripped_lines = [] @@ -2261,30 +2263,41 @@ class Perl6Lexer(ExtendedRegexLexer): return stripped_lines + # XXX handle block comments lines = text.splitlines() lines = strip_pod(lines) text = '\n'.join(lines) - if shebang_matches(text, r'perl6|rakudo|niecza'): + if shebang_matches(text, r'perl6|rakudo|niecza|pugs'): return True - if 'use v6' in text: - return 0.91 # 0.01 greater than Perl says for 'my $' - if re.search(r'[$@%]\*[A-Z]+', text): # Perl 6-style globals ($*OS) - return 0.91 - if re.search(r'[$@%]\?[A-Z]+', text): # Perl 6 compiler variables ($?PACKAGE) - return 0.91 - if re.search(r'[$@%][!.][A-Za-z0-9_-]+', text): # Perl 6 member variables - return 0.91 - - for line in text.splitlines(): - if re.match(r'\s*(?:my|our)?\s*module', line): # module declarations - return 0.91 - if re.match(r'\s*(?:my|our)?\s*role', line): # role declarations - return 0.91 - if re.match(r'\s*(?:my|our)?\s*class\b', line): # class declarations - return 0.91 - return False + saw_perl_decl = False + rating = False + + # check for my/our/has declarations + # copied PERL6_IDENTIFIER_RANGE from above; not happy about that + if re.search("(?:my|our|has)\s+(?:['a-zA-Z0-9_:-]+\s+)?[$@%&(]", text): + rating = 0.8 + saw_perl_decl = True + + for line in lines: + line = re.sub('#.*', '', line) + if re.match('^\s*$', line): + continue + + # match v6; use v6; use v6.0; use v6.0.0; + if re.match('^\s*(?:use\s+)?v6(?:\.\d(?:\.\d)?)?;', line): + return True + # match class, module, role, enum, grammar declarations + class_decl = re.match('^\s*(?:(?P<scope>my|our)\s+)?(?:module|class|role|enum|grammar)', line) + if class_decl: + if saw_perl_decl or class_decl.group('scope') is not None: + return True + rating = 0.05 + continue + break + + return rating def __init__(self, **options): super(Perl6Lexer, self).__init__(**options) @@ -2295,7 +2308,7 @@ class HyLexer(RegexLexer): """ Lexer for `Hy <http://hylang.org/>`_ source code. - *New in Pygments 1.7.* + .. versionadded:: 2.0 """ name = 'Hy' aliases = ['hylang'] diff --git a/pygments/lexers/asm.py b/pygments/lexers/asm.py index 655669bf..2727a55d 100644 --- a/pygments/lexers/asm.py +++ b/pygments/lexers/asm.py @@ -17,7 +17,7 @@ from pygments.token import Text, Name, Number, String, Comment, Punctuation, \ Other, Keyword, Operator __all__ = ['GasLexer', 'ObjdumpLexer','DObjdumpLexer', 'CppObjdumpLexer', - 'CObjdumpLexer', 'LlvmLexer', 'NasmLexer', 'Ca65Lexer'] + 'CObjdumpLexer', 'LlvmLexer', 'NasmLexer', 'NasmObjdumpLexer', 'Ca65Lexer'] class GasLexer(RegexLexer): @@ -96,6 +96,55 @@ class GasLexer(RegexLexer): return 0.1 +def _objdump_lexer_tokens(asm_lexer): + """ + Common objdump lexer tokens to wrap an ASM lexer. + """ + hex_re = r'[0-9A-Za-z]' + return { + 'root': [ + # File name & format: + ('(.*?)(:)( +file format )(.*?)$', + bygroups(Name.Label, Punctuation, Text, String)), + # Section header + ('(Disassembly of section )(.*?)(:)$', + bygroups(Text, Name.Label, Punctuation)), + # Function labels + # (With offset) + ('('+hex_re+'+)( )(<)(.*?)([-+])(0[xX][A-Za-z0-9]+)(>:)$', + bygroups(Number.Hex, Text, Punctuation, Name.Function, + Punctuation, Number.Hex, Punctuation)), + # (Without offset) + ('('+hex_re+'+)( )(<)(.*?)(>:)$', + bygroups(Number.Hex, Text, Punctuation, Name.Function, + Punctuation)), + # Code line with disassembled instructions + ('( *)('+hex_re+r'+:)(\t)((?:'+hex_re+hex_re+' )+)( *\t)([a-zA-Z].*?)$', + bygroups(Text, Name.Label, Text, Number.Hex, Text, + using(asm_lexer))), + # Code line with ascii + ('( *)('+hex_re+r'+:)(\t)((?:'+hex_re+hex_re+' )+)( *)(.*?)$', + bygroups(Text, Name.Label, Text, Number.Hex, Text, String)), + # Continued code line, only raw opcodes without disassembled + # instruction + ('( *)('+hex_re+r'+:)(\t)((?:'+hex_re+hex_re+' )+)$', + bygroups(Text, Name.Label, Text, Number.Hex)), + # Skipped a few bytes + (r'\t\.\.\.$', Text), + # Relocation line + # (With offset) + (r'(\t\t\t)('+hex_re+r'+:)( )([^\t]+)(\t)(.*?)([-+])(0x'+hex_re+'+)$', + bygroups(Text, Name.Label, Text, Name.Property, Text, + Name.Constant, Punctuation, Number.Hex)), + # (Without offset) + (r'(\t\t\t)('+hex_re+r'+:)( )([^\t]+)(\t)(.*?)$', + bygroups(Text, Name.Label, Text, Name.Property, Text, + Name.Constant)), + (r'[^\n]+\n', Other) + ] + } + + class ObjdumpLexer(RegexLexer): """ For the output of 'objdump -dr' @@ -105,50 +154,9 @@ class ObjdumpLexer(RegexLexer): filenames = ['*.objdump'] mimetypes = ['text/x-objdump'] - hex = r'[0-9A-Za-z]' - tokens = { - 'root': [ - # File name & format: - ('(.*?)(:)( +file format )(.*?)$', - bygroups(Name.Label, Punctuation, Text, String)), - # Section header - ('(Disassembly of section )(.*?)(:)$', - bygroups(Text, Name.Label, Punctuation)), - # Function labels - # (With offset) - ('('+hex+'+)( )(<)(.*?)([-+])(0[xX][A-Za-z0-9]+)(>:)$', - bygroups(Number.Hex, Text, Punctuation, Name.Function, - Punctuation, Number.Hex, Punctuation)), - # (Without offset) - ('('+hex+'+)( )(<)(.*?)(>:)$', - bygroups(Number.Hex, Text, Punctuation, Name.Function, - Punctuation)), - # Code line with disassembled instructions - ('( *)('+hex+r'+:)(\t)((?:'+hex+hex+' )+)( *\t)([a-zA-Z].*?)$', - bygroups(Text, Name.Label, Text, Number.Hex, Text, - using(GasLexer))), - # Code line with ascii - ('( *)('+hex+r'+:)(\t)((?:'+hex+hex+' )+)( *)(.*?)$', - bygroups(Text, Name.Label, Text, Number.Hex, Text, String)), - # Continued code line, only raw opcodes without disassembled - # instruction - ('( *)('+hex+r'+:)(\t)((?:'+hex+hex+' )+)$', - bygroups(Text, Name.Label, Text, Number.Hex)), - # Skipped a few bytes - (r'\t\.\.\.$', Text), - # Relocation line - # (With offset) - (r'(\t\t\t)('+hex+r'+:)( )([^\t]+)(\t)(.*?)([-+])(0x' + hex + '+)$', - bygroups(Text, Name.Label, Text, Name.Property, Text, - Name.Constant, Punctuation, Number.Hex)), - # (Without offset) - (r'(\t\t\t)('+hex+r'+:)( )([^\t]+)(\t)(.*?)$', - bygroups(Text, Name.Label, Text, Name.Property, Text, - Name.Constant)), - (r'[^\n]+\n', Other) - ] - } + tokens = _objdump_lexer_tokens(GasLexer) + class DObjdumpLexer(DelegatingLexer): @@ -374,11 +382,25 @@ class NasmLexer(RegexLexer): } +class NasmObjdumpLexer(ObjdumpLexer): + """ + For the output of 'objdump -d -M intel'. + + .. versionadded:: 2.0 + """ + name = 'objdump-nasm' + aliases = ['objdump-nasm'] + filenames = ['*.objdump-intel'] + mimetypes = ['text/x-nasm-objdump'] + + tokens = _objdump_lexer_tokens(NasmLexer) + + class Ca65Lexer(RegexLexer): """ For ca65 assembler sources. - *New in Pygments 1.6.* + .. versionadded:: 1.6 """ name = 'ca65' aliases = ['ca65'] diff --git a/pygments/lexers/compiled.py b/pygments/lexers/compiled.py index 809b1db3..e6e10098 100644 --- a/pygments/lexers/compiled.py +++ b/pygments/lexers/compiled.py @@ -31,7 +31,8 @@ __all__ = ['CLexer', 'CppLexer', 'DLexer', 'DelphiLexer', 'ECLexer', 'FantomLexer', 'RustLexer', 'CudaLexer', 'MonkeyLexer', 'SwigLexer', 'DylanLidLexer', 'DylanConsoleLexer', 'CobolLexer', 'CobolFreeformatLexer', 'LogosLexer', 'ClayLexer', 'PikeLexer', - 'ChapelLexer', 'EiffelLexer'] + 'ChapelLexer', 'EiffelLexer', 'Inform6Lexer', 'Inform7Lexer', + 'Inform6TemplateLexer', 'MqlLexer'] class CFamilyLexer(RegexLexer): @@ -240,7 +241,7 @@ class PikeLexer(CppLexer): """ For `Pike <http://pike.lysator.liu.se/>`_ source code. - *New in Pygments 1.7.* + .. versionadded:: 2.0 """ name = 'Pike' aliases = ['pike'] @@ -279,7 +280,7 @@ class SwigLexer(CppLexer): """ For `SWIG <http://www.swig.org/>`_ source code. - *New in Pygments 1.7.* + .. versionadded:: 2.0 """ name = 'SWIG' aliases = ['swig'] @@ -336,7 +337,7 @@ class ECLexer(CLexer): """ For eC source code with preprocessor directives. - *New in Pygments 1.5.* + .. versionadded:: 1.5 """ name = 'eC' aliases = ['ec'] @@ -372,7 +373,7 @@ class NesCLexer(CLexer): For `nesC <https://github.com/tinyos/nesc>`_ source code with preprocessor directives. - *New in Pygments 1.7.* + .. versionadded:: 2.0 """ name = 'nesC' aliases = ['nesc'] @@ -397,7 +398,7 @@ class ClayLexer(RegexLexer): """ For `Clay <http://claylabs.com/clay/>`_ source. - *New in Pygments 1.7.* + .. versionadded:: 2.0 """ name = 'Clay' filenames = ['*.clay'] @@ -448,7 +449,7 @@ class DLexer(RegexLexer): """ For D source. - *New in Pygments 1.2.* + .. versionadded:: 1.2 """ name = 'D' filenames = ['*.d', '*.di'] @@ -468,20 +469,23 @@ class DLexer(RegexLexer): (r'(abstract|alias|align|asm|assert|auto|body|break|case|cast' r'|catch|class|const|continue|debug|default|delegate|delete' r'|deprecated|do|else|enum|export|extern|finally|final' - r'|foreach_reverse|foreach|for|function|goto|if|import|inout' - r'|interface|invariant|in|is|lazy|mixin|module|new|nothrow|out' + r'|foreach_reverse|foreach|for|function|goto|if|immutable|import' + r'|interface|invariant|inout|in|is|lazy|mixin|module|new|nothrow|out' r'|override|package|pragma|private|protected|public|pure|ref|return' - r'|scope|static|struct|super|switch|synchronized|template|this' + r'|scope|shared|static|struct|super|switch|synchronized|template|this' r'|throw|try|typedef|typeid|typeof|union|unittest|version|volatile' - r'|while|with|__traits)\b', Keyword + r'|while|with|__gshared|__traits|__vector|__parameters)\b', Keyword ), (r'(bool|byte|cdouble|cent|cfloat|char|creal|dchar|double|float' r'|idouble|ifloat|int|ireal|long|real|short|ubyte|ucent|uint|ulong' r'|ushort|void|wchar)\b', Keyword.Type ), (r'(false|true|null)\b', Keyword.Constant), + (r'(__FILE__|__MODULE__|__LINE__|__FUNCTION__|__PRETTY_FUNCTION__' + r'|__DATE__|__EOF__|__TIME__|__TIMESTAMP__|__VENDOR__|__VERSION__)\b', + Keyword.Pseudo), (r'macro\b', Keyword.Reserved), - (r'(string|wstring|dstring)\b', Name.Builtin), + (r'(string|wstring|dstring|size_t|ptrdiff_t)\b', Name.Builtin), # FloatLiteral # -- HexFloat (r'0[xX]([0-9a-fA-F_]*\.[0-9a-fA-F_]+|[0-9a-fA-F_]+)' @@ -527,6 +531,8 @@ class DLexer(RegexLexer): (r'q"(.).*?\1"', String), # -- TokenString (r'q{', String, 'token_string'), + # Attributes + (r'@([a-zA-Z_]\w*)?', Name.Decorator), # Tokens (r'(~=|\^=|%=|\*=|==|!>=|!<=|!<>=|!<>|!<|!>|!=|>>>=|>>>|>>=|>>|>=' r'|<>=|<>|<<=|<<|<=|\+\+|\+=|--|-=|\|\||\|=|&&|&=|\.\.\.|\.\.|/=)' @@ -534,6 +540,8 @@ class DLexer(RegexLexer): ), # Identifier (r'[a-zA-Z_]\w*', Name), + # Line + (r'#line\s.*\n', Comment.Special), ], 'nested_comment': [ (r'[^+/]+', Comment.Multiline), @@ -878,7 +886,7 @@ class DelphiLexer(Lexer): if get_bool_opt(options, 'freepascal', True): self.keywords.update(self.FREE_PASCAL_KEYWORDS) self.builtins = set() - for unit in get_list_opt(options, 'units', self.BUILTIN_UNITS.keys()): + for unit in get_list_opt(options, 'units', list(self.BUILTIN_UNITS)): self.builtins.update(self.BUILTIN_UNITS[unit]) def get_tokens_unprocessed(self, text): @@ -1082,7 +1090,7 @@ class DylanLexer(RegexLexer): """ For the `Dylan <http://www.opendylan.org/>`_ language. - *New in Pygments 0.7.* + .. versionadded:: 0.7 """ name = 'Dylan' @@ -1274,7 +1282,7 @@ class DylanLidLexer(RegexLexer): """ For Dylan LID (Library Interchange Definition) files. - *New in Pygments 1.6.* + .. versionadded:: 1.6 """ name = 'DylanLID' @@ -1312,7 +1320,7 @@ class DylanConsoleLexer(Lexer): This is based on a copy of the RubyConsoleLexer. - *New in Pygments 1.6.* + .. versionadded:: 1.6 """ name = 'Dylan session' aliases = ['dylan-console', 'dylan-repl'] @@ -1422,7 +1430,7 @@ def objective(baselexer): (r'^([-+])(\s*)' # method marker r'(\(.*?\))?(\s*)' # return type r'([a-zA-Z$_][a-zA-Z0-9$_]*:?)', # begin of method name - bygroups(Keyword, Text, using(this), + bygroups(Punctuation, Text, using(this), Text, Name.Function), 'method'), inherit, @@ -1433,8 +1441,8 @@ def objective(baselexer): # discussion in Issue 789 (r',', Punctuation), (r'\.\.\.', Punctuation), - (r'(\(.*?\))([a-zA-Z$_][a-zA-Z0-9$_]*)', bygroups(using(this), - Name.Variable)), + (r'(\(.*?\))(\s*)([a-zA-Z$_][a-zA-Z0-9$_]*)', + bygroups(using(this), Text, Name.Variable)), (r'[a-zA-Z$_][a-zA-Z0-9$_]*:', Name.Function), (';', Punctuation, '#pop'), ('{', Punctuation, 'function'), @@ -1447,6 +1455,8 @@ def objective(baselexer): return 1.0 elif '@"' in text: # strings return 0.8 + elif re.search('@[0-9]+', text): + return 0.7 elif _oc_message.search(text): return 0.8 return 0 @@ -1495,7 +1505,7 @@ class FortranLexer(RegexLexer): """ Lexer for FORTRAN 90 code. - *New in Pygments 0.10.* + .. versionadded:: 0.10 """ name = 'Fortran' aliases = ['fortran'] @@ -1610,7 +1620,7 @@ class GLShaderLexer(RegexLexer): """ GLSL (OpenGL Shader) lexer. - *New in Pygments 1.1.* + .. versionadded:: 1.1 """ name = 'GLSL' aliases = ['glsl'] @@ -1722,7 +1732,7 @@ class CythonLexer(RegexLexer): """ For Pyrex and `Cython <http://cython.org>`_ source code. - *New in Pygments 1.1.* + .. versionadded:: 1.1 """ name = 'Cython' @@ -1888,7 +1898,7 @@ class ValaLexer(RegexLexer): """ For Vala source code with preprocessor directives. - *New in Pygments 1.1.* + .. versionadded:: 1.1 """ name = 'Vala' aliases = ['vala', 'vapi'] @@ -1977,7 +1987,7 @@ class OocLexer(RegexLexer): """ For `Ooc <http://ooc-lang.org/>`_ source code - *New in Pygments 1.2.* + .. versionadded:: 1.2 """ name = 'Ooc' aliases = ['ooc'] @@ -2116,7 +2126,7 @@ class FelixLexer(RegexLexer): """ For `Felix <http://www.felix-lang.org>`_ source code. - *New in Pygments 1.2.* + .. versionadded:: 1.2 """ name = 'Felix' @@ -2370,7 +2380,7 @@ class AdaLexer(RegexLexer): """ For Ada source code. - *New in Pygments 1.3.* + .. versionadded:: 1.3 """ name = 'Ada' @@ -2513,7 +2523,7 @@ class Modula2Lexer(RegexLexer): `gm2ext` Also highlight GNU extensions (default: False). - *New in Pygments 1.3.* + .. versionadded:: 1.3 """ name = 'Modula-2' aliases = ['modula2', 'm2'] @@ -2695,7 +2705,7 @@ class BlitzMaxLexer(RegexLexer): """ For `BlitzMax <http://blitzbasic.com>`_ source code. - *New in Pygments 1.4.* + .. versionadded:: 1.4 """ name = 'BlitzMax' @@ -2789,7 +2799,7 @@ class BlitzBasicLexer(RegexLexer): """ For `BlitzBasic <http://blitzbasic.com>`_ source code. - *New in Pygments 1.7.* + .. versionadded:: 2.0 """ name = 'BlitzBasic' @@ -2871,7 +2881,7 @@ class NimrodLexer(RegexLexer): """ For `Nimrod <http://nimrod-code.org/>`_ source code. - *New in Pygments 1.5.* + .. versionadded:: 1.5 """ name = 'Nimrod' @@ -3012,7 +3022,7 @@ class FantomLexer(RegexLexer): """ For Fantom source code. - *New in Pygments 1.5.* + .. versionadded:: 1.5 """ name = 'Fantom' aliases = ['fan'] @@ -3242,12 +3252,12 @@ class FantomLexer(RegexLexer): class RustLexer(RegexLexer): """ - Lexer for Mozilla's Rust programming language. + Lexer for the Rust programming language (version 0.9). - *New in Pygments 1.6.* + .. versionadded:: 1.6 """ name = 'Rust' - filenames = ['*.rs', '*.rc'] + filenames = ['*.rs'] aliases = ['rust'] mimetypes = ['text/x-rustsrc'] @@ -3256,18 +3266,55 @@ class RustLexer(RegexLexer): # Whitespace and Comments (r'\n', Text), (r'\s+', Text), + (r'//[/!](.*?)\n', Comment.Doc), (r'//(.*?)\n', Comment.Single), (r'/[*](.|\n)*?[*]/', Comment.Multiline), # Keywords - (r'(as|assert|break|const' - r'|copy|do|else|enum|extern|fail' - r'|false|fn|for|if|impl|let|log' - r'|loop|match|mod|move|mut|once|priv|pub|pure' - r'|ref|return|static|struct|trait|true|type|unsafe|use|while' - r'|u8|u16|u32|u64|i8|i16|i32|i64|uint' - r'|int|float|f32|f64|str)\b', Keyword), - + (r'(as|box|break|continue' + r'|do|else|enum|extern' + r'|fn|for|if|impl|in' + r'|loop|match|mut|priv|proc|pub' + r'|ref|return|static|\'static|struct|trait|true|type' + r'|unsafe|while)\b', + Keyword), + (r'(alignof|be|const|offsetof|pure|sizeof|typeof|once|unsized' + r'|yield)\b', Keyword.Reserved), + (r'(mod|use)\b', Keyword.Namespace), + (r'(true|false)\b', Keyword.Constant), + (r'let\b', Keyword.Declaration), + (r'(u8|u16|u32|u64|i8|i16|i32|i64|uint|int|f32|f64' + r'|str|bool)\b', Keyword.Type), + (r'self\b', Name.Builtin.Pseudo), + # Prelude + (r'(Freeze|Pod|Send|Sized|Add|Sub|Mul|Div|Rem|Neg|Not|BitAnd' + r'|BitOr|BitXor|Drop|Shl|Shr|Index|Option|Some|None|Result' + r'|Ok|Err|from_str|range|print|println|Any|AnyOwnExt|AnyRefExt' + r'|AnyMutRefExt|Ascii|AsciiCast|OnwedAsciiCast|AsciiStr' + r'|IntoBytes|Bool|ToCStr|Char|Clone|DeepClone|Eq|ApproxEq' + r'|Ord|TotalEq|Ordering|Less|Equal|Greater|Equiv|Container' + r'|Mutable|Map|MutableMap|Set|MutableSet|Default|FromStr' + r'|Hash|FromIterator|Extendable|Iterator|DoubleEndedIterator' + r'|RandomAccessIterator|CloneableIterator|OrdIterator' + r'|MutableDoubleEndedIterator|ExactSize|Times|Algebraic' + r'|Trigonometric|Exponential|Hyperbolic|Bitwise|BitCount' + r'|Bounded|Integer|Fractional|Real|RealExt|Num|NumCast' + r'|CheckedAdd|CheckedSub|CheckedMul|Orderable|Signed' + r'|Unsigned|Round|Primitive|Int|Float|ToStrRadix' + r'|ToPrimitive|FromPrimitive|GenericPath|Path|PosixPath' + r'|WindowsPath|RawPtr|Buffer|Writer|Reader|Seek' + r'|SendStr|SendStrOwned|SendStrStatic|IntoSendStr|Str' + r'|StrVector|StrSlice|OwnedStr|IterBytes|ToStr|IntoStr' + r'|CopyableTuple|ImmutableTuple|ImmutableTuple\d+' + r'|Tuple\d+|ImmutableEqVector|ImmutableTotalOrdVector' + r'|ImmutableCopyableVector|OwnedVector|OwnedCopyableVector' + r'|OwnedEqVector|MutableVector|MutableTotalOrdVector' + r'|Vector|VectorVector|CopyableVector|ImmutableVector' + r'|Port|Chan|SharedChan|spawn|drop)\b', Name.Builtin), + # Borrowed pointer + (r'(&)(\'[A-Za-z_]\w*)?', bygroups(Operator, Name)), + # Labels + (r'\'[A-Za-z_]\w*:', Name.Label), # Character Literal (r"""'(\\['"\\nrt]|\\x[0-9a-fA-F]{2}|\\[0-7]{1,3}""" r"""|\\u[0-9a-fA-F]{4}|\\U[0-9a-fA-F]{8}|.)'""", @@ -3275,9 +3322,9 @@ class RustLexer(RegexLexer): # Lifetime (r"""'[a-zA-Z_][a-zA-Z0-9_]*""", Name.Label), # Binary Literal - (r'0[Bb][01_]+', Number, 'number_lit'), + (r'0b[01_]+', Number, 'number_lit'), # Octal Literal - (r'0[0-7_]+', Number.Oct, 'number_lit'), + (r'0o[0-7_]+', Number.Oct, 'number_lit'), # Hexadecimal Literal (r'0[xX][0-9a-fA-F_]+', Number.Hex, 'number_lit'), # Decimal Literal @@ -3285,20 +3332,22 @@ class RustLexer(RegexLexer): r'[0-9_]+|\.[0-9_]*|[eE][+\-]?[0-9_]+)?', Number, 'number_lit'), # String Literal (r'"', String, 'string'), + (r'r(#*)".*?"\1', String.Raw), # Operators and Punctuation (r'[{}()\[\],.;]', Punctuation), (r'[+\-*/%&|<>^!~@=:?]', Operator), # Identifier - (r'[a-zA-Z_$][a-zA-Z0-9_]*', Name), + (r'[a-zA-Z_]\w*', Name), # Attributes (r'#\[', Comment.Preproc, 'attribute['), - (r'#\(', Comment.Preproc, 'attribute('), # Macros - (r'[A-Za-z_][A-Za-z0-9_]*!\[', Comment.Preproc, 'attribute['), - (r'[A-Za-z_][A-Za-z0-9_]*!\(', Comment.Preproc, 'attribute('), + (r'([A-Za-z_]\w*)!\s*([A-Za-z_]\w*)?\s*\{', + bygroups(Comment.Preproc, Name), 'macro{'), + (r'([A-Za-z_]\w*)!\s*([A-Za-z_]\w*)?\(', + bygroups(Comment.Preproc, Name), 'macro('), ], 'number_lit': [ (r'(([ui](8|16|32|64)?)|(f(32|64)?))?', Keyword, '#pop'), @@ -3310,6 +3359,14 @@ class RustLexer(RegexLexer): (r'[^\\"]+', String), (r'\\', String), ], + 'macro{': [ + (r'\{', Operator, '#push'), + (r'\}', Operator, '#pop'), + ], + 'macro(': [ + (r'\(', Operator, '#push'), + (r'\)', Operator, '#pop'), + ], 'attribute_common': [ (r'"', String, 'string'), (r'\[', Comment.Preproc, 'attribute['), @@ -3333,7 +3390,7 @@ class CudaLexer(CLexer): For NVIDIA `CUDA™ <http://developer.nvidia.com/category/zone/cuda-zone>`_ source. - *New in Pygments 1.6.* + .. versionadded:: 1.6 """ name = 'CUDA' filenames = ['*.cu', '*.cuh'] @@ -3383,7 +3440,7 @@ class MonkeyLexer(RegexLexer): `Monkey <https://en.wikipedia.org/wiki/Monkey_(programming_language)>`_ source code. - *New in Pygments 1.6.* + .. versionadded:: 1.6 """ name = 'Monkey' @@ -3511,7 +3568,7 @@ class CobolLexer(RegexLexer): """ Lexer for OpenCOBOL code. - *New in Pygments 1.6.* + .. versionadded:: 1.6 """ name = 'COBOL' aliases = ['cobol'] @@ -3705,7 +3762,7 @@ class CobolFreeformatLexer(CobolLexer): """ Lexer for Free format OpenCOBOL code. - *New in Pygments 1.6.* + .. versionadded:: 1.6 """ name = 'COBOLFree' aliases = ['cobolfree'] @@ -3724,7 +3781,7 @@ class LogosLexer(ObjectiveCppLexer): """ For Logos + Objective-C source code with preprocessor directives. - *New in Pygments 1.6.* + .. versionadded:: 1.6 """ name = 'Logos' @@ -3788,7 +3845,7 @@ class ChapelLexer(RegexLexer): """ For `Chapel <http://chapel.cray.com/>`_ source. - *New in Pygments 1.7.* + .. versionadded:: 2.0 """ name = 'Chapel' filenames = ['*.chpl'] @@ -3865,7 +3922,7 @@ class EiffelLexer(RegexLexer): """ For `Eiffel <http://www.eiffel.com>`_ source code. - *New in Pygments 1.7.* + .. versionadded:: 2.0 """ name = 'Eiffel' aliases = ['eiffel'] @@ -3904,3 +3961,1138 @@ class EiffelLexer(RegexLexer): (r'[0-9]+', Number.Integer), ], } + + +class Inform6Lexer(RegexLexer): + """ + For `Inform 6 <http://inform-fiction.org/>`_ source code. + + .. versionadded:: 2.0 + """ + + name = 'Inform 6' + aliases = ['inform6', 'i6'] + filenames = ['*.inf'] + + flags = re.MULTILINE | re.DOTALL | re.UNICODE + + _name = r'[a-zA-Z_][a-zA-Z_0-9]*' + + # Inform 7 maps these four character classes to their ASCII + # equivalents. To support Inform 6 inclusions within Inform 7, + # Inform6Lexer maps them too. + _dash = u'\\-\u2010-\u2014' + _dquote = u'"\u201c\u201d' + _squote = u"'\u2018\u2019" + _newline = u'\\n\u0085\u2028\u2029' + + tokens = { + 'root': [ + (r'(\A(!%%[^%s]*[%s])+)?' % (_newline, _newline), Comment.Preproc, + 'directive') + ], + '_whitespace': [ + (r'\s+', Text), + (r'![^%s]*' % _newline, Comment.Single) + ], + 'default': [ + include('_whitespace'), + (r'\[', Punctuation, 'many-values'), # Array initialization + (r':|(?=;)', Punctuation, '#pop'), + (r'<', Punctuation), # Second angle bracket in an action statement + (r'', Text, ('expression', '_expression')) + ], + + # Expressions + '_expression': [ + include('_whitespace'), + (r'(?=sp\b)', Text, '#pop'), + (r'(?=[%s%s$0-9#a-zA-Z_])' % (_dquote, _squote), Text, + ('#pop', 'value')), + (r'\+\+|[%s]{1,2}(?!>)|~~?' % _dash, Operator), + (r'(?=[()\[%s,?@{:;])' % _dash, Text, '#pop') + ], + 'expression': [ + include('_whitespace'), + (r'\(', Punctuation, ('expression', '_expression')), + (r'\)', Punctuation, '#pop'), + (r'\[', Punctuation, ('#pop', 'statements', 'locals')), + (r'>(?=(\s+|(![^%s]*))*[>;])' % _newline, Punctuation), + (r'\+\+|[%s]{2}(?!>)' % _dash, Operator), + (r',', Punctuation, '_expression'), + (r'&&?|\|\|?|[=~><]?=|[%s]{1,2}>?|\.\.?[&#]?|::|[<>+*/%%]' % _dash, + Operator, '_expression'), + (r'(has|hasnt|in|notin|ofclass|or|provides)\b', Operator.Word, + '_expression'), + (r'sp\b', Name), + (r'\?~?', Name.Label, 'label?'), + (r'[@{]', Error), + (r'', Text, '#pop') + ], + '_assembly-expression': [ + (r'\(', Punctuation, ('#push', '_expression')), + (r'[\[\]]', Punctuation), + (r'[%s]>' % _dash, Punctuation, '_expression'), + (r'sp\b', Keyword.Pseudo), + (r';', Punctuation, '#pop:3'), + include('expression') + ], + '_for-expression': [ + (r'\)', Punctuation, '#pop:2'), + (r':', Punctuation, '#pop'), + include('expression') + ], + '_keyword-expression': [ + (r'(from|near|to)\b', Keyword, '_expression'), + include('expression') + ], + '_list-expression': [ + (r',', Punctuation, '#pop'), + include('expression') + ], + '_object-expression': [ + (r'has\b', Keyword.Declaration, '#pop'), + include('_list-expression') + ], + + # Values + 'value': [ + include('_whitespace'), + # Strings + (r'[%s][^@][%s]' % (_squote, _squote), String.Char, '#pop'), + (r'([%s])(@{[0-9a-fA-F]{1,4}})([%s])' % (_squote, _squote), + bygroups(String.Char, String.Escape, String.Char), '#pop'), + (r'([%s])(@..)([%s])' % (_squote, _squote), + bygroups(String.Char, String.Escape, String.Char), '#pop'), + (r'[%s]' % _squote, String.Single, ('#pop', 'dictionary-word')), + (r'[%s]' % _dquote, String.Double, ('#pop', 'string')), + # Numbers + (r'\$[+%s][0-9]*\.?[0-9]*([eE][+%s]?[0-9]+)?' % (_dash, _dash), + Number.Float, '#pop'), + (r'\$[0-9a-fA-F]+', Number.Hex, '#pop'), + (r'\$\$[01]+', Number, '#pop'), # Binary + (r'[0-9]+', Number.Integer, '#pop'), + # Values prefixed by hashes + (r'(##|#a\$)(%s)' % _name, bygroups(Operator, Name), '#pop'), + (r'(#g\$)(%s)' % _name, + bygroups(Operator, Name.Variable.Global), '#pop'), + (r'#[nw]\$', Operator, ('#pop', 'obsolete-dictionary-word')), + (r'(#r\$)(%s)' % _name, bygroups(Operator, Name.Function), '#pop'), + (r'#', Name.Builtin, ('#pop', 'system-constant')), + # System functions + (r'(child|children|elder|eldest|glk|indirect|metaclass|parent|' + r'random|sibling|younger|youngest)\b', Name.Builtin, '#pop'), + # Metaclasses + (r'(?i)(Class|Object|Routine|String)\b', Name.Builtin, '#pop'), + # Veneer routines + (r'(?i)(Box__Routine|CA__Pr|CDefArt|CInDefArt|Cl__Ms|' + r'Copy__Primitive|CP__Tab|DA__Pr|DB__Pr|DefArt|Dynam__String|' + r'EnglishNumber|Glk__Wrap|IA__Pr|IB__Pr|InDefArt|Main__|' + r'Meta__class|OB__Move|OB__Remove|OC__Cl|OP__Pr|Print__Addr|' + r'Print__PName|PrintShortName|RA__Pr|RA__Sc|RL__Pr|R_Process|' + r'RT__ChG|RT__ChGt|RT__ChLDB|RT__ChLDW|RT__ChPR|RT__ChPrintA|' + r'RT__ChPrintC|RT__ChPrintO|RT__ChPrintS|RT__ChPS|RT__ChR|' + r'RT__ChSTB|RT__ChSTW|RT__ChT|RT__Err|RT__TrPS|RV__Pr|' + r'Symb__Tab|Unsigned__Compare|WV__Pr|Z__Region)\b', Name.Builtin, + '#pop'), + # Other built-in symbols + (r'(?i)(call|copy|create|DEBUG|destroy|DICT_CHAR_SIZE|' + r'DICT_ENTRY_BYTES|DICT_IS_UNICODE|DICT_WORD_SIZE|false|' + r'FLOAT_INFINITY|FLOAT_NAN|FLOAT_NINFINITY|GOBJFIELD_CHAIN|' + r'GOBJFIELD_CHILD|GOBJFIELD_NAME|GOBJFIELD_PARENT|' + r'GOBJFIELD_PROPTAB|GOBJFIELD_SIBLING|GOBJ_EXT_START|' + r'GOBJ_TOTAL_LENGTH|Grammar__Version|INDIV_PROP_START|INFIX|' + r'infix__watching|MODULE_MODE|name|nothing|NUM_ATTR_BYTES|print|' + r'print_to_array|recreate|remaining|self|sender|STRICT_MODE|' + r'sw__var|sys__glob0|sys__glob1|sys__glob2|sys_statusline_flag|' + r'TARGET_GLULX|TARGET_ZCODE|temp__global2|temp__global3|' + r'temp__global4|temp_global|true|USE_MODULES|WORDSIZE)\b', + Name.Builtin, '#pop'), + # Other values + (_name, Name, '#pop') + ], + # Strings + 'dictionary-word': [ + (r'[~^]+', String.Escape), + (r'[^~^\\@({%s]+' % _squote, String.Single), + (r'[({]', String.Single), + (r'@{[0-9a-fA-F]{,4}}', String.Escape), + (r'@..', String.Escape), + (r'[%s]' % _squote, String.Single, '#pop') + ], + 'string': [ + (r'[~^]+', String.Escape), + (r'[^~^\\@({%s]+' % _dquote, String.Double), + (r'[({]', String.Double), + (r'\\', String.Escape), + (r'@(\\\s*[%s]\s*)*@((\\\s*[%s]\s*)*[0-9])*' % + (_newline, _newline), String.Escape), + (r'@(\\\s*[%s]\s*)*{((\\\s*[%s]\s*)*[0-9a-fA-F]){,4}' + r'(\\\s*[%s]\s*)*}' % (_newline, _newline, _newline), + String.Escape), + (r'@(\\\s*[%s]\s*)*.(\\\s*[%s]\s*)*.' % (_newline, _newline), + String.Escape), + (r'[%s]' % _dquote, String.Double, '#pop') + ], + 'plain-string': [ + (r'[^~^\\({\[\]%s]+' % _dquote, String.Double), + (r'[~^({\[\]]', String.Double), + (r'\\', String.Escape), + (r'[%s]' % _dquote, String.Double, '#pop') + ], + # Names + '_constant': [ + include('_whitespace'), + (_name, Name.Constant, '#pop'), + include('value') + ], + '_global': [ + include('_whitespace'), + (_name, Name.Variable.Global, '#pop'), + include('value') + ], + 'label?': [ + include('_whitespace'), + (r'(%s)?' % _name, Name.Label, '#pop') + ], + 'variable?': [ + include('_whitespace'), + (r'(%s)?' % _name, Name.Variable, '#pop') + ], + # Values after hashes + 'obsolete-dictionary-word': [ + (r'\S[a-zA-Z_0-9]*', String.Other, '#pop') + ], + 'system-constant': [ + include('_whitespace'), + (_name, Name.Builtin, '#pop') + ], + + # Directives + 'directive': [ + include('_whitespace'), + (r'#', Punctuation), + (r';', Punctuation, '#pop'), + (r'\[', Punctuation, + ('default', 'statements', 'locals', 'routine-name?')), + (r'(?i)(abbreviate|endif|dictionary|ifdef|iffalse|ifndef|ifnot|' + r'iftrue|ifv3|ifv5|release|serial|switches|system_file|version)' + r'\b', Keyword, 'default'), + (r'(?i)(array|global)\b', Keyword, + ('default', 'directive-keyword?', '_global')), + (r'(?i)attribute\b', Keyword, ('default', 'alias?', '_constant')), + (r'(?i)class\b', Keyword, + ('object-body', 'duplicates', 'class-name')), + (r'(?i)(constant|default)\b', Keyword, + ('default', 'expression', '_constant')), + (r'(?i)(end\b)(.*)', bygroups(Keyword, Text)), + (r'(?i)(extend|verb)\b', Keyword, 'grammar'), + (r'(?i)fake_action\b', Keyword, ('default', '_constant')), + (r'(?i)import\b', Keyword, 'manifest'), + (r'(?i)(include|link)\b', Keyword, + ('default', 'before-plain-string')), + (r'(?i)(lowstring|undef)\b', Keyword, ('default', '_constant')), + (r'(?i)message\b', Keyword, ('default', 'diagnostic')), + (r'(?i)(nearby|object)\b', Keyword, + ('object-body', '_object-head')), + (r'(?i)property\b', Keyword, + ('default', 'alias?', '_constant', 'property-keyword*')), + (r'(?i)replace\b', Keyword, + ('default', 'routine-name?', 'routine-name?')), + (r'(?i)statusline\b', Keyword, ('default', 'directive-keyword?')), + (r'(?i)stub\b', Keyword, ('default', 'routine-name?')), + (r'(?i)trace\b', Keyword, + ('default', 'trace-keyword?', 'trace-keyword?')), + (r'(?i)zcharacter\b', Keyword, + ('default', 'directive-keyword?', 'directive-keyword?')), + (_name, Name.Class, ('object-body', '_object-head')) + ], + # [, Replace, Stub + 'routine-name?': [ + include('_whitespace'), + (r'(%s)?' % _name, Name.Function, '#pop') + ], + 'locals': [ + include('_whitespace'), + (r';', Punctuation, '#pop'), + (r'\*', Punctuation), + (_name, Name.Variable) + ], + # Array + 'many-values': [ + include('_whitespace'), + (r';', Punctuation), + (r'\]', Punctuation, '#pop'), + (r':', Error), + (r'', Text, ('expression', '_expression')) + ], + # Attribute, Property + 'alias?': [ + include('_whitespace'), + (r'alias\b', Keyword, ('#pop', '_constant')), + (r'', Text, '#pop') + ], + # Class, Object, Nearby + 'class-name': [ + include('_whitespace'), + (r'(?=[,;]|(class|has|private|with)\b)', Text, '#pop'), + (_name, Name.Class, '#pop') + ], + 'duplicates': [ + include('_whitespace'), + (r'\(', Punctuation, ('#pop', 'expression', '_expression')), + (r'', Text, '#pop') + ], + '_object-head': [ + (r'[%s]>' % _dash, Punctuation), + (r'(class|has|private|with)\b', Keyword.Declaration, '#pop'), + include('_global') + ], + 'object-body': [ + include('_whitespace'), + (r';', Punctuation, '#pop:2'), + (r',', Punctuation), + (r'class\b', Keyword.Declaration, 'class-segment'), + (r'(has|private|with)\b', Keyword.Declaration), + (r':', Error), + (r'', Text, ('_object-expression', '_expression')) + ], + 'class-segment': [ + include('_whitespace'), + (r'(?=[,;]|(class|has|private|with)\b)', Text, '#pop'), + (_name, Name.Class), + (r'', Text, 'value') + ], + # Extend, Verb + 'grammar': [ + include('_whitespace'), + (r'=', Punctuation, ('#pop', 'default')), + (r'\*', Punctuation, ('#pop', 'grammar-line')), + (r'', Text, '_directive-keyword') + ], + 'grammar-line': [ + include('_whitespace'), + (r';', Punctuation, '#pop'), + (r'[/*]', Punctuation), + (r'[%s]>' % _dash, Punctuation, 'value'), + (r'(noun|scope)\b', Keyword, '=routine'), + (r'', Text, '_directive-keyword') + ], + '=routine': [ + include('_whitespace'), + (r'=', Punctuation, 'routine-name?'), + (r'', Text, '#pop') + ], + # Import + 'manifest': [ + include('_whitespace'), + (r';', Punctuation, '#pop'), + (r',', Punctuation), + (r'(?i)(global\b)?', Keyword, '_global') + ], + # Include, Link, Message + 'diagnostic': [ + include('_whitespace'), + (r'[%s]' % _dquote, String.Double, ('#pop', 'message-string')), + (r'', Text, ('#pop', 'before-plain-string', 'directive-keyword?')) + ], + 'before-plain-string': [ + include('_whitespace'), + (r'[%s]' % _dquote, String.Double, ('#pop', 'plain-string')) + ], + 'message-string': [ + (r'[~^]+', String.Escape), + include('plain-string') + ], + + # Keywords used in directives + '_directive-keyword!': [ + include('_whitespace'), + (r'(additive|alias|buffer|class|creature|data|error|fatalerror|' + r'first|has|held|initial|initstr|last|long|meta|multi|' + r'multiexcept|multiheld|multiinside|noun|number|only|private|' + r'replace|reverse|scope|score|special|string|table|terminating|' + r'time|topic|warning|with)\b', Keyword, '#pop'), + (r'[%s]{1,2}>|[+=]' % _dash, Punctuation, '#pop') + ], + '_directive-keyword': [ + include('_directive-keyword!'), + include('value') + ], + 'directive-keyword?': [ + include('_directive-keyword!'), + (r'', Text, '#pop') + ], + 'property-keyword*': [ + include('_whitespace'), + (r'(additive|long)\b', Keyword), + (r'', Text, '#pop') + ], + 'trace-keyword?': [ + include('_whitespace'), + (r'(assembly|dictionary|expressions|lines|linker|objects|off|on|' + r'symbols|tokens|verbs)\b', Keyword, '#pop'), + (r'', Text, '#pop') + ], + + # Statements + 'statements': [ + include('_whitespace'), + (r'\]', Punctuation, '#pop'), + (r'[;{}]', Punctuation), + (r'(box|break|continue|default|give|inversion|new_line|quit|read|' + r'remove|return|rfalse|rtrue|spaces|string|until)\b', Keyword, + 'default'), + (r'(do|else)\b', Keyword), + (r'(font|style)\b', Keyword, + ('default', 'miscellaneous-keyword?')), + (r'for\b', Keyword, ('for', '(?')), + (r'(if|switch|while)', Keyword, + ('expression', '_expression', '(?')), + (r'(jump|save|restore)\b', Keyword, ('default', 'label?')), + (r'objectloop\b', Keyword, + ('_keyword-expression', 'variable?', '(?')), + (r'print(_ret)?\b|(?=[%s])' % _dquote, Keyword, 'print-list'), + (r'\.', Name.Label, 'label?'), + (r'@', Keyword, 'opcode'), + (r'#(?![agrnw]\$|#)', Punctuation, 'directive'), + (r'<', Punctuation, 'default'), + (r'(move\b)?', Keyword, + ('default', '_keyword-expression', '_expression')) + ], + 'miscellaneous-keyword?': [ + include('_whitespace'), + (r'(bold|fixed|from|near|off|on|reverse|roman|to|underline)\b', + Keyword, '#pop'), + (r'(a|A|an|address|char|name|number|object|property|string|the|' + r'The)\b(?=(\s+|(![^%s]*))*\))' % _newline, Keyword.Pseudo, + '#pop'), + (r'%s(?=(\s+|(![^%s]*))*\))' % (_name, _newline), Name.Function, + '#pop'), + (r'', Text, '#pop') + ], + '(?': [ + include('_whitespace'), + (r'\(?', Punctuation, '#pop') + ], + 'for': [ + include('_whitespace'), + (r';?', Punctuation, ('_for-expression', '_expression')) + ], + 'print-list': [ + include('_whitespace'), + (r';', Punctuation, '#pop'), + (r':', Error), + (r'', Text, + ('_list-expression', '_expression', '_list-expression', 'form')) + ], + 'form': [ + include('_whitespace'), + (r'\(', Punctuation, ('#pop', 'miscellaneous-keyword?')), + (r'', Text, '#pop') + ], + + # Assembly + 'opcode': [ + include('_whitespace'), + (r'[%s]' % _dquote, String.Double, ('operands', 'plain-string')), + (_name, Keyword, 'operands') + ], + 'operands': [ + (r':', Error), + (r'', Text, ('_assembly-expression', '_expression')) + ] + } + + def get_tokens_unprocessed(self, text): + # 'in' is either a keyword or an operator. + # If the token two tokens after 'in' is ')', 'in' is a keyword: + # objectloop(a in b) + # Otherwise, it is an operator: + # objectloop(a in b && true) + objectloop_queue = [] + objectloop_token_count = -1 + previous_token = None + for index, token, value in RegexLexer.get_tokens_unprocessed(self, + text): + if previous_token is Name.Variable and value == 'in': + objectloop_queue = [[index, token, value]] + objectloop_token_count = 2 + elif objectloop_token_count > 0: + if token not in Comment and token not in Text: + objectloop_token_count -= 1 + objectloop_queue.append((index, token, value)) + else: + if objectloop_token_count == 0: + if objectloop_queue[-1][2] == ')': + objectloop_queue[0][1] = Keyword + while objectloop_queue: + yield objectloop_queue.pop(0) + objectloop_token_count = -1 + yield index, token, value + if token not in Comment and token not in Text: + previous_token = token + while objectloop_queue: + yield objectloop_queue.pop(0) + + +class Inform7Lexer(RegexLexer): + """ + For `Inform 7 <http://inform7.com/>`_ source code. + + .. versionadded:: 2.0 + """ + + name = 'Inform 7' + aliases = ['inform7', 'i7'] + filenames = ['*.ni', '*.i7x'] + + flags = re.MULTILINE | re.DOTALL | re.UNICODE + + _dash = Inform6Lexer._dash + _dquote = Inform6Lexer._dquote + _newline = Inform6Lexer._newline + _start = r'\A|(?<=[%s])' % _newline + + # There are three variants of Inform 7, differing in how to + # interpret at signs and braces in I6T. In top-level inclusions, at + # signs in the first column are inweb syntax. In phrase definitions + # and use options, tokens in braces are treated as I7. Use options + # also interpret "{N}". + tokens = {} + token_variants = ['+i6t-not-inline', '+i6t-inline', '+i6t-use-option'] + + for level in token_variants: + tokens[level] = { + '+i6-root': list(Inform6Lexer.tokens['root']), + '+i6t-root': [ # For Inform6TemplateLexer + (r'[^%s]*' % Inform6Lexer._newline, Comment.Preproc, + ('directive', '+p')) + ], + 'root': [ + (r'(\|?\s)+', Text), + (r'\[', Comment.Multiline, '+comment'), + (r'[%s]' % _dquote, Generic.Heading, + ('+main', '+titling', '+titling-string')), + (r'', Text, ('+main', '+heading?')) + ], + '+titling-string': [ + (r'[^%s]+' % _dquote, Generic.Heading), + (r'[%s]' % _dquote, Generic.Heading, '#pop') + ], + '+titling': [ + (r'\[', Comment.Multiline, '+comment'), + (r'[^%s.;:|%s]+' % (_dquote, _newline), Generic.Heading), + (r'[%s]' % _dquote, Generic.Heading, '+titling-string'), + (r'[%s]{2}|(?<=[\s%s])\|[\s%s]' % (_newline, _dquote, _dquote), + Text, ('#pop', '+heading?')), + (r'[.;:]|(?<=[\s%s])\|' % _dquote, Text, '#pop'), + (r'[|%s]' % _newline, Generic.Heading) + ], + '+main': [ + (r'(?i)[^%s:a\[(|%s]+' % (_dquote, _newline), Text), + (r'[%s]' % _dquote, String.Double, '+text'), + (r':', Text, '+phrase-definition'), + (r'(?i)\bas\b', Text, '+use-option'), + (r'\[', Comment.Multiline, '+comment'), + (r'(\([%s])(.*?)([%s]\))' % (_dash, _dash), + bygroups(Punctuation, + using(this, state=('+i6-root', 'directive'), + i6t='+i6t-not-inline'), Punctuation)), + (r'(%s|(?<=[\s;:.%s]))\|\s|[%s]{2,}' % + (_start, _dquote, _newline), Text, '+heading?'), + (r'(?i)[a(|%s]' % _newline, Text) + ], + '+phrase-definition': [ + (r'\s+', Text), + (r'\[', Comment.Multiline, '+comment'), + (r'(\([%s])(.*?)([%s]\))' % (_dash, _dash), + bygroups(Punctuation, + using(this, state=('+i6-root', 'directive', + 'default', 'statements'), + i6t='+i6t-inline'), Punctuation), '#pop'), + (r'', Text, '#pop') + ], + '+use-option': [ + (r'\s+', Text), + (r'\[', Comment.Multiline, '+comment'), + (r'(\([%s])(.*?)([%s]\))' % (_dash, _dash), + bygroups(Punctuation, + using(this, state=('+i6-root', 'directive'), + i6t='+i6t-use-option'), Punctuation), '#pop'), + (r'', Text, '#pop') + ], + '+comment': [ + (r'[^\[\]]+', Comment.Multiline), + (r'\[', Comment.Multiline, '#push'), + (r'\]', Comment.Multiline, '#pop') + ], + '+text': [ + (r'[^\[%s]+' % _dquote, String.Double), + (r'\[.*?\]', String.Interpol), + (r'[%s]' % _dquote, String.Double, '#pop') + ], + '+heading?': [ + (r'(\|?\s)+', Text), + (r'\[', Comment.Multiline, '+comment'), + (r'[%s]{4}\s+' % _dash, Text, '+documentation-heading'), + (r'[%s]{1,3}' % _dash, Text), + (r'(?i)(volume|book|part|chapter|section)\b[^%s]*' % _newline, + Generic.Heading, '#pop'), + (r'', Text, '#pop') + ], + '+documentation-heading': [ + (r'\s+', Text), + (r'\[', Comment.Multiline, '+comment'), + (r'(?i)documentation\s+', Text, '+documentation-heading2'), + (r'', Text, '#pop') + ], + '+documentation-heading2': [ + (r'\s+', Text), + (r'\[', Comment.Multiline, '+comment'), + (r'[%s]{4}\s' % _dash, Text, '+documentation'), + (r'', Text, '#pop:2') + ], + '+documentation': [ + (r'(?i)(%s)\s*(chapter|example)\s*:[^%s]*' % + (_start, _newline), Generic.Heading), + (r'(?i)(%s)\s*section\s*:[^%s]*' % (_start, _newline), + Generic.Subheading), + (r'((%s)\t.*?[%s])+' % (_start, _newline), + using(this, state='+main')), + (r'[^%s\[]+|[%s\[]' % (_newline, _newline), Text), + (r'\[', Comment.Multiline, '+comment'), + ], + '+i6t-not-inline': [ + (r'(%s)@c( .*?)?([%s]|\Z)' % (_start, _newline), + Comment.Preproc), + (r'(%s)@([%s]+|Purpose:)[^%s]*' % (_start, _dash, _newline), + Comment.Preproc), + (r'(%s)@p( .*?)?([%s]|\Z)' % (_start, _newline), + Generic.Heading, '+p') + ], + '+i6t-use-option': [ + include('+i6t-not-inline'), + (r'({)(N)(})', bygroups(Punctuation, Text, Punctuation)) + ], + '+i6t-inline': [ + (r'({)(\S[^}]*)?(})', + bygroups(Punctuation, using(this, state='+main'), + Punctuation)) + ], + '+i6t': [ + (r'({[%s])(![^}]*)(}?)' % _dash, + bygroups(Punctuation, Comment.Single, Punctuation)), + (r'({[%s])(lines)(:)([^}]*)(}?)' % _dash, + bygroups(Punctuation, Keyword, Punctuation, Text, + Punctuation), '+lines'), + (r'({[%s])([^:}]*)(:?)([^}]*)(}?)' % _dash, + bygroups(Punctuation, Keyword, Punctuation, Text, + Punctuation)), + (r'(\(\+)(.*?)(\+\)|\Z)', + bygroups(Punctuation, using(this, state='+main'), + Punctuation)) + ], + '+p': [ + (r'[^@]+', Comment.Preproc), + (r'(%s)@c( .*?)?([%s]|\Z)' % (_start, _newline), + Comment.Preproc, '#pop'), + (r'(%s)@([%s]|Purpose:)' % (_start, _dash), Comment.Preproc), + (r'(%s)@p( .*?)?([%s]|\Z)' % (_start, _newline), + Generic.Heading), + (r'@', Comment.Preproc) + ], + '+lines': [ + (r'(%s)@c( .*?)?([%s]|\Z)' % (_start, _newline), + Comment.Preproc), + (r'(%s)@([%s]|Purpose:)[^%s]*' % (_start, _dash, _newline), + Comment.Preproc), + (r'(%s)@p( .*?)?([%s]|\Z)' % (_start, _newline), + Generic.Heading, '+p'), + (r'(%s)@[a-zA-Z_0-9]*[ %s]' % (_start, _newline), Keyword), + (r'![^%s]*' % _newline, Comment.Single), + (r'({)([%s]endlines)(})' % _dash, + bygroups(Punctuation, Keyword, Punctuation), '#pop'), + (r'[^@!{]+?([%s]|\Z)|.' % _newline, Text) + ] + } + # Inform 7 can include snippets of Inform 6 template language, + # so all of Inform6Lexer's states are copied here, with + # modifications to account for template syntax. Inform7Lexer's + # own states begin with '+' to avoid name conflicts. Some of + # Inform6Lexer's states begin with '_': these are not modified. + # They deal with template syntax either by including modified + # states, or by matching r'' then pushing to modified states. + for token in Inform6Lexer.tokens: + if token == 'root': + continue + tokens[level][token] = list(Inform6Lexer.tokens[token]) + if not token.startswith('_'): + tokens[level][token][:0] = [include('+i6t'), include(level)] + + def __init__(self, **options): + level = options.get('i6t', '+i6t-not-inline') + if level not in self._all_tokens: + self._tokens = self.__class__.process_tokendef(level) + else: + self._tokens = self._all_tokens[level] + RegexLexer.__init__(self, **options) + + +class Inform6TemplateLexer(Inform7Lexer): + """ + For `Inform 6 template + <http://inform7.com/sources/src/i6template/Woven/index.html>`_ code. + + .. versionadded:: 2.0 + """ + + name = 'Inform 6 template' + aliases = ['i6t'] + filenames = ['*.i6t'] + + def get_tokens_unprocessed(self, text, stack=('+i6t-root',)): + return Inform7Lexer.get_tokens_unprocessed(self, text, stack) + + +class MqlLexer(CppLexer): + """ + For `MQL4 <http://docs.mql4.com/>`_ and + `MQL5 <http://www.mql5.com/en/docs>`_ source code. + + .. versionadded:: 2.0 + """ + name = 'MQL' + aliases = ['mql', 'mq4', 'mq5', 'mql4', 'mql5'] + filenames = ['*.mq4', '*.mq5', '*.mqh'] + mimetypes = ['text/x-mql'] + + tokens = { + 'statements': [ + (r'(input|_Digits|_Point|_LastError|_Period|_RandomSeed|' + r'_StopFlag|_Symbol|_UninitReason|' + r'Ask|Bars|Bid|Close|Digits|High|Low|Open|Point|Time|Volume)\b', + Keyword), + (r'(void|char|uchar|bool|short|ushort|int|uint|color|long|ulong|datetime|' + r'float|double|string)\b', + Keyword.Type), + (r'(Alert|CheckPointer|Comment|DebugBreak|ExpertRemove|' + r'GetPointer|GetTickCount|MessageBox|PeriodSeconds|PlaySound|' + r'Print|PrintFormat|ResetLastError|ResourceCreate|ResourceFree|' + r'ResourceReadImage|ResourceSave|SendFTP|SendMail|SendNotification|' + r'Sleep|TerminalClose|TesterStatistics|ZeroMemory|' + r'ArrayBsearch|ArrayCopy|ArrayCompare|ArrayFree|ArrayGetAsSeries|' + r'ArrayInitialize|ArrayFill|ArrayIsSeries|ArrayIsDynamic|' + r'ArrayMaximum|ArrayMinimum|ArrayRange|ArrayResize|' + r'ArraySetAsSeries|ArraySize|ArraySort|ArrayCopyRates|' + r'ArrayCopySeries|ArrayDimension|' + r'CharToString|DoubleToString|EnumToString|NormalizeDouble|' + r'StringToDouble|StringToInteger|StringToTime|TimeToString|' + r'IntegerToString|ShortToString|ShortArrayToString|' + r'StringToShortArray|CharArrayToString|StringToCharArray|' + r'ColorToARGB|ColorToString|StringToColor|StringFormat|' + r'CharToStr|DoubleToStr|StrToDouble|StrToInteger|StrToTime|TimeToStr|' + r'MathAbs|MathArccos|MathArcsin|MathArctan|MathCeil|MathCos|MathExp|' + r'MathFloor|MathLog|MathMax|MathMin|MathMod|MathPow|MathRand|' + r'MathRound|MathSin|MathSqrt|MathSrand|MathTan|MathIsValidNumber|' + r'StringAdd|StringBufferLen|StringCompare|StringConcatenate|StringFill|' + r'StringFind|StringGetCharacter|StringInit|StringLen|StringReplace|' + r'StringSetCharacter|StringSplit|StringSubstr|StringToLower|StringToUpper|' + r'StringTrimLeft|StringTrimRight|StringGetChar|StringSetChar|' + r'TimeCurrent|TimeTradeServer|TimeLocal|TimeGMT|TimeDaylightSavings|' + r'TimeGMTOffset|TimeToStruct|StructToTime|Day|DayOfWeek|DayOfYear|' + r'Hour|Minute|Month|Seconds|TimeDay|TimeDayOfWeek|TimeDayOfYear|TimeHour|' + r'TimeMinute|TimeMonth|TimeSeconds|TimeYear|Year|' + r'AccountInfoDouble|AccountInfoInteger|AccountInfoString|AccountBalance|' + r'AccountCredit|AccountCompany|AccountCurrency|AccountEquity|' + r'AccountFreeMargin|AccountFreeMarginCheck|AccountFreeMarginMode|' + r'AccountLeverage|AccountMargin|AccountName|AccountNumber|AccountProfit|' + r'AccountServer|AccountStopoutLevel|AccountStopoutMode|' + r'GetLastError|IsStopped|UninitializeReason|MQLInfoInteger|MQLInfoString|' + r'Symbol|Period|Digits|Point|IsConnected|IsDemo|IsDllsAllowed|' + r'IsExpertEnabled|IsLibrariesAllowed|IsOptimization|IsTesting|' + r'IsTradeAllowed|' + r'IsTradeContextBusy|IsVisualMode|TerminalCompany|TerminalName|' + r'TerminalPath|' + r'SymbolsTotal|SymbolName|SymbolSelect|SymbolIsSynchronized|' + r'SymbolInfoDouble|' + r'SymbolInfoInteger|SymbolInfoString|SymbolInfoTick|' + r'SymbolInfoSessionQuote|' + r'SymbolInfoSessionTrade|MarketInfo|' + r'SeriesInfoInteger|CopyRates|CopyTime|CopyOpen|' + r'CopyHigh|CopyLow|CopyClose|' + r'CopyTickVolume|CopyRealVolume|CopySpread|iBars|iBarShift|iClose|' + r'iHigh|iHighest|iLow|iLowest|iOpen|iTime|iVolume|' + r'HideTestIndicators|Period|RefreshRates|Symbol|WindowBarsPerChart|' + r'WindowExpertName|WindowFind|WindowFirstVisibleBar|WindowHandle|' + r'WindowIsVisible|WindowOnDropped|WindowPriceMax|WindowPriceMin|' + r'WindowPriceOnDropped|WindowRedraw|WindowScreenShot|' + r'WindowTimeOnDropped|WindowsTotal|WindowXOnDropped|WindowYOnDropped|' + r'OrderClose|OrderCloseBy|OrderClosePrice|OrderCloseTime|OrderComment|' + r'OrderCommission|OrderDelete|OrderExpiration|OrderLots|OrderMagicNumber|' + r'OrderModify|OrderOpenPrice|OrderOpenTime|OrderPrint|OrderProfit|' + r'OrderSelect|OrderSend|OrdersHistoryTotal|OrderStopLoss|OrdersTotal|' + r'OrderSwap|OrderSymbol|OrderTakeProfit|OrderTicket|OrderType|' + r'GlobalVariableCheck|GlobalVariableTime|' + r'GlobalVariableDel|GlobalVariableGet|GlobalVariableName|' + r'GlobalVariableSet|GlobalVariablesFlush|GlobalVariableTemp|' + r'GlobalVariableSetOnCondition|GlobalVariablesDeleteAll|' + r'GlobalVariablesTotal|GlobalVariableCheck|GlobalVariableTime|' + r'GlobalVariableDel|GlobalVariableGet|' + r'GlobalVariableName|GlobalVariableSet|GlobalVariablesFlush|' + r'GlobalVariableTemp|GlobalVariableSetOnCondition|' + r'GlobalVariablesDeleteAll|GlobalVariablesTotal|' + r'GlobalVariableCheck|GlobalVariableTime|GlobalVariableDel|' + r'GlobalVariableGet|GlobalVariableName|GlobalVariableSet|' + r'GlobalVariablesFlush|GlobalVariableTemp|' + r'GlobalVariableSetOnCondition|GlobalVariablesDeleteAll|' + r'GlobalVariablesTotal|' + r'FileFindFirst|FileFindNext|FileFindClose|FileOpen|FileDelete|' + r'FileFlush|FileGetInteger|FileIsEnding|FileIsLineEnding|' + r'FileClose|FileIsExist|FileCopy|FileMove|FileReadArray|' + r'FileReadBool|FileReadDatetime|FileReadDouble|FileReadFloat|' + r'FileReadInteger|FileReadLong|FileReadNumber|FileReadString|' + r'FileReadStruct|FileSeek|FileSize|FileTell|FileWrite|' + r'FileWriteArray|FileWriteDouble|FileWriteFloat|FileWriteInteger|' + r'FileWriteLong|FileWriteString|FileWriteStruct|FolderCreate|' + r'FolderDelete|FolderClean|FileOpenHistory|' + r'IndicatorSetDouble|IndicatorSetInteger|IndicatorSetString|' + r'SetIndexBuffer|IndicatorBuffers|IndicatorCounted|IndicatorDigits|' + r'IndicatorShortName|SetIndexArrow|SetIndexDrawBegin|' + r'SetIndexEmptyValue|SetIndexLabel|SetIndexShift|' + r'SetIndexStyle|SetLevelStyle|SetLevelValue|' + r'ObjectCreate|ObjectName|ObjectDelete|ObjectsDeleteAll|' + r'ObjectFind|ObjectGetTimeByValue|ObjectGetValueByTime|' + r'ObjectMove|ObjectsTotal|ObjectGetDouble|ObjectGetInteger|' + r'ObjectGetString|ObjectSetDouble|ObjectSetInteger|' + r'ObjectSetString|TextSetFont|TextOut|TextGetSize|' + r'ObjectDescription|ObjectGet|ObjectGetFiboDescription|' + r'ObjectGetShiftByValue|ObjectGetValueByShift|ObjectSet|' + r'ObjectSetFiboDescription|ObjectSetText|ObjectType|' + r'iAC|iAD|iADX|iAlligator|iAO|iATR|iBearsPower|' + r'iBands|iBandsOnArray|iBullsPower|iCCI|iCCIOnArray|' + r'iCustom|iDeMarker|iEnvelopes|iEnvelopesOnArray|' + r'iForce|iFractals|iGator|iIchimoku|iBWMFI|iMomentum|' + r'iMomentumOnArray|iMFI|iMA|iMAOnArray|iOsMA|iMACD|' + r'iOBV|iSAR|iRSI|iRSIOnArray|iRVI|iStdDev|iStdDevOnArray|' + r'iStochastic|iWPR|' + r'EventSetMillisecondTimer|EventSetTimer|' + r'EventKillTimer|EventChartCustom)\b', Name.Function), + (r'(CHARTEVENT_KEYDOWN|CHARTEVENT_MOUSE_MOVE|' + r'CHARTEVENT_OBJECT_CREATE|' + r'CHARTEVENT_OBJECT_CHANGE|CHARTEVENT_OBJECT_DELETE|' + r'CHARTEVENT_CLICK|' + r'CHARTEVENT_OBJECT_CLICK|CHARTEVENT_OBJECT_DRAG|' + r'CHARTEVENT_OBJECT_ENDEDIT|' + r'CHARTEVENT_CHART_CHANGE|CHARTEVENT_CUSTOM|' + r'CHARTEVENT_CUSTOM_LAST|' + r'PERIOD_CURRENT|PERIOD_M1|PERIOD_M2|PERIOD_M3|' + r'PERIOD_M4|PERIOD_M5|' + r'PERIOD_M6|PERIOD_M10|PERIOD_M12|PERIOD_M15|' + r'PERIOD_M20|PERIOD_M30|' + r'PERIOD_H1|PERIOD_H2|PERIOD_H3|PERIOD_H4|' + r'PERIOD_H6|PERIOD_H8|' + r'PERIOD_H12|PERIOD_D1|PERIOD_W1|PERIOD_MN1|' + r'CHART_IS_OBJECT|CHART_BRING_TO_TOP|' + r'CHART_MOUSE_SCROLL|CHART_EVENT_MOUSE_MOVE|' + r'CHART_EVENT_OBJECT_CREATE|' + r'CHART_EVENT_OBJECT_DELETE|CHART_MODE|CHART_FOREGROUND|' + r'CHART_SHIFT|' + r'CHART_AUTOSCROLL|CHART_SCALE|CHART_SCALEFIX|' + r'CHART_SCALEFIX_11|' + r'CHART_SCALE_PT_PER_BAR|CHART_SHOW_OHLC|' + r'CHART_SHOW_BID_LINE|' + r'CHART_SHOW_ASK_LINE|CHART_SHOW_LAST_LINE|' + r'CHART_SHOW_PERIOD_SEP|' + r'CHART_SHOW_GRID|CHART_SHOW_VOLUMES|' + r'CHART_SHOW_OBJECT_DESCR|' + r'CHART_VISIBLE_BARS|CHART_WINDOWS_TOTAL|' + r'CHART_WINDOW_IS_VISIBLE|' + r'CHART_WINDOW_HANDLE|CHART_WINDOW_YDISTANCE|' + r'CHART_FIRST_VISIBLE_BAR|' + r'CHART_WIDTH_IN_BARS|CHART_WIDTH_IN_PIXELS|' + r'CHART_HEIGHT_IN_PIXELS|' + r'CHART_COLOR_BACKGROUND|CHART_COLOR_FOREGROUND|' + r'CHART_COLOR_GRID|' + r'CHART_COLOR_VOLUME|CHART_COLOR_CHART_UP|' + r'CHART_COLOR_CHART_DOWN|' + r'CHART_COLOR_CHART_LINE|CHART_COLOR_CANDLE_BULL|' + r'CHART_COLOR_CANDLE_BEAR|' + r'CHART_COLOR_BID|CHART_COLOR_ASK|CHART_COLOR_LAST|' + r'CHART_COLOR_STOP_LEVEL|' + r'CHART_SHOW_TRADE_LEVELS|CHART_DRAG_TRADE_LEVELS|' + r'CHART_SHOW_DATE_SCALE|' + r'CHART_SHOW_PRICE_SCALE|CHART_SHIFT_SIZE|' + r'CHART_FIXED_POSITION|' + r'CHART_FIXED_MAX|CHART_FIXED_MIN|CHART_POINTS_PER_BAR|' + r'CHART_PRICE_MIN|' + r'CHART_PRICE_MAX|CHART_COMMENT|CHART_BEGIN|' + r'CHART_CURRENT_POS|CHART_END|' + r'CHART_BARS|CHART_CANDLES|CHART_LINE|CHART_VOLUME_HIDE|' + r'CHART_VOLUME_TICK|CHART_VOLUME_REAL|' + r'OBJ_VLINE|OBJ_HLINE|OBJ_TREND|OBJ_TRENDBYANGLE|OBJ_CYCLES|' + r'OBJ_CHANNEL|OBJ_STDDEVCHANNEL|OBJ_REGRESSION|OBJ_PITCHFORK|' + r'OBJ_GANNLINE|OBJ_GANNFAN|OBJ_GANNGRID|OBJ_FIBO|' + r'OBJ_FIBOTIMES|OBJ_FIBOFAN|OBJ_FIBOARC|OBJ_FIBOCHANNEL|' + r'OBJ_EXPANSION|OBJ_RECTANGLE|OBJ_TRIANGLE|OBJ_ELLIPSE|' + r'OBJ_ARROW_THUMB_UP|OBJ_ARROW_THUMB_DOWN|' + r'OBJ_ARROW_UP|OBJ_ARROW_DOWN|' + r'OBJ_ARROW_STOP|OBJ_ARROW_CHECK|OBJ_ARROW_LEFT_PRICE|' + r'OBJ_ARROW_RIGHT_PRICE|OBJ_ARROW_BUY|OBJ_ARROW_SELL|' + r'OBJ_ARROW|' + r'OBJ_TEXT|OBJ_LABEL|OBJ_BUTTON|OBJ_BITMAP|' + r'OBJ_BITMAP_LABEL|' + r'OBJ_EDIT|OBJ_EVENT|OBJ_RECTANGLE_LABEL|' + r'OBJPROP_TIME1|OBJPROP_PRICE1|OBJPROP_TIME2|' + r'OBJPROP_PRICE2|OBJPROP_TIME3|' + r'OBJPROP_PRICE3|OBJPROP_COLOR|OBJPROP_STYLE|' + r'OBJPROP_WIDTH|' + r'OBJPROP_BACK|OBJPROP_RAY|OBJPROP_ELLIPSE|' + r'OBJPROP_SCALE|' + r'OBJPROP_ANGLE|OBJPROP_ARROWCODE|OBJPROP_TIMEFRAMES|' + r'OBJPROP_DEVIATION|OBJPROP_FONTSIZE|OBJPROP_CORNER|' + r'OBJPROP_XDISTANCE|OBJPROP_YDISTANCE|OBJPROP_FIBOLEVELS|' + r'OBJPROP_LEVELCOLOR|OBJPROP_LEVELSTYLE|OBJPROP_LEVELWIDTH|' + r'OBJPROP_FIRSTLEVEL|OBJPROP_COLOR|OBJPROP_STYLE|OBJPROP_WIDTH|' + r'OBJPROP_BACK|OBJPROP_ZORDER|OBJPROP_FILL|OBJPROP_HIDDEN|' + r'OBJPROP_SELECTED|OBJPROP_READONLY|OBJPROP_TYPE|OBJPROP_TIME|' + r'OBJPROP_SELECTABLE|OBJPROP_CREATETIME|OBJPROP_LEVELS|' + r'OBJPROP_LEVELCOLOR|OBJPROP_LEVELSTYLE|OBJPROP_LEVELWIDTH|' + r'OBJPROP_ALIGN|OBJPROP_FONTSIZE|OBJPROP_RAY_RIGHT|OBJPROP_RAY|' + r'OBJPROP_ELLIPSE|OBJPROP_ARROWCODE|OBJPROP_TIMEFRAMES|OBJPROP_ANCHOR|' + r'OBJPROP_XDISTANCE|OBJPROP_YDISTANCE|OBJPROP_DRAWLINES|OBJPROP_STATE|' + r'OBJPROP_CHART_ID|OBJPROP_XSIZE|OBJPROP_YSIZE|OBJPROP_XOFFSET|' + r'OBJPROP_YOFFSET|OBJPROP_PERIOD|OBJPROP_DATE_SCALE|OBJPROP_PRICE_SCALE|' + r'OBJPROP_CHART_SCALE|OBJPROP_BGCOLOR|OBJPROP_CORNER|OBJPROP_BORDER_TYPE|' + r'OBJPROP_BORDER_COLOR|OBJPROP_PRICE|OBJPROP_LEVELVALUE|OBJPROP_SCALE|' + r'OBJPROP_ANGLE|OBJPROP_DEVIATION|' + r'OBJPROP_NAME|OBJPROP_TEXT|OBJPROP_TOOLTIP|OBJPROP_LEVELTEXT|' + r'OBJPROP_FONT|OBJPROP_BMPFILE|OBJPROP_SYMBOL|' + r'BORDER_FLAT|BORDER_RAISED|BORDER_SUNKEN|ALIGN_LEFT|ALIGN_CENTER|' + r'ALIGN_RIGHT|ANCHOR_LEFT_UPPER|ANCHOR_LEFT|ANCHOR_LEFT_LOWER|' + r'ANCHOR_LOWER|ANCHOR_RIGHT_LOWER|ANCHOR_RIGHT|ANCHOR_RIGHT_UPPER|' + r'ANCHOR_UPPER|ANCHOR_CENTER|ANCHOR_TOP|ANCHOR_BOTTOM|' + r'CORNER_LEFT_UPPER|CORNER_LEFT_LOWER|CORNER_RIGHT_LOWER|' + r'CORNER_RIGHT_UPPER|' + r'OBJ_NO_PERIODS|EMPTY|OBJ_PERIOD_M1|OBJ_PERIOD_M5|OBJ_PERIOD_M15|' + r'OBJ_PERIOD_M30|OBJ_PERIOD_H1|OBJ_PERIOD_H4|OBJ_PERIOD_D1|' + r'OBJ_PERIOD_W1|OBJ_PERIOD_MN1|OBJ_ALL_PERIODS|' + r'GANN_UP_TREND|GANN_DOWN_TREND|' + r'((clr)?(Black|DarkGreen|DarkSlateGray|Olive|' + r'Green|Teal|Navy|Purple|' + r'Maroon|Indigo|MidnightBlue|DarkBlue|' + r'DarkOliveGreen|SaddleBrown|' + r'ForestGreen|OliveDrab|SeaGreen|' + r'DarkGoldenrod|DarkSlateBlue|' + r'Sienna|MediumBlue|Brown|DarkTurquoise|' + r'DimGray|LightSeaGreen|' + r'DarkViolet|FireBrick|MediumVioletRed|' + r'MediumSeaGreen|Chocolate|' + r'Crimson|SteelBlue|Goldenrod|MediumSpringGreen|' + r'LawnGreen|CadetBlue|' + r'DarkOrchid|YellowGreen|LimeGreen|OrangeRed|' + r'DarkOrange|Orange|' + r'Gold|Yellow|Chartreuse|Lime|SpringGreen|' + r'Aqua|DeepSkyBlue|Blue|' + r'Magenta|Red|Gray|SlateGray|Peru|BlueViolet|' + r'LightSlateGray|DeepPink|' + r'MediumTurquoise|DodgerBlue|Turquoise|RoyalBlue|' + r'SlateBlue|DarkKhaki|' + r'IndianRed|MediumOrchid|GreenYellow|' + r'MediumAquamarine|DarkSeaGreen|' + r'Tomato|RosyBrown|Orchid|MediumPurple|' + r'PaleVioletRed|Coral|CornflowerBlue|' + r'DarkGray|SandyBrown|MediumSlateBlue|' + r'Tan|DarkSalmon|BurlyWood|' + r'HotPink|Salmon|Violet|LightCoral|SkyBlue|' + r'LightSalmon|Plum|' + r'Khaki|LightGreen|Aquamarine|Silver|' + r'LightSkyBlue|LightSteelBlue|' + r'LightBlue|PaleGreen|Thistle|PowderBlue|' + r'PaleGoldenrod|PaleTurquoise|' + r'LightGray|Wheat|NavajoWhite|Moccasin|' + r'LightPink|Gainsboro|PeachPuff|' + r'Pink|Bisque|LightGoldenrod|BlanchedAlmond|' + r'LemonChiffon|Beige|' + r'AntiqueWhite|PapayaWhip|Cornsilk|' + r'LightYellow|LightCyan|Linen|' + r'Lavender|MistyRose|OldLace|WhiteSmoke|' + r'Seashell|Ivory|Honeydew|' + r'AliceBlue|LavenderBlush|MintCream|Snow|White))|' + r'SYMBOL_THUMBSUP|SYMBOL_THUMBSDOWN|' + r'SYMBOL_ARROWUP|SYMBOL_ARROWDOWN|' + r'SYMBOL_STOPSIGN|SYMBOL_CHECKSIGN|' + r'SYMBOL_LEFTPRICE|SYMBOL_RIGHTPRICE|' + r'PRICE_CLOSE|PRICE_OPEN|PRICE_HIGH|PRICE_LOW|' + r'PRICE_MEDIAN|PRICE_TYPICAL|PRICE_WEIGHTED|' + r'VOLUME_TICK|VOLUME_REAL|' + r'STO_LOWHIGH|STO_CLOSECLOSE|' + r'MODE_OPEN|MODE_LOW|MODE_HIGH|MODE_CLOSE|MODE_VOLUME|MODE_TIME|' + r'MODE_SMA|MODE_EMA|MODE_SMMA|MODE_LWMA|' + r'MODE_MAIN|MODE_SIGNAL|MODE_MAIN|' + r'MODE_PLUSDI|MODE_MINUSDI|MODE_UPPER|' + r'MODE_LOWER|MODE_GATORJAW|MODE_GATORTEETH|' + r'MODE_GATORLIPS|MODE_TENKANSEN|' + r'MODE_KIJUNSEN|MODE_SENKOUSPANA|' + r'MODE_SENKOUSPANB|MODE_CHINKOUSPAN|' + r'DRAW_LINE|DRAW_SECTION|DRAW_HISTOGRAM|' + r'DRAW_ARROW|DRAW_ZIGZAG|DRAW_NONE|' + r'STYLE_SOLID|STYLE_DASH|STYLE_DOT|' + r'STYLE_DASHDOT|STYLE_DASHDOTDOT|' + r'DRAW_NONE|DRAW_LINE|DRAW_SECTION|DRAW_HISTOGRAM|' + r'DRAW_ARROW|DRAW_ZIGZAG|DRAW_FILLING|' + r'INDICATOR_DATA|INDICATOR_COLOR_INDEX|' + r'INDICATOR_CALCULATIONS|INDICATOR_DIGITS|' + r'INDICATOR_HEIGHT|INDICATOR_LEVELS|' + r'INDICATOR_LEVELCOLOR|INDICATOR_LEVELSTYLE|' + r'INDICATOR_LEVELWIDTH|INDICATOR_MINIMUM|' + r'INDICATOR_MAXIMUM|INDICATOR_LEVELVALUE|' + r'INDICATOR_SHORTNAME|INDICATOR_LEVELTEXT|' + r'TERMINAL_BUILD|TERMINAL_CONNECTED|' + r'TERMINAL_DLLS_ALLOWED|TERMINAL_TRADE_ALLOWED|' + r'TERMINAL_EMAIL_ENABLED|' + r'TERMINAL_FTP_ENABLED|TERMINAL_MAXBARS|' + r'TERMINAL_CODEPAGE|TERMINAL_CPU_CORES|' + r'TERMINAL_DISK_SPACE|TERMINAL_MEMORY_PHYSICAL|' + r'TERMINAL_MEMORY_TOTAL|' + r'TERMINAL_MEMORY_AVAILABLE|TERMINAL_MEMORY_USED|' + r'TERMINAL_X64|' + r'TERMINAL_OPENCL_SUPPORT|TERMINAL_LANGUAGE|' + r'TERMINAL_COMPANY|TERMINAL_NAME|' + r'TERMINAL_PATH|TERMINAL_DATA_PATH|' + r'TERMINAL_COMMONDATA_PATH|' + r'MQL_PROGRAM_TYPE|MQL_DLLS_ALLOWED|' + r'MQL_TRADE_ALLOWED|MQL_DEBUG|' + r'MQL_PROFILER|MQL_TESTER|MQL_OPTIMIZATION|' + r'MQL_VISUAL_MODE|' + r'MQL_FRAME_MODE|MQL_LICENSE_TYPE|MQL_PROGRAM_NAME|' + r'MQL_PROGRAM_PATH|' + r'PROGRAM_SCRIPT|PROGRAM_EXPERT|' + r'PROGRAM_INDICATOR|LICENSE_FREE|' + r'LICENSE_DEMO|LICENSE_FULL|LICENSE_TIME|' + r'MODE_LOW|MODE_HIGH|MODE_TIME|MODE_BID|' + r'MODE_ASK|MODE_POINT|' + r'MODE_DIGITS|MODE_SPREAD|MODE_STOPLEVEL|' + r'MODE_LOTSIZE|MODE_TICKVALUE|' + r'MODE_TICKSIZE|MODE_SWAPLONG|' + r'MODE_SWAPSHORT|MODE_STARTING|' + r'MODE_EXPIRATION|MODE_TRADEALLOWED|' + r'MODE_MINLOT|MODE_LOTSTEP|MODE_MAXLOT|' + r'MODE_SWAPTYPE|MODE_PROFITCALCMODE|' + r'MODE_MARGINCALCMODE|MODE_MARGININIT|' + r'MODE_MARGINMAINTENANCE|MODE_MARGINHEDGED|' + r'MODE_MARGINREQUIRED|MODE_FREEZELEVEL|' + r'SUNDAY|MONDAY|TUESDAY|WEDNESDAY|THURSDAY|' + r'FRIDAY|SATURDAY|' + r'ACCOUNT_LOGIN|ACCOUNT_TRADE_MODE|' + r'ACCOUNT_LEVERAGE|' + r'ACCOUNT_LIMIT_ORDERS|ACCOUNT_MARGIN_SO_MODE|' + r'ACCOUNT_TRADE_ALLOWED|ACCOUNT_TRADE_EXPERT|' + r'ACCOUNT_BALANCE|' + r'ACCOUNT_CREDIT|ACCOUNT_PROFIT|ACCOUNT_EQUITY|' + r'ACCOUNT_MARGIN|' + r'ACCOUNT_FREEMARGIN|ACCOUNT_MARGIN_LEVEL|' + r'ACCOUNT_MARGIN_SO_CALL|' + r'ACCOUNT_MARGIN_SO_SO|ACCOUNT_NAME|' + r'ACCOUNT_SERVER|ACCOUNT_CURRENCY|' + r'ACCOUNT_COMPANY|ACCOUNT_TRADE_MODE_DEMO|' + r'ACCOUNT_TRADE_MODE_CONTEST|' + r'ACCOUNT_TRADE_MODE_REAL|ACCOUNT_STOPOUT_MODE_PERCENT|' + r'ACCOUNT_STOPOUT_MODE_MONEY|' + r'STAT_INITIAL_DEPOSIT|STAT_WITHDRAWAL|STAT_PROFIT|' + r'STAT_GROSS_PROFIT|' + r'STAT_GROSS_LOSS|STAT_MAX_PROFITTRADE|' + r'STAT_MAX_LOSSTRADE|STAT_CONPROFITMAX|' + r'STAT_CONPROFITMAX_TRADES|STAT_MAX_CONWINS|' + r'STAT_MAX_CONPROFIT_TRADES|' + r'STAT_CONLOSSMAX|STAT_CONLOSSMAX_TRADES|' + r'STAT_MAX_CONLOSSES|' + r'STAT_MAX_CONLOSS_TRADES|STAT_BALANCEMIN|' + r'STAT_BALANCE_DD|' + r'STAT_BALANCEDD_PERCENT|STAT_BALANCE_DDREL_PERCENT|' + r'STAT_BALANCE_DD_RELATIVE|STAT_EQUITYMIN|' + r'STAT_EQUITY_DD|' + r'STAT_EQUITYDD_PERCENT|STAT_EQUITY_DDREL_PERCENT|' + r'STAT_EQUITY_DD_RELATIVE|STAT_EXPECTED_PAYOFF|' + r'STAT_PROFIT_FACTOR|' + r'STAT_RECOVERY_FACTOR|STAT_SHARPE_RATIO|' + r'STAT_MIN_MARGINLEVEL|' + r'STAT_CUSTOM_ONTESTER|STAT_DEALS|STAT_TRADES|' + r'STAT_PROFIT_TRADES|' + r'STAT_LOSS_TRADES|STAT_SHORT_TRADES|STAT_LONG_TRADES|' + r'STAT_PROFIT_SHORTTRADES|STAT_PROFIT_LONGTRADES|' + r'STAT_PROFITTRADES_AVGCON|STAT_LOSSTRADES_AVGCON|' + r'SERIES_BARS_COUNT|SERIES_FIRSTDATE|SERIES_LASTBAR_DATE|' + r'SERIES_SERVER_FIRSTDATE|SERIES_TERMINAL_FIRSTDATE|' + r'SERIES_SYNCHRONIZED|' + r'OP_BUY|OP_SELL|OP_BUYLIMIT|OP_SELLLIMIT|' + r'OP_BUYSTOP|OP_SELLSTOP|' + r'TRADE_ACTION_DEAL|TRADE_ACTION_PENDING|' + r'TRADE_ACTION_SLTP|' + r'TRADE_ACTION_MODIFY|TRADE_ACTION_REMOVE|' + r'__DATE__|__DATETIME__|__LINE__|__FILE__|' + r'__PATH__|__FUNCTION__|' + r'__FUNCSIG__|__MQLBUILD__|__MQL4BUILD__|' + r'M_E|M_LOG2E|M_LOG10E|M_LN2|M_LN10|' + r'M_PI|M_PI_2|M_PI_4|M_1_PI|' + r'M_2_PI|M_2_SQRTPI|M_SQRT2|M_SQRT1_2|' + r'CHAR_MIN|CHAR_MAX|UCHAR_MAX|' + r'SHORT_MIN|SHORT_MAX|USHORT_MAX|' + r'INT_MIN|INT_MAX|UINT_MAX|' + r'LONG_MIN|LONG_MAX|ULONG_MAX|' + r'DBL_MIN|DBL_MAX|DBL_EPSILON|DBL_DIG|DBL_MANT_DIG|' + r'DBL_MAX_10_EXP|DBL_MAX_EXP|DBL_MIN_10_EXP|DBL_MIN_EXP|' + r'FLT_MIN|FLT_MAX|FLT_EPSILON|' + r'FLT_DIG|FLT_MANT_DIG|FLT_MAX_10_EXP|' + r'FLT_MAX_EXP|FLT_MIN_10_EXP|FLT_MIN_EXP|REASON_PROGRAM' + r'REASON_REMOVE|REASON_RECOMPILE|' + r'REASON_CHARTCHANGE|REASON_CHARTCLOSE|' + r'REASON_PARAMETERS|REASON_ACCOUNT|' + r'REASON_TEMPLATE|REASON_INITFAILED|' + r'REASON_CLOSE|POINTER_INVALID' + r'POINTER_DYNAMIC|POINTER_AUTOMATIC|' + r'NULL|EMPTY|EMPTY_VALUE|CLR_NONE|WHOLE_ARRAY|' + r'CHARTS_MAX|clrNONE|EMPTY_VALUE|INVALID_HANDLE|' + r'IS_DEBUG_MODE|IS_PROFILE_MODE|NULL|WHOLE_ARRAY|WRONG_VALUE|' + r'ERR_NO_ERROR|ERR_NO_RESULT|ERR_COMMON_ERROR|' + r'ERR_INVALID_TRADE_PARAMETERS|' + r'ERR_SERVER_BUSY|ERR_OLD_VERSION|ERR_NO_CONNECTION|' + r'ERR_NOT_ENOUGH_RIGHTS|' + r'ERR_TOO_FREQUENT_REQUESTS|ERR_MALFUNCTIONAL_TRADE|' + r'ERR_ACCOUNT_DISABLED|' + r'ERR_INVALID_ACCOUNT|ERR_TRADE_TIMEOUT|' + r'ERR_INVALID_PRICE|ERR_INVALID_STOPS|' + r'ERR_INVALID_TRADE_VOLUME|ERR_MARKET_CLOSED|' + r'ERR_TRADE_DISABLED|' + r'ERR_NOT_ENOUGH_MONEY|ERR_PRICE_CHANGED|' + r'ERR_OFF_QUOTES|ERR_BROKER_BUSY|' + r'ERR_REQUOTE|ERR_ORDER_LOCKED|' + r'ERR_LONG_POSITIONS_ONLY_ALLOWED|ERR_TOO_MANY_REQUESTS|' + r'ERR_TRADE_MODIFY_DENIED|ERR_TRADE_CONTEXT_BUSY|' + r'ERR_TRADE_EXPIRATION_DENIED|' + r'ERR_TRADE_TOO_MANY_ORDERS|ERR_TRADE_HEDGE_PROHIBITED|' + r'ERR_TRADE_PROHIBITED_BY_FIFO|' + r'FILE_READ|FILE_WRITE|FILE_BIN|FILE_CSV|FILE_TXT|' + r'FILE_ANSI|FILE_UNICODE|' + r'FILE_SHARE_READ|FILE_SHARE_WRITE|FILE_REWRITE|' + r'FILE_COMMON|FILE_EXISTS|' + r'FILE_CREATE_DATE|FILE_MODIFY_DATE|' + r'FILE_ACCESS_DATE|FILE_SIZE|FILE_POSITION|' + r'FILE_END|FILE_LINE_END|FILE_IS_COMMON|' + r'FILE_IS_TEXT|FILE_IS_BINARY|' + r'FILE_IS_CSV|FILE_IS_ANSI|FILE_IS_READABLE|FILE_IS_WRITABLE|' + r'SEEK_SET|SEEK_CUR|SEEK_END|CP_ACP|' + r'CP_OEMCP|CP_MACCP|CP_THREAD_ACP|' + r'CP_SYMBOL|CP_UTF7|CP_UTF8|IDOK|IDCANCEL|IDABORT|' + r'IDRETRY|IDIGNORE|IDYES|IDNO|IDTRYAGAIN|IDCONTINUE|' + r'MB_OK|MB_OKCANCEL|MB_ABORTRETRYIGNORE|MB_YESNOCANCEL|' + r'MB_YESNO|MB_RETRYCANCEL|' + r'MB_CANCELTRYCONTINUE|MB_ICONSTOP|MB_ICONERROR|' + r'MB_ICONHAND|MB_ICONQUESTION|' + r'MB_ICONEXCLAMATION|MB_ICONWARNING|' + r'MB_ICONINFORMATION|MB_ICONASTERISK|' + r'MB_DEFBUTTON1|MB_DEFBUTTON2|MB_DEFBUTTON3|MB_DEFBUTTON4)\b', + Name.Constant), + inherit, + ], + } diff --git a/pygments/lexers/dalvik.py b/pygments/lexers/dalvik.py index 07d97eb6..901b7c5a 100644 --- a/pygments/lexers/dalvik.py +++ b/pygments/lexers/dalvik.py @@ -21,7 +21,7 @@ class SmaliLexer(RegexLexer): For `Smali <http://code.google.com/p/smali/>`_ (Android/Dalvik) assembly code. - *New in Pygments 1.6.* + .. versionadded:: 1.6 """ name = 'Smali' aliases = ['smali'] diff --git a/pygments/lexers/dotnet.py b/pygments/lexers/dotnet.py index 5bda5c2d..0754ba02 100644 --- a/pygments/lexers/dotnet.py +++ b/pygments/lexers/dotnet.py @@ -14,7 +14,7 @@ from pygments.lexer import RegexLexer, DelegatingLexer, bygroups, include, \ using, this from pygments.token import Punctuation, \ Text, Comment, Operator, Keyword, Name, String, Number, Literal, Other -from pygments.util import get_choice_opt +from pygments.util import get_choice_opt, iteritems from pygments import unistring as uni from pygments.lexers.web import XmlLexer @@ -44,7 +44,7 @@ class CSharpLexer(RegexLexer): The default value is ``basic``. - *New in Pygments 0.8.* + .. versionadded:: 0.8 """ name = 'C#' @@ -71,7 +71,7 @@ class CSharpLexer(RegexLexer): tokens = {} token_variants = True - for levelname, cs_ident in levels.items(): + for levelname, cs_ident in iteritems(levels): tokens[levelname] = { 'root': [ # method names @@ -126,7 +126,7 @@ class CSharpLexer(RegexLexer): } def __init__(self, **options): - level = get_choice_opt(options, 'unicodelevel', self.tokens.keys(), 'basic') + level = get_choice_opt(options, 'unicodelevel', list(self.tokens), 'basic') if level not in self._all_tokens: # compile the regexes now self._tokens = self.__class__.process_tokendef(level) @@ -156,7 +156,7 @@ class NemerleLexer(RegexLexer): The default value is ``basic``. - *New in Pygments 1.5.* + .. versionadded:: 1.5 """ name = 'Nemerle' @@ -183,7 +183,7 @@ class NemerleLexer(RegexLexer): tokens = {} token_variants = True - for levelname, cs_ident in levels.items(): + for levelname, cs_ident in iteritems(levels): tokens[levelname] = { 'root': [ # method names @@ -284,7 +284,7 @@ class NemerleLexer(RegexLexer): } def __init__(self, **options): - level = get_choice_opt(options, 'unicodelevel', self.tokens.keys(), + level = get_choice_opt(options, 'unicodelevel', list(self.tokens), 'basic') if level not in self._all_tokens: # compile the regexes now @@ -531,7 +531,7 @@ class FSharpLexer(RegexLexer): """ For the F# language (version 3.0). - *New in Pygments 1.5.* + .. versionadded:: 1.5 """ name = 'FSharp' @@ -638,6 +638,8 @@ class FSharpLexer(RegexLexer): (r'[A-Z][A-Za-z0-9_\']*(?=\s*\.)', Name.Namespace), (r'[A-Z][A-Za-z0-9_\']*', Name, '#pop'), (r'[a-z_][A-Za-z0-9_\']*', Name, '#pop'), + # e.g. dictionary index access + (r'', Text, '#pop'), ], 'comment': [ (r'[^(*)@"]+', Comment), diff --git a/pygments/lexers/foxpro.py b/pygments/lexers/foxpro.py index bc6cc296..99a65ce7 100644 --- a/pygments/lexers/foxpro.py +++ b/pygments/lexers/foxpro.py @@ -24,7 +24,7 @@ class FoxProLexer(RegexLexer): FoxPro syntax allows to shorten all keywords and function names to 4 characters. Shortened forms are not recognized by this lexer. - *New in Pygments 1.6.* + .. versionadded:: 1.6 """ name = 'FoxPro' diff --git a/pygments/lexers/functional.py b/pygments/lexers/functional.py index 9df88d97..122114fa 100644 --- a/pygments/lexers/functional.py +++ b/pygments/lexers/functional.py @@ -19,7 +19,7 @@ __all__ = ['RacketLexer', 'SchemeLexer', 'CommonLispLexer', 'HaskellLexer', 'AgdaLexer', 'LiterateHaskellLexer', 'LiterateAgdaLexer', 'SMLLexer', 'OcamlLexer', 'ErlangLexer', 'ErlangShellLexer', 'OpaLexer', 'CoqLexer', 'NewLispLexer', 'NixLexer', 'ElixirLexer', - 'ElixirConsoleLexer', 'KokaLexer'] + 'ElixirConsoleLexer', 'KokaLexer', 'IdrisLexer', 'LiterateIdrisLexer'] line_re = re.compile('.*?\n') @@ -30,7 +30,7 @@ class RacketLexer(RegexLexer): Lexer for `Racket <http://racket-lang.org/>`_ source code (formerly known as PLT Scheme). - *New in Pygments 1.6.* + .. versionadded:: 1.6 """ name = 'Racket' @@ -599,7 +599,7 @@ class SchemeLexer(RegexLexer): It supports the full Scheme syntax as defined in R5RS. - *New in Pygments 0.6.* + .. versionadded:: 0.6 """ name = 'Scheme' aliases = ['scheme', 'scm'] @@ -720,10 +720,10 @@ class CommonLispLexer(RegexLexer): """ A Common Lisp lexer. - *New in Pygments 0.9.* + .. versionadded:: 0.9 """ name = 'Common Lisp' - aliases = ['common-lisp', 'cl', 'lisp'] + aliases = ['common-lisp', 'cl', 'lisp', 'elisp', 'emacs'] filenames = ['*.cl', '*.lisp', '*.el'] # use for Elisp too mimetypes = ['text/x-common-lisp'] @@ -898,7 +898,7 @@ class HaskellLexer(RegexLexer): """ A Haskell lexer based on the lexemes defined in the Haskell 98 Report. - *New in Pygments 0.8.* + .. versionadded:: 0.8 """ name = 'Haskell' aliases = ['haskell', 'hs'] @@ -996,7 +996,7 @@ class HaskellLexer(RegexLexer): ], 'character': [ # Allows multi-chars, incorrectly. - (r"[^\\']", String.Char), + (r"[^\\']'", String.Char, '#pop'), (r"\\", String.Escape, 'escape'), ("'", String.Char, '#pop'), ], @@ -1017,12 +1017,121 @@ class HaskellLexer(RegexLexer): } +class IdrisLexer(RegexLexer): + """ + A lexer for the dependently typed programming language Idris. + + Based on the Haskell and Agda Lexer. + + .. versionadded:: 2.0 + """ + name = 'Idris' + aliases = ['idris', 'idr'] + filenames = ['*.idr'] + mimetypes = ['text/x-idris'] + + reserved = ['case','class','data','default','using','do','else', + 'if','in','infix[lr]?','instance','rewrite','auto', + 'namespace','codata','mutual','private','public','abstract', + 'total','partial', + 'let','proof','of','then','static','where','_','with', + 'pattern', 'term', 'syntax','prefix', + 'postulate','parameters','record','dsl','impossible','implicit', + 'tactics','intros','intro','compute','refine','exaxt','trivial'] + + ascii = ['NUL','SOH','[SE]TX','EOT','ENQ','ACK', + 'BEL','BS','HT','LF','VT','FF','CR','S[OI]','DLE', + 'DC[1-4]','NAK','SYN','ETB','CAN', + 'EM','SUB','ESC','[FGRU]S','SP','DEL'] + + annotations = ['assert_total','lib','link','include','provide','access', + 'default'] + + tokens = { + 'root': [ + # Declaration + (r'^(\s*)([^\s\(\)\{\}]+)(\s*)(:)(\s*)', + bygroups(Text, Name.Function, Text, Operator.Word, Text)), + # Comments + (r'^(\s*)(%%%s)' % '|'.join(annotations), + bygroups(Text, Keyword.Reserved)), + (r'--(?![!#$%&*+./<=>?@\^|_~:\\]).*?$', Comment.Single), + (r'{-', Comment.Multiline, 'comment'), + # Identifiers + (r'\b(%s)(?!\')\b' % '|'.join(reserved), Keyword.Reserved), + (r'(import|module)(\s+)', bygroups(Keyword.Reserved, Text), 'module'), + (r"('')?[A-Z][\w\']*", Keyword.Type), + (r'[a-z][A-Za-z0-9_\']*', Text), + # Special Symbols + (r'(<-|::|->|=>|=)', Operator.Word), # specials + (r'([\(\)\{\}\[\]:!#$%&*+.\\/<=>?@^|~-]+)', Operator.Word), # specials + # Numbers + (r'\d+[eE][+-]?\d+', Number.Float), + (r'\d+\.\d+([eE][+-]?\d+)?', Number.Float), + (r'0[xX][\da-fA-F]+', Number.Hex), + (r'\d+', Number.Integer), + # Strings + (r"'", String.Char, 'character'), + (r'"', String, 'string'), + (r'[^\s\(\)\{\}]+', Text), + (r'\s+?', Text), # Whitespace + ], + 'module': [ + (r'\s+', Text), + (r'([A-Z][a-zA-Z0-9_.]*)(\s+)(\()', + bygroups(Name.Namespace, Text, Punctuation), 'funclist'), + (r'[A-Z][a-zA-Z0-9_.]*', Name.Namespace, '#pop'), + ], + 'funclist': [ + (r'\s+', Text), + (r'[A-Z][a-zA-Z0-9_]*', Keyword.Type), + (r'(_[\w\']+|[a-z][\w\']*)', Name.Function), + (r'--.*$', Comment.Single), + (r'{-', Comment.Multiline, 'comment'), + (r',', Punctuation), + (r'[:!#$%&*+.\\/<=>?@^|~-]+', Operator), + # (HACK, but it makes sense to push two instances, believe me) + (r'\(', Punctuation, ('funclist', 'funclist')), + (r'\)', Punctuation, '#pop:2'), + ], + # NOTE: the next four states are shared in the AgdaLexer; make sure + # any change is compatible with Agda as well or copy over and change + 'comment': [ + # Multiline Comments + (r'[^-{}]+', Comment.Multiline), + (r'{-', Comment.Multiline, '#push'), + (r'-}', Comment.Multiline, '#pop'), + (r'[-{}]', Comment.Multiline), + ], + 'character': [ + # Allows multi-chars, incorrectly. + (r"[^\\']", String.Char), + (r"\\", String.Escape, 'escape'), + ("'", String.Char, '#pop'), + ], + 'string': [ + (r'[^\\"]+', String), + (r"\\", String.Escape, 'escape'), + ('"', String, '#pop'), + ], + 'escape': [ + (r'[abfnrtv"\'&\\]', String.Escape, '#pop'), + (r'\^[][A-Z@\^_]', String.Escape, '#pop'), + ('|'.join(ascii), String.Escape, '#pop'), + (r'o[0-7]+', String.Escape, '#pop'), + (r'x[\da-fA-F]+', String.Escape, '#pop'), + (r'\d+', String.Escape, '#pop'), + (r'\s+\\', String.Escape, '#pop') + ], + } + + class AgdaLexer(RegexLexer): """ For the `Agda <http://wiki.portal.chalmers.se/agda/pmwiki.php>`_ dependently typed functional programming language and proof assistant. - *New in Pygments 1.7.* + .. versionadded:: 2.0 """ name = 'Agda' @@ -1049,12 +1158,12 @@ class AgdaLexer(RegexLexer): (r'{!', Comment.Directive, 'hole'), # Lexemes: # Identifiers - (ur'\b(%s)(?!\')\b' % '|'.join(reserved), Keyword.Reserved), + (r'\b(%s)(?!\')\b' % '|'.join(reserved), Keyword.Reserved), (r'(import|module)(\s+)', bygroups(Keyword.Reserved, Text), 'module'), (r'\b(Set|Prop)\b', Keyword.Type), # Special Symbols (r'(\(|\)|\{|\})', Operator), - (ur'(\.{1,3}|\||[\u039B]|[\u2200]|[\u2192]|:|=|->)', Operator.Word), + (u'(\\.{1,3}|\\||[\u039B]|[\u2200]|[\u2192]|:|=|->)', Operator.Word), # Numbers (r'\d+[eE][+-]?\d+', Number.Float), (r'\d+\.\d+([eE][+-]?\d+)?', Number.Float), @@ -1161,7 +1270,7 @@ class LiterateHaskellLexer(LiterateLexer): is autodetected: if the first non-whitespace character in the source is a backslash or percent character, LaTeX is assumed, else Bird. - *New in Pygments 0.9.* + .. versionadded:: 0.9 """ name = 'Literate Haskell' aliases = ['lhs', 'literate-haskell', 'lhaskell'] @@ -1173,6 +1282,29 @@ class LiterateHaskellLexer(LiterateLexer): LiterateLexer.__init__(self, hslexer, **options) +class LiterateIdrisLexer(LiterateLexer): + """ + For Literate Idris (Bird-style or LaTeX) source. + + Additional options accepted: + + `litstyle` + If given, must be ``"bird"`` or ``"latex"``. If not given, the style + is autodetected: if the first non-whitespace character in the source + is a backslash or percent character, LaTeX is assumed, else Bird. + + .. versionadded:: 2.0 + """ + name = 'Literate Idris' + aliases = ['lidr', 'literate-idris', 'lidris'] + filenames = ['*.lidr'] + mimetypes = ['text/x-literate-idris'] + + def __init__(self, **options): + hslexer = IdrisLexer(**options) + LiterateLexer.__init__(self, hslexer, **options) + + class LiterateAgdaLexer(LiterateLexer): """ For Literate Agda source. @@ -1184,7 +1316,7 @@ class LiterateAgdaLexer(LiterateLexer): is autodetected: if the first non-whitespace character in the source is a backslash or percent character, LaTeX is assumed, else Bird. - *New in Pygments 1.7.* + .. versionadded:: 2.0 """ name = 'Literate Agda' aliases = ['lagda', 'literate-agda'] @@ -1200,7 +1332,7 @@ class SMLLexer(RegexLexer): """ For the Standard ML language. - *New in Pygments 1.5.* + .. versionadded:: 1.5 """ name = 'Standard ML' @@ -1526,7 +1658,7 @@ class OcamlLexer(RegexLexer): """ For the OCaml language. - *New in Pygments 0.7.* + .. versionadded:: 0.7 """ name = 'OCaml' @@ -1620,7 +1752,7 @@ class ErlangLexer(RegexLexer): Blame Jeremy Thurgood (http://jerith.za.net/). - *New in Pygments 0.9.* + .. versionadded:: 0.9 """ name = 'Erlang' @@ -1725,7 +1857,7 @@ class ErlangShellLexer(Lexer): """ Shell sessions in erl (for Erlang code). - *New in Pygments 1.1.* + .. versionadded:: 1.1 """ name = 'Erlang erl session' aliases = ['erl'] @@ -1768,7 +1900,7 @@ class OpaLexer(RegexLexer): """ Lexer for the Opa language (http://opalang.org). - *New in Pygments 1.5.* + .. versionadded:: 1.5 """ name = 'Opa' @@ -2091,7 +2223,7 @@ class CoqLexer(RegexLexer): """ For the `Coq <http://coq.inria.fr/>`_ theorem prover. - *New in Pygments 1.5.* + .. versionadded:: 1.5 """ name = 'Coq' @@ -2232,7 +2364,7 @@ class NewLispLexer(RegexLexer): """ For `newLISP. <www.newlisp.org>`_ source code (version 10.3.0). - *New in Pygments 1.5.* + .. versionadded:: 1.5 """ name = 'NewLisp' @@ -2363,7 +2495,7 @@ class NixLexer(RegexLexer): """ For the `Nix language <http://nixos.org/nix/>`_. - *New in Pygments 1.7.* + .. versionadded:: 2.0 """ name = 'Nix' @@ -2400,7 +2532,7 @@ class NixLexer(RegexLexer): ('(%s)' % '|'.join(re.escape(entry) + '\\b' for entry in builtins), Name.Builtin), - (r'\b(true|false)\b', Name.Constant), + (r'\b(true|false|null)\b', Name.Constant), # operators ('(%s)' % '|'.join(re.escape(entry) for entry in operators), @@ -2427,7 +2559,8 @@ class NixLexer(RegexLexer): (r'[a-zA-Z][a-zA-Z0-9\+\-\.]*\:[a-zA-Z0-9%/?:@&=+$,\\_.!~*\'-]+', Literal), # names of variables - (r'[a-zA-Z_][a-zA-Z0-9_\'-]*', String.Symbol), + (r'[a-zA-Z0-9-_]+\s*=', String.Symbol), + (r'[a-zA-Z_][a-zA-Z0-9_\'-]*', Text), ], 'comment': [ @@ -2484,7 +2617,7 @@ class ElixirLexer(RegexLexer): """ For the `Elixir language <http://elixir-lang.org>`_. - *New in Pygments 1.5.* + .. versionadded:: 1.5 """ name = 'Elixir' @@ -2595,7 +2728,7 @@ class ElixirConsoleLexer(Lexer): iex> length [head | tail] 3 - *New in Pygments 1.5.* + .. versionadded:: 1.5 """ name = 'Elixir iex session' @@ -2641,7 +2774,7 @@ class KokaLexer(RegexLexer): Lexer for the `Koka <http://koka.codeplex.com>`_ language. - *New in Pygments 1.6.* + .. versionadded:: 1.6 """ name = 'Koka' diff --git a/pygments/lexers/hdl.py b/pygments/lexers/hdl.py index 0ea9a7c5..1ebe4e5c 100644 --- a/pygments/lexers/hdl.py +++ b/pygments/lexers/hdl.py @@ -22,7 +22,7 @@ class VerilogLexer(RegexLexer): """ For verilog source code with preprocessor directives. - *New in Pygments 1.4.* + .. versionadded:: 1.4 """ name = 'verilog' aliases = ['verilog', 'v'] @@ -134,7 +134,7 @@ class SystemVerilogLexer(RegexLexer): Extends verilog lexer to recognise all SystemVerilog keywords from IEEE 1800-2009 standard. - *New in Pygments 1.5.* + .. versionadded:: 1.5 """ name = 'systemverilog' aliases = ['systemverilog', 'sv'] @@ -274,7 +274,7 @@ class VhdlLexer(RegexLexer): """ For VHDL source code. - *New in Pygments 1.5.* + .. versionadded:: 1.5 """ name = 'vhdl' aliases = ['vhdl'] diff --git a/pygments/lexers/jvm.py b/pygments/lexers/jvm.py index c4029822..64c47b6e 100644 --- a/pygments/lexers/jvm.py +++ b/pygments/lexers/jvm.py @@ -19,8 +19,9 @@ from pygments import unistring as uni __all__ = ['JavaLexer', 'ScalaLexer', 'GosuLexer', 'GosuTemplateLexer', - 'GroovyLexer', 'IokeLexer', 'ClojureLexer', 'KotlinLexer', - 'XtendLexer', 'AspectJLexer', 'CeylonLexer'] + 'GroovyLexer', 'IokeLexer', 'ClojureLexer', 'ClojureScriptLexer', + 'KotlinLexer', 'XtendLexer', 'AspectJLexer', 'CeylonLexer', + 'PigLexer'] class JavaLexer(RegexLexer): @@ -66,7 +67,7 @@ class JavaLexer(RegexLexer): (r'[~\^\*!%&\[\]\(\)\{\}<>\|+=:;,./?-]', Operator), (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float), (r'0x[0-9a-fA-F]+', Number.Hex), - (r'[0-9]+L?', Number.Integer), + (r'[0-9]+(_+[0-9]+)*L?', Number.Integer), (r'\n', Text) ], 'class': [ @@ -82,7 +83,7 @@ class AspectJLexer(JavaLexer): """ For `AspectJ <http://www.eclipse.org/aspectj/>`_ source code. - *New in Pygments 1.6.* + .. versionadded:: 1.6 """ name = 'AspectJ' @@ -242,25 +243,25 @@ class ScalaLexer(RegexLexer): u'\ua760\ua762\ua764\ua766\ua768\ua76a\ua76c\ua76e\ua779\ua77b' u'\ua77d-\ua77e\ua780\ua782\ua784\ua786\ua78b\uff21-\uff3a]') - idrest = ur'%s(?:%s|[0-9])*(?:(?<=_)%s)?' % (letter, letter, op) + idrest = u'%s(?:%s|[0-9])*(?:(?<=_)%s)?' % (letter, letter, op) tokens = { 'root': [ # method names (r'(class|trait|object)(\s+)', bygroups(Keyword, Text), 'class'), - (ur"'%s" % idrest, Text.Symbol), + (u"'%s" % idrest, Text.Symbol), (r'[^\S\n]+', Text), (r'//.*?\n', Comment.Single), (r'/\*', Comment.Multiline, 'comment'), - (ur'@%s' % idrest, Name.Decorator), - (ur'(abstract|ca(?:se|tch)|d(?:ef|o)|e(?:lse|xtends)|' - ur'f(?:inal(?:ly)?|or(?:Some)?)|i(?:f|mplicit)|' - ur'lazy|match|new|override|pr(?:ivate|otected)' - ur'|re(?:quires|turn)|s(?:ealed|uper)|' - ur't(?:h(?:is|row)|ry)|va[lr]|w(?:hile|ith)|yield)\b|' + (u'@%s' % idrest, Name.Decorator), + (u'(abstract|ca(?:se|tch)|d(?:ef|o)|e(?:lse|xtends)|' + u'f(?:inal(?:ly)?|or(?:Some)?)|i(?:f|mplicit)|' + u'lazy|match|new|override|pr(?:ivate|otected)' + u'|re(?:quires|turn)|s(?:ealed|uper)|' + u't(?:h(?:is|row)|ry)|va[lr]|w(?:hile|ith)|yield)\\b|' u'(<[%:-]|=>|>:|[#=@_\u21D2\u2190])(\\b|(?=\\s)|$)', Keyword), - (ur':(?!%s)' % op, Keyword, 'type'), - (ur'%s%s\b' % (upper, idrest), Name.Class), + (u':(?!%s)' % op, Keyword, 'type'), + (u'%s%s\\b' % (upper, idrest), Name.Class), (r'(true|false|null)\b', Keyword.Constant), (r'(import|package)(\s+)', bygroups(Keyword, Text), 'import'), (r'(type)(\s+)', bygroups(Keyword, Text), 'type'), @@ -281,34 +282,34 @@ class ScalaLexer(RegexLexer): (r'\n', Text) ], 'class': [ - (ur'(%s|%s|`[^`]+`)(\s*)(\[)' % (idrest, op), + (u'(%s|%s|`[^`]+`)(\\s*)(\\[)' % (idrest, op), bygroups(Name.Class, Text, Operator), 'typeparam'), (r'\s+', Text), (r'{', Operator, '#pop'), (r'\(', Operator, '#pop'), (r'//.*?\n', Comment.Single, '#pop'), - (ur'%s|%s|`[^`]+`' % (idrest, op), Name.Class, '#pop'), + (u'%s|%s|`[^`]+`' % (idrest, op), Name.Class, '#pop'), ], 'type': [ (r'\s+', Text), (u'<[%:]|>:|[#_\u21D2]|forSome|type', Keyword), (r'([,\);}]|=>|=)(\s*)', bygroups(Operator, Text), '#pop'), (r'[\(\{]', Operator, '#push'), - (ur'((?:%s|%s|`[^`]+`)(?:\.(?:%s|%s|`[^`]+`))*)(\s*)(\[)' % + (u'((?:%s|%s|`[^`]+`)(?:\\.(?:%s|%s|`[^`]+`))*)(\\s*)(\\[)' % (idrest, op, idrest, op), bygroups(Keyword.Type, Text, Operator), ('#pop', 'typeparam')), - (ur'((?:%s|%s|`[^`]+`)(?:\.(?:%s|%s|`[^`]+`))*)(\s*)$' % + (u'((?:%s|%s|`[^`]+`)(?:\\.(?:%s|%s|`[^`]+`))*)(\\s*)$' % (idrest, op, idrest, op), bygroups(Keyword.Type, Text), '#pop'), (r'//.*?\n', Comment.Single, '#pop'), - (ur'\.|%s|%s|`[^`]+`' % (idrest, op), Keyword.Type) + (u'\\.|%s|%s|`[^`]+`' % (idrest, op), Keyword.Type) ], 'typeparam': [ (r'[\s,]+', Text), (u'<[%:]|=>|>:|[#_\u21D2]|forSome|type', Keyword), (r'([\]\)\}])', Operator, '#pop'), (r'[\(\[\{]', Operator, '#push'), - (ur'\.|%s|%s|`[^`]+`' % (idrest, op), Keyword.Type) + (u'\\.|%s|%s|`[^`]+`' % (idrest, op), Keyword.Type) ], 'comment': [ (r'[^/\*]+', Comment.Multiline), @@ -317,7 +318,7 @@ class ScalaLexer(RegexLexer): (r'[*/]', Comment.Multiline) ], 'import': [ - (ur'(%s|\.)+' % idrest, Name.Namespace, '#pop') + (u'(%s|\\.)+' % idrest, Name.Namespace, '#pop') ], } @@ -326,7 +327,7 @@ class GosuLexer(RegexLexer): """ For Gosu source code. - *New in Pygments 1.5.* + .. versionadded:: 1.5 """ name = 'Gosu' @@ -405,7 +406,7 @@ class GosuTemplateLexer(Lexer): """ For Gosu templates. - *New in Pygments 1.5.* + .. versionadded:: 1.5 """ name = 'Gosu Template' @@ -424,7 +425,7 @@ class GroovyLexer(RegexLexer): """ For `Groovy <http://groovy.codehaus.org/>`_ source code. - *New in Pygments 1.5.* + .. versionadded:: 1.5 """ name = 'Groovy' @@ -486,7 +487,7 @@ class IokeLexer(RegexLexer): For `Ioke <http://ioke.org/>`_ (a strongly typed, dynamic, prototype based programming language) source. - *New in Pygments 1.4.* + .. versionadded:: 1.4 """ name = 'Ioke' filenames = ['*.ik'] @@ -638,9 +639,9 @@ class IokeLexer(RegexLexer): r'System|Text|Tuple)(?![a-zA-Z0-9!:_?])', Name.Builtin), # functions - (ur'(generateMatchMethod|aliasMethod|\u03bb|\u028E|fnx|fn|method|' - ur'dmacro|dlecro|syntax|macro|dlecrox|lecrox|lecro|syntax)' - ur'(?![a-zA-Z0-9!:_?])', Name.Function), + (u'(generateMatchMethod|aliasMethod|\u03bb|\u028E|fnx|fn|method|' + u'dmacro|dlecro|syntax|macro|dlecrox|lecrox|lecro|syntax)' + u'(?![a-zA-Z0-9!:_?])', Name.Function), # Numbers (r'-?0[xX][0-9a-fA-F]+', Number.Hex), @@ -650,13 +651,13 @@ class IokeLexer(RegexLexer): (r'#\(', Punctuation), # Operators - (ur'(&&>>|\|\|>>|\*\*>>|:::|::|\.\.\.|===|\*\*>|\*\*=|&&>|&&=|' - ur'\|\|>|\|\|=|\->>|\+>>|!>>|<>>>|<>>|&>>|%>>|#>>|@>>|/>>|\*>>|' - ur'\?>>|\|>>|\^>>|~>>|\$>>|=>>|<<=|>>=|<=>|<\->|=~|!~|=>|\+\+|' - ur'\-\-|<=|>=|==|!=|&&|\.\.|\+=|\-=|\*=|\/=|%=|&=|\^=|\|=|<\-|' - ur'\+>|!>|<>|&>|%>|#>|\@>|\/>|\*>|\?>|\|>|\^>|~>|\$>|<\->|\->|' - ur'<<|>>|\*\*|\?\||\?&|\|\||>|<|\*|\/|%|\+|\-|&|\^|\||=|\$|!|~|' - ur'\?|#|\u2260|\u2218|\u2208|\u2209)', Operator), + (r'(&&>>|\|\|>>|\*\*>>|:::|::|\.\.\.|===|\*\*>|\*\*=|&&>|&&=|' + r'\|\|>|\|\|=|\->>|\+>>|!>>|<>>>|<>>|&>>|%>>|#>>|@>>|/>>|\*>>|' + r'\?>>|\|>>|\^>>|~>>|\$>>|=>>|<<=|>>=|<=>|<\->|=~|!~|=>|\+\+|' + r'\-\-|<=|>=|==|!=|&&|\.\.|\+=|\-=|\*=|\/=|%=|&=|\^=|\|=|<\-|' + r'\+>|!>|<>|&>|%>|#>|\@>|\/>|\*>|\?>|\|>|\^>|~>|\$>|<\->|\->|' + r'<<|>>|\*\*|\?\||\?&|\|\||>|<|\*|\/|%|\+|\-|&|\^|\||=|\$|!|~|' + u'\\?|#|\u2260|\u2218|\u2208|\u2209)', Operator), (r'(and|nand|or|xor|nor|return|import)(?![a-zA-Z0-9_!?])', Operator), @@ -676,7 +677,7 @@ class ClojureLexer(RegexLexer): """ Lexer for `Clojure <http://clojure.org/>`_ source code. - *New in Pygments 0.11.* + .. versionadded:: 0.11 """ name = 'Clojure' aliases = ['clojure', 'clj'] @@ -813,12 +814,25 @@ class ClojureLexer(RegexLexer): } +class ClojureScriptLexer(ClojureLexer): + """ + Lexer for `ClojureScript <http://clojure.org/clojurescript>`_ + source code. + + .. versionadded:: 2.0 + """ + name = 'ClojureScript' + aliases = ['clojurescript', 'cljs'] + filenames = ['*.cljs'] + mimetypes = ['text/x-clojurescript', 'application/x-clojurescript'] + + class TeaLangLexer(RegexLexer): """ For `Tea <http://teatrove.org/>`_ source code. Only used within a TeaTemplateLexer. - *New in Pygments 1.5.* + .. versionadded:: 1.5 """ flags = re.MULTILINE | re.DOTALL @@ -864,7 +878,7 @@ class CeylonLexer(RegexLexer): """ For `Ceylon <http://ceylon-lang.org/>`_ source code. - *New in Pygments 1.6.* + .. versionadded:: 1.6 """ name = 'Ceylon' @@ -945,7 +959,7 @@ class KotlinLexer(RegexLexer): For `Kotlin <http://kotlin.jetbrains.org/>`_ source code. - *New in Pygments 1.5.* + .. versionadded:: 1.5 """ name = 'Kotlin' @@ -1007,7 +1021,7 @@ class XtendLexer(RegexLexer): """ For `Xtend <http://xtend-lang.org/>`_ source code. - *New in Pygments 1.6.* + .. versionadded:: 1.6 """ name = 'Xtend' @@ -1043,7 +1057,7 @@ class XtendLexer(RegexLexer): 'class'), (r'(import)(\s+)', bygroups(Keyword.Namespace, Text), 'import'), (r"(''')", String, 'template'), - (ur"(\u00BB)", String, 'template'), + (u'(\u00BB)', String, 'template'), (r'"(\\\\|\\"|[^"])*"', String), (r"'(\\\\|\\'|[^'])*'", String), (r'[a-zA-Z_][a-zA-Z0-9_]*:', Name.Label), @@ -1062,7 +1076,73 @@ class XtendLexer(RegexLexer): ], 'template': [ (r"'''", String, '#pop'), - (ur"\u00AB", String, '#pop'), + (u'\u00AB', String, '#pop'), (r'.', String) ], } + +class PigLexer(RegexLexer): + """ + For `Pig Latin <https://pig.apache.org/>`_ source code. + + .. versionadded:: 2.0 + """ + + name = 'Pig' + aliases = ['pig'] + filenames = ['*.pig'] + mimetypes = ['text/x-pig'] + + flags = re.MULTILINE | re.IGNORECASE + + tokens = { + 'root': [ + (r'\s+', Text), + (r'--.*', Comment), + (r'/\*[\w\W]*?\*/', Comment.Multiline), + (r'\\\n', Text), + (r'\\', Text), + (r'\'(?:\\[ntbrf\\\']|\\u[0-9a-f]{4}|[^\'\\\n\r])*\'', String), + include('keywords'), + include('types'), + include('builtins'), + include('punct'), + include('operators'), + (r'[0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float), + (r'0x[0-9a-f]+', Number.Hex), + (r'[0-9]+L?', Number.Integer), + (r'\n', Text), + (r'([a-z_][a-z0-9_]*)(\s*)(\()', + bygroups(Name.Function, Text, Punctuation)), + (r'[()#:]', Text), + (r'[^(:#\'\")\s]+', Text), + (r'\S+\s+', Text) # TODO: make tests pass without \s+ + ], + 'keywords': [ + (r'(assert|and|any|all|arrange|as|asc|bag|by|cache|CASE|cat|cd|cp|' + r'%declare|%default|define|dense|desc|describe|distinct|du|dump|' + r'eval|exex|explain|filter|flatten|foreach|full|generate|group|' + r'help|if|illustrate|import|inner|input|into|is|join|kill|left|' + r'limit|load|ls|map|matches|mkdir|mv|not|null|onschema|or|order|' + r'outer|output|parallel|pig|pwd|quit|register|returns|right|rm|' + r'rmf|rollup|run|sample|set|ship|split|stderr|stdin|stdout|store|' + r'stream|through|union|using|void)\b', Keyword) + ], + 'builtins': [ + (r'(AVG|BinStorage|cogroup|CONCAT|copyFromLocal|copyToLocal|COUNT|' + r'cross|DIFF|MAX|MIN|PigDump|PigStorage|SIZE|SUM|TextLoader|' + r'TOKENIZE)\b', Name.Builtin) + ], + 'types': [ + (r'(bytearray|BIGINTEGER|BIGDECIMAL|chararray|datetime|double|float|' + r'int|long|tuple)\b', Keyword.Type) + ], + 'punct': [ + (r'[;(){}\[\]]', Punctuation), + ], + 'operators': [ + (r'[#=,./%+\-?]', Operator), + (r'(eq|gt|lt|gte|lte|neq|matches)\b', Operator), + (r'(==|<=|<|>=|>|!=)', Operator), + ], + } diff --git a/pygments/lexers/math.py b/pygments/lexers/math.py index 93c7cbd6..e7a8948b 100644 --- a/pygments/lexers/math.py +++ b/pygments/lexers/math.py @@ -9,6 +9,8 @@ :license: BSD, see LICENSE for details. """ +from __future__ import print_function + import re from pygments.util import shebang_matches @@ -24,14 +26,14 @@ from pygments.lexers import _stan_builtins __all__ = ['JuliaLexer', 'JuliaConsoleLexer', 'MuPADLexer', 'MatlabLexer', 'MatlabSessionLexer', 'OctaveLexer', 'ScilabLexer', 'NumPyLexer', 'RConsoleLexer', 'SLexer', 'JagsLexer', 'BugsLexer', 'StanLexer', - 'IDLLexer', 'RdLexer', 'IgorLexer', 'MathematicaLexer'] + 'IDLLexer', 'RdLexer', 'IgorLexer', 'MathematicaLexer', 'GAPLexer'] class JuliaLexer(RegexLexer): """ For `Julia <http://julialang.org/>`_ source code. - *New in Pygments 1.6.* + .. versionadded:: 1.6 """ name = 'Julia' aliases = ['julia','jl'] @@ -151,7 +153,7 @@ class JuliaConsoleLexer(Lexer): """ For Julia console sessions. Modeled after MatlabSessionLexer. - *New in Pygments 1.6.* + .. versionadded:: 1.6 """ name = 'Julia console' aliases = ['jlcon'] @@ -200,7 +202,7 @@ class MuPADLexer(RegexLexer): A `MuPAD <http://www.mupad.com>`_ lexer. Contributed by Christopher Creutzig <christopher@creutzig.de>. - *New in Pygments 0.8.* + .. versionadded:: 0.8 """ name = 'MuPAD' aliases = ['mupad'] @@ -270,7 +272,7 @@ class MatlabLexer(RegexLexer): """ For Matlab source code. - *New in Pygments 0.10.* + .. versionadded:: 0.10 """ name = 'Matlab' aliases = ['matlab'] @@ -388,7 +390,7 @@ class MatlabSessionLexer(Lexer): For Matlab sessions. Modeled after PythonConsoleLexer. Contributed by Ken Schutte <kschutte@csail.mit.edu>. - *New in Pygments 0.10.* + .. versionadded:: 0.10 """ name = 'Matlab session' aliases = ['matlabsession'] @@ -431,7 +433,7 @@ class MatlabSessionLexer(Lexer): yield match.start(), Generic.Output, line - print insertions + print(insertions) if curcode: # or item: for item in do_insertions( insertions, mlexer.get_tokens_unprocessed(curcode)): @@ -442,7 +444,7 @@ class OctaveLexer(RegexLexer): """ For GNU Octave source code. - *New in Pygments 1.5.* + .. versionadded:: 1.5 """ name = 'Octave' aliases = ['octave'] @@ -833,7 +835,7 @@ class ScilabLexer(RegexLexer): """ For Scilab source code. - *New in Pygments 1.5.* + .. versionadded:: 1.5 """ name = 'Scilab' aliases = ['scilab'] @@ -899,7 +901,7 @@ class NumPyLexer(PythonLexer): """ A Python lexer recognizing Numerical Python builtins. - *New in Pygments 0.10.* + .. versionadded:: 0.10 """ name = 'NumPy' @@ -1040,7 +1042,7 @@ class SLexer(RegexLexer): """ For S, S-plus, and R source code. - *New in Pygments 0.10.* + .. versionadded:: 0.10 """ name = 'S' @@ -1128,7 +1130,7 @@ class BugsLexer(RegexLexer): Pygments Lexer for `OpenBugs <http://www.openbugs.info/w/>`_ and WinBugs models. - *New in Pygments 1.6.* + .. versionadded:: 1.6 """ name = 'BUGS' @@ -1223,7 +1225,7 @@ class JagsLexer(RegexLexer): """ Pygments Lexer for JAGS. - *New in Pygments 1.6.* + .. versionadded:: 1.6 """ name = 'JAGS' @@ -1313,7 +1315,7 @@ class StanLexer(RegexLexer): Modeling Language Manual* `pdf <https://github.com/stan-dev/stan/releases/download/v2.0.1/stan-reference-2.0.1.pdf>`__ - *New in Pygments 1.6.* + .. versionadded:: 1.6 """ name = 'Stan' @@ -1386,7 +1388,7 @@ class IDLLexer(RegexLexer): """ Pygments Lexer for IDL (Interactive Data Language). - *New in Pygments 1.6.* + .. versionadded:: 1.6 """ name = 'IDL' aliases = ['idl'] @@ -1632,7 +1634,7 @@ class RdLexer(RegexLexer): Extensions <http://cran.r-project.org/doc/manuals/R-exts.html>`_ and `Parsing Rd files <developer.r-project.org/parseRd.pdf>`_. - *New in Pygments 1.6.* + .. versionadded:: 1.6 """ name = 'Rd' aliases = ['rd'] @@ -1667,7 +1669,7 @@ class IgorLexer(RegexLexer): Pygments Lexer for Igor Pro procedure files (.ipf). See http://www.wavemetrics.com/ and http://www.igorexchange.com/. - *New in Pygments 1.7.* + .. versionadded:: 2.0 """ name = 'Igor' @@ -1929,7 +1931,7 @@ class MathematicaLexer(RegexLexer): """ Lexer for `Mathematica <http://www.wolfram.com/mathematica/>`_ source code. - *New in Pygments 1.7.* + .. versionadded:: 2.0 """ name = 'Mathematica' aliases = ['mathematica', 'mma', 'nb'] @@ -1970,3 +1972,53 @@ class MathematicaLexer(RegexLexer): (r'\s+', Text.Whitespace), ], } + +class GAPLexer(RegexLexer): + """ + For `GAP <http://www.gap-system.org>`_ source code. + + .. versionadded:: 2.0 + """ + name = 'GAP' + aliases = ['gap'] + filenames = ['*.g', '*.gd', '*.gi', '*.gap'] + + tokens = { + 'root' : [ + (r'#.*$', Comment.Single), + (r'"(?:[^"\\]|\\.)*"', String), + (r'\(|\)|\[|\]|\{|\}', Punctuation), + (r'''(?x)\b(?: + if|then|elif|else|fi| + for|while|do|od| + repeat|until| + break|continue| + function|local|return|end| + rec| + quit|QUIT| + IsBound|Unbind| + TryNextMethod| + Info|Assert + )\b''', Keyword), + (r'''(?x)\b(?: + true|false|fail|infinity + )\b''', + Name.Constant), + (r'''(?x)\b(?: + (Declare|Install)([A-Z][A-Za-z]+)| + BindGlobal|BIND_GLOBAL + )\b''', + Name.Builtin), + (r'\.|,|:=|;|=|\+|-|\*|/|\^|>|<', Operator), + (r'''(?x)\b(?: + and|or|not|mod|in + )\b''', + Operator.Word), + (r'''(?x) + (?:[a-zA-Z_0-9]+|`[^`]*`) + (?:::[a-zA-Z_0-9]+|`[^`]*`)*''', Name.Variable), + (r'[0-9]+(?:\.[0-9]*)?(?:e[0-9]+)?', Number), + (r'\.[0-9]+(?:e[0-9]+)?', Number), + (r'.', Text) + ] + } diff --git a/pygments/lexers/other.py b/pygments/lexers/other.py index dc16954a..ba777e28 100644 --- a/pygments/lexers/other.py +++ b/pygments/lexers/other.py @@ -36,7 +36,7 @@ __all__ = ['BrainfuckLexer', 'BefungeLexer', 'RedcodeLexer', 'MOOCodeLexer', 'ECLLexer', 'UrbiscriptLexer', 'OpenEdgeLexer', 'BroLexer', 'MscgenLexer', 'KconfigLexer', 'VGLLexer', 'SourcePawnLexer', 'RobotFrameworkLexer', 'PuppetLexer', 'NSISLexer', 'RPMSpecLexer', - 'CbmBasicV2Lexer', 'AutoItLexer', 'RexxLexer'] + 'CbmBasicV2Lexer', 'AutoItLexer', 'RexxLexer', 'APLLexer'] class ECLLexer(RegexLexer): @@ -45,7 +45,7 @@ class ECLLexer(RegexLexer): <http://hpccsystems.com/community/docs/ecl-language-reference/html>`_ language. - *New in Pygments 1.5.* + .. versionadded:: 1.5 """ name = 'ECL' @@ -176,7 +176,7 @@ class BefungeLexer(RegexLexer): Lexer for the esoteric `Befunge <http://en.wikipedia.org/wiki/Befunge>`_ language. - *New in Pygments 0.7.* + .. versionadded:: 0.7 """ name = 'Befunge' aliases = ['befunge'] @@ -206,7 +206,7 @@ class RedcodeLexer(RegexLexer): A simple Redcode lexer based on ICWS'94. Contributed by Adam Blinkinsop <blinks@acm.org>. - *New in Pygments 0.8.* + .. versionadded:: 0.8 """ name = 'Redcode' aliases = ['redcode'] @@ -242,7 +242,7 @@ class MOOCodeLexer(RegexLexer): For `MOOCode <http://www.moo.mud.org/>`_ (the MOO scripting language). - *New in Pygments 0.9.* + .. versionadded:: 0.9 """ name = 'MOOCode' filenames = ['*.moo'] @@ -286,7 +286,7 @@ class SmalltalkLexer(RegexLexer): Contributed by Stefan Matthias Aust. Rewritten by Nils Winter. - *New in Pygments 0.10.* + .. versionadded:: 0.10 """ name = 'Smalltalk' filenames = ['*.st'] @@ -405,7 +405,7 @@ class LogtalkLexer(RegexLexer): """ For `Logtalk <http://logtalk.org/>`_ source code. - *New in Pygments 0.10.* + .. versionadded:: 0.10 """ name = 'Logtalk' @@ -633,7 +633,7 @@ class GnuplotLexer(RegexLexer): """ For `Gnuplot <http://gnuplot.info/>`_ plotting scripts. - *New in Pygments 0.11.* + .. versionadded:: 0.11 """ name = 'Gnuplot' @@ -792,7 +792,7 @@ class PovrayLexer(RegexLexer): """ For `Persistence of Vision Raytracer <http://www.povray.org/>`_ files. - *New in Pygments 0.11.* + .. versionadded:: 0.11 """ name = 'POVRay' aliases = ['pov'] @@ -1150,18 +1150,18 @@ class AppleScriptLexer(RegexLexer): tokens = { 'root': [ (r'\s+', Text), - (ur'¬\n', String.Escape), + (u'¬\\n', String.Escape), (r"'s\s+", Text), # This is a possessive, consider moving (r'(--|#).*?$', Comment), (r'\(\*', Comment.Multiline, 'comment'), (r'[\(\){}!,.:]', Punctuation), - (ur'(«)([^»]+)(»)', + (u'(«)([^»]+)(»)', bygroups(Text, Name.Builtin, Text)), (r'\b((?:considering|ignoring)\s*)' r'(application responses|case|diacriticals|hyphens|' r'numeric strings|punctuation|white space)', bygroups(Keyword, Name.Builtin)), - (ur'(-|\*|\+|&|≠|>=?|<=?|=|≥|≤|/|÷|\^)', Operator), + (u'(-|\\*|\\+|&|≠|>=?|<=?|=|≥|≤|/|÷|\\^)', Operator), (r"\b(%s)\b" % '|'.join(Operators), Operator.Word), (r'^(\s*(?:on|end)\s+)' r'(%s)' % '|'.join(StudioEvents[::-1]), @@ -1198,7 +1198,7 @@ class ModelicaLexer(RegexLexer): """ For `Modelica <http://www.modelica.org/>`_ source code. - *New in Pygments 1.1.* + .. versionadded:: 1.1 """ name = 'Modelica' aliases = ['modelica'] @@ -1217,23 +1217,27 @@ class ModelicaLexer(RegexLexer): ], 'statements': [ (r'"', String, 'string'), - (r'\'', Name, 'quoted_ident'), (r'(\d+\.\d*|\.\d+|\d+|\d.)[eE][+-]?\d+[lL]?', Number.Float), (r'(\d+\.\d*|\.\d+)', Number.Float), (r'\d+[Ll]?', Number.Integer), (r'[~!%^&*+=|?:<>/-]', Operator), - (r'[()\[\]{},.;]', Punctuation), (r'(true|false|NULL|Real|Integer|Boolean)\b', Name.Builtin), - (r'([a-zA-Z_][\w\[\]]*|\'[a-zA-Z_\+\-\*\/\^][\w]*\')' - r'(\.([a-zA-Z_\][\w\[\]]*|\'[a-zA-Z_\+\-\*\/\^][\w]*\'))+', Name.Class), + (r'([a-zA-Z_][\w]*|[\'][^\']+[\'])' + r'([\[\d,:\]]*)' + r'(\.([a-zA-Z_][\w]*|[\'][^\']+[\']))+' + r'([\[\d,:\]]*)', Name.Class), + (r'([a-zA-Z_][\w]*|[\'][^\']+[\'])' + r'([\[\d,:\]]+)', Name.Class), (r'(\'[\w\+\-\*\/\^]+\'|\w+)', Name), + (r'[()\[\]{},.;]', Punctuation), + (r'\'', Name, 'quoted_ident'), ], 'root': [ include('whitespace'), include('keywords'), + include('classes'), include('functions'), include('operators'), - include('classes'), (r'("<html>|<html>)', Name.Tag, 'html-content'), include('statements'), ], @@ -1253,15 +1257,16 @@ class ModelicaLexer(RegexLexer): r'tanh|zeros)\b', Name.Function), ], 'operators': [ - (r'(actualStream|and|assert|cardinality|change|Clock|delay|der|edge|' - r'hold|homotopy|initial|inStream|noEvent|not|or|pre|previous|reinit|' - r'return|sample|smooth|spatialDistribution|subSample|terminal|' + (r'(actualStream|and|assert|backSample|cardinality|change|Clock|' + r'delay|der|edge|hold|homotopy|initial|inStream|noClock|noEvent|' + r'not|or|pre|previous|reinit|return|sample|smooth|' + r'spatialDistribution|shiftSample|subSample|superSample|terminal|' r'terminate)\b', Name.Builtin), ], 'classes': [ - (r'(block|class|connector|end|function|model|package|' + (r'(operator)?(\s+)?(block|class|connector|end|function|model|operator|package|' r'record|type)(\s+)((?!if|when|while)[A-Za-z_]\w*|[\'][^\']+[\'])([;]?)', - bygroups(Keyword, Text, Name.Class, Text)) + bygroups(Keyword, Text, Keyword, Text, Name.Class, Text)) ], 'quoted_ident': [ (r'\'', Name, '#pop'), @@ -1286,7 +1291,7 @@ class RebolLexer(RegexLexer): """ A `REBOL <http://www.rebol.com/>`_ lexer. - *New in Pygments 1.1.* + .. versionadded:: 1.1 """ name = 'REBOL' aliases = ['rebol'] @@ -1514,7 +1519,7 @@ class ABAPLexer(RegexLexer): """ Lexer for ABAP, SAP's integrated language. - *New in Pygments 1.1.* + .. versionadded:: 1.1 """ name = 'ABAP' aliases = ['abap'] @@ -1762,17 +1767,17 @@ class GherkinLexer(RegexLexer): """ For `Gherkin <http://github.com/aslakhellesoy/gherkin/>` syntax. - *New in Pygments 1.2.* + .. versionadded:: 1.2 """ name = 'Gherkin' aliases = ['cucumber', 'gherkin'] filenames = ['*.feature'] mimetypes = ['text/x-gherkin'] - feature_keywords = ur'^(기능|機能|功能|フィーチャ|خاصية|תכונה|Функціонал|Функционалност|Функционал|Фича|Особина|Могућност|Özellik|Właściwość|Tính năng|Trajto|Savybė|Požiadavka|Požadavek|Osobina|Ominaisuus|Omadus|OH HAI|Mogućnost|Mogucnost|Jellemző|Fīča|Funzionalità|Funktionalität|Funkcionalnost|Funkcionalitāte|Funcționalitate|Functionaliteit|Functionalitate|Funcionalitat|Funcionalidade|Fonctionnalité|Fitur|Feature|Egenskap|Egenskab|Crikey|Característica|Arwedd)(:)(.*)$' - feature_element_keywords = ur'^(\s*)(시나리오 개요|시나리오|배경|背景|場景大綱|場景|场景大纲|场景|劇本大綱|劇本|テンプレ|シナリオテンプレート|シナリオテンプレ|シナリオアウトライン|シナリオ|سيناريو مخطط|سيناريو|الخلفية|תרחיש|תבנית תרחיש|רקע|Тарих|Сценарій|Сценарио|Сценарий структураси|Сценарий|Структура сценарію|Структура сценарија|Структура сценария|Скица|Рамка на сценарий|Пример|Предыстория|Предистория|Позадина|Передумова|Основа|Концепт|Контекст|Założenia|Wharrimean is|Tình huống|The thing of it is|Tausta|Taust|Tapausaihio|Tapaus|Szenariogrundriss|Szenario|Szablon scenariusza|Stsenaarium|Struktura scenarija|Skica|Skenario konsep|Skenario|Situācija|Senaryo taslağı|Senaryo|Scénář|Scénario|Schema dello scenario|Scenārijs pēc parauga|Scenārijs|Scenár|Scenaro|Scenariusz|Scenariul de şablon|Scenariul de sablon|Scenariu|Scenario Outline|Scenario Amlinellol|Scenario|Scenarijus|Scenarijaus šablonas|Scenarij|Scenarie|Rerefons|Raamstsenaarium|Primer|Pozadí|Pozadina|Pozadie|Plan du scénario|Plan du Scénario|Osnova scénáře|Osnova|Náčrt Scénáře|Náčrt Scenáru|Mate|MISHUN SRSLY|MISHUN|Kịch bản|Konturo de la scenaro|Kontext|Konteksts|Kontekstas|Kontekst|Koncept|Khung tình huống|Khung kịch bản|Háttér|Grundlage|Geçmiş|Forgatókönyv vázlat|Forgatókönyv|Fono|Esquema do Cenário|Esquema do Cenario|Esquema del escenario|Esquema de l\'escenari|Escenario|Escenari|Dis is what went down|Dasar|Contexto|Contexte|Contesto|Condiţii|Conditii|Cenário|Cenario|Cefndir|Bối cảnh|Blokes|Bakgrunn|Bakgrund|Baggrund|Background|B4|Antecedents|Antecedentes|All y\'all|Achtergrond|Abstrakt Scenario|Abstract Scenario)(:)(.*)$' - examples_keywords = ur'^(\s*)(예|例子|例|サンプル|امثلة|דוגמאות|Сценарији|Примери|Приклади|Мисоллар|Значения|Örnekler|Voorbeelden|Variantai|Tapaukset|Scenarios|Scenariji|Scenarijai|Příklady|Példák|Príklady|Przykłady|Primjeri|Primeri|Piemēri|Pavyzdžiai|Paraugs|Juhtumid|Exemplos|Exemples|Exemplele|Exempel|Examples|Esempi|Enghreifftiau|Ekzemploj|Eksempler|Ejemplos|EXAMPLZ|Dữ liệu|Contoh|Cobber|Beispiele)(:)(.*)$' - step_keywords = ur'^(\s*)(하지만|조건|먼저|만일|만약|단|그리고|그러면|那麼|那么|而且|當|当|前提|假設|假如|但是|但し|並且|もし|ならば|ただし|しかし|かつ|و |متى |لكن |عندما |ثم |بفرض |اذاً |כאשר |וגם |בהינתן |אזי |אז |אבל |Якщо |Унда |То |Припустимо, що |Припустимо |Онда |Но |Нехай |Лекин |Когато |Када |Кад |К тому же |И |Задато |Задати |Задате |Если |Допустим |Дадено |Ва |Бирок |Аммо |Али |Але |Агар |А |І |Și |És |Zatati |Zakładając |Zadato |Zadate |Zadano |Zadani |Zadan |Youse know when youse got |Youse know like when |Yna |Ya know how |Ya gotta |Y |Wun |Wtedy |When y\'all |When |Wenn |WEN |Và |Ve |Und |Un |Thì |Then y\'all |Then |Tapi |Tak |Tada |Tad |Så |Stel |Soit |Siis |Si |Sed |Se |Quando |Quand |Quan |Pryd |Pokud |Pokiaľ |Però |Pero |Pak |Oraz |Onda |Ond |Oletetaan |Og |Och |O zaman |Når |När |Niin |Nhưng |N |Mutta |Men |Mas |Maka |Majd |Mais |Maar |Ma |Lorsque |Lorsqu\'|Kun |Kuid |Kui |Khi |Keď |Ketika |Když |Kaj |Kai |Kada |Kad |Jeżeli |Ja |Ir |I CAN HAZ |I |Ha |Givun |Givet |Given y\'all |Given |Gitt |Gegeven |Gegeben sei |Fakat |Eğer ki |Etant donné |Et |Então |Entonces |Entao |En |Eeldades |E |Duota |Dun |Donitaĵo |Donat |Donada |Do |Diyelim ki |Dengan |Den youse gotta |De |Dato |Dar |Dann |Dan |Dado |Dacă |Daca |DEN |Când |Cuando |Cho |Cept |Cand |Cal |But y\'all |But |Buh |Biết |Bet |BUT |Atès |Atunci |Atesa |Anrhegedig a |Angenommen |And y\'all |And |An |Ama |Als |Alors |Allora |Ali |Aleshores |Ale |Akkor |Aber |AN |A také |A |\* )' + feature_keywords = u'^(기능|機能|功能|フィーチャ|خاصية|תכונה|Функціонал|Функционалност|Функционал|Фича|Особина|Могућност|Özellik|Właściwość|Tính năng|Trajto|Savybė|Požiadavka|Požadavek|Osobina|Ominaisuus|Omadus|OH HAI|Mogućnost|Mogucnost|Jellemző|Fīča|Funzionalità|Funktionalität|Funkcionalnost|Funkcionalitāte|Funcționalitate|Functionaliteit|Functionalitate|Funcionalitat|Funcionalidade|Fonctionnalité|Fitur|Feature|Egenskap|Egenskab|Crikey|Característica|Arwedd)(:)(.*)$' + feature_element_keywords = u'^(\\s*)(시나리오 개요|시나리오|배경|背景|場景大綱|場景|场景大纲|场景|劇本大綱|劇本|テンプレ|シナリオテンプレート|シナリオテンプレ|シナリオアウトライン|シナリオ|سيناريو مخطط|سيناريو|الخلفية|תרחיש|תבנית תרחיש|רקע|Тарих|Сценарій|Сценарио|Сценарий структураси|Сценарий|Структура сценарію|Структура сценарија|Структура сценария|Скица|Рамка на сценарий|Пример|Предыстория|Предистория|Позадина|Передумова|Основа|Концепт|Контекст|Założenia|Wharrimean is|Tình huống|The thing of it is|Tausta|Taust|Tapausaihio|Tapaus|Szenariogrundriss|Szenario|Szablon scenariusza|Stsenaarium|Struktura scenarija|Skica|Skenario konsep|Skenario|Situācija|Senaryo taslağı|Senaryo|Scénář|Scénario|Schema dello scenario|Scenārijs pēc parauga|Scenārijs|Scenár|Scenaro|Scenariusz|Scenariul de şablon|Scenariul de sablon|Scenariu|Scenario Outline|Scenario Amlinellol|Scenario|Scenarijus|Scenarijaus šablonas|Scenarij|Scenarie|Rerefons|Raamstsenaarium|Primer|Pozadí|Pozadina|Pozadie|Plan du scénario|Plan du Scénario|Osnova scénáře|Osnova|Náčrt Scénáře|Náčrt Scenáru|Mate|MISHUN SRSLY|MISHUN|Kịch bản|Konturo de la scenaro|Kontext|Konteksts|Kontekstas|Kontekst|Koncept|Khung tình huống|Khung kịch bản|Háttér|Grundlage|Geçmiş|Forgatókönyv vázlat|Forgatókönyv|Fono|Esquema do Cenário|Esquema do Cenario|Esquema del escenario|Esquema de l\'escenari|Escenario|Escenari|Dis is what went down|Dasar|Contexto|Contexte|Contesto|Condiţii|Conditii|Cenário|Cenario|Cefndir|Bối cảnh|Blokes|Bakgrunn|Bakgrund|Baggrund|Background|B4|Antecedents|Antecedentes|All y\'all|Achtergrond|Abstrakt Scenario|Abstract Scenario)(:)(.*)$' + examples_keywords = u'^(\\s*)(예|例子|例|サンプル|امثلة|דוגמאות|Сценарији|Примери|Приклади|Мисоллар|Значения|Örnekler|Voorbeelden|Variantai|Tapaukset|Scenarios|Scenariji|Scenarijai|Příklady|Példák|Príklady|Przykłady|Primjeri|Primeri|Piemēri|Pavyzdžiai|Paraugs|Juhtumid|Exemplos|Exemples|Exemplele|Exempel|Examples|Esempi|Enghreifftiau|Ekzemploj|Eksempler|Ejemplos|EXAMPLZ|Dữ liệu|Contoh|Cobber|Beispiele)(:)(.*)$' + step_keywords = u'^(\\s*)(하지만|조건|먼저|만일|만약|단|그리고|그러면|那麼|那么|而且|當|当|前提|假設|假如|但是|但し|並且|もし|ならば|ただし|しかし|かつ|و |متى |لكن |عندما |ثم |بفرض |اذاً |כאשר |וגם |בהינתן |אזי |אז |אבל |Якщо |Унда |То |Припустимо, що |Припустимо |Онда |Но |Нехай |Лекин |Когато |Када |Кад |К тому же |И |Задато |Задати |Задате |Если |Допустим |Дадено |Ва |Бирок |Аммо |Али |Але |Агар |А |І |Și |És |Zatati |Zakładając |Zadato |Zadate |Zadano |Zadani |Zadan |Youse know when youse got |Youse know like when |Yna |Ya know how |Ya gotta |Y |Wun |Wtedy |When y\'all |When |Wenn |WEN |Và |Ve |Und |Un |Thì |Then y\'all |Then |Tapi |Tak |Tada |Tad |Så |Stel |Soit |Siis |Si |Sed |Se |Quando |Quand |Quan |Pryd |Pokud |Pokiaľ |Però |Pero |Pak |Oraz |Onda |Ond |Oletetaan |Og |Och |O zaman |Når |När |Niin |Nhưng |N |Mutta |Men |Mas |Maka |Majd |Mais |Maar |Ma |Lorsque |Lorsqu\'|Kun |Kuid |Kui |Khi |Keď |Ketika |Když |Kaj |Kai |Kada |Kad |Jeżeli |Ja |Ir |I CAN HAZ |I |Ha |Givun |Givet |Given y\'all |Given |Gitt |Gegeven |Gegeben sei |Fakat |Eğer ki |Etant donné |Et |Então |Entonces |Entao |En |Eeldades |E |Duota |Dun |Donitaĵo |Donat |Donada |Do |Diyelim ki |Dengan |Den youse gotta |De |Dato |Dar |Dann |Dan |Dado |Dacă |Daca |DEN |Când |Cuando |Cho |Cept |Cand |Cal |But y\'all |But |Buh |Biết |Bet |BUT |Atès |Atunci |Atesa |Anrhegedig a |Angenommen |And y\'all |And |An |Ama |Als |Alors |Allora |Ali |Aleshores |Ale |Akkor |Aber |AN |A také |A |\* )' tokens = { 'comments': [ @@ -1874,7 +1879,7 @@ class AsymptoteLexer(RegexLexer): """ For `Asymptote <http://asymptote.sf.net/>`_ source code. - *New in Pygments 1.2.* + .. versionadded:: 1.2 """ name = 'Asymptote' aliases = ['asy', 'asymptote'] @@ -1995,7 +2000,7 @@ class PostScriptLexer(RegexLexer): <http://partners.adobe.com/public/developer/en/ps/PLRM.pdf> is the authority for this. - *New in Pygments 1.4.* + .. versionadded:: 1.4 """ name = 'PostScript' aliases = ['postscript', 'postscr'] @@ -2083,7 +2088,7 @@ class AutohotkeyLexer(RegexLexer): """ For `autohotkey <http://www.autohotkey.com/>`_ source code. - *New in Pygments 1.4.* + .. versionadded:: 1.4 """ name = 'autohotkey' aliases = ['ahk', 'autohotkey'] @@ -2263,7 +2268,7 @@ class MaqlLexer(RegexLexer): <https://secure.gooddata.com/docs/html/advanced.metric.tutorial.html>`_ scripts. - *New in Pygments 1.4.* + .. versionadded:: 1.4 """ name = 'MAQL' @@ -2322,7 +2327,7 @@ class GoodDataCLLexer(RegexLexer): Lexer for `GoodData-CL <http://github.com/gooddata/GoodData-CL/raw/master/cli/src/main/resources/com/gooddata/processor/COMMANDS.txt>`_ script files. - *New in Pygments 1.4.* + .. versionadded:: 1.4 """ name = 'GoodData-CL' @@ -2367,7 +2372,7 @@ class ProtoBufLexer(RegexLexer): Lexer for `Protocol Buffer <http://code.google.com/p/protobuf/>`_ definition files. - *New in Pygments 1.4.* + .. versionadded:: 1.4 """ name = 'Protocol Buffer' @@ -2419,7 +2424,7 @@ class HybrisLexer(RegexLexer): """ For `Hybris <http://www.hybris-lang.org>`_ source code. - *New in Pygments 1.4.* + .. versionadded:: 1.4 """ name = 'Hybris' @@ -2497,7 +2502,7 @@ class AwkLexer(RegexLexer): """ For Awk scripts. - *New in Pygments 1.5.* + .. versionadded:: 1.5 """ name = 'Awk' @@ -2523,11 +2528,11 @@ class AwkLexer(RegexLexer): 'root': [ (r'^(?=\s|/)', Text, 'slashstartsregex'), include('commentsandwhitespace'), - (r'\+\+|--|\|\||&&|in|\$|!?~|' + (r'\+\+|--|\|\||&&|in\b|\$|!?~|' r'(\*\*|[-<>+*%\^/!=])=?', Operator, 'slashstartsregex'), (r'[{(\[;,]', Punctuation, 'slashstartsregex'), (r'[})\].]', Punctuation), - (r'(break|continue|do|while|exit|for|if|' + (r'(break|continue|do|while|exit|for|if|else|' r'return)\b', Keyword, 'slashstartsregex'), (r'function\b', Keyword.Declaration, 'slashstartsregex'), (r'(atan2|cos|exp|int|log|rand|sin|sqrt|srand|gensub|gsub|index|' @@ -2551,7 +2556,7 @@ class Cfengine3Lexer(RegexLexer): """ Lexer for `CFEngine3 <http://cfengine.org>`_ policy files. - *New in Pygments 1.5.* + .. versionadded:: 1.5 """ name = 'CFEngine3' @@ -2615,7 +2620,7 @@ class SnobolLexer(RegexLexer): Recognizes the common ASCII equivalents of the original SNOBOL4 operators. Does not require spaces around binary operators. - *New in Pygments 1.5.* + .. versionadded:: 1.5 """ name = "Snobol" @@ -2679,7 +2684,7 @@ class UrbiscriptLexer(ExtendedRegexLexer): """ For UrbiScript source code. - *New in Pygments 1.5.* + .. versionadded:: 1.5 """ name = 'UrbiScript' @@ -2786,7 +2791,7 @@ class OpenEdgeLexer(RegexLexer): Lexer for `OpenEdge ABL (formerly Progress) <http://web.progress.com/en/openedge/abl.html>`_ source code. - *New in Pygments 1.5.* + .. versionadded:: 1.5 """ name = 'OpenEdge ABL' aliases = ['openedge', 'abl', 'progress'] @@ -2838,7 +2843,7 @@ class BroLexer(RegexLexer): """ For `Bro <http://bro-ids.org/>`_ scripts. - *New in Pygments 1.5.* + .. versionadded:: 1.5 """ name = 'Bro' aliases = ['bro'] @@ -2916,7 +2921,7 @@ class CbmBasicV2Lexer(RegexLexer): """ For CBM BASIC V2 sources. - *New in Pygments 1.6.* + .. versionadded:: 1.6 """ name = 'CBM BASIC V2' aliases = ['cbmbas'] @@ -2954,7 +2959,7 @@ class MscgenLexer(RegexLexer): """ For `Mscgen <http://www.mcternan.me.uk/mscgen/>`_ files. - *New in Pygments 1.6.* + .. versionadded:: 1.6 """ name = 'Mscgen' aliases = ['mscgen', 'msc'] @@ -3015,7 +3020,7 @@ class KconfigLexer(RegexLexer): """ For Linux-style Kconfig files. - *New in Pygments 1.6.* + .. versionadded:: 1.6 """ name = 'Kconfig' @@ -3090,7 +3095,7 @@ class VGLLexer(RegexLexer): For `SampleManager VGL <http://www.thermoscientific.com/samplemanager>`_ source code. - *New in Pygments 1.6.* + .. versionadded:: 1.6 """ name = 'VGL' aliases = ['vgl'] @@ -3123,7 +3128,7 @@ class SourcePawnLexer(RegexLexer): """ For SourcePawn source code with preprocessor directives. - *New in Pygments 1.6.* + .. versionadded:: 1.6 """ name = 'SourcePawn' aliases = ['sp'] @@ -3232,7 +3237,7 @@ class PuppetLexer(RegexLexer): """ For `Puppet <http://puppetlabs.com/>`__ configuration DSL. - *New in Pygments 1.6.* + .. versionadded:: 1.6 """ name = 'Puppet' aliases = ['puppet'] @@ -3313,7 +3318,7 @@ class NSISLexer(RegexLexer): """ For `NSIS <http://nsis.sourceforge.net/>`_ scripts. - *New in Pygments 1.6.* + .. versionadded:: 1.6 """ name = 'NSIS' aliases = ['nsis', 'nsi', 'nsh'] @@ -3435,9 +3440,9 @@ class NSISLexer(RegexLexer): class RPMSpecLexer(RegexLexer): """ - For RPM *.spec files + For RPM ``.spec`` files. - *New in Pygments 1.6.* + .. versionadded:: 1.6 """ name = 'RPMSpec' @@ -3513,7 +3518,7 @@ class AutoItLexer(RegexLexer): AutoIt is a freeware BASIC-like scripting language designed for automating the Windows GUI and general scripting - *New in Pygments 1.6.* + .. versionadded:: 1.6 """ name = 'AutoIt' aliases = ['autoit'] @@ -3693,7 +3698,7 @@ class RexxLexer(RegexLexer): systems. It is popular for I/O- and data based tasks and can act as glue language to bind different applications together. - *New in Pygments 1.7.* + .. versionadded:: 2.0 """ name = 'Rexx' aliases = ['rexx', 'arexx'] @@ -3736,9 +3741,9 @@ class RexxLexer(RegexLexer): r'while)\b', Keyword.Reserved), ], 'operator': [ - (ur'(-|//|/|\(|\)|\*\*|\*|\\<<|\\<|\\==|\\=|\\>>|\\>|\\|\|\||\||' - ur'&&|&|%|\+|<<=|<<|<=|<>|<|==|=|><|>=|>>=|>>|>|¬<<|¬<|¬==|¬=|' - ur'¬>>|¬>|¬|\.|,)', Operator), + (r'(-|//|/|\(|\)|\*\*|\*|\\<<|\\<|\\==|\\=|\\>>|\\>|\\|\|\||\||' + r'&&|&|%|\+|<<=|<<|<=|<>|<|==|=|><|>=|>>=|>>|>|¬<<|¬<|¬==|¬=|' + r'¬>>|¬>|¬|\.|,)', Operator), ], 'string_double': [ (r'[^"\n]+', String), @@ -3794,3 +3799,88 @@ class RexxLexer(RegexLexer): for (pattern, weight) in RexxLexer.PATTERNS_AND_WEIGHTS if pattern.search(lowerText)) + 0.01 return min(result, 1.0) + + +class APLLexer(RegexLexer): + """ + A simple APL lexer. + + .. versionadded:: 2.0 + """ + name = 'APL' + aliases = ['apl'] + filenames = ['*.apl'] + + tokens = { + 'root': [ + # Whitespace + # ========== + (r'\s+', Text), + # + # Comment + # ======= + # '⍝' is traditional; '#' is supported by GNU APL and NGN (but not Dyalog) + (u'[⍝#].*$', Comment.Single), + # + # Strings + # ======= + (r'\'((\'\')|[^\'])*\'', String.Single), + (r'"(("")|[^"])*"', String.Double), # supported by NGN APL + # + # Punctuation + # =========== + # This token type is used for diamond and parenthesis + # but not for bracket and ; (see below) + (u'[⋄◇()]', Punctuation), + # + # Array indexing + # ============== + # Since this token type is very important in APL, it is not included in + # the punctuation token type but rather in the following one + (r'[\[\];]', String.Regex), + # + # Distinguished names + # =================== + # following IBM APL2 standard + (u'⎕[A-Za-zΔ∆⍙][A-Za-zΔ∆⍙_¯0-9]*', Name.Function), + # + # Labels + # ====== + # following IBM APL2 standard + # (u'[A-Za-zΔ∆⍙][A-Za-zΔ∆⍙_¯0-9]*:', Name.Label), + # + # Variables + # ========= + # following IBM APL2 standard + (u'[A-Za-zΔ∆⍙][A-Za-zΔ∆⍙_¯0-9]*', Name.Variable), + # + # Numbers + # ======= + (u'¯?(0[Xx][0-9A-Fa-f]+|[0-9]*\.?[0-9]+([Ee][+¯]?[0-9]+)?|¯|∞)' + u'([Jj]¯?(0[Xx][0-9A-Fa-f]+|[0-9]*\.?[0-9]+([Ee][+¯]?[0-9]+)?|¯|∞))?', + Number), + # + # Operators + # ========== + (u'[\.\\\/⌿⍀¨⍣⍨⍠⍤∘]', Name.Attribute), # closest token type + (u'[+\-×÷⌈⌊∣|⍳?*⍟○!⌹<≤=>≥≠≡≢∊⍷∪∩~∨∧⍱⍲⍴,⍪⌽⊖⍉↑↓⊂⊃⌷⍋⍒⊤⊥⍕⍎⊣⊢⍁⍂≈⌸⍯↗]', + Operator), + # + # Constant + # ======== + (u'⍬', Name.Constant), + # + # Quad symbol + # =========== + (u'[⎕⍞]', Name.Variable.Global), + # + # Arrows left/right + # ================= + (u'[←→]', Keyword.Declaration), + # + # D-Fn + # ==== + (u'[⍺⍵⍶⍹∇:]', Name.Builtin.Pseudo), + (r'[{}]', Keyword.Type), + ], + } diff --git a/pygments/lexers/parsers.py b/pygments/lexers/parsers.py index 80d52ac4..fc8cbb6f 100644 --- a/pygments/lexers/parsers.py +++ b/pygments/lexers/parsers.py @@ -38,7 +38,7 @@ class RagelLexer(RegexLexer): fragments of Ragel. For ``.rl`` files, use RagelEmbeddedLexer instead (or one of the language-specific subclasses). - *New in Pygments 1.1.* + .. versionadded:: 1.1 """ name = 'Ragel' @@ -132,7 +132,7 @@ class RagelEmbeddedLexer(RegexLexer): This will only highlight Ragel statements. If you want host language highlighting then call the language-specific Ragel lexer. - *New in Pygments 1.1.* + .. versionadded:: 1.1 """ name = 'Embedded Ragel' @@ -212,7 +212,7 @@ class RagelRubyLexer(DelegatingLexer): """ A lexer for `Ragel`_ in a Ruby host file. - *New in Pygments 1.1.* + .. versionadded:: 1.1 """ name = 'Ragel in Ruby Host' @@ -231,7 +231,7 @@ class RagelCLexer(DelegatingLexer): """ A lexer for `Ragel`_ in a C host file. - *New in Pygments 1.1.* + .. versionadded:: 1.1 """ name = 'Ragel in C Host' @@ -250,7 +250,7 @@ class RagelDLexer(DelegatingLexer): """ A lexer for `Ragel`_ in a D host file. - *New in Pygments 1.1.* + .. versionadded:: 1.1 """ name = 'Ragel in D Host' @@ -268,7 +268,7 @@ class RagelCppLexer(DelegatingLexer): """ A lexer for `Ragel`_ in a CPP host file. - *New in Pygments 1.1.* + .. versionadded:: 1.1 """ name = 'Ragel in CPP Host' @@ -286,7 +286,7 @@ class RagelObjectiveCLexer(DelegatingLexer): """ A lexer for `Ragel`_ in an Objective C host file. - *New in Pygments 1.1.* + .. versionadded:: 1.1 """ name = 'Ragel in Objective C Host' @@ -306,7 +306,7 @@ class RagelJavaLexer(DelegatingLexer): """ A lexer for `Ragel`_ in a Java host file. - *New in Pygments 1.1.* + .. versionadded:: 1.1 """ name = 'Ragel in Java Host' @@ -327,7 +327,7 @@ class AntlrLexer(RegexLexer): Should not be called directly, instead use DelegatingLexer for your target language. - *New in Pygments 1.1.* + .. versionadded:: 1.1 .. _ANTLR: http://www.antlr.org/ """ @@ -524,7 +524,7 @@ class AntlrLexer(RegexLexer): # """ # ANTLR with C Target # -# *New in Pygments 1.1* +# .. versionadded:: 1.1 # """ # # name = 'ANTLR With C Target' @@ -541,7 +541,7 @@ class AntlrCppLexer(DelegatingLexer): """ `ANTLR`_ with CPP Target - *New in Pygments 1.1.* + .. versionadded:: 1.1 """ name = 'ANTLR With CPP Target' @@ -560,7 +560,7 @@ class AntlrObjectiveCLexer(DelegatingLexer): """ `ANTLR`_ with Objective-C Target - *New in Pygments 1.1.* + .. versionadded:: 1.1 """ name = 'ANTLR With ObjectiveC Target' @@ -580,7 +580,7 @@ class AntlrCSharpLexer(DelegatingLexer): """ `ANTLR`_ with C# Target - *New in Pygments 1.1.* + .. versionadded:: 1.1 """ name = 'ANTLR With C# Target' @@ -600,7 +600,7 @@ class AntlrPythonLexer(DelegatingLexer): """ `ANTLR`_ with Python Target - *New in Pygments 1.1.* + .. versionadded:: 1.1 """ name = 'ANTLR With Python Target' @@ -620,7 +620,7 @@ class AntlrJavaLexer(DelegatingLexer): """ `ANTLR`_ with Java Target - *New in Pygments 1.1* + .. versionadded:: 1. """ name = 'ANTLR With Java Target' @@ -640,7 +640,7 @@ class AntlrRubyLexer(DelegatingLexer): """ `ANTLR`_ with Ruby Target - *New in Pygments 1.1.* + .. versionadded:: 1.1 """ name = 'ANTLR With Ruby Target' @@ -660,7 +660,7 @@ class AntlrPerlLexer(DelegatingLexer): """ `ANTLR`_ with Perl Target - *New in Pygments 1.1.* + .. versionadded:: 1.1 """ name = 'ANTLR With Perl Target' @@ -680,7 +680,7 @@ class AntlrActionScriptLexer(DelegatingLexer): """ `ANTLR`_ with ActionScript Target - *New in Pygments 1.1.* + .. versionadded:: 1.1 """ name = 'ANTLR With ActionScript Target' @@ -700,7 +700,7 @@ class TreetopBaseLexer(RegexLexer): A base lexer for `Treetop <http://treetop.rubyforge.org/>`_ grammars. Not for direct use; use TreetopLexer instead. - *New in Pygments 1.6.* + .. versionadded:: 1.6 """ tokens = { @@ -767,7 +767,7 @@ class TreetopLexer(DelegatingLexer): """ A lexer for `Treetop <http://treetop.rubyforge.org/>`_ grammars. - *New in Pygments 1.6.* + .. versionadded:: 1.6 """ name = 'Treetop' diff --git a/pygments/lexers/shell.py b/pygments/lexers/shell.py index 4376611a..b069b375 100644 --- a/pygments/lexers/shell.py +++ b/pygments/lexers/shell.py @@ -27,7 +27,7 @@ class BashLexer(RegexLexer): """ Lexer for (ba|k|)sh shell scripts. - *New in Pygments 0.6.* + .. versionadded:: 0.6 """ name = 'Bash' @@ -111,7 +111,7 @@ class BashSessionLexer(Lexer): """ Lexer for simplistic shell sessions. - *New in Pygments 1.1.* + .. versionadded:: 1.1 """ name = 'Bash Session' @@ -162,7 +162,7 @@ class ShellSessionLexer(Lexer): """ Lexer for shell sessions that works with different command prompts - *New in Pygments 1.6.* + .. versionadded:: 1.6 """ name = 'Shell Session' @@ -208,7 +208,7 @@ class BatchLexer(RegexLexer): """ Lexer for the DOS/Windows Batch file format. - *New in Pygments 0.7.* + .. versionadded:: 0.7 """ name = 'Batchfile' aliases = ['bat', 'batch', 'dosbatch', 'winbatch'] @@ -264,7 +264,7 @@ class TcshLexer(RegexLexer): """ Lexer for tcsh scripts. - *New in Pygments 0.10.* + .. versionadded:: 0.10 """ name = 'Tcsh' @@ -331,7 +331,7 @@ class PowerShellLexer(RegexLexer): """ For Windows PowerShell code. - *New in Pygments 1.5.* + .. versionadded:: 1.5 """ name = 'PowerShell' aliases = ['powershell', 'posh', 'ps1', 'psm1'] diff --git a/pygments/lexers/special.py b/pygments/lexers/special.py index d7fe6b53..9ea2e22c 100644 --- a/pygments/lexers/special.py +++ b/pygments/lexers/special.py @@ -10,11 +10,10 @@ """ import re -import cStringIO from pygments.lexer import Lexer from pygments.token import Token, Error, Text -from pygments.util import get_choice_opt, b +from pygments.util import get_choice_opt, text_type, BytesIO __all__ = ['TextLexer', 'RawTokenLexer'] @@ -35,7 +34,7 @@ class TextLexer(Lexer): _ttype_cache = {} -line_re = re.compile(b('.*?\n')) +line_re = re.compile(b'.*?\n') class RawTokenLexer(Lexer): """ @@ -60,12 +59,12 @@ class RawTokenLexer(Lexer): Lexer.__init__(self, **options) def get_tokens(self, text): - if isinstance(text, unicode): + if isinstance(text, text_type): # raw token stream never has any non-ASCII characters text = text.encode('ascii') if self.compress == 'gz': import gzip - gzipfile = gzip.GzipFile('', 'rb', 9, cStringIO.StringIO(text)) + gzipfile = gzip.GzipFile('', 'rb', 9, BytesIO(text)) text = gzipfile.read() elif self.compress == 'bz2': import bz2 @@ -73,7 +72,7 @@ class RawTokenLexer(Lexer): # do not call Lexer.get_tokens() because we do not want Unicode # decoding to occur, and stripping is not optional. - text = text.strip(b('\n')) + b('\n') + text = text.strip(b'\n') + b'\n' for i, t, v in self.get_tokens_unprocessed(text): yield t, v @@ -81,7 +80,7 @@ class RawTokenLexer(Lexer): length = 0 for match in line_re.finditer(text): try: - ttypestr, val = match.group().split(b('\t'), 1) + ttypestr, val = match.group().split(b'\t', 1) except ValueError: val = match.group().decode(self.encoding) ttype = Error diff --git a/pygments/lexers/sql.py b/pygments/lexers/sql.py index 94a131ff..73180772 100644 --- a/pygments/lexers/sql.py +++ b/pygments/lexers/sql.py @@ -42,8 +42,9 @@ import re from pygments.lexer import Lexer, RegexLexer, do_insertions, bygroups from pygments.token import Punctuation, \ - Text, Comment, Operator, Keyword, Name, String, Number, Generic + Text, Comment, Operator, Keyword, Name, String, Number, Generic from pygments.lexers import get_lexer_by_name, ClassNotFound +from pygments.util import iteritems from pygments.lexers._postgres_builtins import KEYWORDS, DATATYPES, \ PSEUDO_TYPES, PLPGSQL_KEYWORDS @@ -124,7 +125,7 @@ class PostgresLexer(PostgresBase, RegexLexer): """ Lexer for the PostgreSQL dialect of SQL. - *New in Pygments 1.5.* + .. versionadded:: 1.5 """ name = 'PostgreSQL SQL dialect' @@ -169,14 +170,14 @@ class PlPgsqlLexer(PostgresBase, RegexLexer): """ Handle the extra syntax in Pl/pgSQL language. - *New in Pygments 1.5.* + .. versionadded:: 1.5 """ name = 'PL/pgSQL' aliases = ['plpgsql'] mimetypes = ['text/x-plpgsql'] flags = re.IGNORECASE - tokens = dict((k, l[:]) for (k, l) in PostgresLexer.tokens.iteritems()) + tokens = dict((k, l[:]) for (k, l) in iteritems(PostgresLexer.tokens)) # extend the keywords list for i, pattern in enumerate(tokens['root']): @@ -210,7 +211,7 @@ class PsqlRegexLexer(PostgresBase, RegexLexer): aliases = [] # not public flags = re.IGNORECASE - tokens = dict((k, l[:]) for (k, l) in PostgresLexer.tokens.iteritems()) + tokens = dict((k, l[:]) for (k, l) in iteritems(PostgresLexer.tokens)) tokens['root'].append( (r'\\[^\s]+', Keyword.Pseudo, 'psql-command')) @@ -244,19 +245,20 @@ class lookahead(object): def send(self, i): self._nextitem = i return i - def next(self): + def __next__(self): if self._nextitem is not None: ni = self._nextitem self._nextitem = None return ni - return self.iter.next() + return next(self.iter) + next = __next__ class PostgresConsoleLexer(Lexer): """ Lexer for psql sessions. - *New in Pygments 1.5.* + .. versionadded:: 1.5 """ name = 'PostgreSQL console (psql)' @@ -277,7 +279,7 @@ class PostgresConsoleLexer(Lexer): insertions = [] while 1: try: - line = lines.next() + line = next(lines) except StopIteration: # allow the emission of partially collected items # the repl loop will be broken below @@ -314,7 +316,7 @@ class PostgresConsoleLexer(Lexer): # Emit the output lines out_token = Generic.Output while 1: - line = lines.next() + line = next(lines) mprompt = re_prompt.match(line) if mprompt is not None: # push the line back to have it processed by the prompt @@ -523,7 +525,7 @@ class SqliteConsoleLexer(Lexer): """ Lexer for example sessions using sqlite3. - *New in Pygments 0.11.* + .. versionadded:: 0.11 """ name = 'sqlite3con' diff --git a/pygments/lexers/templates.py b/pygments/lexers/templates.py index 987faee8..72f81d63 100644 --- a/pygments/lexers/templates.py +++ b/pygments/lexers/templates.py @@ -36,9 +36,10 @@ __all__ = ['HtmlPhpLexer', 'XmlPhpLexer', 'CssPhpLexer', 'MakoCssLexer', 'JspLexer', 'CheetahLexer', 'CheetahHtmlLexer', 'CheetahXmlLexer', 'CheetahJavascriptLexer', 'EvoqueLexer', 'EvoqueHtmlLexer', 'EvoqueXmlLexer', 'ColdfusionLexer', - 'ColdfusionHtmlLexer', 'VelocityLexer', 'VelocityHtmlLexer', - 'VelocityXmlLexer', 'SspLexer', 'TeaTemplateLexer', 'LassoHtmlLexer', - 'LassoXmlLexer', 'LassoCssLexer', 'LassoJavascriptLexer'] + 'ColdfusionHtmlLexer', 'ColdfusionCFCLexer', 'VelocityLexer', + 'VelocityHtmlLexer', 'VelocityXmlLexer', 'SspLexer', + 'TeaTemplateLexer', 'LassoHtmlLexer', 'LassoXmlLexer', + 'LassoCssLexer', 'LassoJavascriptLexer'] class ErbLexer(Lexer): @@ -399,7 +400,7 @@ class MyghtyLexer(RegexLexer): Generic `myghty templates`_ lexer. Code that isn't Myghty markup is yielded as `Token.Other`. - *New in Pygments 0.6.* + .. versionadded:: 0.6 .. _myghty templates: http://www.myghty.org/ """ @@ -447,7 +448,7 @@ class MyghtyHtmlLexer(DelegatingLexer): Subclass of the `MyghtyLexer` that highlights unlexer data with the `HtmlLexer`. - *New in Pygments 0.6.* + .. versionadded:: 0.6 """ name = 'HTML+Myghty' @@ -464,7 +465,7 @@ class MyghtyXmlLexer(DelegatingLexer): Subclass of the `MyghtyLexer` that highlights unlexer data with the `XmlLexer`. - *New in Pygments 0.6.* + .. versionadded:: 0.6 """ name = 'XML+Myghty' @@ -481,7 +482,7 @@ class MyghtyJavascriptLexer(DelegatingLexer): Subclass of the `MyghtyLexer` that highlights unlexer data with the `JavascriptLexer`. - *New in Pygments 0.6.* + .. versionadded:: 0.6 """ name = 'JavaScript+Myghty' @@ -500,7 +501,7 @@ class MyghtyCssLexer(DelegatingLexer): Subclass of the `MyghtyLexer` that highlights unlexer data with the `CssLexer`. - *New in Pygments 0.6.* + .. versionadded:: 0.6 """ name = 'CSS+Myghty' @@ -519,7 +520,7 @@ class MasonLexer(RegexLexer): .. _mason templates: http://www.masonhq.com/ - *New in Pygments 1.4.* + .. versionadded:: 1.4 """ name = 'Mason' aliases = ['mason'] @@ -572,7 +573,7 @@ class MakoLexer(RegexLexer): Generic `mako templates`_ lexer. Code that isn't Mako markup is yielded as `Token.Other`. - *New in Pygments 0.7.* + .. versionadded:: 0.7 .. _mako templates: http://www.makotemplates.org/ """ @@ -640,7 +641,7 @@ class MakoHtmlLexer(DelegatingLexer): Subclass of the `MakoLexer` that highlights unlexed data with the `HtmlLexer`. - *New in Pygments 0.7.* + .. versionadded:: 0.7 """ name = 'HTML+Mako' @@ -656,7 +657,7 @@ class MakoXmlLexer(DelegatingLexer): Subclass of the `MakoLexer` that highlights unlexer data with the `XmlLexer`. - *New in Pygments 0.7.* + .. versionadded:: 0.7 """ name = 'XML+Mako' @@ -672,7 +673,7 @@ class MakoJavascriptLexer(DelegatingLexer): Subclass of the `MakoLexer` that highlights unlexer data with the `JavascriptLexer`. - *New in Pygments 0.7.* + .. versionadded:: 0.7 """ name = 'JavaScript+Mako' @@ -690,7 +691,7 @@ class MakoCssLexer(DelegatingLexer): Subclass of the `MakoLexer` that highlights unlexer data with the `CssLexer`. - *New in Pygments 0.7.* + .. versionadded:: 0.7 """ name = 'CSS+Mako' @@ -1343,7 +1344,7 @@ class JspRootLexer(RegexLexer): Base for the `JspLexer`. Yields `Token.Other` for area outside of JSP tags. - *New in Pygments 0.7.* + .. versionadded:: 0.7 """ tokens = { @@ -1367,7 +1368,7 @@ class JspLexer(DelegatingLexer): """ Lexer for Java Server Pages. - *New in Pygments 0.7.* + .. versionadded:: 0.7 """ name = 'Java Server Page' aliases = ['jsp'] @@ -1390,7 +1391,7 @@ class EvoqueLexer(RegexLexer): """ For files using the Evoque templating system. - *New in Pygments 1.1.* + .. versionadded:: 1.1 """ name = 'Evoque' aliases = ['evoque'] @@ -1443,7 +1444,7 @@ class EvoqueHtmlLexer(DelegatingLexer): Subclass of the `EvoqueLexer` that highlights unlexed data with the `HtmlLexer`. - *New in Pygments 1.1.* + .. versionadded:: 1.1 """ name = 'HTML+Evoque' aliases = ['html+evoque'] @@ -1459,7 +1460,7 @@ class EvoqueXmlLexer(DelegatingLexer): Subclass of the `EvoqueLexer` that highlights unlexed data with the `XmlLexer`. - *New in Pygments 1.1.* + .. versionadded:: 1.1 """ name = 'XML+Evoque' aliases = ['xml+evoque'] @@ -1478,23 +1479,28 @@ class ColdfusionLexer(RegexLexer): aliases = ['cfs'] filenames = [] mimetypes = [] - flags = re.IGNORECASE | re.MULTILINE + flags = re.IGNORECASE tokens = { 'root': [ - (r'//.*', Comment), + (r'//.*?\n', Comment.Single), + (r'/\*(?:.|\n)*?\*/', Comment.Multiline), (r'\+\+|--', Operator), (r'[-+*/^&=!]', Operator), - (r'<=|>=|<|>', Operator), + (r'<=|>=|<|>|==', Operator), (r'mod\b', Operator), (r'(eq|lt|gt|lte|gte|not|is|and|or)\b', Operator), (r'\|\||&&', Operator), + (r'\?', Operator), (r'"', String.Double, 'string'), # There is a special rule for allowing html in single quoted # strings, evidently. (r"'.*?'", String.Single), (r'\d+', Number), - (r'(if|else|len|var|case|default|break|switch)\b', Keyword), + (r'(if|else|len|var|case|default|break|switch|component|property|function|do|try|catch|in|continue|for|return|while)\b', Keyword), + (r'(required|any|array|binary|boolean|component|date|guid|numeric|query|string|struct|uuid|xml)\b', Keyword), + (r'(true|false|null)\b', Keyword.Constant), + (r'(application|session|client|cookie|super|this|variables|arguments)\b', Name.Constant), (r'([A-Za-z_$][A-Za-z0-9_.]*)(\s*)(\()', bygroups(Name.Function, Text, Punctuation)), (r'[A-Za-z_$][A-Za-z0-9_.]*', Name.Variable), @@ -1558,7 +1564,7 @@ class ColdfusionHtmlLexer(DelegatingLexer): """ name = 'Coldfusion HTML' aliases = ['cfm'] - filenames = ['*.cfm', '*.cfml', '*.cfc'] + filenames = ['*.cfm', '*.cfml'] mimetypes = ['application/x-coldfusion'] def __init__(self, **options): @@ -1566,11 +1572,25 @@ class ColdfusionHtmlLexer(DelegatingLexer): **options) +class ColdfusionCFCLexer(DelegatingLexer): + """ + Coldfusion markup/script components + """ + name = 'Coldfusion CFC' + aliases = ['cfc'] + filenames = ['*.cfc'] + mimetypes = [] + + def __init__(self, **options): + super(ColdfusionCFCLexer, self).__init__(ColdfusionHtmlLexer, ColdfusionLexer, + **options) + + class SspLexer(DelegatingLexer): """ Lexer for Scalate Server Pages. - *New in Pygments 1.4.* + .. versionadded:: 1.4 """ name = 'Scalate Server Page' aliases = ['ssp'] @@ -1596,7 +1616,7 @@ class TeaTemplateRootLexer(RegexLexer): Base for the `TeaTemplateLexer`. Yields `Token.Other` for area outside of code blocks. - *New in Pygments 1.5.* + .. versionadded:: 1.5 """ tokens = { @@ -1617,7 +1637,7 @@ class TeaTemplateLexer(DelegatingLexer): """ Lexer for `Tea Templates <http://teatrove.org/>`_. - *New in Pygments 1.5.* + .. versionadded:: 1.5 """ name = 'Tea' aliases = ['tea'] @@ -1644,7 +1664,7 @@ class LassoHtmlLexer(DelegatingLexer): Nested JavaScript and CSS is also highlighted. - *New in Pygments 1.6.* + .. versionadded:: 1.6 """ name = 'HTML+Lasso' @@ -1672,7 +1692,7 @@ class LassoXmlLexer(DelegatingLexer): Subclass of the `LassoLexer` which highlights unhandled data with the `XmlLexer`. - *New in Pygments 1.6.* + .. versionadded:: 1.6 """ name = 'XML+Lasso' @@ -1696,7 +1716,7 @@ class LassoCssLexer(DelegatingLexer): Subclass of the `LassoLexer` which highlights unhandled data with the `CssLexer`. - *New in Pygments 1.6.* + .. versionadded:: 1.6 """ name = 'CSS+Lasso' @@ -1722,7 +1742,7 @@ class LassoJavascriptLexer(DelegatingLexer): Subclass of the `LassoLexer` which highlights unhandled data with the `JavascriptLexer`. - *New in Pygments 1.6.* + .. versionadded:: 1.6 """ name = 'JavaScript+Lasso' diff --git a/pygments/lexers/text.py b/pygments/lexers/text.py index e4c5656b..82cc82b3 100644 --- a/pygments/lexers/text.py +++ b/pygments/lexers/text.py @@ -62,7 +62,7 @@ class RegeditLexer(RegexLexer): <http://en.wikipedia.org/wiki/Windows_Registry#.REG_files>`_ files produced by regedit. - *New in Pygments 1.6.* + .. versionadded:: 1.6 """ name = 'reg' @@ -103,7 +103,7 @@ class PropertiesLexer(RegexLexer): """ Lexer for configuration files in Java's properties format. - *New in Pygments 1.4.* + .. versionadded:: 1.4 """ name = 'Properties' @@ -125,7 +125,7 @@ class SourcesListLexer(RegexLexer): """ Lexer that highlights debian sources.list files. - *New in Pygments 0.7.* + .. versionadded:: 0.7 """ name = 'Debian Sourcelist' @@ -218,7 +218,7 @@ class BaseMakefileLexer(RegexLexer): """ Lexer for simple Makefiles (no preprocessing). - *New in Pygments 0.10.* + .. versionadded:: 0.10 """ name = 'Base Makefile' @@ -228,8 +228,10 @@ class BaseMakefileLexer(RegexLexer): tokens = { 'root': [ + # recipes (need to allow spaces because of expandtabs) (r'^(?:[\t ]+.*\n|\n)+', using(BashLexer)), - (r'\$\((?:.*\\\n|.*\n)+', using(BashLexer)), + # special variables + (r'\$[<@$+%?|*]', Keyword), (r'\s+', Text), (r'#.*?\n', Comment), (r'(export)(\s+)(?=[a-zA-Z0-9_${}\t -]+\n)', @@ -244,7 +246,15 @@ class BaseMakefileLexer(RegexLexer): # targets (r'([^\n:]+)(:+)([ \t]*)', bygroups(Name.Function, Operator, Text), 'block-header'), - # TODO: add paren handling (grr) + # expansions + (r'\$\(', Keyword, 'expansion'), + ], + 'expansion': [ + (r'[^$a-zA-Z_)]+', Text), + (r'[a-zA-Z_]+', Name.Variable), + (r'\$', Keyword), + (r'\(', Keyword, '#push'), + (r'\)', Keyword, '#pop'), ], 'export': [ (r'[a-zA-Z0-9_${}-]+', Name.Variable), @@ -252,12 +262,13 @@ class BaseMakefileLexer(RegexLexer): (r'\s+', Text), ], 'block-header': [ - (r'[^,\\\n#]+', Number), - (r',', Punctuation), - (r'#.*?\n', Comment), + (r'[,|]', Punctuation), + (r'#.*?\n', Comment, '#pop'), (r'\\\n', Text), # line continuation - (r'\\.', Text), - (r'(?:[\t ]+.*\n|\n)+', using(BashLexer), '#pop'), + (r'\$\(', Keyword, 'expansion'), + (r'[a-zA-Z_]+', Name), + (r'\n', Text, '#pop'), + (r'.', Text), ], } @@ -303,7 +314,7 @@ class DarcsPatchLexer(RegexLexer): format. Examples of this format are derived by commands such as ``darcs annotate --patch`` and ``darcs send``. - *New in Pygments 0.10.* + .. versionadded:: 0.10 """ name = 'Darcs Patch' aliases = ['dpatch'] @@ -416,7 +427,7 @@ class BBCodeLexer(RegexLexer): """ A lexer that highlights BBCode(-like) syntax. - *New in Pygments 0.6.* + .. versionadded:: 0.6 """ name = 'BBCode' @@ -507,7 +518,7 @@ class GroffLexer(RegexLexer): Lexer for the (g)roff typesetting language, supporting groff extensions. Mainly useful for highlighting manpage sources. - *New in Pygments 0.6.* + .. versionadded:: 0.6 """ name = 'Groff' @@ -562,7 +573,7 @@ class ApacheConfLexer(RegexLexer): Lexer for configuration files following the Apache config file format. - *New in Pygments 0.6.* + .. versionadded:: 0.6 """ name = 'ApacheConf' @@ -601,7 +612,7 @@ class MoinWikiLexer(RegexLexer): """ For MoinMoin (and Trac) Wiki markup. - *New in Pygments 0.7.* + .. versionadded:: 0.7 """ name = 'MoinMoin/Trac Wiki markup' @@ -646,7 +657,7 @@ class RstLexer(RegexLexer): """ For `reStructuredText <http://docutils.sf.net/rst.html>`_ markup. - *New in Pygments 0.7.* + .. versionadded:: 0.7 Additional options accepted: @@ -654,7 +665,9 @@ class RstLexer(RegexLexer): Highlight the contents of ``.. sourcecode:: language``, ``.. code:: language`` and ``.. code-block:: language`` directives with a lexer for the given language (default: - ``True``). *New in Pygments 0.8.* + ``True``). + + .. versionadded:: 0.8 """ name = 'reStructuredText' aliases = ['rst', 'rest', 'restructuredtext'] @@ -813,7 +826,7 @@ class VimLexer(RegexLexer): """ Lexer for VimL script files. - *New in Pygments 0.8.* + .. versionadded:: 0.8 """ name = 'VimL' aliases = ['vim'] @@ -830,7 +843,7 @@ class VimLexer(RegexLexer): # TODO: regexes can have other delims (r'/(\\\\|\\/|[^\n/])*/', String.Regex), (r'"(\\\\|\\"|[^\n"])*"', String.Double), - (r"'(\\\\|\\'|[^\n'])*'", String.Single), + (r"'(''|[^\n'])*'", String.Single), # Who decided that doublequote was a good comment character?? (r'(?<=\s)"[^\-:.%#=*].*', Comment), @@ -897,7 +910,7 @@ class GettextLexer(RegexLexer): """ Lexer for Gettext catalog files. - *New in Pygments 0.9.* + .. versionadded:: 0.9 """ name = 'Gettext Catalog' aliases = ['pot', 'po'] @@ -925,7 +938,7 @@ class SquidConfLexer(RegexLexer): """ Lexer for `squid <http://www.squid-cache.org/>`_ configuration files. - *New in Pygments 0.9.* + .. versionadded:: 0.9 """ name = 'SquidConf' @@ -1057,7 +1070,7 @@ class DebianControlLexer(RegexLexer): """ Lexer for Debian ``control`` files and ``apt-cache show <pkg>`` outputs. - *New in Pygments 0.9.* + .. versionadded:: 0.9 """ name = 'Debian Control file' aliases = ['control', 'debcontrol'] @@ -1127,7 +1140,7 @@ class YamlLexer(ExtendedRegexLexer): Lexer for `YAML <http://yaml.org/>`_, a human-friendly data serialization language. - *New in Pygments 0.11.* + .. versionadded:: 0.11 """ name = 'YAML' @@ -1529,7 +1542,7 @@ class LighttpdConfLexer(RegexLexer): """ Lexer for `Lighttpd <http://lighttpd.net/>`_ configuration files. - *New in Pygments 0.11.* + .. versionadded:: 0.11 """ name = 'Lighttpd configuration file' aliases = ['lighty', 'lighttpd'] @@ -1557,7 +1570,7 @@ class NginxConfLexer(RegexLexer): """ Lexer for `Nginx <http://nginx.net/>`_ configuration files. - *New in Pygments 0.11.* + .. versionadded:: 0.11 """ name = 'Nginx configuration file' aliases = ['nginx'] @@ -1603,7 +1616,7 @@ class CMakeLexer(RegexLexer): """ Lexer for `CMake <http://cmake.org/Wiki/CMake>`_ files. - *New in Pygments 1.2.* + .. versionadded:: 1.2 """ name = 'CMake' aliases = ['cmake'] @@ -1673,7 +1686,7 @@ class HttpLexer(RegexLexer): """ Lexer for HTTP sessions. - *New in Pygments 1.5.* + .. versionadded:: 1.5 """ name = 'HTTP' @@ -1742,7 +1755,7 @@ class PyPyLogLexer(RegexLexer): """ Lexer for PyPy log files. - *New in Pygments 1.5.* + .. versionadded:: 1.5 """ name = "PyPy Log" aliases = ["pypylog", "pypy"] @@ -1814,7 +1827,7 @@ class HxmlLexer(RegexLexer): """ Lexer for `haXe build <http://haxe.org/doc/compiler>`_ files. - *New in Pygments 1.6.* + .. versionadded:: 1.6 """ name = 'Hxml' aliases = ['haxeml', 'hxml'] @@ -1857,7 +1870,7 @@ class EbnfLexer(RegexLexer): <http://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_Form>`_ grammars. - *New in Pygments 1.7.* + .. versionadded:: 2.0 """ name = 'EBNF' diff --git a/pygments/lexers/web.py b/pygments/lexers/web.py index 10522aa4..7d3073f1 100644 --- a/pygments/lexers/web.py +++ b/pygments/lexers/web.py @@ -13,11 +13,11 @@ import re import copy from pygments.lexer import RegexLexer, ExtendedRegexLexer, bygroups, using, \ - include, this + include, this from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ - Number, Other, Punctuation, Literal + Number, Other, Punctuation, Literal from pygments.util import get_bool_opt, get_list_opt, looks_like_xml, \ - html_doctype_matches, unirange + html_doctype_matches, unirange, iteritems from pygments.lexers.agile import RubyLexer from pygments.lexers.compiled import ScalaLexer @@ -28,7 +28,7 @@ __all__ = ['HtmlLexer', 'XmlLexer', 'JavascriptLexer', 'JsonLexer', 'CssLexer', 'ObjectiveJLexer', 'CoffeeScriptLexer', 'LiveScriptLexer', 'DuelLexer', 'ScamlLexer', 'JadeLexer', 'XQueryLexer', 'DtdLexer', 'DartLexer', 'LassoLexer', 'QmlLexer', 'TypeScriptLexer', - 'KalLexer'] + 'KalLexer', 'CirruLexer', 'MaskLexer', 'ZephirLexer'] class JavascriptLexer(RegexLexer): @@ -95,7 +95,7 @@ class JsonLexer(RegexLexer): """ For JSON data structures. - *New in Pygments 1.5.* + .. versionadded:: 1.5 """ name = 'JSON' @@ -178,7 +178,7 @@ class ActionScriptLexer(RegexLexer): """ For ActionScript source code. - *New in Pygments 0.9.* + .. versionadded:: 0.9 """ name = 'ActionScript' @@ -262,7 +262,7 @@ class ActionScript3Lexer(RegexLexer): """ For ActionScript 3 source code. - *New in Pygments 0.11.* + .. versionadded:: 0.11 """ name = 'ActionScript 3' @@ -484,7 +484,7 @@ class ObjectiveJLexer(RegexLexer): """ For Objective-J source code with preprocessor directives. - *New in Pygments 1.3.* + .. versionadded:: 1.3 """ name = 'Objective-J' @@ -891,7 +891,7 @@ class PhpLexer(RegexLexer): self._functions = set() if self.funcnamehighlighting: from pygments.lexers._phpbuiltins import MODULES - for key, value in MODULES.iteritems(): + for key, value in iteritems(MODULES): if key not in self.disabledmodules: self._functions.update(value) RegexLexer.__init__(self, **options) @@ -921,7 +921,7 @@ class DtdLexer(RegexLexer): """ A lexer for DTDs (Document Type Definitions). - *New in Pygments 1.5.* + .. versionadded:: 1.5 """ flags = re.MULTILINE | re.DOTALL @@ -1056,7 +1056,7 @@ class XsltLexer(XmlLexer): ''' A lexer for XSLT. - *New in Pygments 0.10.* + .. versionadded:: 0.10 ''' name = 'XSLT' @@ -1094,7 +1094,7 @@ class MxmlLexer(RegexLexer): For MXML markup. Nested AS3 in <script> tags is highlighted by the appropriate lexer. - *New in Pygments 1.1.* + .. versionadded:: 1.1 """ flags = re.MULTILINE | re.DOTALL name = 'MXML' @@ -1137,7 +1137,7 @@ class HaxeLexer(ExtendedRegexLexer): """ For Haxe source code (http://haxe.org/). - *New in Pygments 1.3.* + .. versionadded:: 1.3 """ name = 'Haxe' @@ -2007,7 +2007,7 @@ class HamlLexer(ExtendedRegexLexer): """ For Haml markup. - *New in Pygments 1.3.* + .. versionadded:: 1.3 """ name = 'Haml' @@ -2284,7 +2284,7 @@ class SassLexer(ExtendedRegexLexer): """ For Sass stylesheets. - *New in Pygments 1.3.* + .. versionadded:: 1.3 """ name = 'Sass' @@ -2355,7 +2355,7 @@ class SassLexer(ExtendedRegexLexer): (r"\*/", Comment, '#pop'), ], } - for group, common in common_sass_tokens.iteritems(): + for group, common in iteritems(common_sass_tokens): tokens[group] = copy.copy(common) tokens['value'].append((r'\n', Text, 'root')) tokens['selector'].append((r'\n', Text, 'root')) @@ -2402,7 +2402,7 @@ class ScssLexer(RegexLexer): (r"\*/", Comment, '#pop'), ], } - for group, common in common_sass_tokens.iteritems(): + for group, common in iteritems(common_sass_tokens): tokens[group] = copy.copy(common) tokens['value'].extend([(r'\n', Text), (r'[;{}]', Punctuation, 'root')]) tokens['selector'].extend([(r'\n', Text), (r'[;{}]', Punctuation, 'root')]) @@ -2414,7 +2414,7 @@ class CoffeeScriptLexer(RegexLexer): .. _CoffeeScript: http://coffeescript.org - *New in Pygments 1.3.* + .. versionadded:: 1.3 """ name = 'CoffeeScript' @@ -2521,7 +2521,7 @@ class KalLexer(RegexLexer): .. _Kal: http://rzimmerman.github.io/kal - *New in Pygments 1.7.* + .. versionadded:: 2.0 """ name = 'Kal' @@ -2748,7 +2748,7 @@ class DuelLexer(RegexLexer): See http://duelengine.org/. See http://jsonml.org/jbst/. - *New in Pygments 1.4.* + .. versionadded:: 1.4 """ name = 'Duel' @@ -2779,7 +2779,7 @@ class ScamlLexer(ExtendedRegexLexer): """ For `Scaml markup <http://scalate.fusesource.org/>`_. Scaml is Haml for Scala. - *New in Pygments 1.4.* + .. versionadded:: 1.4 """ name = 'Scaml' @@ -2893,7 +2893,7 @@ class JadeLexer(ExtendedRegexLexer): Jade is a variant of Scaml, see: http://scalate.fusesource.org/documentation/scaml-reference.html - *New in Pygments 1.4.* + .. versionadded:: 1.4 """ name = 'Jade' @@ -3001,7 +3001,7 @@ class XQueryLexer(ExtendedRegexLexer): An XQuery lexer, parsing a stream and outputting the tokens needed to highlight xquery code. - *New in Pygments 1.4.* + .. versionadded:: 1.4 """ name = 'XQuery' aliases = ['xquery', 'xqy', 'xq', 'xql', 'xqm'] @@ -3406,7 +3406,7 @@ class XQueryLexer(ExtendedRegexLexer): 'xml_comment': [ (r'(-->)', popstate_xmlcomment_callback), (r'[^-]{1,2}', Literal), - (ur'\t|\r|\n|[\u0020-\uD7FF]|[\uE000-\uFFFD]|' + + (u'\\t|\\r|\\n|[\u0020-\uD7FF]|[\uE000-\uFFFD]|' + unirange(0x10000, 0x10ffff), Literal), ], 'processing_instruction': [ @@ -3416,12 +3416,12 @@ class XQueryLexer(ExtendedRegexLexer): ], 'processing_instruction_content': [ (r'\?>', String.Doc, '#pop'), - (ur'\t|\r|\n|[\u0020-\uD7FF]|[\uE000-\uFFFD]|' + + (u'\\t|\\r|\\n|[\u0020-\uD7FF]|[\uE000-\uFFFD]|' + unirange(0x10000, 0x10ffff), Literal), ], 'cdata_section': [ (r']]>', String.Doc, '#pop'), - (ur'\t|\r|\n|[\u0020-\uD7FF]|[\uE000-\uFFFD]|' + + (u'\\t|\\r|\\n|[\u0020-\uD7FF]|[\uE000-\uFFFD]|' + unirange(0x10000, 0x10ffff), Literal), ], 'start_tag': [ @@ -3490,7 +3490,7 @@ class XQueryLexer(ExtendedRegexLexer): ], 'pragmacontents': [ (r'#\)', Punctuation, 'operator'), - (ur'\t|\r|\n|[\u0020-\uD7FF]|[\uE000-\uFFFD]|' + + (u'\\t|\\r|\\n|[\u0020-\uD7FF]|[\uE000-\uFFFD]|' + unirange(0x10000, 0x10ffff), Literal), (r'(\s+)', Text), ], @@ -3663,7 +3663,7 @@ class DartLexer(RegexLexer): """ For `Dart <http://dartlang.org/>`_ source code. - *New in Pygments 1.5.* + .. versionadded:: 1.5 """ name = 'Dart' @@ -3691,7 +3691,7 @@ class DartLexer(RegexLexer): r'native|operator|set|static|typedef|var)\b', Keyword.Declaration), (r'\b(bool|double|Dynamic|int|num|Object|String|void)\b', Keyword.Type), (r'\b(false|null|true)\b', Keyword.Constant), - (r'[~!%^&*+=|?:<>/-]|as', Operator), + (r'[~!%^&*+=|?:<>/-]|as\b', Operator), (r'[a-zA-Z_$][a-zA-Z0-9_]*:', Name.Label), (r'[a-zA-Z_$][a-zA-Z0-9_]*', Name), (r'[(){}\[\],.;]', Punctuation), @@ -3763,7 +3763,7 @@ class TypeScriptLexer(RegexLexer): """ For `TypeScript <http://typescriptlang.org/>`_ source code. - *New in Pygments 1.6.* + .. versionadded:: 1.6 """ name = 'TypeScript' @@ -3850,7 +3850,7 @@ class LassoLexer(RegexLexer): If given and ``True``, only highlight code between delimiters as Lasso (default: ``False``). - *New in Pygments 1.6.* + .. versionadded:: 1.6 """ name = 'Lasso' @@ -4063,9 +4063,9 @@ class LassoLexer(RegexLexer): self._members = set() if self.builtinshighlighting: from pygments.lexers._lassobuiltins import BUILTINS, MEMBERS - for key, value in BUILTINS.iteritems(): + for key, value in iteritems(BUILTINS): self._builtins.update(value) - for key, value in MEMBERS.iteritems(): + for key, value in iteritems(MEMBERS): self._members.update(value) RegexLexer.__init__(self, **options) @@ -4099,7 +4099,7 @@ class QmlLexer(RegexLexer): """ For QML files. See http://doc.qt.digia.com/4.7/qdeclarativeintroduction.html. - *New in Pygments 1.6.* + .. versionadded:: 1.6 """ # QML is based on javascript, so much of this is taken from the @@ -4166,3 +4166,238 @@ class QmlLexer(RegexLexer): (r"'(\\\\|\\'|[^'])*'", String.Single), ] } + + +class CirruLexer(RegexLexer): + """ + Syntax rules of Cirru can be found at: + http://grammar.cirru.org/ + + * using ``()`` to markup blocks, but limited in the same line + * using ``""`` to markup strings, allow ``\`` to escape + * using ``$`` as a shorthand for ``()`` till indentation end or ``)`` + * using indentations for create nesting + + .. versionadded:: 2.0 + """ + + name = 'Cirru' + aliases = ['cirru'] + filenames = ['*.cirru', '*.cr'] + mimetypes = ['text/x-cirru'] + flags = re.MULTILINE + + tokens = { + 'string': [ + (r'[^"\\\n]', String), + (r'\\', String.Escape, 'escape'), + (r'"', String, '#pop'), + ], + 'escape': [ + (r'.', String.Escape, '#pop'), + ], + 'function': [ + (r'[\w-][^\s\(\)\"]*', Name.Function, '#pop'), + (r'\)', Operator, '#pop'), + (r'(?=\n)', Text, '#pop'), + (r'\(', Operator, '#push'), + (r'"', String, ('#pop', 'string')), + (r'\s+', Text.Whitespace), + (r'\,', Operator, '#pop'), + ], + 'line': [ + (r'^\B', Text.Whitespace, 'function'), + (r'\$', Operator, 'function'), + (r'\(', Operator, 'function'), + (r'\)', Operator), + (r'(?=\n)', Text, '#pop'), + (r'\n', Text, '#pop'), + (r'"', String, 'string'), + (r'\s+', Text.Whitespace), + (r'[\d\.]+', Number), + (r'[\w-][^\"\(\)\s]*', Name.Variable), + (r'--', Comment.Single) + ], + 'root': [ + (r'^\s*', Text.Whitespace, ('line', 'function')), + (r'^\s+$', Text.Whitespace), + ] + } + + +class MaskLexer(RegexLexer): + """ + For `Mask <http://github.com/atmajs/MaskJS>`__ markup. + + .. versionadded:: 2.0 + """ + name = 'Mask' + aliases = ['mask'] + filenames = ['*.mask'] + mimetypes = ['text/x-mask'] + + flags = re.MULTILINE | re.IGNORECASE | re.DOTALL + tokens = { + 'root': [ + (r'\s+', Text), + (r'//.*?\n', Comment.Single), + (r'/\*.*?\*/', Comment.Multiline), + (r'[\{\};>]', Punctuation), + (r"'''", String, 'string-trpl-single'), + (r'"""', String, 'string-trpl-double'), + (r"'", String, 'string-single'), + (r'"', String, 'string-double'), + (r'([\w-]+)', Name.Tag, 'node'), + (r'([^\.#;{>\s]+)', Name.Class, 'node'), + (r'(#[\w_-]+)', Name.Function, 'node'), + (r'(\.[\w_-]+)', Name.Variable.Class, 'node') + ], + 'string-base': [ + (r'\\.', String.Escape), + (r'~\[', String.Interpol, 'interpolation'), + (r'.', String.Single), + ], + 'string-single':[ + (r"'", String.Single, '#pop'), + include('string-base') + ], + 'string-double':[ + (r'"', String.Single, '#pop'), + include('string-base') + ], + 'string-trpl-single':[ + (r"'''", String.Single, '#pop'), + include('string-base') + ], + 'string-trpl-double':[ + (r'"""', String.Single, '#pop'), + include('string-base') + ], + 'interpolation': [ + (r'\]', String.Interpol, '#pop'), + (r'\s*:', String.Interpol, 'expression'), + (r'\s*\w+:', Name.Other), + (r'[^\]]+', String.Interpol) + ], + 'expression': [ + (r'[^\]]+', using(JavascriptLexer), '#pop') + ], + 'node': [ + (r'\s+', Text), + (r'\.', Name.Variable.Class, 'node-class'), + (r'\#', Name.Function, 'node-id'), + (r'style[ \t]*=', Name.Attribute, 'node-attr-style-value'), + (r'[\w_:-]+[ \t]*=', Name.Attribute, 'node-attr-value'), + (r'[\w_:-]+', Name.Attribute), + (r'[>{;]', Punctuation, '#pop') + ], + 'node-class': [ + (r'[\w-]+', Name.Variable.Class), + (r'~\[', String.Interpol, 'interpolation'), + (r'', Text, '#pop') + ], + 'node-id': [ + (r'[\w-]+', Name.Function), + (r'~\[', String.Interpol, 'interpolation'), + (r'', Text, '#pop') + ], + 'node-attr-value':[ + (r'\s+', Text), + (r'[\w_]+', Name.Variable, '#pop'), + (r"'", String, 'string-single-pop2'), + (r'"', String, 'string-double-pop2'), + (r'', Text, '#pop') + ], + 'node-attr-style-value':[ + (r'\s+', Text), + (r"'", String.Single, 'css-single-end'), + (r'"', String.Single, 'css-double-end'), + include('node-attr-value') + ], + 'css-base': [ + (r'\s+', Text), + (r"[;]", Punctuation), + (r"[\w\-_]+\s*:", Name.Builtin) + ], + 'css-single-end': [ + include('css-base'), + (r"'", String.Single, '#pop:2'), + (r"[^;']+", Name.Entity) + ], + 'css-double-end': [ + include('css-base'), + (r'"', String.Single, '#pop:2'), + (r"[^;\"]+", Name.Entity) + ], + 'string-single-pop2':[ + (r"'", String.Single, '#pop:2'), + include('string-base') + ], + 'string-double-pop2':[ + (r'"', String.Single, '#pop:2'), + include('string-base') + ], + } + + +class ZephirLexer(RegexLexer): + """ + For `Zephir language <http://zephir-lang.com/>`_ source code. + + Zephir is a compiled high level language aimed + to the creation of C-extensions for PHP. + + .. versionadded:: 2.0 + """ + + name = 'Zephir' + aliases = ['zephir'] + filenames = ['*.zep'] + + zephir_keywords = [ 'fetch', 'echo', 'isset', 'empty'] + zephir_type = [ 'bit', 'bits' , 'string' ] + + flags = re.DOTALL | re.MULTILINE + + tokens = { + 'commentsandwhitespace': [ + (r'\s+', Text), + (r'//.*?\n', Comment.Single), + (r'/\*.*?\*/', Comment.Multiline) + ], + 'slashstartsregex': [ + include('commentsandwhitespace'), + (r'/(\\.|[^[/\\\n]|\[(\\.|[^\]\\\n])*])+/' + r'([gim]+\b|\B)', String.Regex, '#pop'), + (r'', Text, '#pop') + ], + 'badregex': [ + (r'\n', Text, '#pop') + ], + 'root': [ + (r'^(?=\s|/|<!--)', Text, 'slashstartsregex'), + include('commentsandwhitespace'), + (r'\+\+|--|~|&&|\?|:|\|\||\\(?=\n)|' + r'(<<|>>>?|==?|!=?|->|[-<>+*%&\|\^/])=?', Operator, 'slashstartsregex'), + (r'[{(\[;,]', Punctuation, 'slashstartsregex'), + (r'[})\].]', Punctuation), + (r'(for|in|while|do|break|return|continue|switch|case|default|if|else|loop|require|inline|' + r'throw|try|catch|finally|new|delete|typeof|instanceof|void|namespace|use|extends|' + r'this|fetch|isset|unset|echo|fetch|likely|unlikely|empty)\b', Keyword, 'slashstartsregex'), + (r'(var|let|with|function)\b', Keyword.Declaration, 'slashstartsregex'), + (r'(abstract|boolean|bool|char|class|const|double|enum|export|' + r'extends|final|float|goto|implements|import|int|string|interface|long|ulong|char|uchar|native|unsigned|' + r'private|protected|public|short|static|self|throws|reverse|' + r'transient|volatile)\b', Keyword.Reserved), + (r'(true|false|null|undefined)\b', Keyword.Constant), + (r'(Array|Boolean|Date|_REQUEST|_COOKIE|_SESSION|' + r'_GET|_POST|_SERVER|this|stdClass|range|count|iterator|' + r'window)\b', Name.Builtin), + (r'[$a-zA-Z_][a-zA-Z0-9_\\]*', Name.Other), + (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float), + (r'0x[0-9a-fA-F]+', Number.Hex), + (r'[0-9]+', Number.Integer), + (r'"(\\\\|\\"|[^"])*"', String.Double), + (r"'(\\\\|\\'|[^'])*'", String.Single), + ] + } diff --git a/pygments/sphinxext.py b/pygments/sphinxext.py new file mode 100644 index 00000000..5ab8f060 --- /dev/null +++ b/pygments/sphinxext.py @@ -0,0 +1,153 @@ +# -*- coding: utf-8 -*- +""" + pygments.sphinxext + ~~~~~~~~~~~~~~~~~~ + + Sphinx extension to generate automatic documentation of lexers, + formatters and filters. + + :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from __future__ import print_function + +import sys + +from docutils import nodes +from docutils.statemachine import ViewList +from sphinx.util.compat import Directive +from sphinx.util.nodes import nested_parse_with_titles + + +MODULEDOC = ''' +.. module:: %s + +%s +%s +''' + +LEXERDOC = ''' +.. class:: %s + + :Short names: %s + :Filenames: %s + :MIME types: %s + + %s + +''' + +FMTERDOC = ''' +.. class:: %s + + :Short names: %s + :Filenames: %s + + %s + +''' + +FILTERDOC = ''' +.. class:: %s + + :Name: %s + + %s + +''' + +class PygmentsDoc(Directive): + """ + A directive to collect all lexers/formatters/filters and generate + autoclass directives for them. + """ + has_content = False + required_arguments = 1 + optional_arguments = 0 + final_argument_whitespace = False + option_spec = {} + + def run(self): + self.filenames = set() + if self.arguments[0] == 'lexers': + out = self.document_lexers() + elif self.arguments[0] == 'formatters': + out = self.document_formatters() + elif self.arguments[0] == 'filters': + out = self.document_filters() + else: + raise Exception('invalid argument for "pygmentsdoc" directive') + node = nodes.compound() + vl = ViewList(out.split('\n'), source='') + nested_parse_with_titles(self.state, vl, node) + for fn in self.filenames: + self.state.document.settings.record_dependencies.add(fn) + return node.children + + def document_lexers(self): + from pygments.lexers._mapping import LEXERS + out = [] + modules = {} + moduledocstrings = {} + for classname, data in sorted(LEXERS.items(), key=lambda x: x[0]): + module = data[0] + mod = __import__(module, None, None, [classname]) + self.filenames.add(mod.__file__) + cls = getattr(mod, classname) + if not cls.__doc__: + print("Warning: %s does not have a docstring." % classname) + docstring = cls.__doc__ + if isinstance(docstring, bytes): + docstring = docstring.decode('utf8') + modules.setdefault(module, []).append(( + classname, + ', '.join(data[2]) or 'None', + ', '.join(data[3]).replace('*', '\\*').replace('_', '\\') or 'None', + ', '.join(data[4]) or 'None', + docstring)) + if module not in moduledocstrings: + moddoc = mod.__doc__ + if isinstance(moddoc, bytes): + moddoc = moddoc.decode('utf8') + moduledocstrings[module] = moddoc + + for module, lexers in sorted(modules.items(), key=lambda x: x[0]): + heading = moduledocstrings[module].splitlines()[4].strip().rstrip('.') + out.append(MODULEDOC % (module, heading, '-'*len(heading))) + for data in lexers: + out.append(LEXERDOC % data) + + return ''.join(out) + + def document_formatters(self): + from pygments.formatters import FORMATTERS + + out = [] + for cls, data in sorted(FORMATTERS.items(), + key=lambda x: x[0].__name__): + self.filenames.add(sys.modules[cls.__module__].__file__) + docstring = cls.__doc__ + if isinstance(docstring, bytes): + docstring = docstring.decode('utf8') + heading = cls.__name__ + out.append(FMTERDOC % (heading, ', '.join(data[1]) or 'None', + ', '.join(data[2]).replace('*', '\\*') or 'None', + docstring)) + return ''.join(out) + + def document_filters(self): + from pygments.filters import FILTERS + + out = [] + for name, cls in FILTERS.items(): + self.filenames.add(sys.modules[cls.__module__].__file__) + docstring = cls.__doc__ + if isinstance(docstring, bytes): + docstring = docstring.decode('utf8') + out.append(FILTERDOC % (cls.__name__, name, docstring)) + return ''.join(out) + + +def setup(app): + app.add_directive('pygmentsdoc', PygmentsDoc) diff --git a/pygments/style.py b/pygments/style.py index d0fc26be..bb54377c 100644 --- a/pygments/style.py +++ b/pygments/style.py @@ -10,6 +10,7 @@ """ from pygments.token import Token, STANDARD_TYPES +from pygments.util import add_metaclass class StyleMeta(type): @@ -104,8 +105,8 @@ class StyleMeta(type): return len(cls._styles) +@add_metaclass(StyleMeta) class Style(object): - __metaclass__ = StyleMeta #: overall background color (``None`` means transparent) background_color = '#ffffff' diff --git a/pygments/token.py b/pygments/token.py index f6c3066d..c40ffd33 100644 --- a/pygments/token.py +++ b/pygments/token.py @@ -49,6 +49,7 @@ Token = _TokenType() # Special token types Text = Token.Text Whitespace = Text.Whitespace +Escape = Token.Escape Error = Token.Error # Text that doesn't belong to this lexer (e.g. HTML in PHP) Other = Token.Other @@ -116,6 +117,7 @@ STANDARD_TYPES = { Text: '', Whitespace: 'w', + Escape: 'esc', Error: 'err', Other: 'x', diff --git a/pygments/unistring.py b/pygments/unistring.py index da87b6df..2752037f 100644 --- a/pygments/unistring.py +++ b/pygments/unistring.py @@ -11,7 +11,8 @@ :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ -from pygments.util import u_prefix + +from pygments.util import u_prefix, unichr Cc = u'\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f' @@ -133,7 +134,7 @@ except UnicodeDecodeError: Cs = '' # Jython can't handle isolated surrogates\n\n""" % repr(val).lstrip('u')) else: f.write('%s = %r\n\n' % (cat, val)) - f.write('cats = %r\n\n' % sorted(categories.keys())) + f.write('cats = %r\n\n' % sorted(categories)) f.write('# Generated from unidata %s\n\n' % (unicodedata.unidata_version,)) f.write(footer) diff --git a/pygments/util.py b/pygments/util.py index d40a88c8..c302900f 100644 --- a/pygments/util.py +++ b/pygments/util.py @@ -11,7 +11,6 @@ import re import sys -import codecs split_path_re = re.compile(r'[/\\ ]') @@ -52,7 +51,7 @@ def get_bool_opt(options, optname, default=None): return string elif isinstance(string, int): return bool(string) - elif not isinstance(string, basestring): + elif not isinstance(string, string_types): raise OptionError('Invalid type %r for option %s; use ' '1/0, yes/no, true/false, on/off' % ( string, optname)) @@ -82,7 +81,7 @@ def get_int_opt(options, optname, default=None): def get_list_opt(options, optname, default=None): val = options.get(optname, default) - if isinstance(val, basestring): + if isinstance(val, string_types): return val.split() elif isinstance(val, (list, tuple)): return list(val) @@ -253,25 +252,35 @@ def unirange(a, b): # Python 2/3 compatibility -if sys.version_info < (3,0): - b = bytes = str +if sys.version_info < (3, 0): + unichr = unichr + xrange = xrange + string_types = (str, unicode) + text_type = unicode u_prefix = 'u' + iteritems = dict.iteritems + itervalues = dict.itervalues import StringIO, cStringIO - BytesIO = cStringIO.StringIO + # unfortunately, io.StringIO in Python 2 doesn't accept str at all StringIO = StringIO.StringIO - uni_open = codecs.open + BytesIO = cStringIO.StringIO else: - import builtins - bytes = builtins.bytes + unichr = chr + xrange = range + string_types = (str,) + text_type = str u_prefix = '' - def b(s): - if isinstance(s, str): - return bytes(map(ord, s)) - elif isinstance(s, bytes): - return s - else: - raise TypeError("Invalid argument %r for b()" % (s,)) - import io - BytesIO = io.BytesIO - StringIO = io.StringIO - uni_open = builtins.open + iteritems = dict.items + itervalues = dict.values + from io import StringIO, BytesIO + +def add_metaclass(metaclass): + """Class decorator for creating a class with a metaclass.""" + def wrapper(cls): + orig_vars = cls.__dict__.copy() + orig_vars.pop('__dict__', None) + orig_vars.pop('__weakref__', None) + for slots_var in orig_vars.get('__slots__', ()): + orig_vars.pop(slots_var) + return metaclass(cls.__name__, cls.__bases__, orig_vars) + return wrapper |