From 8c38291427992de658273cf75bd86d54084e6a83 Mon Sep 17 00:00:00 2001 From: Camil Staps Date: Thu, 14 Jul 2016 14:40:56 +0200 Subject: Complete rewrite of the Clean lexer --- pygments/lexers/clean.py | 325 +++++++++++------------------------------------ 1 file changed, 77 insertions(+), 248 deletions(-) diff --git a/pygments/lexers/clean.py b/pygments/lexers/clean.py index b87ff99e..c41fa400 100644 --- a/pygments/lexers/clean.py +++ b/pygments/lexers/clean.py @@ -9,10 +9,9 @@ :license: BSD, see LICENSE for details. """ -from pygments.lexer import ExtendedRegexLexer, LexerContext, \ - bygroups, words, include, default -from pygments.token import Comment, Keyword, Literal, Name, Number, Operator, \ - Punctuation, String, Text, Whitespace +from pygments.lexer import ExtendedRegexLexer, words, include +from pygments.token import Comment, Error, Keyword, Literal, Name, Number, \ + Operator, Punctuation, String, Whitespace __all__ = ['CleanLexer'] @@ -28,261 +27,91 @@ class CleanLexer(ExtendedRegexLexer): aliases = ['clean'] filenames = ['*.icl', '*.dcl'] - def get_tokens_unprocessed(self, text=None, context=None): - ctx = LexerContext(text, 0) - ctx.indent = 0 - return ExtendedRegexLexer.get_tokens_unprocessed(self, text, context=ctx) + keywords = ( + 'case', 'ccall', 'class', 'code', 'derive', 'export', 'foreign', + 'from', 'generic', 'if', 'import', 'in', 'infix', 'infixl', 'infixr', + 'inline', 'instance', 'let', 'of', 'otherwise', 'qualified', 'special', + 'stdcall', 'where', 'with') - def check_class_not_import(lexer, match, ctx): - if match.group(0) == 'import': - yield match.start(), Keyword.Namespace, match.group(0) - ctx.stack = ctx.stack[:-1] + ['fromimportfunc'] - else: - yield match.start(), Name.Class, match.group(0) - ctx.pos = match.end() + modulewords = ('implementation', 'definition', 'system') - def check_instance_class(lexer, match, ctx): - if match.group(0) == 'instance' or match.group(0) == 'class': - yield match.start(), Keyword, match.group(0) - else: - yield match.start(), Name.Function, match.group(0) - ctx.stack = ctx.stack + ['fromimportfunctype'] - ctx.pos = match.end() - - @staticmethod - def indent_len(text): - # Tabs are four spaces: - # https://svn.cs.ru.nl/repos/clean-platform/trunk/doc/STANDARDS.txt - text = text.replace('\n', '') - return len(text.replace('\t', ' ')), len(text) - - def store_indent(lexer, match, ctx): - ctx.indent, _ = CleanLexer.indent_len(match.group(0)) - ctx.pos = match.end() - yield match.start(), Text, match.group(0) - - def check_indent1(lexer, match, ctx): - indent, reallen = CleanLexer.indent_len(match.group(0)) - if indent > ctx.indent: - yield match.start(), Whitespace, match.group(0) - ctx.pos = match.start() + reallen + 1 - else: - ctx.indent = 0 - ctx.pos = match.start() - ctx.stack = ctx.stack[:-1] - yield match.start(), Whitespace, match.group(0)[1:] - - def check_indent2(lexer, match, ctx): - indent, reallen = CleanLexer.indent_len(match.group(0)) - if indent > ctx.indent: - yield match.start(), Whitespace, match.group(0) - ctx.pos = match.start() + reallen + 1 - else: - ctx.indent = 0 - ctx.pos = match.start() - ctx.stack = ctx.stack[:-2] - - def check_indent3(lexer, match, ctx): - indent, reallen = CleanLexer.indent_len(match.group(0)) - if indent > ctx.indent: - yield match.start(), Whitespace, match.group(0) - ctx.pos = match.start() + reallen + 1 - else: - ctx.indent = 0 - ctx.pos = match.start() - ctx.stack = ctx.stack[:-3] - yield match.start(), Whitespace, match.group(0)[1:] - if match.group(0) == '\n\n': - ctx.pos = ctx.pos + 1 - - def skip(lexer, match, ctx): - ctx.stack = ctx.stack[:-1] - ctx.pos = match.end() - yield match.start(), Comment, match.group(0) - - keywords = ('class', 'instance', 'where', 'with', 'let', 'let!', - 'in', 'case', 'of', 'infix', 'infixr', 'infixl', 'generic', - 'derive', 'otherwise', 'code', 'inline') + lowerId = r'[a-z][\w\d`]*' + upperId = r'[A-Z][\w\d`]*' + funnyId = r'[~@#\$%\^?!+\-*<>\\/|&=:]+' + scoreUpperId = r'_' + upperId + scoreLowerId = r'_' + lowerId tokens = { - 'common': [ - (r';', Punctuation, '#pop'), - (r'//', Comment, 'singlecomment'), - ], 'root': [ - # Comments - (r'//.*\n', Comment.Single), - (r'(?s)/\*\*.*?\*/', Comment.Special), - (r'(?s)/\*.*?\*/', Comment.Multi), - - # Modules, imports, etc. - (r'\b((?:implementation|definition|system)\s+)?(module)(\s+)([\w`\.]+)', - bygroups(Keyword.Namespace, Keyword.Namespace, Text, Name.Class)), - (r'(?<=\n)import(?=\s)', Keyword.Namespace, 'import'), - (r'(?<=\n)from(?=\s)', Keyword.Namespace, 'fromimport'), - - # Keywords - # We cannot use (?s)^|(?<=\s) as prefix, so need to repeat this - (words(keywords, prefix=r'(?<=\s)', suffix=r'(?=\s)'), Keyword), - (words(keywords, prefix=r'^', suffix=r'(?=\s)'), Keyword), - - # Function definitions - (r'(?=\{\|)', Whitespace, 'genericfunction'), - (r'(?<=\n)([ \t]*)([\w`$()=\-<>~*\^|+&%]+)((?:\s+\w)*)(\s*)(::)', - bygroups(store_indent, Name.Function, Keyword.Type, Whitespace, - Punctuation), - 'functiondefargs'), - - # Type definitions - (r'(?<=\n)([ \t]*)(::)', bygroups(store_indent, Punctuation), 'typedef'), - (r'^([ \t]*)(::)', bygroups(store_indent, Punctuation), 'typedef'), - - # Literals - (r'\'\\?.(?|&~*\^/]', Operator), - (r'\\\\', Operator), - - # Lambda expressions - (r'\\.*?(->|\.|=)', Name.Function), - - # Whitespace - (r'\s', Whitespace), - - include('common'), - ], - 'fromimport': [ - include('common'), - (r'([\w`\.]+)', check_class_not_import), - (r'\n', Whitespace, '#pop'), - (r'\s', Whitespace), - ], - 'fromimportfunc': [ - include('common'), - (r'(::)\s+([^,\s]+)', bygroups(Punctuation, Keyword.Type)), - (r'([\w`$()=\-<>~*\^|+&%\/]+)', check_instance_class), - (r',', Punctuation), - (r'\n', Whitespace, '#pop'), - (r'\s', Whitespace), - ], - 'fromimportfunctype': [ - include('common'), - (r'[{(\[]', Punctuation, 'combtype'), - (r',', Punctuation, '#pop'), - (r'[:;.#]', Punctuation), - (r'\n', Whitespace, '#pop:2'), - (r'[^\S\n]+', Whitespace), - (r'\S+', Keyword.Type), - ], - 'combtype': [ - include('common'), - (r'[})\]]', Punctuation, '#pop'), - (r'[{(\[]', Punctuation, '#pop'), - (r'[,:;.#]', Punctuation), + include('comments'), + include('keywords'), + include('module'), + include('whitespace'), + include('literals'), + include('operators'), + include('delimiters'), + include('names'), + ], + 'whitespace': [ (r'\s+', Whitespace), - (r'\S+', Keyword.Type), - ], - 'import': [ - include('common'), - (words(('from', 'import', 'as', 'qualified'), - prefix='(?<=\s)', suffix='(?=\s)'), Keyword.Namespace), - (r'[\w`\.]+', Name.Class), - (r'\n', Whitespace, '#pop'), - (r',', Punctuation), - (r'[^\S\n]+', Whitespace), - ], - 'singlecomment': [ - (r'(.)(?=\n)', skip), - (r'.+(?!\n)', Comment), ], - 'doubleqstring': [ - (r'[^\\"]+', String.Double), + 'comments': [ + (r'//.*\n', Comment.Single), + (r'/\*', Comment.Multi, 'comments.in'), + (r'/\*\*', Comment.Special, 'comments.in'), + ], + 'comments.in': [ + (r'\*\/', Comment.Multi, '#pop'), + (r'/\*', Comment.Multi, '#push'), + (r'[^*/]+', Comment.Multi), + (r'\*(?!/)', Comment.Multi), + (r'/', Comment.Multi), + ], + 'keywords': [ + (words(keywords, prefix=r'\b', suffix=r'\b'), Keyword), + ], + 'module': [ + include('comments'), + (words(modulewords, prefix=r'\b', suffix=r'\b'), Keyword.Namespace), + (r'\bmodule\b', Keyword.Namespace, 'module.name'), + include('whitespace'), + ], + 'module.name': [ + include('whitespace'), + (lowerId, Name.Class, '#pop'), + (upperId, Name.Class, '#pop'), + (funnyId, Name.Class, '#pop'), + (scoreLowerId, Name.Class, '#pop'), + (scoreUpperId, Name.Class, '#pop'), + ], + 'literals': [ + (r'\'([^\'\\]|\\(x[\da-fA-F]+|\d+|.))\'', Literal.Char), + (r'[+~-]?0[0-7]+\b', Number.Oct), + (r'[+~-]?\d+\.\d+(E[+-]?\d+)?', Number.Float), + (r'[+~-]?\d+\b', Number.Integer), + (r'[+~-]?0x[\da-fA-F]+\b', Number.Hex), + (r'True|False', Literal), + (r'"', String.Double, 'literals.stringd'), + #(r'\[\'', Literal. + ], + 'literals.stringd': [ + (r'[^\\"$\n]+', String.Double), (r'"', String.Double, '#pop'), (r'\\.', String.Double), + (r'[$\n]', Error, '#pop'), ], - 'typedef': [ - include('common'), - (r'[\w`]+', Keyword.Type), - (r'[:=|(),\[\]{}!*]', Punctuation), - (r'->', Punctuation), - (r'\n(?=[^\s|])', Whitespace, '#pop'), - (r'\s', Whitespace), - (r'.', Keyword.Type), - ], - 'genericfunction': [ - include('common'), - (r'\{\|', Punctuation), - (r'\|\}', Punctuation, '#pop'), - (r',', Punctuation), - (r'->', Punctuation), - (r'(\s+of\s+)(\{)', bygroups(Keyword, Punctuation), 'genericftypes'), - (r'\s', Whitespace), - (r'[\w`\[\]{}!]+', Keyword.Type), - (r'[*()]', Punctuation), - ], - 'genericftypes': [ - include('common'), - (r'[\w`]+', Keyword.Type), - (r',', Punctuation), - (r'\s', Whitespace), - (r'\}', Punctuation, '#pop'), - ], - 'functiondefargs': [ - include('common'), - (r'\n(\s*)', check_indent1), - (r'[!{}()\[\],:;.#]', Punctuation), - (r'->', Punctuation, 'functiondefres'), - (r'^(?=\S)', Whitespace, '#pop'), - (r'\S', Keyword.Type), - (r'\s', Whitespace), + 'operators': [ + (r'[-~@#\$%\^?!+*<>\\/|&=:\.]+', Operator), + (r'\b_+\b', Operator), ], - 'functiondefres': [ - include('common'), - (r'\n(\s*)', check_indent2), - (r'^(?=\S)', Whitespace, '#pop:2'), - (r'[!{}()\[\],:;.#]', Punctuation), - (r'\|', Punctuation, 'functiondefclasses'), - (r'\S', Keyword.Type), - (r'\s', Whitespace), - ], - 'functiondefclasses': [ - include('common'), - (r'\n(\s*)', check_indent3), - (r'^(?=\S)', Whitespace, '#pop:3'), - (r'[,&]', Punctuation), - (r'\[', Punctuation, 'functiondefuniquneq'), - (r'[\w`$()=\-<>~*\^|+&%\/{}\[\]@]', Name.Function, 'functionname'), - (r'\s+', Whitespace), - ], - 'functiondefuniquneq': [ - include('common'), - (r'[a-z]+', Keyword.Type), - (r'\s+', Whitespace), - (r'<=|,', Punctuation), - (r'\]', Punctuation, '#pop') + 'delimiters': [ + (r'[,;(){}\[\]]', Punctuation) ], - 'functionname': [ - include('common'), - (r'[\w`$()=\-<>~*\^|+&%\/]+', Name.Function), - (r'(?=\{\|)', Punctuation, 'genericfunction'), - default('#pop'), + 'names': [ + (r'\b' + lowerId, Name), + (scoreLowerId, Name), + (r'\b' + funnyId, Name.Function), + (r'\b' + upperId, Name.Class), + (scoreUpperId, Name.Class), ] } -- cgit v1.2.1 From 836d98cc163ea0dcb1b60ef1e536fdacb351d78a Mon Sep 17 00:00:00 2001 From: Benoit Vey Date: Mon, 25 Jul 2016 07:10:17 +0200 Subject: Add lexer for the Pony language --- pygments/lexers/_mapping.py | 1 + pygments/lexers/pony.py | 92 +++++++++++++++++++++++++++++++++++++++++ tests/examplefiles/example.pony | 18 ++++++++ 3 files changed, 111 insertions(+) create mode 100644 pygments/lexers/pony.py create mode 100644 tests/examplefiles/example.pony diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py index a6097b1c..847cc0bf 100644 --- a/pygments/lexers/_mapping.py +++ b/pygments/lexers/_mapping.py @@ -316,6 +316,7 @@ LEXERS = { 'PikeLexer': ('pygments.lexers.c_like', 'Pike', ('pike',), ('*.pike', '*.pmod'), ('text/x-pike',)), 'PkgConfigLexer': ('pygments.lexers.configs', 'PkgConfig', ('pkgconfig',), ('*.pc',), ()), 'PlPgsqlLexer': ('pygments.lexers.sql', 'PL/pgSQL', ('plpgsql',), (), ('text/x-plpgsql',)), + 'PonyLexer': ('pygments.lexers.pony', 'Pony', ('pony',), ('*.pony',), ()), 'PostScriptLexer': ('pygments.lexers.graphics', 'PostScript', ('postscript', 'postscr'), ('*.ps', '*.eps'), ('application/postscript',)), 'PostgresConsoleLexer': ('pygments.lexers.sql', 'PostgreSQL console (psql)', ('psql', 'postgresql-console', 'postgres-console'), (), ('text/x-postgresql-psql',)), 'PostgresLexer': ('pygments.lexers.sql', 'PostgreSQL SQL dialect', ('postgresql', 'postgres'), (), ('text/x-postgresql',)), diff --git a/pygments/lexers/pony.py b/pygments/lexers/pony.py new file mode 100644 index 00000000..486e404b --- /dev/null +++ b/pygments/lexers/pony.py @@ -0,0 +1,92 @@ +# -*- coding: utf-8 -*- +""" + pygments.lexers.pony + ~~~~~~~~~~~~~~~~~~~~ + + Lexers for Pony and related languages. + + :copyright: Copyright 2006-2016 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.lexer import RegexLexer, bygroups, words +from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ + Number, Punctuation + +__all__ = ['PonyLexer'] + + +class PonyLexer(RegexLexer): + """ + For Pony source code. + """ + + name = 'Pony' + aliases = ['pony'] + filenames = ['*.pony'] + + _caps = r'(iso|trn|ref|val|box|tag)' + + tokens = { + 'root': [ + (r'\n', Text), + (r'[^\S\n]+', Text), + (r'//.*\n', Comment.Single), + (r'/\*', Comment.Multiline, 'nested_comment'), + (r'"""(?:.|\n)*?"""', String.Doc), + (r'"', String, 'string'), + (r'\'.*\'', String.Char), + (r'=>|[]{}:().~;,|&!^?[]', Punctuation), + (words(( + 'addressof', 'and', 'as', 'consume', 'digestof', 'is', 'isnt', + 'not', 'or'), + suffix=r'\b'), + Operator.Word), + (r'!=|==|<<|>>|[-+/*%=<>]', Operator), + (words(( + 'box', 'break', 'compile_error', 'compile_intrinsic', + 'continue', 'do', 'else', 'elseif', 'embed', 'end', 'error', + 'for', 'if', 'ifdef', 'in', 'iso', 'lambda', 'let', 'match', + 'object', 'recover', 'ref', 'repeat', 'return', 'tag', 'then', + 'this', 'trn', 'try', 'until', 'use', 'var', 'val', 'where', + 'while', 'with', '#any', '#read', '#send', '#share'), + suffix=r'\b'), + Keyword), + (r'(actor|class|struct|primitive|interface|trait|type)((?:\s)+)', + bygroups(Keyword, Text), 'typename'), + (r'(new|fun|be)((?:\s)+)', bygroups(Keyword, Text), 'methodname'), + (words(( + 'I8', 'U8', 'I16', 'U16', 'I32', 'U32', 'I64', 'U64', 'I128', + 'U128', 'ILong', 'ULong', 'ISize', 'USize', 'F32', 'F64', + 'Bool', 'Pointer', 'None', 'Any', 'Array', 'String', + 'Iterator'), + suffix=r'\b'), + Name.Builtin.Type), + (r'_?[A-Z]\w*', Name.Type), + (r'(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+', Number.Float), + (r'0x[0-9a-fA-F]+', Number.Hex), + (r'\d+', Number.Integer), + (r'(true|false)\b', Name.Builtin), + (r'_\d*', Name), + (r'_?[a-z][\w\'_]*', Name) + ], + 'typename': [ + (_caps + r'?((?:\s)*)(_?[A-Z]\w*)', + bygroups(Keyword, Text, Name.Class), '#pop') + ], + 'methodname': [ + (_caps + r'?((?:\s)*)(_?[a-z]\w*)', + bygroups(Keyword, Text, Name.Function), '#pop') + ], + 'nested_comment': [ + (r'[^*/]+', Comment.Multiline), + (r'/\*', Comment.Multiline, '#push'), + (r'\*/', Comment.Multiline, '#pop'), + (r'[*/]', Comment.Multiline) + ], + 'string': [ + (r'"', String, '#pop'), + (r'\\"', String), + (r'[^\\"]+', String) + ] + } diff --git a/tests/examplefiles/example.pony b/tests/examplefiles/example.pony new file mode 100644 index 00000000..654f2376 --- /dev/null +++ b/tests/examplefiles/example.pony @@ -0,0 +1,18 @@ +use "somepkg" + +/* + /* Nested */ +*/ + +class trn Foo[A: Stringable ref] is Stringable + let _x = "\"" + + fun val dofoo() => + """ + DocString + """ + (U64(2), "foo")._2 + +actor Main + new create(env: Env) => + env.out.print("Hello world") -- cgit v1.2.1 From cdbc000e03a5a92ea2a28020e71218cae5010a5d Mon Sep 17 00:00:00 2001 From: Thomas Aglassinger Date: Sat, 1 Oct 2016 18:17:05 +0200 Subject: Added analyse_text() that attempts to detect MySQL and Transact-SQL. --- pygments/lexers/sql.py | 52 ++++++++++++++++++++++++++++++++++++++++++++++++-- tests/test_sql.py | 46 +++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 95 insertions(+), 3 deletions(-) diff --git a/pygments/lexers/sql.py b/pygments/lexers/sql.py index e225a66e..0bc33493 100644 --- a/pygments/lexers/sql.py +++ b/pygments/lexers/sql.py @@ -59,7 +59,14 @@ line_re = re.compile('.*?\n') language_re = re.compile(r"\s+LANGUAGE\s+'?(\w+)'?", re.IGNORECASE) -do_re = re.compile(r'\bDO\b', re.IGNORECASE) +do_re = re.compile(r'\bDO\b', re.IGNORECASE) + +# Regular expressions for analyse_text() +name_between_bracket_re = re.compile(r'\[[a-zA-Z_]\w*\]') +name_between_backtick_re = re.compile(r'`[a-zA-Z_]\w*`') +tsql_go_re = re.compile(r'\bgo\b', re.IGNORECASE) +tsql_declare_re = re.compile(r'\bdeclare\s+@', re.IGNORECASE) +tsql_variable_re = re.compile(r'@[a-zA-Z_]\w*\b') def language_callback(lexer, match): @@ -82,7 +89,7 @@ def language_callback(lexer, match): lexer.text[max(0, match.start()-25):match.start()])) if m: l = lexer._get_lexer('plpgsql') - + # 1 = $, 2 = delimiter, 3 = $ yield (match.start(1), String, match.group(1)) yield (match.start(2), String.Delimiter, match.group(2)) @@ -480,6 +487,9 @@ class SqlLexer(RegexLexer): ] } + def analyse_text(text): + return 0.01 + class TransactSqlLexer(RegexLexer): """ @@ -536,6 +546,29 @@ class TransactSqlLexer(RegexLexer): ] } + def analyse_text(text): + rating = 0 + if tsql_declare_re.search(text): + # Found T-SQL variable declaration. + rating = 1.0 + else: + name_between_backtick_count = len( + name_between_backtick_re.findall((text))) + name_between_bracket_count = len( + name_between_bracket_re.findall(text)) + if name_between_bracket_count >= 2 * name_between_backtick_count: + # Found at least twice as many [name] as `name`. + rating += 0.5 + elif name_between_bracket_count > name_between_backtick_count: + rating += 0.2 + elif name_between_bracket_count > 0: + rating += 0.1 + if tsql_variable_re.search(text) is not None: + rating += 0.1 + if tsql_go_re.search(text) is not None: + rating += 0.1 + return rating + class MySqlLexer(RegexLexer): """ @@ -609,6 +642,21 @@ class MySqlLexer(RegexLexer): ] } + def analyse_text(text): + rating = 0 + name_between_backtick_count = len( + name_between_backtick_re.findall((text))) + name_between_bracket_count = len( + name_between_bracket_re.findall(text)) + if name_between_backtick_count >= 2 * name_between_bracket_count: + # Found at least twice as many `name` as [name]. + rating += 0.5 + elif name_between_backtick_count > name_between_bracket_count: + rating += 0.2 + elif name_between_backtick_count > 0: + rating += 0.1 + return rating + class SqliteConsoleLexer(Lexer): """ diff --git a/tests/test_sql.py b/tests/test_sql.py index c5f5c758..6be34006 100644 --- a/tests/test_sql.py +++ b/tests/test_sql.py @@ -8,7 +8,10 @@ """ import unittest -from pygments.lexers.sql import TransactSqlLexer +from pygments.lexers.sql import name_between_bracket_re, \ + name_between_backtick_re, tsql_go_re, tsql_declare_re, \ + tsql_variable_re, MySqlLexer, SqlLexer, TransactSqlLexer + from pygments.token import Comment, Name, Number, Punctuation, Whitespace @@ -72,3 +75,44 @@ class TransactSqlLexerTest(unittest.TestCase): (Comment.Multiline, '*/'), (Comment.Multiline, '*/'), )) + + +class SqlAnalyzeTextTest(unittest.TestCase): + def test_can_match_analyze_text_res(self): + self.assertEqual(['`a`', '`bc`'], + name_between_backtick_re.findall('select `a`, `bc` from some')) + self.assertEqual(['[a]', '[bc]'], + name_between_bracket_re.findall('select [a], [bc] from some')) + self.assertTrue(tsql_declare_re.search('--\nDeClaRe @some int;')) + self.assertTrue(tsql_go_re.search('select 1\ngo\n--')) + self.assertTrue(tsql_variable_re.search( + 'create procedure dbo.usp_x @a int, @b int')) + + def test_can_analyze_text(self): + mysql_lexer = MySqlLexer() + sql_lexer = SqlLexer() + tsql_lexer = TransactSqlLexer() + code_to_expected_lexer_map = { + 'select `a`, `bc` from some': mysql_lexer, + 'select a, bc from some': sql_lexer, + 'select [a], [bc] from some': tsql_lexer, + '-- `a`, `bc`\nselect [a], [bc] from some': tsql_lexer, + '-- `a`, `bc`\nselect [a], [bc] from some; go': tsql_lexer, + } + sql_lexers = set(code_to_expected_lexer_map.values()) + for code, expected_lexer in code_to_expected_lexer_map.items(): + ratings_and_lexers = list((lexer.analyse_text(code), lexer.name) for lexer in sql_lexers) + best_rating, best_lexer_name = sorted(ratings_and_lexers, reverse=True)[0] + expected_rating = expected_lexer.analyse_text(code) + message = ( + 'lexer must be %s (rating %.2f) instead of ' + '%s (rating %.2f) for analyse_text() on code:\n%s') % ( + expected_lexer.name, + expected_rating, + best_lexer_name, + best_rating, + code + ) + self.assertEqual( + expected_lexer.name, best_lexer_name, message + ) -- cgit v1.2.1 From bc4380207f90b0c3f8edb77ad0b32ef1d701079f Mon Sep 17 00:00:00 2001 From: Thomas Aglassinger Date: Sun, 2 Oct 2016 12:55:12 +0200 Subject: Added lexer for VBScript. --- AUTHORS | 3 +- pygments/lexers/_mapping.py | 1 + pygments/lexers/_vbscript_builtins.py | 275 ++++++++++++++++++++++++++++++++++ pygments/lexers/basic.py | 70 ++++++++- tests/examplefiles/example.vbs | 55 +++++++ tests/test_basic.py | 74 +++++++++ 6 files changed, 474 insertions(+), 4 deletions(-) create mode 100644 pygments/lexers/_vbscript_builtins.py create mode 100644 tests/examplefiles/example.vbs create mode 100644 tests/test_basic.py diff --git a/AUTHORS b/AUTHORS index 66377ead..915a747d 100644 --- a/AUTHORS +++ b/AUTHORS @@ -7,7 +7,8 @@ Other contributors, listed alphabetically, are: * Sam Aaron -- Ioke lexer * Ali Afshar -- image formatter -* Thomas Aglassinger -- Easytrieve, JCL, Rexx and Transact-SQL lexers +* Thomas Aglassinger -- Easytrieve, JCL, Rexx, Transact-SQL and VBScript + lexers * Muthiah Annamalai -- Ezhil lexer * Kumar Appaiah -- Debian control lexer * Andreas Amann -- AppleScript lexer diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py index a6097b1c..882ec292 100644 --- a/pygments/lexers/_mapping.py +++ b/pygments/lexers/_mapping.py @@ -422,6 +422,7 @@ LEXERS = { 'TypoScriptHtmlDataLexer': ('pygments.lexers.typoscript', 'TypoScriptHtmlData', ('typoscripthtmldata',), (), ()), 'TypoScriptLexer': ('pygments.lexers.typoscript', 'TypoScript', ('typoscript',), ('*.ts', '*.txt'), ('text/x-typoscript',)), 'UrbiscriptLexer': ('pygments.lexers.urbi', 'UrbiScript', ('urbiscript',), ('*.u',), ('application/x-urbiscript',)), + 'VBScriptLexer': ('pygments.lexers.basic', 'VBScript', (), ('*.vbs', '*.VBS'), ()), 'VCLLexer': ('pygments.lexers.varnish', 'VCL', ('vcl',), ('*.vcl',), ('text/x-vclsrc',)), 'VCLSnippetLexer': ('pygments.lexers.varnish', 'VCLSnippets', ('vclsnippets', 'vclsnippet'), (), ('text/x-vclsnippet',)), 'VCTreeStatusLexer': ('pygments.lexers.console', 'VCTreeStatus', ('vctreestatus',), (), ()), diff --git a/pygments/lexers/_vbscript_builtins.py b/pygments/lexers/_vbscript_builtins.py new file mode 100644 index 00000000..493e5c7c --- /dev/null +++ b/pygments/lexers/_vbscript_builtins.py @@ -0,0 +1,275 @@ +# -*- coding: utf-8 -*- +""" + pygments.lexers._vbscript_builtins + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + These are manually translated lists from + http://www.indusoft.com/pdf/VBScript%20Reference.pdf. + + :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" +KEYWORDS = [ + # dim: special rule + 'call', + 'case', + 'class', + # const: special rule + 'do', + 'each', + 'else', + 'elseif', + 'end', + 'erase', + 'execute', + 'function', + 'exit', + 'for', + 'function', + 'GetRef', + 'global', + 'if', + 'let', + 'loop', + 'next', + 'new', + # option: special rule + 'private', + 'public', + 'redim', + 'select', + 'set', + 'sub', + 'then', + 'wend', + 'while', + 'with', +] + +BUILTIN_FUNCTIONS = [ + 'Abs', + 'Array', + 'Asc', + 'Atn', + 'CBool', + 'CByte', + 'CCur', + 'CDate', + 'CDbl', + 'Chr', + 'CInt', + 'CLng', + 'Cos', + 'CreateObject', + 'CSng', + 'CStr', + 'Date', + 'DateAdd', + 'DateDiff', + 'DatePart', + 'DateSerial', + 'DateValue', + 'Day', + 'Eval', + 'Exp', + 'Filter', + 'Fix', + 'FormatCurrency', + 'FormatDateTime', + 'FormatNumber', + 'FormatPercent', + 'GetObject', + 'GetLocale', + 'Hex', + 'Hour', + 'InStr', + 'inStrRev', + 'Int', + 'IsArray', + 'IsDate', + 'IsEmpty', + 'IsNull', + 'IsNumeric', + 'IsObject', + 'Join', + 'LBound', + 'LCase', + 'Left', + 'Len', + 'LoadPicture', + 'Log', + 'LTrim', + 'Mid', + 'Minute', + 'Month', + 'MonthName', + 'MsgBox', + 'Now', + 'Oct', + 'Randomize', + 'RegExp', + 'Replace', + 'RGB', + 'Right', + 'Rnd', + 'Round', + 'RTrim', + 'ScriptEngine', + 'ScriptEngineBuildVersion', + 'ScriptEngineMajorVersion', + 'ScriptEngineMinorVersion', + 'Second', + 'SetLocale', + 'Sgn', + 'Space', + 'Split', + 'Sqr', + 'StrComp', + 'String', + 'StrReverse', + 'Tan', + 'Time', + 'Timer', + 'TimeSerial', + 'TimeValue', + 'Trim', + 'TypeName', + 'UBound', + 'UCase', + 'VarType', + 'Weekday', + 'WeekdayName', + 'Year', +] + +BUILTIN_VARIABLES = [ + 'Debug', + 'Dictionary', + 'Drive', + 'Drives', + 'Err', + 'File', + 'Files', + 'FileSystemObject', + 'Folder', + 'Folders', + 'Match', + 'Matches', + 'RegExp', + 'Submatches', + 'TextStream', +] + +OPERATORS = [ + '+', + '-', + '*', + '/', + '\\', + '^', + '|', + '<', + '<=', + '>', + '>=', + '=', + '<>', + '&', + '$', +] + +OPERATOR_WORDS = [ + 'mod', + 'and', + 'or', + 'xor', + 'eqv', + 'imp', + 'is', + 'not', +] + +BUILTIN_CONSTANTS = [ + 'vbAbort', + 'vbAbortRetryIgnore', + 'vbApplicationModal', + 'vbArray', + 'vbBinaryCompare', + 'vbBlack', + 'vbBlue', + 'vbBoole', + 'vbByte', + 'vbCancel', + 'vbCr', + 'vbCritical', + 'vbCrLf', + 'vbCurrency', + 'vbCyan', + 'vbDataObject', + 'vbDate', + 'vbDefaultButton1', + 'vbDefaultButton2', + 'vbDefaultButton3', + 'vbDefaultButton4', + 'vbDouble', + 'vbEmpty', + 'vbError', + 'vbExclamation', + 'vbFalse', + 'vbFirstFullWeek', + 'vbFirstJan1', + 'vbFormFeed', + 'vbFriday', + 'vbGeneralDate', + 'vbGreen', + 'vbIgnore', + 'vbInformation', + 'vbInteger', + 'vbLf', + 'vbLong', + 'vbLongDate', + 'vbLongTime', + 'vbMagenta', + 'vbMonday', + 'vbMsgBoxHelpButton', + 'vbMsgBoxRight', + 'vbMsgBoxRtlReading', + 'vbMsgBoxSetForeground', + 'vbNewLine', + 'vbNo', + 'vbNull', + 'vbNullChar', + 'vbNullString', + 'vbObject', + 'vbObjectError', + 'vbOK', + 'vbOKCancel', + 'vbOKOnly', + 'vbQuestion', + 'vbRed', + 'vbRetry', + 'vbRetryCancel', + 'vbSaturday', + 'vbShortDate', + 'vbShortTime', + 'vbSingle', + 'vbString', + 'vbSunday', + 'vbSystemModal', + 'vbTab', + 'vbTextCompare', + 'vbThursday', + 'vbTrue', + 'vbTuesday', + 'vbUseDefault', + 'vbUseSystem', + 'vbUseSystem', + 'vbVariant', + 'vbVerticalTab', + 'vbWednesday', + 'vbWhite', + 'vbYellow', + 'vbYes', + 'vbYesNo', + 'vbYesNoCancel', +] \ No newline at end of file diff --git a/pygments/lexers/basic.py b/pygments/lexers/basic.py index a73ad8b4..09d9bfb9 100644 --- a/pygments/lexers/basic.py +++ b/pygments/lexers/basic.py @@ -12,11 +12,13 @@ import re from pygments.lexer import RegexLexer, bygroups, default, words, include -from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ - Number, Punctuation +from pygments.token import Comment, Error, Keyword, Name, Number, \ + Punctuation, Operator, String, Text, Whitespace +from pygments.lexers import _vbscript_builtins + __all__ = ['BlitzBasicLexer', 'BlitzMaxLexer', 'MonkeyLexer', 'CbmBasicV2Lexer', - 'QBasicLexer'] + 'QBasicLexer', 'VBScriptLexer'] class BlitzMaxLexer(RegexLexer): @@ -498,3 +500,65 @@ class QBasicLexer(RegexLexer): def analyse_text(text): if '$DYNAMIC' in text or '$STATIC' in text: return 0.9 + + +class VBScriptLexer(RegexLexer): + """ + VBScript is scripting language that is modeled on Visual Basic. + + .. versionadded:: 2.2 + """ + name = 'VBScript' + aliases = [] + filenames = ['*.vbs', '*.VBS'] + flags = re.IGNORECASE + + tokens = { + 'root': [ + (r"'[^\n]*", Comment.Single), + (r'\s+', Whitespace), + ('"', String.Double, 'string'), + ('&h[0-9a-f]+', Number.Hex), + # Float variant 1, for example: 1., 1.e2, 1.2e3 + (r'[0-9]+\.[0-9]*(e[+-]?[0-9]+)?', Number.Float), + (r'\.[0-9]+(e[+-]?[0-9]+)?', Number.Float), # Float variant 2, for example: .1, .1e2 + (r'[0-9]+e[+-]?[0-9]+', Number.Float), # Float variant 3, for example: 123e45 + ('\d+', Number.Integer), + ('#.+#', String), # date or time value + (r'(dim)(\s+)([a-z_][a-z0-9_]*)', + bygroups(Keyword.Declaration, Whitespace, Name.Variable), 'dim_more'), + (r'(function|sub)(\s+)([a-z_][a-z0-9_]*)', + bygroups(Keyword.Declaration, Whitespace, Name.Function)), + (r'(class)(\s+)([a-z_][a-z0-9_]*)', bygroups(Keyword.Declaration, Whitespace, Name.Class)), + (r'(const)(\s+)([a-z_][a-z0-9_]*)', bygroups(Keyword.Declaration, Whitespace, Name.Constant)), + (r'(end)(\s+)(class|function|if|property|sub|with)', bygroups(Keyword, Whitespace, Keyword)), + (r'(on)(\s+)(error)(\s+)(goto)(\s+)(0)', + bygroups(Keyword, Whitespace, Keyword, Whitespace, Keyword, Whitespace, Number.Integer)), + (r'(on)(\s+)(error)(\s+)(resume)(\s+)(next)', + bygroups(Keyword, Whitespace, Keyword, Whitespace, Keyword, Whitespace, Keyword)), + (r'(option)(\s+)(explicit)', bygroups(Keyword, Whitespace, Keyword)), + (r'(property)(\s+)(get|let|set)(\s+)([a-z_][a-z0-9_]*)', + bygroups(Keyword.Declaration, Whitespace, Keyword.Declaration, Whitespace, Name.Property)), + (r'rem\s.*[^\n]*', Comment.Single), + (words(_vbscript_builtins.KEYWORDS, suffix=r'\b'), Keyword), + (words(_vbscript_builtins.OPERATORS), Operator), + (words(_vbscript_builtins.OPERATOR_WORDS, suffix=r'\b'), Operator.Word), + (words(_vbscript_builtins.BUILTIN_CONSTANTS, suffix=r'\b'), Name.Constant), + (words(_vbscript_builtins.BUILTIN_FUNCTIONS, suffix=r'\b'), Name.Builtin), + (words(_vbscript_builtins.BUILTIN_VARIABLES, suffix=r'\b'), Name.Builtin), + (r'[a-z_][a-z0-9_]*', Name), + (r'\b_\n', Operator), + (words(r'(),.:'), Punctuation), + ('.+(\n)?', Error) + ], + 'dim_more': [ + (r'(\s*)(,)(\s*)([a-z_][a-z0-9]*)', bygroups(Whitespace, Punctuation, Whitespace, Name.Variable)), + default('#pop'), + ], + 'string': [ + (r'[^"\n]+', String.Double), + (r'\"\"', String.Double), + (r'"', String.Double, '#pop'), + (r'\n', Error, '#pop'), # Unterminated string + ], + } \ No newline at end of file diff --git a/tests/examplefiles/example.vbs b/tests/examplefiles/example.vbs new file mode 100644 index 00000000..d962b73d --- /dev/null +++ b/tests/examplefiles/example.vbs @@ -0,0 +1,55 @@ +rem VBScript examples + +' Various constants of different types +const someText = "some " & """text""" +const someInt = 123 +const someHex = &h3110c0d3 +const someFloat = 123.45e-67 +const someDate = #1/2/2016# +const someTime = #12:34:56 AM# +const someBool = vbTrue ' -1 + +' Do some math. +radius = 1.e2 +area = radius ^ 2 * 3.1315 +a = 17 : b = 23 +c = sqr(a ^2 + b ^ 2) + +' Write 10 files. +For i = 1 to 10 + createFile( i ) +Next + +Public Sub createFile(a) + Dim fso, TargetFile + TargetPath = "C:\some_" & a & ".tmp" + Set fso = CreateObject("Scripting.FileSystemObject") + Set TargetFile = fso.CreateTextFile(TargetPath) + TargetFile.WriteLine("Hello " & vbCrLf & "world!") + TargetFile.Close +End Sub + +' Define a class with a property. +Class Customer + Private m_CustomerName + + Private Sub Class_Initialize + m_CustomerName = "" + End Sub + + ' CustomerName property. + Public Property Get CustomerName + CustomerName = m_CustomerName + End Property + + Public Property Let CustomerName(custname) + m_CustomerName = custname + End Property +End Class + +' Special constructs +Option Explicit +On Error Resume Next +On Error Goto 0 + +' Comment without terminating CR/LF. \ No newline at end of file diff --git a/tests/test_basic.py b/tests/test_basic.py new file mode 100644 index 00000000..03d10cd2 --- /dev/null +++ b/tests/test_basic.py @@ -0,0 +1,74 @@ +# -*- coding: utf-8 -*- +""" + Pygments Basic lexers tests + ~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + :copyright: Copyright 2006-2016 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" +import unittest + +from pygments.lexers.basic import VBScriptLexer +from pygments.token import Error, Name, Number, Punctuation, String, Whitespace + + +class VBScriptLexerTest(unittest.TestCase): + + def setUp(self): + self.lexer = VBScriptLexer() + + def _assert_are_tokens_of_type(self, examples, expected_token_type): + for test_number, example in enumerate(examples.split(), 1): + token_count = 0 + for token_type, token_value in self.lexer.get_tokens(example): + if token_type != Whitespace: + token_count += 1 + self.assertEqual( + token_type, expected_token_type, + 'token_type #%d for %s is be %s but must be %s' % + (test_number, token_value, token_type, expected_token_type)) + self.assertEqual( + token_count, 1, + '%s must yield exactly 1 token instead of %d' % + (example, token_count)) + + def _assert_tokens_match(self, text, expected_tokens_without_trailing_newline): + actual_tokens = tuple(self.lexer.get_tokens(text)) + if (len(actual_tokens) >= 1) and (actual_tokens[-1] == (Whitespace, '\n')): + actual_tokens = tuple(actual_tokens[:-1]) + self.assertEqual( + expected_tokens_without_trailing_newline, actual_tokens, + 'text must yield expected tokens: %s' % text) + + def test_can_lex_float(self): + self._assert_are_tokens_of_type( + '1. 1.e1 .1 1.2 1.2e3 1.2e+3 1.2e-3 1e2', Number.Float) + self._assert_tokens_match( + '1e2.1e2', + ((Number.Float, '1e2'), (Number.Float, '.1e2')) + ) + + def test_can_reject_almost_float(self): + self._assert_tokens_match( + '.e1', + ((Punctuation, '.'), (Name, 'e1'))) + + def test_can_lex_integer(self): + self._assert_are_tokens_of_type( + '1 23 456', Number.Integer) + + def test_can_lex_names(self): + self._assert_are_tokens_of_type( + u'thingy thingy123 _thingy _123', Name) + + def test_can_recover_after_unterminated_string(self): + self._assert_tokens_match( + '"x\nx', + ((String.Double, '"'), (String.Double, 'x'), (Error, '\n'), (Name, 'x')) + ) + + def test_can_recover_from_invalid_character(self): + self._assert_tokens_match( + 'a;bc\nd', + ((Name, 'a'), (Error, ';bc\n'), (Name, 'd')) + ) -- cgit v1.2.1 From 590580e08ee53b7934a7886bf43f42878d77ce5c Mon Sep 17 00:00:00 2001 From: Thomas Aglassinger Date: Sun, 2 Oct 2016 15:41:33 +0200 Subject: Added a few missing constants and keywords for VBScript. --- pygments/lexers/_vbscript_builtins.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pygments/lexers/_vbscript_builtins.py b/pygments/lexers/_vbscript_builtins.py index 493e5c7c..7d514790 100644 --- a/pygments/lexers/_vbscript_builtins.py +++ b/pygments/lexers/_vbscript_builtins.py @@ -10,6 +10,8 @@ :license: BSD, see LICENSE for details. """ KEYWORDS = [ + 'ByRef', + 'ByVal', # dim: special rule 'call', 'case', @@ -190,6 +192,8 @@ OPERATOR_WORDS = [ ] BUILTIN_CONSTANTS = [ + 'False', + 'True', 'vbAbort', 'vbAbortRetryIgnore', 'vbApplicationModal', -- cgit v1.2.1 From ae448d124630513e63151850e8a458b68ece33b6 Mon Sep 17 00:00:00 2001 From: Nathan Reed Date: Wed, 5 Oct 2016 17:14:50 -0700 Subject: Add HLSL lexer --- pygments/lexers/_mapping.py | 1 + pygments/lexers/graphics.py | 147 +++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 147 insertions(+), 1 deletion(-) diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py index a6097b1c..6412d567 100644 --- a/pygments/lexers/_mapping.py +++ b/pygments/lexers/_mapping.py @@ -174,6 +174,7 @@ LEXERS = { 'GosuTemplateLexer': ('pygments.lexers.jvm', 'Gosu Template', ('gst',), ('*.gst',), ('text/x-gosu-template',)), 'GroffLexer': ('pygments.lexers.markup', 'Groff', ('groff', 'nroff', 'man'), ('*.[1234567]', '*.man'), ('application/x-troff', 'text/troff')), 'GroovyLexer': ('pygments.lexers.jvm', 'Groovy', ('groovy',), ('*.groovy', '*.gradle'), ('text/x-groovy',)), + 'HLSLShaderLexer': ('pygments.lexers.graphics', 'HLSL', ('hlsl',), ('*.hlsl', '*.hlsli'), ('text/x-hlsl',)), 'HamlLexer': ('pygments.lexers.html', 'Haml', ('haml',), ('*.haml',), ('text/x-haml',)), 'HandlebarsHtmlLexer': ('pygments.lexers.templates', 'HTML+Handlebars', ('html+handlebars',), ('*.handlebars', '*.hbs'), ('text/html+handlebars', 'text/x-handlebars-template')), 'HandlebarsLexer': ('pygments.lexers.templates', 'Handlebars', ('handlebars',), (), ()), diff --git a/pygments/lexers/graphics.py b/pygments/lexers/graphics.py index b40e0286..a5da2375 100644 --- a/pygments/lexers/graphics.py +++ b/pygments/lexers/graphics.py @@ -15,7 +15,7 @@ from pygments.token import Text, Comment, Operator, Keyword, Name, \ Number, Punctuation, String __all__ = ['GLShaderLexer', 'PostScriptLexer', 'AsymptoteLexer', 'GnuplotLexer', - 'PovrayLexer'] + 'PovrayLexer', 'HLSLShaderLexer'] class GLShaderLexer(RegexLexer): @@ -75,6 +75,151 @@ class GLShaderLexer(RegexLexer): } +class HLSLShaderLexer(RegexLexer): + """ + HLSL (Microsoft Direct3D Shader) lexer. + + .. versionadded:: 2.2 + """ + name = 'HLSL' + aliases = ['hlsl'] + filenames = ['*.hlsl', '*.hlsli'] + mimetypes = ['text/x-hlsl'] + + tokens = { + 'root': [ + (r'^#.*', Comment.Preproc), + (r'//.*', Comment.Single), + (r'/(\\\n)?[*](.|\n)*?[*](\\\n)?/', Comment.Multiline), + (r'\+|-|~|!=?|\*|/|%|<<|>>|<=?|>=?|==?|&&?|\^|\|\|?', + Operator), + (r'[?:]', Operator), # quick hack for ternary + (r'\bdefined\b', Operator), + (r'[;{}(),.\[\]]', Punctuation), + # FIXME when e is present, no decimal point needed + (r'[+-]?\d*\.\d+([eE][-+]?\d+)?f?', Number.Float), + (r'[+-]?\d+\.\d*([eE][-+]?\d+)?f?', Number.Float), + (r'0[xX][0-9a-fA-F]*', Number.Hex), + (r'0[0-7]*', Number.Oct), + (r'[1-9][0-9]*', Number.Integer), + (words(( + 'asm','asm_fragment','break','case','cbuffer','centroid','class', + 'column_major','compile','compile_fragment','const','continue', + 'default','discard','do','else','export','extern','for','fxgroup', + 'globallycoherent','groupshared','if','in','inline','inout', + 'interface','line','lineadj','linear','namespace','nointerpolation', + 'noperspective','NULL','out','packoffset','pass','pixelfragment', + 'point','precise','return','register','row_major','sample', + 'sampler','shared','stateblock','stateblock_state','static', + 'struct','switch','tbuffer','technique','technique10', + 'technique11','texture','typedef','triangle','triangleadj', + 'uniform','vertexfragment','volatile','while'), + prefix=r'\b', suffix=r'\b'), + Keyword), + (words(('true','false'), prefix=r'\b', suffix=r'\b'), + Keyword.Constant), + (words(( + 'auto','catch','char','const_cast','delete','dynamic_cast','enum', + 'explicit','friend','goto','long','mutable','new','operator', + 'private','protected','public','reinterpret_cast','short','signed', + 'sizeof','static_cast','template','this','throw','try','typename', + 'union','unsigned','using','virtual'), + prefix=r'\b', suffix=r'\b'), + Keyword.Reserved), + (words(( + 'dword','matrix','snorm','string','unorm','unsigned','void','vector', + 'BlendState','Buffer','ByteAddressBuffer','ComputeShader', + 'DepthStencilState','DepthStencilView','DomainShader', + 'GeometryShader','HullShader','InputPatch','LineStream', + 'OutputPatch','PixelShader','PointStream','RasterizerState', + 'RenderTargetView','RasterizerOrderedBuffer', + 'RasterizerOrderedByteAddressBuffer', + 'RasterizerOrderedStructuredBuffer','RasterizerOrderedTexture1D', + 'RasterizerOrderedTexture1DArray','RasterizerOrderedTexture2D', + 'RasterizerOrderedTexture2DArray','RasterizerOrderedTexture3D', + 'RWBuffer','RWByteAddressBuffer','RWStructuredBuffer', + 'RWTexture1D','RWTexture1DArray','RWTexture2D','RWTexture2DArray', + 'RWTexture3D','SamplerState','SamplerComparisonState', + 'StructuredBuffer','Texture1D','Texture1DArray','Texture2D', + 'Texture2DArray','Texture2DMS','Texture2DMSArray','Texture3D', + 'TextureCube','TextureCubeArray','TriangleStream','VertexShader'), + prefix=r'\b', suffix=r'\b'), + Keyword.Type), + (words(( + 'bool','double','float','int','half','min16float','min10float', + 'min16int','min12int','min16uint','uint'), + prefix=r'\b', suffix=r'([1-4](x[1-4])?)?\b'), + Keyword.Type), # vector and matrix types + (words(( + 'abort','abs','acos','all','AllMemoryBarrier', + 'AllMemoryBarrierWithGroupSync','any','AppendStructuredBuffer', + 'asdouble','asfloat','asin','asint','asuint','asuint','atan', + 'atan2','ceil','CheckAccessFullyMapped','clamp','clip', + 'CompileShader','ConsumeStructuredBuffer','cos','cosh','countbits', + 'cross','D3DCOLORtoUBYTE4','ddx','ddx_coarse','ddx_fine','ddy', + 'ddy_coarse','ddy_fine','degrees','determinant', + 'DeviceMemoryBarrier','DeviceMemoryBarrierWithGroupSync','distance', + 'dot','dst','errorf','EvaluateAttributeAtCentroid', + 'EvaluateAttributeAtSample','EvaluateAttributeSnapped','exp', + 'exp2','f16tof32','f32tof16','faceforward','firstbithigh', + 'firstbitlow','floor','fma','fmod','frac','frexp','fwidth', + 'GetRenderTargetSampleCount','GetRenderTargetSamplePosition', + 'GlobalOrderedCountIncrement','GroupMemoryBarrier', + 'GroupMemoryBarrierWithGroupSync','InterlockedAdd','InterlockedAnd', + 'InterlockedCompareExchange','InterlockedCompareStore', + 'InterlockedExchange','InterlockedMax','InterlockedMin', + 'InterlockedOr','InterlockedXor','isfinite','isinf','isnan', + 'ldexp','length','lerp','lit','log','log10','log2','mad','max', + 'min','modf','msad4','mul','noise','normalize','pow','printf', + 'Process2DQuadTessFactorsAvg','Process2DQuadTessFactorsMax', + 'Process2DQuadTessFactorsMin','ProcessIsolineTessFactors', + 'ProcessQuadTessFactorsAvg','ProcessQuadTessFactorsMax', + 'ProcessQuadTessFactorsMin','ProcessTriTessFactorsAvg', + 'ProcessTriTessFactorsMax','ProcessTriTessFactorsMin', + 'QuadReadLaneAt','QuadSwapX','QuadSwapY','radians','rcp', + 'reflect','refract','reversebits','round','rsqrt','saturate', + 'sign','sin','sincos','sinh','smoothstep','sqrt','step','tan', + 'tanh','tex1D','tex1D','tex1Dbias','tex1Dgrad','tex1Dlod', + 'tex1Dproj','tex2D','tex2D','tex2Dbias','tex2Dgrad','tex2Dlod', + 'tex2Dproj','tex3D','tex3D','tex3Dbias','tex3Dgrad','tex3Dlod', + 'tex3Dproj','texCUBE','texCUBE','texCUBEbias','texCUBEgrad', + 'texCUBElod','texCUBEproj','transpose','trunc','WaveAllBitAnd', + 'WaveAllMax','WaveAllMin','WaveAllBitOr','WaveAllBitXor', + 'WaveAllEqual','WaveAllProduct','WaveAllSum','WaveAllTrue', + 'WaveAnyTrue','WaveBallot','WaveGetLaneCount','WaveGetLaneIndex', + 'WaveGetOrderedIndex','WaveIsHelperLane','WaveOnce', + 'WavePrefixProduct','WavePrefixSum','WaveReadFirstLane', + 'WaveReadLaneAt'), + prefix=r'\b', suffix=r'\b'), + Name.Builtin), # built-in functions + (words(( + 'SV_ClipDistance','SV_ClipDistance0','SV_ClipDistance1', + 'SV_Culldistance','SV_CullDistance0','SV_CullDistance1', + 'SV_Coverage','SV_Depth','SV_DepthGreaterEqual', + 'SV_DepthLessEqual','SV_DispatchThreadID','SV_DomainLocation', + 'SV_GroupID','SV_GroupIndex','SV_GroupThreadID','SV_GSInstanceID', + 'SV_InnerCoverage','SV_InsideTessFactor','SV_InstanceID', + 'SV_IsFrontFace','SV_OutputControlPointID','SV_Position', + 'SV_PrimitiveID','SV_RenderTargetArrayIndex','SV_SampleIndex', + 'SV_StencilRef','SV_TessFactor','SV_VertexID', + 'SV_ViewportArrayIndex'), + prefix=r'\b', suffix=r'\b'), + Name.Decorator), # system-value semantics + (r'\bSV_Target[0-7]?\b', Name.Decorator), + (words(( + 'allow_uav_condition','branch','call','domain','earlydepthstencil', + 'fastopt','flatten','forcecase','instance','loop','maxtessfactor', + 'numthreads','outputcontrolpoints','outputtopology','partitioning', + 'patchconstantfunc','unroll'), + prefix=r'\b', suffix=r'\b'), + Name.Decorator), # attributes + (r'[a-zA-Z_]\w*', Name), + (r'\\$', Comment.Preproc), # backslash at end of line -- usually macro continuation + (r'\s+', Text), + ], + } + + class PostScriptLexer(RegexLexer): """ Lexer for PostScript files. -- cgit v1.2.1 From c22e58c9e4c2fd191782fc9659c0044a9c13ead3 Mon Sep 17 00:00:00 2001 From: Nathan Reed Date: Wed, 5 Oct 2016 17:15:01 -0700 Subject: Add myself to AUTHORS --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 66377ead..18051bb4 100644 --- a/AUTHORS +++ b/AUTHORS @@ -159,6 +159,7 @@ Other contributors, listed alphabetically, are: * Elias Rabel -- Fortran fixed form lexer * raichoo -- Idris lexer * Kashif Rasul -- CUDA lexer +* Nathan Reed -- HLSL lexer * Justin Reidy -- MXML lexer * Norman Richards -- JSON lexer * Corey Richardson -- Rust lexer updates -- cgit v1.2.1 From c8f8ac0ef923681e47ff852809c28978fa790176 Mon Sep 17 00:00:00 2001 From: Nathan Reed Date: Wed, 5 Oct 2016 17:36:45 -0700 Subject: Add example file for HLSL --- tests/examplefiles/example.hlsl | 157 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 157 insertions(+) create mode 100644 tests/examplefiles/example.hlsl diff --git a/tests/examplefiles/example.hlsl b/tests/examplefiles/example.hlsl new file mode 100644 index 00000000..77f1fa49 --- /dev/null +++ b/tests/examplefiles/example.hlsl @@ -0,0 +1,157 @@ +// A few random snippets of HLSL shader code I gathered... + +[numthreads(256, 1, 1)] +void cs_main(uint3 threadId : SV_DispatchThreadID) +{ + // Seed the PRNG using the thread ID + rng_state = threadId.x; + + // Generate a few numbers... + uint r0 = rand_xorshift(); + uint r1 = rand_xorshift(); + // Do some stuff with them... + + // Generate a random float in [0, 1)... + float f0 = float(rand_xorshift()) * (1.0 / 4294967296.0); + + // ...etc. +} + +// Constant buffer of parameters +cbuffer IntegratorParams : register(b0) +{ + float2 specPow; // Spec powers in XY directions (equal for isotropic BRDFs) + float3 L; // Unit vector toward light + int2 cThread; // Total threads launched in XY dimensions + int2 xyOutput; // Where in the output buffer to store the result +} + +static const float pi = 3.141592654; + +float AshikhminShirleyNDF(float3 H) +{ + float normFactor = sqrt((specPow.x + 2.0f) * (specPow.y + 2.0)) * (0.5f / pi); + float NdotH = H.z; + float2 Hxy = normalize(H.xy); + return normFactor * pow(NdotH, dot(specPow, Hxy * Hxy)); +} + +float BeckmannNDF(float3 H) +{ + float glossFactor = specPow.x * 0.5f + 1.0f; // This is 1/m^2 in the usual Beckmann formula + float normFactor = glossFactor * (1.0f / pi); + float NdotHSq = H.z * H.z; + return normFactor / (NdotHSq * NdotHSq) * exp(glossFactor * (1.0f - 1.0f / NdotHSq)); +} + +// Output buffer for compute shader (actually float, but must be declared as uint +// for atomic operations to work) +globallycoherent RWTexture2D o_data : register(u0); + +// Sum up the outputs of all threads and store to the output location +static const uint threadGroupSize2D = 16; +static const uint threadGroupSize1D = threadGroupSize2D * threadGroupSize2D; +groupshared float g_partialSums[threadGroupSize1D]; +void SumAcrossThreadsAndStore(float value, uint iThreadInGroup) +{ + // First reduce within the threadgroup: partial sums of 2, 4, 8... elements + // are calculated by 1/2, 1/4, 1/8... of the threads, always keeping the + // active threads at the front of the group to minimize divergence. + + // NOTE: there are faster ways of doing this...but this is simple to code + // and good enough. + + g_partialSums[iThreadInGroup] = value; + GroupMemoryBarrierWithGroupSync(); + + [unroll] for (uint i = threadGroupSize1D / 2; i > 0; i /= 2) + { + if (iThreadInGroup < i) + { + g_partialSums[iThreadInGroup] += g_partialSums[iThreadInGroup + i]; + } + GroupMemoryBarrierWithGroupSync(); + } + + // Then reduce across threadgroups: one thread from each group adds the group + // total to the final output location, using a software transactional memory + // style since D3D11 doesn't support atomic add on floats. + // (Assumes the output value has been cleared to zero beforehand.) + + if (iThreadInGroup == 0) + { + float threadGroupSum = g_partialSums[0]; + uint outputValueRead = o_data[xyOutput]; + while (true) + { + uint newOutputValue = asuint(asfloat(outputValueRead) + threadGroupSum); + uint previousOutputValue; + InterlockedCompareExchange( + o_data[xyOutput], outputValueRead, newOutputValue, previousOutputValue); + if (previousOutputValue == outputValueRead) + break; + outputValueRead = previousOutputValue; + } + } +} + +void main( + in Vertex i_vtx, + out Vertex o_vtx, + out float3 o_vecCamera : CAMERA, + out float4 o_uvzwShadow : UVZW_SHADOW, + out float4 o_posClip : SV_Position) +{ + o_vtx = i_vtx; + o_vecCamera = g_posCamera - i_vtx.m_pos; + o_uvzwShadow = mul(float4(i_vtx.m_pos, 1.0), g_matWorldToUvzwShadow); + o_posClip = mul(float4(i_vtx.m_pos, 1.0), g_matWorldToClip); +} + +#pragma pack_matrix(row_major) + +struct Vertex +{ + float3 m_pos : POSITION; + float3 m_normal : NORMAL; + float2 m_uv : UV; +}; + +cbuffer CBFrame : CB_FRAME // matches struct CBFrame in test.cpp +{ + float4x4 g_matWorldToClip; + float4x4 g_matWorldToUvzwShadow; + float3x3 g_matWorldToUvzShadowNormal; + float3 g_posCamera; + + float3 g_vecDirectionalLight; + float3 g_rgbDirectionalLight; + + float2 g_dimsShadowMap; + float g_normalOffsetShadow; + float g_shadowSharpening; + + float g_exposure; // Exposure multiplier +} + +Texture2D g_texDiffuse : register(t0); +SamplerState g_ss : register(s0); + +void main( + in Vertex i_vtx, + in float3 i_vecCamera : CAMERA, + in float4 i_uvzwShadow : UVZW_SHADOW, + out float3 o_rgb : SV_Target) +{ + float3 normal = normalize(i_vtx.m_normal); + + // Sample shadow map + float shadow = EvaluateShadow(i_uvzwShadow, normal); + + // Evaluate diffuse lighting + float3 diffuseColor = g_texDiffuse.Sample(g_ss, i_vtx.m_uv); + float3 diffuseLight = g_rgbDirectionalLight * (shadow * saturate(dot(normal, g_vecDirectionalLight))); + diffuseLight += SimpleAmbient(normal); + + o_rgb = diffuseColor * diffuseLight; +} -- cgit v1.2.1 From 5c31893c2af08aa03dc37ca18db213dbfb244c79 Mon Sep 17 00:00:00 2001 From: Jim Hester Date: Tue, 22 Nov 2016 10:14:00 -0500 Subject: SLexer improvements - Remove list of base functions in favor of classifying all calls as Name.Function - Rewrite regex for variable name detection - Correctly classify backtick variable names as Name - Add a few tests for the above --- pygments/lexers/r.py | 274 ++------------------------------------------------- tests/test_r.py | 61 ++++++++++++ 2 files changed, 68 insertions(+), 267 deletions(-) create mode 100644 tests/test_r.py diff --git a/pygments/lexers/r.py b/pygments/lexers/r.py index 1a47ca26..b03e4798 100644 --- a/pygments/lexers/r.py +++ b/pygments/lexers/r.py @@ -11,7 +11,7 @@ import re -from pygments.lexer import Lexer, RegexLexer, include, words, do_insertions +from pygments.lexer import Lexer, RegexLexer, include, do_insertions, bygroups from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ Number, Punctuation, Generic @@ -80,286 +80,25 @@ class SLexer(RegexLexer): mimetypes = ['text/S-plus', 'text/S', 'text/x-r-source', 'text/x-r', 'text/x-R', 'text/x-r-history', 'text/x-r-profile'] - builtins_base = ( - 'Arg', 'Conj', 'Cstack_info', 'Encoding', 'FALSE', - 'Filter', 'Find', 'I', 'ISOdate', 'ISOdatetime', 'Im', 'Inf', - 'La.svd', 'Map', 'Math.Date', 'Math.POSIXt', 'Math.data.frame', - 'Math.difftime', 'Math.factor', 'Mod', 'NA_character_', - 'NA_complex_', 'NA_real_', 'NCOL', 'NROW', 'NULLNA_integer_', 'NaN', - 'Negate', 'NextMethod', 'Ops.Date', 'Ops.POSIXt', 'Ops.data.frame', - 'Ops.difftime', 'Ops.factor', 'Ops.numeric_version', 'Ops.ordered', - 'Position', 'R.Version', 'R.home', 'R.version', 'R.version.string', - 'RNGkind', 'RNGversion', 'R_system_version', 'Re', 'Recall', - 'Reduce', 'Summary.Date', 'Summary.POSIXct', 'Summary.POSIXlt', - 'Summary.data.frame', 'Summary.difftime', 'Summary.factor', - 'Summary.numeric_version', 'Summary.ordered', 'Sys.Date', - 'Sys.chmod', 'Sys.getenv', 'Sys.getlocale', 'Sys.getpid', - 'Sys.glob', 'Sys.info', 'Sys.localeconv', 'Sys.readlink', - 'Sys.setFileTime', 'Sys.setenv', 'Sys.setlocale', 'Sys.sleep', - 'Sys.time', 'Sys.timezone', 'Sys.umask', 'Sys.unsetenv', - 'Sys.which', 'TRUE', 'UseMethod', 'Vectorize', 'abbreviate', 'abs', - 'acos', 'acosh', 'addNA', 'addTaskCallback', 'agrep', 'alist', - 'all', 'all.equal', 'all.equal.POSIXct', 'all.equal.character', - 'all.equal.default', 'all.equal.factor', 'all.equal.formula', - 'all.equal.language', 'all.equal.list', 'all.equal.numeric', - 'all.equal.raw', 'all.names', 'all.vars', 'any', 'anyDuplicated', - 'anyDuplicated.array', 'anyDuplicated.data.frame', - 'anyDuplicated.default', 'anyDuplicated.matrix', 'aperm', - 'aperm.default', 'aperm.table', 'append', 'apply', 'args', - 'arrayInd', 'as.Date', 'as.Date.POSIXct', 'as.Date.POSIXlt', - 'as.Date.character', 'as.Date.date', 'as.Date.dates', - 'as.Date.default', 'as.Date.factor', 'as.Date.numeric', - 'as.POSIXct', 'as.POSIXct.Date', 'as.POSIXct.POSIXlt', - 'as.POSIXct.date', 'as.POSIXct.dates', 'as.POSIXct.default', - 'as.POSIXct.numeric', 'as.POSIXlt', 'as.POSIXlt.Date', - 'as.POSIXlt.POSIXct', 'as.POSIXlt.character', 'as.POSIXlt.date', - 'as.POSIXlt.dates', 'as.POSIXlt.default', 'as.POSIXlt.factor', - 'as.POSIXlt.numeric', 'as.array', 'as.array.default', 'as.call', - 'as.character', 'as.character.Date', 'as.character.POSIXt', - 'as.character.condition', 'as.character.default', - 'as.character.error', 'as.character.factor', 'as.character.hexmode', - 'as.character.numeric_version', 'as.character.octmode', - 'as.character.srcref', 'as.complex', 'as.data.frame', - 'as.data.frame.AsIs', 'as.data.frame.Date', 'as.data.frame.POSIXct', - 'as.data.frame.POSIXlt', 'as.data.frame.array', - 'as.data.frame.character', 'as.data.frame.complex', - 'as.data.frame.data.frame', 'as.data.frame.default', - 'as.data.frame.difftime', 'as.data.frame.factor', - 'as.data.frame.integer', 'as.data.frame.list', - 'as.data.frame.logical', 'as.data.frame.matrix', - 'as.data.frame.model.matrix', 'as.data.frame.numeric', - 'as.data.frame.numeric_version', 'as.data.frame.ordered', - 'as.data.frame.raw', 'as.data.frame.table', 'as.data.frame.ts', - 'as.data.frame.vector', 'as.difftime', 'as.double', - 'as.double.POSIXlt', 'as.double.difftime', 'as.environment', - 'as.expression', 'as.expression.default', 'as.factor', - 'as.function', 'as.function.default', 'as.hexmode', 'as.integer', - 'as.list', 'as.list.Date', 'as.list.POSIXct', 'as.list.data.frame', - 'as.list.default', 'as.list.environment', 'as.list.factor', - 'as.list.function', 'as.list.numeric_version', 'as.logical', - 'as.logical.factor', 'as.matrix', 'as.matrix.POSIXlt', - 'as.matrix.data.frame', 'as.matrix.default', 'as.matrix.noquote', - 'as.name', 'as.null', 'as.null.default', 'as.numeric', - 'as.numeric_version', 'as.octmode', 'as.ordered', - 'as.package_version', 'as.pairlist', 'as.qr', 'as.raw', 'as.single', - 'as.single.default', 'as.symbol', 'as.table', 'as.table.default', - 'as.vector', 'as.vector.factor', 'asNamespace', 'asS3', 'asS4', - 'asin', 'asinh', 'assign', 'atan', 'atan2', 'atanh', - 'attachNamespace', 'attr', 'attr.all.equal', 'attributes', - 'autoload', 'autoloader', 'backsolve', 'baseenv', 'basename', - 'besselI', 'besselJ', 'besselK', 'besselY', 'beta', - 'bindingIsActive', 'bindingIsLocked', 'bindtextdomain', 'bitwAnd', - 'bitwNot', 'bitwOr', 'bitwShiftL', 'bitwShiftR', 'bitwXor', 'body', - 'bquote', 'browser', 'browserCondition', 'browserSetDebug', - 'browserText', 'builtins', 'by', 'by.data.frame', 'by.default', - 'bzfile', 'c.Date', 'c.POSIXct', 'c.POSIXlt', 'c.noquote', - 'c.numeric_version', 'call', 'callCC', 'capabilities', 'casefold', - 'cat', 'category', 'cbind', 'cbind.data.frame', 'ceiling', - 'char.expand', 'charToRaw', 'charmatch', 'chartr', 'check_tzones', - 'chol', 'chol.default', 'chol2inv', 'choose', 'class', - 'clearPushBack', 'close', 'close.connection', 'close.srcfile', - 'close.srcfilealias', 'closeAllConnections', 'col', 'colMeans', - 'colSums', 'colnames', 'commandArgs', 'comment', 'computeRestarts', - 'conditionCall', 'conditionCall.condition', 'conditionMessage', - 'conditionMessage.condition', 'conflicts', 'contributors', 'cos', - 'cosh', 'crossprod', 'cummax', 'cummin', 'cumprod', 'cumsum', 'cut', - 'cut.Date', 'cut.POSIXt', 'cut.default', 'dQuote', 'data.class', - 'data.matrix', 'date', 'debug', 'debugonce', - 'default.stringsAsFactors', 'delayedAssign', 'deparse', 'det', - 'determinant', 'determinant.matrix', 'dget', 'diag', 'diff', - 'diff.Date', 'diff.POSIXt', 'diff.default', 'difftime', 'digamma', - 'dim', 'dim.data.frame', 'dimnames', 'dimnames.data.frame', 'dir', - 'dir.create', 'dirname', 'do.call', 'dput', 'drop', 'droplevels', - 'droplevels.data.frame', 'droplevels.factor', 'dump', 'duplicated', - 'duplicated.POSIXlt', 'duplicated.array', 'duplicated.data.frame', - 'duplicated.default', 'duplicated.matrix', - 'duplicated.numeric_version', 'dyn.load', 'dyn.unload', 'eapply', - 'eigen', 'else', 'emptyenv', 'enc2native', 'enc2utf8', - 'encodeString', 'enquote', 'env.profile', 'environment', - 'environmentIsLocked', 'environmentName', 'eval', 'eval.parent', - 'evalq', 'exists', 'exp', 'expand.grid', 'expm1', 'expression', - 'factor', 'factorial', 'fifo', 'file', 'file.access', 'file.append', - 'file.choose', 'file.copy', 'file.create', 'file.exists', - 'file.info', 'file.link', 'file.path', 'file.remove', 'file.rename', - 'file.show', 'file.symlink', 'find.package', 'findInterval', - 'findPackageEnv', 'findRestart', 'floor', 'flush', - 'flush.connection', 'force', 'formals', 'format', - 'format.AsIs', 'format.Date', 'format.POSIXct', 'format.POSIXlt', - 'format.data.frame', 'format.default', 'format.difftime', - 'format.factor', 'format.hexmode', 'format.info', - 'format.libraryIQR', 'format.numeric_version', 'format.octmode', - 'format.packageInfo', 'format.pval', 'format.summaryDefault', - 'formatC', 'formatDL', 'forwardsolve', 'gamma', 'gc', 'gc.time', - 'gcinfo', 'gctorture', 'gctorture2', 'get', 'getAllConnections', - 'getCallingDLL', 'getCallingDLLe', 'getConnection', - 'getDLLRegisteredRoutines', 'getDLLRegisteredRoutines.DLLInfo', - 'getDLLRegisteredRoutines.character', 'getElement', - 'getExportedValue', 'getHook', 'getLoadedDLLs', 'getNamespace', - 'getNamespaceExports', 'getNamespaceImports', 'getNamespaceInfo', - 'getNamespaceName', 'getNamespaceUsers', 'getNamespaceVersion', - 'getNativeSymbolInfo', 'getOption', 'getRversion', 'getSrcLines', - 'getTaskCallbackNames', 'geterrmessage', 'gettext', 'gettextf', - 'getwd', 'gl', 'globalenv', 'gregexpr', 'grep', 'grepRaw', 'grepl', - 'gsub', 'gzcon', 'gzfile', 'head', 'iconv', 'iconvlist', - 'icuSetCollate', 'identical', 'identity', 'ifelse', 'importIntoEnv', - 'in', 'inherits', 'intToBits', 'intToUtf8', 'interaction', 'interactive', - 'intersect', 'inverse.rle', 'invisible', 'invokeRestart', - 'invokeRestartInteractively', 'is.R', 'is.array', 'is.atomic', - 'is.call', 'is.character', 'is.complex', 'is.data.frame', - 'is.double', 'is.element', 'is.environment', 'is.expression', - 'is.factor', 'is.finite', 'is.function', 'is.infinite', - 'is.integer', 'is.language', 'is.list', 'is.loaded', 'is.logical', - 'is.matrix', 'is.na', 'is.na.POSIXlt', 'is.na.data.frame', - 'is.na.numeric_version', 'is.name', 'is.nan', 'is.null', - 'is.numeric', 'is.numeric.Date', 'is.numeric.POSIXt', - 'is.numeric.difftime', 'is.numeric_version', 'is.object', - 'is.ordered', 'is.package_version', 'is.pairlist', 'is.primitive', - 'is.qr', 'is.raw', 'is.recursive', 'is.single', 'is.symbol', - 'is.table', 'is.unsorted', 'is.vector', 'isBaseNamespace', - 'isIncomplete', 'isNamespace', 'isOpen', 'isRestart', 'isS4', - 'isSeekable', 'isSymmetric', 'isSymmetric.matrix', 'isTRUE', - 'isatty', 'isdebugged', 'jitter', 'julian', 'julian.Date', - 'julian.POSIXt', 'kappa', 'kappa.default', 'kappa.lm', 'kappa.qr', - 'kronecker', 'l10n_info', 'labels', 'labels.default', 'lapply', - 'lazyLoad', 'lazyLoadDBexec', 'lazyLoadDBfetch', 'lbeta', 'lchoose', - 'length', 'length.POSIXlt', 'letters', 'levels', 'levels.default', - 'lfactorial', 'lgamma', 'library.dynam', 'library.dynam.unload', - 'licence', 'license', 'list.dirs', 'list.files', 'list2env', 'load', - 'loadNamespace', 'loadedNamespaces', 'loadingNamespaceInfo', - 'local', 'lockBinding', 'lockEnvironment', 'log', 'log10', 'log1p', - 'log2', 'logb', 'lower.tri', 'ls', 'make.names', 'make.unique', - 'makeActiveBinding', 'mapply', 'margin.table', 'mat.or.vec', - 'match', 'match.arg', 'match.call', 'match.fun', 'max', 'max.col', - 'mean', 'mean.Date', 'mean.POSIXct', 'mean.POSIXlt', 'mean.default', - 'mean.difftime', 'mem.limits', 'memCompress', 'memDecompress', - 'memory.profile', 'merge', 'merge.data.frame', 'merge.default', - 'message', 'mget', 'min', 'missing', 'mode', 'month.abb', - 'month.name', 'months', 'months.Date', 'months.POSIXt', - 'months.abb', 'months.nameletters', 'names', 'names.POSIXlt', - 'namespaceExport', 'namespaceImport', 'namespaceImportClasses', - 'namespaceImportFrom', 'namespaceImportMethods', 'nargs', 'nchar', - 'ncol', 'new.env', 'ngettext', 'nlevels', 'noquote', 'norm', - 'normalizePath', 'nrow', 'numeric_version', 'nzchar', 'objects', - 'oldClass', 'on.exit', 'open', 'open.connection', 'open.srcfile', - 'open.srcfilealias', 'open.srcfilecopy', 'options', 'order', - 'ordered', 'outer', 'packBits', 'packageEvent', - 'packageHasNamespace', 'packageStartupMessage', 'package_version', - 'pairlist', 'parent.env', 'parent.frame', 'parse', - 'parseNamespaceFile', 'paste', 'paste0', 'path.expand', - 'path.package', 'pipe', 'pmatch', 'pmax', 'pmax.int', 'pmin', - 'pmin.int', 'polyroot', 'pos.to.env', 'pretty', 'pretty.default', - 'prettyNum', 'print', 'print.AsIs', 'print.DLLInfo', - 'print.DLLInfoList', 'print.DLLRegisteredRoutines', 'print.Date', - 'print.NativeRoutineList', 'print.POSIXct', 'print.POSIXlt', - 'print.by', 'print.condition', 'print.connection', - 'print.data.frame', 'print.default', 'print.difftime', - 'print.factor', 'print.function', 'print.hexmode', - 'print.libraryIQR', 'print.listof', 'print.noquote', - 'print.numeric_version', 'print.octmode', 'print.packageInfo', - 'print.proc_time', 'print.restart', 'print.rle', - 'print.simple.list', 'print.srcfile', 'print.srcref', - 'print.summary.table', 'print.summaryDefault', 'print.table', - 'print.warnings', 'prmatrix', 'proc.time', 'prod', 'prop.table', - 'provideDimnames', 'psigamma', 'pushBack', 'pushBackLength', 'q', - 'qr', 'qr.Q', 'qr.R', 'qr.X', 'qr.coef', 'qr.default', 'qr.fitted', - 'qr.qty', 'qr.qy', 'qr.resid', 'qr.solve', 'quarters', - 'quarters.Date', 'quarters.POSIXt', 'quit', 'quote', 'range', - 'range.default', 'rank', 'rapply', 'raw', 'rawConnection', - 'rawConnectionValue', 'rawShift', 'rawToBits', 'rawToChar', 'rbind', - 'rbind.data.frame', 'rcond', 'read.dcf', 'readBin', 'readChar', - 'readLines', 'readRDS', 'readRenviron', 'readline', 'reg.finalizer', - 'regexec', 'regexpr', 'registerS3method', 'registerS3methods', - 'regmatches', 'remove', 'removeTaskCallback', 'rep', 'rep.Date', - 'rep.POSIXct', 'rep.POSIXlt', 'rep.factor', 'rep.int', - 'rep.numeric_version', 'rep_len', 'replace', 'replicate', - 'requireNamespace', 'restartDescription', 'restartFormals', - 'retracemem', 'rev', 'rev.default', 'rle', 'rm', 'round', - 'round.Date', 'round.POSIXt', 'row', 'row.names', - 'row.names.data.frame', 'row.names.default', 'rowMeans', 'rowSums', - 'rownames', 'rowsum', 'rowsum.data.frame', 'rowsum.default', - 'sQuote', 'sample', 'sample.int', 'sapply', 'save', 'save.image', - 'saveRDS', 'scale', 'scale.default', 'scan', 'search', - 'searchpaths', 'seek', 'seek.connection', 'seq', 'seq.Date', - 'seq.POSIXt', 'seq.default', 'seq.int', 'seq_along', 'seq_len', - 'sequence', 'serialize', 'set.seed', 'setHook', 'setNamespaceInfo', - 'setSessionTimeLimit', 'setTimeLimit', 'setdiff', 'setequal', - 'setwd', 'shQuote', 'showConnections', 'sign', 'signalCondition', - 'signif', 'simpleCondition', 'simpleError', 'simpleMessage', - 'simpleWarning', 'simplify2array', 'sin', 'single', - 'sinh', 'sink', 'sink.number', 'slice.index', 'socketConnection', - 'socketSelect', 'solve', 'solve.default', 'solve.qr', 'sort', - 'sort.POSIXlt', 'sort.default', 'sort.int', 'sort.list', 'split', - 'split.Date', 'split.POSIXct', 'split.data.frame', 'split.default', - 'sprintf', 'sqrt', 'srcfile', 'srcfilealias', 'srcfilecopy', - 'srcref', 'standardGeneric', 'stderr', 'stdin', 'stdout', 'stop', - 'stopifnot', 'storage.mode', 'strftime', 'strptime', 'strsplit', - 'strtoi', 'strtrim', 'structure', 'strwrap', 'sub', 'subset', - 'subset.data.frame', 'subset.default', 'subset.matrix', - 'substitute', 'substr', 'substring', 'sum', 'summary', - 'summary.Date', 'summary.POSIXct', 'summary.POSIXlt', - 'summary.connection', 'summary.data.frame', 'summary.default', - 'summary.factor', 'summary.matrix', 'summary.proc_time', - 'summary.srcfile', 'summary.srcref', 'summary.table', - 'suppressMessages', 'suppressPackageStartupMessages', - 'suppressWarnings', 'svd', 'sweep', 'sys.call', 'sys.calls', - 'sys.frame', 'sys.frames', 'sys.function', 'sys.load.image', - 'sys.nframe', 'sys.on.exit', 'sys.parent', 'sys.parents', - 'sys.save.image', 'sys.source', 'sys.status', 'system', - 'system.file', 'system.time', 'system2', 't', 't.data.frame', - 't.default', 'table', 'tabulate', 'tail', 'tan', 'tanh', 'tapply', - 'taskCallbackManager', 'tcrossprod', 'tempdir', 'tempfile', - 'testPlatformEquivalence', 'textConnection', 'textConnectionValue', - 'toString', 'toString.default', 'tolower', 'topenv', 'toupper', - 'trace', 'traceback', 'tracemem', 'tracingState', 'transform', - 'transform.data.frame', 'transform.default', 'trigamma', 'trunc', - 'trunc.Date', 'trunc.POSIXt', 'truncate', 'truncate.connection', - 'try', 'tryCatch', 'typeof', 'unclass', 'undebug', 'union', - 'unique', 'unique.POSIXlt', 'unique.array', 'unique.data.frame', - 'unique.default', 'unique.matrix', 'unique.numeric_version', - 'units', 'units.difftime', 'unix.time', 'unlink', 'unlist', - 'unloadNamespace', 'unlockBinding', 'unname', 'unserialize', - 'unsplit', 'untrace', 'untracemem', 'unz', 'upper.tri', 'url', - 'utf8ToInt', 'vapply', 'version', 'warning', 'warnings', 'weekdays', - 'weekdays.Date', 'weekdays.POSIXt', 'which', 'which.max', - 'which.min', 'with', 'with.default', 'withCallingHandlers', - 'withRestarts', 'withVisible', 'within', 'within.data.frame', - 'within.list', 'write', 'write.dcf', 'writeBin', 'writeChar', - 'writeLines', 'xor', 'xor.hexmode', 'xor.octmode', - 'xpdrows.data.frame', 'xtfrm', 'xtfrm.AsIs', 'xtfrm.Date', - 'xtfrm.POSIXct', 'xtfrm.POSIXlt', 'xtfrm.Surv', 'xtfrm.default', - 'xtfrm.difftime', 'xtfrm.factor', 'xtfrm.numeric_version', 'xzfile', - 'zapsmall' - ) - + valid_name = r'`[^`]+`|(?:(?:[a-zA-z]|[_.][^0-9])[\w_.]*)' tokens = { 'comments': [ (r'#.*$', Comment.Single), ], 'valid_name': [ - (r'[a-zA-Z][\w.]*', Text), - # can begin with ., but not if that is followed by a digit - (r'\.[a-zA-Z_][\w.]*', Text), + (valid_name, Name), ], 'punctuation': [ (r'\[{1,2}|\]{1,2}|\(|\)|;|,', Punctuation), ], 'keywords': [ - (words(builtins_base, suffix=r'(?![\w. =])'), - Keyword.Pseudo), (r'(if|else|for|while|repeat|in|next|break|return|switch|function)' r'(?![\w.])', Keyword.Reserved), - (r'(array|category|character|complex|double|function|integer|list|' - r'logical|matrix|numeric|vector|data.frame|c)' - r'(?![\w.])', - Keyword.Type), - (r'(library|require|attach|detach|source)' - r'(?![\w.])', - Keyword.Namespace) ], 'operators': [ (r'<>?|-|==|<=|>=|<|>|&&?|!=|\|\|?|\?', Operator), - (r'\*|\+|\^|/|!|%[^%]*%|=|~|\$|@|:{1,3}', Operator) + (r'\*|\+|\^|/|!|%[^%]*%|=|~|\$|@|:{1,3}', Operator), ], 'builtin_symbols': [ (r'(NULL|NA(_(integer|real|complex|character)_)?|' @@ -379,17 +118,18 @@ class SLexer(RegexLexer): include('comments'), # whitespaces (r'\s+', Text), - (r'`.*?`', String.Backtick), (r'\'', String, 'string_squote'), (r'\"', String, 'string_dquote'), include('builtin_symbols'), + include('valid_name'), include('numbers'), include('keywords'), include('punctuation'), include('operators'), - include('valid_name'), ], 'root': [ + # calls: + ('r%s\s*(?=\()' % valid_name, Name.Function), include('statements'), # blocks: (r'\{|\}', Punctuation), diff --git a/tests/test_r.py b/tests/test_r.py new file mode 100644 index 00000000..a6464480 --- /dev/null +++ b/tests/test_r.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- +""" + R Tests + ~~~~~~~~~ + + :copyright: Copyright 2006-2016 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import unittest + +from pygments.lexers import SLexer +from pygments.token import Token, Name, Punctuation + + +class RTest(unittest.TestCase): + def setUp(self): + self.lexer = SLexer() + + def testCall(self): + fragment = u'f(1, a)\n' + tokens = [ + (Name.Function, u'f'), + (Punctuation, u'('), + (Token.Literal.Number, u'1'), + (Punctuation, u','), + (Token.Text, u' '), + (Token.Name, u'a'), + (Punctuation, u')'), + (Token.Text, u'\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testName1(self): + fragment = u'._a_2.c' + tokens = [ + (Name, u'._a_2.c'), + (Token.Text, u'\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testName2(self): + # Invalid names are valid if backticks are used + fragment = u'`.1 blah`' + tokens = [ + (Name, u'`.1 blah`'), + (Token.Text, u'\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testCustomOperator(self): + fragment = u'7 % and % 8' + tokens = [ + (Token.Literal.Number, u'7'), + (Token.Text, u' '), + (Token.Operator, u'% and %'), + (Token.Text, u' '), + (Token.Literal.Number, u'8'), + (Token.Text, u'\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) -- cgit v1.2.1 From e028505914e2e201bb2e346e32d5d29453151750 Mon Sep 17 00:00:00 2001 From: Jim Hester Date: Tue, 22 Nov 2016 11:23:14 -0500 Subject: Support escaping internal backticks --- pygments/lexers/r.py | 2 +- tests/test_r.py | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/pygments/lexers/r.py b/pygments/lexers/r.py index b03e4798..64cf8bf1 100644 --- a/pygments/lexers/r.py +++ b/pygments/lexers/r.py @@ -80,7 +80,7 @@ class SLexer(RegexLexer): mimetypes = ['text/S-plus', 'text/S', 'text/x-r-source', 'text/x-r', 'text/x-R', 'text/x-r-history', 'text/x-r-profile'] - valid_name = r'`[^`]+`|(?:(?:[a-zA-z]|[_.][^0-9])[\w_.]*)' + valid_name = r'(?:`[^`\\]*(?:\\.[^`\\]*)*`)|(?:(?:[a-zA-z]|[_.][^0-9])[\w_.]*)' tokens = { 'comments': [ (r'#.*$', Comment.Single), diff --git a/tests/test_r.py b/tests/test_r.py index a6464480..d0e9090b 100644 --- a/tests/test_r.py +++ b/tests/test_r.py @@ -48,6 +48,15 @@ class RTest(unittest.TestCase): ] self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + def testName3(self): + # Internal backticks can be escaped + fragment = u'`.1 \` blah`' + tokens = [ + (Name, u'`.1 \` blah`'), + (Token.Text, u'\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + def testCustomOperator(self): fragment = u'7 % and % 8' tokens = [ -- cgit v1.2.1 From 4c6730a5dd1f37bb1155b7789bc2779d30a7cd30 Mon Sep 17 00:00:00 2001 From: Nathan Whetsell Date: Sun, 11 Dec 2016 12:10:04 -0500 Subject: Fix capitalization of Csound --- pygments/lexers/csound.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pygments/lexers/csound.py b/pygments/lexers/csound.py index 95ee73d8..f67c12bb 100644 --- a/pygments/lexers/csound.py +++ b/pygments/lexers/csound.py @@ -3,7 +3,7 @@ pygments.lexers.csound ~~~~~~~~~~~~~~~~~~~~~~ - Lexers for CSound languages. + Lexers for Csound languages. :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. -- cgit v1.2.1 From c0310d50f7f372c4f8bde9ec27d6c7c0c3396dd2 Mon Sep 17 00:00:00 2001 From: Nathan Whetsell Date: Sun, 11 Dec 2016 12:10:54 -0500 Subject: Update copyright year --- pygments/lexers/csound.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pygments/lexers/csound.py b/pygments/lexers/csound.py index f67c12bb..f4e06531 100644 --- a/pygments/lexers/csound.py +++ b/pygments/lexers/csound.py @@ -5,7 +5,7 @@ Lexers for Csound languages. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2016 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ -- cgit v1.2.1 From b817bdd7cf272e8e4f57068e4b8f8e07889c5f20 Mon Sep 17 00:00:00 2001 From: Nathan Whetsell Date: Sun, 11 Dec 2016 12:13:23 -0500 Subject: Make Csound #include string delimiters match preprocessor --- pygments/lexers/csound.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pygments/lexers/csound.py b/pygments/lexers/csound.py index f4e06531..8ddca2c4 100644 --- a/pygments/lexers/csound.py +++ b/pygments/lexers/csound.py @@ -25,7 +25,6 @@ newline = (r'((?:(?:;|//).*)*)(\n)', bygroups(Comment.Single, Text)) class CsoundLexer(RegexLexer): - # Subclasses must define a 'single-line string' state. tokens = { 'whitespace': [ (r'[ \t]+', Text), @@ -57,7 +56,7 @@ class CsoundLexer(RegexLexer): 'include': [ include('whitespace'), - (r'"', String, 'single-line string') + (r'([^ \t]).*?\1', String, '#pop') ], 'macro name': [ -- cgit v1.2.1 From c51b6c0ebadd7330a91f7240b707bec6bbd9da26 Mon Sep 17 00:00:00 2001 From: Nathan Whetsell Date: Sun, 11 Dec 2016 12:19:16 -0500 Subject: Update Csound macro nomenclature --- pygments/lexers/csound.py | 50 +++++++++++++++++++++++------------------------ 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/pygments/lexers/csound.py b/pygments/lexers/csound.py index 8ddca2c4..7976dd22 100644 --- a/pygments/lexers/csound.py +++ b/pygments/lexers/csound.py @@ -32,19 +32,19 @@ class CsoundLexer(RegexLexer): (r'/[*](.|\n)*?[*]/', Comment.Multiline) ], - 'macro call': [ + 'macro use': [ (r'(\$\w+\.?)(\()', bygroups(Comment.Preproc, Punctuation), - 'function macro call'), + 'function macro use'), (r'\$\w+(\.|\b)', Comment.Preproc) ], - 'function macro call': [ + 'function macro use': [ (r"((?:\\['\)]|[^'\)])+)(')", bygroups(Comment.Preproc, Punctuation)), (r"([^'\)]+)(\))", bygroups(Comment.Preproc, Punctuation), '#pop') ], - 'whitespace or macro call': [ + 'whitespace or macro use': [ include('whitespace'), - include('macro call') + include('macro use') ], 'preprocessor directives': [ @@ -62,7 +62,7 @@ class CsoundLexer(RegexLexer): 'macro name': [ include('whitespace'), (r'(\w+)(\()', bygroups(Comment.Preproc, Text), - 'function macro argument list'), + 'function macro parameter list'), (r'\w+', Comment.Preproc, 'object macro definition after name') ], 'object macro definition after name': [ @@ -73,12 +73,12 @@ class CsoundLexer(RegexLexer): (r'(\\#|[^#])+', Comment.Preproc), (r'#', Punctuation, '#pop:3') ], - 'function macro argument list': [ + 'function macro parameter list': [ (r"(\w+)(['#])", bygroups(Comment.Preproc, Punctuation)), (r'(\w+)(\))', bygroups(Comment.Preproc, Punctuation), - 'function macro definition after name') + 'function macro definition after parameter list') ], - 'function macro definition after name': [ + 'function macro definition after parameter list': [ (r'[ \t]+', Text), (r'#', Punctuation, 'function macro replacement text') ], @@ -113,14 +113,14 @@ class CsoundScoreLexer(CsoundLexer): ], 'statement': [ - include('whitespace or macro call'), + include('whitespace or macro use'), newline + ('#pop',), include('partial statement') ], 'root': [ newline, - include('whitespace or macro call'), + include('whitespace or macro use'), (r'[{}]', Punctuation, 'statement'), (r'[abefimq-tv-z]|[nN][pP]?', Keyword, 'statement') ], @@ -201,14 +201,14 @@ class CsoundOrchestraLexer(CsoundLexer): ], 'expression': [ - include('whitespace or macro call'), + include('whitespace or macro use'), newline + ('#pop',), include('partial expression') ], 'root': [ newline, - include('whitespace or macro call'), + include('whitespace or macro use'), (r'\binstr\b', Keyword, ('instrument block', 'instrument name list')), (r'\bopcode\b', Keyword, ('opcode block', 'opcode parameter list', 'opcode types', 'opcode types', 'opcode name')), @@ -217,53 +217,53 @@ class CsoundOrchestraLexer(CsoundLexer): ], 'instrument name list': [ - include('whitespace or macro call'), + include('whitespace or macro use'), (r'\d+|\+?[a-zA-Z_]\w*', Name.Function), (r',', Punctuation), newline + ('#pop',) ], 'instrument block': [ newline, - include('whitespace or macro call'), + include('whitespace or macro use'), (r'\bendin\b', Keyword, '#pop'), include('label'), default('expression') ], 'opcode name': [ - include('whitespace or macro call'), + include('whitespace or macro use'), (r'[a-zA-Z_]\w*', opcode_name_callback, '#pop') ], 'opcode types': [ - include('whitespace or macro call'), + include('whitespace or macro use'), (r'0|[]afijkKoOpPStV[]+', Keyword.Type, '#pop'), (r',', Punctuation) ], 'opcode parameter list': [ - include('whitespace or macro call'), + include('whitespace or macro use'), newline + ('#pop',) ], 'opcode block': [ newline, - include('whitespace or macro call'), + include('whitespace or macro use'), (r'\bendop\b', Keyword, '#pop'), include('label'), default('expression') ], 'goto label': [ - include('whitespace or macro call'), + include('whitespace or macro use'), (r'\w+', Name.Label, '#pop'), default('#pop') ], 'goto expression': [ - include('whitespace or macro call'), + include('whitespace or macro use'), (r',', Punctuation, '#pop'), include('partial expression') ], 'single-line string': [ - include('macro call'), + include('macro use'), (r'"', String, '#pop'), # From https://github.com/csound/csound/blob/develop/Opcodes/fout.c#L1405 (r'%\d*(\.\d+)?[cdhilouxX]', String.Interpol), @@ -277,7 +277,7 @@ class CsoundOrchestraLexer(CsoundLexer): ], 'scoreline opcode': [ - include('whitespace or macro call'), + include('whitespace or macro use'), (r'\{\{', String, 'scoreline'), default('#pop') ], @@ -287,7 +287,7 @@ class CsoundOrchestraLexer(CsoundLexer): ], 'python opcode': [ - include('whitespace or macro call'), + include('whitespace or macro use'), (r'\{\{', String, 'python'), default('#pop') ], @@ -297,7 +297,7 @@ class CsoundOrchestraLexer(CsoundLexer): ], 'lua opcode': [ - include('whitespace or macro call'), + include('whitespace or macro use'), (r'"', String, 'single-line string'), (r'\{\{', String, 'lua'), (r',', Punctuation), -- cgit v1.2.1 From 89e31c41d2c472f84c768d4fecebd8f01c4cacb0 Mon Sep 17 00:00:00 2001 From: Nathan Whetsell Date: Sun, 11 Dec 2016 12:22:55 -0500 Subject: Match Csound preprocessor --- pygments/lexers/csound.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/pygments/lexers/csound.py b/pygments/lexers/csound.py index 7976dd22..36003603 100644 --- a/pygments/lexers/csound.py +++ b/pygments/lexers/csound.py @@ -33,9 +33,9 @@ class CsoundLexer(RegexLexer): ], 'macro use': [ - (r'(\$\w+\.?)(\()', bygroups(Comment.Preproc, Punctuation), + (r'(\$[A-Z_a-z]\w*\.?)(\()', bygroups(Comment.Preproc, Punctuation), 'function macro use'), - (r'\$\w+(\.|\b)', Comment.Preproc) + (r'\$[A-Z_a-z]\w*(\.|\b)', Comment.Preproc) ], 'function macro use': [ (r"((?:\\['\)]|[^'\)])+)(')", bygroups(Comment.Preproc, Punctuation)), @@ -49,8 +49,8 @@ class CsoundLexer(RegexLexer): 'preprocessor directives': [ (r'#(e(nd(if)?|lse)|ifn?def|undef)\b|##', Comment.Preproc), - (r'#include\b', Comment.Preproc, 'include'), - (r'#[ \t]*define\b', Comment.Preproc, 'macro name'), + (r'#include', Comment.Preproc, 'include'), + (r'#[ \t]*define', Comment.Preproc, 'macro name'), (r'@+[ \t]*\d*', Comment.Preproc) ], @@ -61,9 +61,9 @@ class CsoundLexer(RegexLexer): 'macro name': [ include('whitespace'), - (r'(\w+)(\()', bygroups(Comment.Preproc, Text), + (r'([A-Z_a-z]\w*)(\()', bygroups(Comment.Preproc, Text), 'function macro parameter list'), - (r'\w+', Comment.Preproc, 'object macro definition after name') + (r'[A-Z_a-z]\w*', Comment.Preproc, 'object macro definition after name') ], 'object macro definition after name': [ include('whitespace'), @@ -74,8 +74,9 @@ class CsoundLexer(RegexLexer): (r'#', Punctuation, '#pop:3') ], 'function macro parameter list': [ - (r"(\w+)(['#])", bygroups(Comment.Preproc, Punctuation)), - (r'(\w+)(\))', bygroups(Comment.Preproc, Punctuation), + include('whitespace'), + (r"([A-Z_a-z]\w*)(['#])", bygroups(Comment.Preproc, Punctuation)), + (r'([A-Z_a-z]\w*)(\))', bygroups(Comment.Preproc, Punctuation), 'function macro definition after parameter list') ], 'function macro definition after parameter list': [ -- cgit v1.2.1 From 0d6d92cc895e325bc7afcad219a6cba523ab3ca9 Mon Sep 17 00:00:00 2001 From: Nathan Whetsell Date: Sun, 11 Dec 2016 12:23:55 -0500 Subject: Fix incorrect token --- pygments/lexers/csound.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pygments/lexers/csound.py b/pygments/lexers/csound.py index 36003603..080cead8 100644 --- a/pygments/lexers/csound.py +++ b/pygments/lexers/csound.py @@ -61,7 +61,7 @@ class CsoundLexer(RegexLexer): 'macro name': [ include('whitespace'), - (r'([A-Z_a-z]\w*)(\()', bygroups(Comment.Preproc, Text), + (r'([A-Z_a-z]\w*)(\()', bygroups(Comment.Preproc, Punctuation), 'function macro parameter list'), (r'[A-Z_a-z]\w*', Comment.Preproc, 'object macro definition after name') ], -- cgit v1.2.1 From a56edd5e77a551a3e310e25e1f204756b67d6730 Mon Sep 17 00:00:00 2001 From: Nathan Whetsell Date: Sun, 11 Dec 2016 12:29:56 -0500 Subject: Update for Csound 6.08.0 --- pygments/lexers/_csound_builtins.py | 201 ++++++++++++++++++++---------------- pygments/lexers/csound.py | 63 +++++++---- tests/examplefiles/test.csd | 32 +++--- tests/examplefiles/test.orc | 32 +++--- tests/examplefiles/test.sco | 2 +- 5 files changed, 180 insertions(+), 150 deletions(-) diff --git a/pygments/lexers/_csound_builtins.py b/pygments/lexers/_csound_builtins.py index a88e0a83..c74e66ab 100644 --- a/pygments/lexers/_csound_builtins.py +++ b/pygments/lexers/_csound_builtins.py @@ -3,48 +3,37 @@ pygments.lexers._csound_builtins ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2016 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ -# Opcodes in Csound 6.05 from -# csound --list-opcodes -# except -# cggoto -# cigoto -# cingoto (undocumented) -# ckgoto -# cngoto -# endin -# igoto -# instr -# kgoto -# loop_ge -# loop_gt -# loop_le -# loop_lt -# opcode -# return -# rigoto -# tigoto -# timout -# which are treated as keywords; the scoreline opcodes -# scoreline -# scoreline_i -# which allow Csound Score highlighting; the pyrun opcodes -# -# pylrun -# pylruni -# pylrunt -# pyrun -# pyruni -# pyrunt -# which allow Python highlighting; and the Lua opcodes -# lua_exec -# lua_opdef -# which allow Lua highlighting. +# Opcodes in Csound 6.08.0 at commit a1159a1 from +# csound --list-opcodes3 +# except for +# cggoto +# cigoto +# cingoto (undocumented) +# ckgoto +# cngoto +# cnkgoto (undocumented) +# endin +# igoto +# instr +# kgoto +# loop_ge +# loop_gt +# loop_le +# loop_lt +# opcode +# return +# rireturn +# rigoto +# tigoto +# timout +# which are treated as keywords in csound.py. + OPCODES = set(( 'ATSadd', 'ATSaddnz', @@ -133,8 +122,10 @@ OPCODES = set(( 'MixerSetLevel', 'MixerSetLevel_i', 'OSCinit', + 'OSCinitM', 'OSClisten', 'OSCsend', + 'S', 'a', 'abs', 'active', @@ -151,7 +142,6 @@ OPCODES = set(( 'ampmidid', 'areson', 'aresonk', - 'array', 'atone', 'atonek', 'atonex', @@ -163,9 +153,7 @@ OPCODES = set(( 'bbcuts', 'betarand', 'bexprnd', - 'bformdec', 'bformdec1', - 'bformenc', 'bformenc1', 'binit', 'biquad', @@ -192,9 +180,11 @@ OPCODES = set(( 'cent', 'centroid', 'ceps', + 'cepsinv', #'cggoto', 'chanctrl', 'changed', + 'changed2', 'chani', 'chano', 'chebyshevpoly', @@ -219,18 +209,18 @@ OPCODES = set(( 'clockon', 'cmplxprod', #'cngoto', + #'cnkgoto', 'comb', 'combinv', 'compilecsd', 'compileorc', 'compilestr', 'compress', + 'compress2', 'connect', 'control', 'convle', 'convolve', - 'copy2ftab', - 'copy2ttab', 'copya2ftab', 'copyf2array', 'cos', @@ -273,6 +263,8 @@ OPCODES = set(( 'dcblock', 'dcblock2', 'dconv', + 'dct', + 'dctinv', 'delay', 'delay1', 'delayk', @@ -286,6 +278,7 @@ OPCODES = set(( 'deltapxw', 'denorm', 'diff', + 'directory', 'diskgrain', 'diskin', 'diskin2', @@ -338,6 +331,7 @@ OPCODES = set(( 'filelen', 'filenchnls', 'filepeak', + 'filescal', 'filesr', 'filevalid', 'fillarray', @@ -385,6 +379,7 @@ OPCODES = set(( 'fprints', 'frac', 'fractalnoise', + 'framebuffer', 'freeverb', 'ftchnls', 'ftconv', @@ -400,6 +395,7 @@ OPCODES = set(( 'ftmorf', 'ftresize', 'ftresizei', + 'ftsamplebank', 'ftsave', 'ftsavek', 'ftsr', @@ -416,7 +412,9 @@ OPCODES = set(( 'gendyx', 'getcfg', 'getcol', + 'getftargs', 'getrow', + 'getseed', 'gogobel', #'goto', 'grain', @@ -432,7 +430,6 @@ OPCODES = set(( 'hdf5write', 'hilbert', 'hrtfearly', - 'hrtfer', 'hrtfmove', 'hrtfmove2', 'hrtfreverb', @@ -442,7 +439,6 @@ OPCODES = set(( 'hvs2', 'hvs3', 'i', - 'iceps', #'igoto', 'ihold', 'imagecreate', @@ -483,9 +479,7 @@ OPCODES = set(( 'jspline', 'k', #'kgoto', - 'ktableseg', 'lenarray', - 'lentab', 'lfo', 'limit', 'line', @@ -533,9 +527,9 @@ OPCODES = set(( 'lpshold', 'lpsholdp', 'lpslot', - #'lua_exec', + 'lua_exec', 'lua_ikopcall', - #'lua_opdef', + 'lua_opdef', 'mac', 'maca', 'madsr', @@ -553,12 +547,12 @@ OPCODES = set(( 'maxaccum', 'maxalloc', 'maxarray', - 'maxtab', 'mclock', 'mdelay', 'median', 'mediank', 'metro', + 'mfb', 'midglobal', 'midic14', 'midic21', @@ -590,13 +584,13 @@ OPCODES = set(( 'minaccum', 'minarray', 'mincer', - 'mintab', 'mirror', 'mode', 'modmatrix', 'monitor', 'moog', 'moogladder', + 'moogladder2', 'moogvcf', 'moogvcf2', 'moscil', @@ -604,12 +598,19 @@ OPCODES = set(( 'mp3in', 'mp3len', 'mp3nchnls', + 'mp3scal', 'mp3sr', 'mpulse', 'mrtmsg', 'multitap', 'mute', + 'mvchpf', + 'mvclpf1', + 'mvclpf2', + 'mvclpf3', + 'mvclpf4', 'mxadsr', + 'nchnls_hw', 'nestedap', 'nlalp', 'nlfilt', @@ -626,12 +627,14 @@ OPCODES = set(( 'nstance', 'nstrnum', 'ntrpol', + 'nxtpow2', 'octave', 'octcps', 'octmidi', 'octmidib', 'octmidinn', 'octpch', + 'olabuffer', #'opcode', 'oscbnk', 'oscil', @@ -684,12 +687,14 @@ OPCODES = set(( 'pan', 'pan2', 'pareq', + 'part2txt', 'partials', 'partikkel', 'partikkelget', 'partikkelset', 'partikkelsync', 'passign', + 'paulstretch', 'pcauchy', 'pchbend', 'pchmidi', @@ -723,8 +728,6 @@ OPCODES = set(( 'pol2rect', 'polyaft', 'polynomial', - 'pop', - 'pop_f', 'port', 'portk', 'poscil', @@ -732,6 +735,7 @@ OPCODES = set(( 'pow', 'powershape', 'powoftwo', + 'pows', 'prealloc', 'prepiano', 'print', @@ -751,8 +755,6 @@ OPCODES = set(( 'ptableiw', 'ptablew', 'ptrack', - 'push', - 'push_f', 'puts', 'pvadd', 'pvbufread', @@ -885,12 +887,12 @@ OPCODES = set(( 'pylexec', 'pylexeci', 'pylexect', - #'pylrun', - #'pylruni', - #'pylrunt', - #'pyrun', - #'pyruni', - #'pyrunt', + 'pylrun', + 'pylruni', + 'pylrunt', + 'pyrun', + 'pyruni', + 'pyrunt', 'qinf', 'qnan', 'r2c', @@ -934,7 +936,7 @@ OPCODES = set(( 'rfft', 'rifft', #'rigoto', - 'rireturn', + #'rireturn', 'rms', 'rnd', 'rnd31', @@ -947,7 +949,6 @@ OPCODES = set(( 'sandpaper', 'scale', 'scalearray', - 'scalet', 'scanhammer', 'scans', 'scantable', @@ -956,8 +957,8 @@ OPCODES = set(( 'schedkwhennamed', 'schedule', 'schedwhen', - #'scoreline', - #'scoreline_i', + 'scoreline', + 'scoreline_i', 'seed', 'sekere', 'semitone', @@ -1019,7 +1020,6 @@ OPCODES = set(( 'slider8table', 'slider8tablef', 'sliderKawai', - 'sndload', 'sndloop', 'sndwarp', 'sndwarpst', @@ -1028,28 +1028,16 @@ OPCODES = set(( 'socksend', 'socksends', 'soundin', - 'soundout', - 'soundouts', 'space', 'spat3d', 'spat3di', 'spat3dt', 'spdist', - 'specaddm', - 'specdiff', - 'specdisp', - 'specfilt', - 'spechist', - 'specptrk', - 'specscal', - 'specsum', - 'spectrum', 'splitrig', 'sprintf', 'sprintfk', 'spsend', 'sqrt', - 'stack', 'statevar', 'stix', 'strcat', @@ -1086,7 +1074,6 @@ OPCODES = set(( 'subinstrinit', 'sum', 'sumarray', - 'sumtab', 'svfilter', 'syncgrain', 'syncloop', @@ -1096,7 +1083,7 @@ OPCODES = set(( 'tab', 'tab2pvs', 'tab_i', - 'tabgen', + 'tabifd', 'table', 'table3', 'table3kt', @@ -1122,15 +1109,12 @@ OPCODES = set(( 'tablewkt', 'tablexkt', 'tablexseg', - 'tabmap', - 'tabmap_i', 'tabmorph', 'tabmorpha', 'tabmorphak', 'tabmorphi', 'tabplay', 'tabrec', - 'tabslice', 'tabsum', 'tabw', 'tabw_i', @@ -1210,6 +1194,7 @@ OPCODES = set(( 'unirand', 'unwrap', 'upsamp', + 'urandom', 'urd', 'vactrol', 'vadd', @@ -1220,11 +1205,6 @@ OPCODES = set(( 'valpass', 'vaset', 'vbap', - 'vbap16', - 'vbap4', - 'vbap4move', - 'vbap8', - 'vbap8move', 'vbapg', 'vbapgmove', 'vbaplsinit', @@ -1326,7 +1306,6 @@ OPCODES = set(( 'xscansmap', 'xscanu', 'xtratim', - 'xyin', 'zacl', 'zakinit', 'zamod', @@ -1344,3 +1323,45 @@ OPCODES = set(( 'zkw', 'zkwm' )) + +DEPRECATED_OPCODES = set(( + 'array', + 'bformdec', + 'bformenc', + 'copy2ftab', + 'copy2ttab', + 'hrtfer', + 'ktableseg', + 'lentab', + 'maxtab', + 'mintab', + 'pop', + 'pop_f', + 'push', + 'push_f', + 'scalet', + 'sndload', + 'soundout', + 'soundouts', + 'specaddm', + 'specdiff', + 'specdisp', + 'specfilt', + 'spechist', + 'specptrk', + 'specscal', + 'specsum', + 'spectrum', + 'stack', + 'sumtab', + 'tabgen', + 'tabmap', + 'tabmap_i', + 'tabslice', + 'vbap16', + 'vbap4', + 'vbap4move', + 'vbap8', + 'vbap8move', + 'xyin' +)) diff --git a/pygments/lexers/csound.py b/pygments/lexers/csound.py index 080cead8..4d9ae398 100644 --- a/pygments/lexers/csound.py +++ b/pygments/lexers/csound.py @@ -14,7 +14,7 @@ import re from pygments.lexer import RegexLexer, bygroups, default, include, using, words from pygments.token import Comment, Keyword, Name, Number, Operator, Punctuation, \ String, Text -from pygments.lexers._csound_builtins import OPCODES +from pygments.lexers._csound_builtins import OPCODES, DEPRECATED_OPCODES from pygments.lexers.html import HtmlLexer from pygments.lexers.python import PythonLexer from pygments.lexers.scripting import LuaLexer @@ -104,8 +104,8 @@ class CsoundScoreLexer(CsoundLexer): tokens = { 'partial statement': [ include('preprocessor directives'), - (r'\d+e[+-]?\d+|(\d+\.\d*|\d*\.\d+)(e[+-]?\d+)?', Number.Float), - (r'0[xX][a-fA-F0-9]+', Number.Hex), + (r'\d+[Ee][+-]?\d+|(\d+\.\d*|\d*\.\d+)([Ee][+-]?\d+)?', Number.Float), + (r'0[Xx][0-9A-Fa-f]+', Number.Hex), (r'\d+', Number.Integer), (r'"', String, 'single-line string'), (r'[+\-*/%^!=<>|&#~.]', Operator), @@ -152,9 +152,12 @@ class CsoundOrchestraLexer(CsoundLexer): yield match.start(), Name.Function, opcode def name_callback(lexer, match): - name = match.group(0) - if re.match('p\d+$', name) or name in OPCODES: + name = match.group(1) + if name in OPCODES or name in DEPRECATED_OPCODES: yield match.start(), Name.Builtin, name + if match.group(2): + yield match.start(2), Punctuation, match.group(2) + yield match.start(3), Keyword.Type, match.group(3) elif name in lexer.user_defined_opcodes: yield match.start(), Name.Function, name else: @@ -167,14 +170,14 @@ class CsoundOrchestraLexer(CsoundLexer): tokens = { 'label': [ - (r'\b(\w+)(:)', bygroups(Name.Label, Punctuation)) + (r'^([ \t]*)(\w+)(:)', bygroups(Text, Name.Label, Punctuation)) ], 'partial expression': [ include('preprocessor directives'), - (r'\b(0dbfs|k(r|smps)|nchnls(_i)?|sr)\b', Name.Variable.Global), - (r'\d+e[+-]?\d+|(\d+\.\d*|\d*\.\d+)(e[+-]?\d+)?', Number.Float), - (r'0[xX][a-fA-F0-9]+', Number.Hex), + (r'\b(0dbfs|A4|k(r|smps)|nchnls(_i)?|sr)\b', Name.Variable.Global), + (r'\d+[Ee][+-]?\d+|(\d+\.\d*|\d*\.\d+)([Ee][+-]?\d+)?', Number.Float), + (r'0[Xx][0-9A-Fa-f]+', Number.Hex), (r'\d+', Number.Integer), (r'"', String, 'single-line string'), (r'\{\{', String, 'multi-line string'), @@ -185,20 +188,24 @@ class CsoundOrchestraLexer(CsoundLexer): 'do', 'else', 'elseif', 'endif', 'enduntil', 'fi', 'if', 'ithen', 'kthen', 'od', 'then', 'until', 'while', # Opcodes that act as control structures - 'return', 'timout' + 'return', 'rireturn' ), prefix=r'\b', suffix=r'\b'), Keyword), - (words(('goto', 'igoto', 'kgoto', 'rigoto', 'tigoto'), + (words(('goto', 'igoto', 'kgoto', 'reinit', 'rigoto', 'tigoto'), prefix=r'\b', suffix=r'\b'), Keyword, 'goto label'), - (words(('cggoto', 'cigoto', 'cingoto', 'ckgoto', 'cngoto'), + (words(('cggoto', 'cigoto', 'cingoto', 'ckgoto', 'cngoto', 'cnkgoto'), prefix=r'\b', suffix=r'\b'), Keyword, ('goto label', 'goto expression')), + (r'\btimout\b', Keyword, + ('goto label', 'goto expression', 'goto expression')), (words(('loop_ge', 'loop_gt', 'loop_le', 'loop_lt'), prefix=r'\b', suffix=r'\b'), Keyword, ('goto label', 'goto expression', 'goto expression', 'goto expression')), (r'\bscoreline(_i)?\b', Name.Builtin, 'scoreline opcode'), + (r'\bprintk?s\b', Name.Builtin, 'prints opcode'), (r'\bpyl?run[it]?\b', Name.Builtin, 'python opcode'), (r'\blua_(exec|opdef)\b', Name.Builtin, 'lua opcode'), - (r'\b[a-zA-Z_]\w*\b', name_callback) + (r'\bp\d+\b', Name.Builtin), + (r'\b([A-Z_a-z]\w*)(?:(:)([A-Za-z]))?\b', name_callback) ], 'expression': [ @@ -219,8 +226,8 @@ class CsoundOrchestraLexer(CsoundLexer): 'instrument name list': [ include('whitespace or macro use'), - (r'\d+|\+?[a-zA-Z_]\w*', Name.Function), - (r',', Punctuation), + (r'\d+|[A-Z_a-z]\w*', Name.Function), + (r'[+,]', Punctuation), newline + ('#pop',) ], 'instrument block': [ @@ -233,7 +240,7 @@ class CsoundOrchestraLexer(CsoundLexer): 'opcode name': [ include('whitespace or macro use'), - (r'[a-zA-Z_]\w*', opcode_name_callback, '#pop') + (r'[A-Z_a-z]\w*', opcode_name_callback, '#pop') ], 'opcode types': [ include('whitespace or macro use'), @@ -266,11 +273,10 @@ class CsoundOrchestraLexer(CsoundLexer): 'single-line string': [ include('macro use'), (r'"', String, '#pop'), - # From https://github.com/csound/csound/blob/develop/Opcodes/fout.c#L1405 - (r'%\d*(\.\d+)?[cdhilouxX]', String.Interpol), - (r'%[!%nNrRtT]|[~^]|\\([\\aAbBnNrRtT"]|[0-7]{1,3})', String.Escape), - (r'[^\\"~$%\^\n]+', String), - (r'[\\"~$%\^\n]', String) + # https://github.com/csound/csound/search?q=unquote_string+path%3AEngine+filename%3Acsound_orc_compile.c + (r'\\([\\abnrt"]|[0-7]{1,3})', String.Escape), + (r'[^\\"$\n]+', String), + (r'[\\"$\n]', String) ], 'multi-line string': [ (r'\}\}', String, '#pop'), @@ -287,6 +293,21 @@ class CsoundOrchestraLexer(CsoundLexer): (r'([^}]+)|\}(?!\})', using(CsoundScoreLexer)) ], + 'prints opcode': [ + include('whitespace or macro use'), + (r'"', String, 'prints'), + default('#pop') + ], + 'prints': [ + include('macro use'), + (r'"', String, '#pop'), + # https://github.com/csound/csound/search?q=sprints1+path%3AOpcodes+filename%3Afout.c + (r'%\d*(\.\d+)?[cdhilouxX]', String.Interpol), + (r'%[!%nNrRtT]|[~^]|\\([\\aAbBnNrRtT"]|[0-7]{1,3})', String.Escape), + (r'[^\\"~$%\^\n]+', String), + (r'[\\"~$%\^\n]', String) + ], + 'python opcode': [ include('whitespace or macro use'), (r'\{\{', String, 'python'), diff --git a/tests/examplefiles/test.csd b/tests/examplefiles/test.csd index 9122309b..3f43bede 100644 --- a/tests/examplefiles/test.csd +++ b/tests/examplefiles/test.csd @@ -13,6 +13,7 @@ nchnls_i = 1 sr = 44100 0dbfs = 1 ksmps = 10 +A4 = 440 // The control rate kr = sr / ksmps can be omitted when the number of audio // samples in a control period (ksmps) is set, but kr may appear in older @@ -80,9 +81,8 @@ instr TestOscillator outc(anOscillator(0dbfs, 110)) endin -// Python can be executed in Csound -// . So can Lua -// . +// You can execute Python and +// Lua in Csound. pyruni {{ import random @@ -106,13 +106,13 @@ def get_number_from_pool(n, p): #define A_HZ #440# // This is a function-like macro: -#define OSCIL_MACRO(VOLUME'FREQUENCY'TABLE) #oscil $VOLUME, $FREQUENCY, $TABLE# +#define OSCIL_MACRO(VOLUME' FREQUENCY' TABLE) #oscil $VOLUME, $FREQUENCY, $TABLE# // Bodies of macros are enclosed in # and can contain newlines. The arguments of // function-like macros are separated by single-quotes. Uses of macros are // prefixed with a dollar sign. instr TestMacro - aSignal $OSCIL_MACRO(1'$A_HZ'1) + aSignal $OSCIL_MACRO(1' $A_HZ' 1) // Not unlike PHP, macros expand in double-quoted strings. prints "The frequency of the oscillator is $A_HZ Hz.\n" out aSignal @@ -129,7 +129,7 @@ instr TestBitwiseNOT endin // Csound uses # for bitwise XOR, which the Csound manual calls bitwise -// non-equivalence . +// non-equivalence . instr TestBitwiseXOR print 0 # 0 print 0 # 1 @@ -178,10 +178,8 @@ loop_lt_label: od endin -// The prints and printks opcodes -// , arguably -// the primary methods of logging output, treat certain sequences of characters -// different from printf in C. +// The prints and printks opcodes, arguably the primary methods of logging +// output, treat certain sequences of characters different from printf in C. instr TestPrints // ^ prints an ESCAPE character (U+001B), not a CIRCUMFLEX ACCENT character // (U+005E). ^^ prints a CIRCUMFLEX ACCENT. @@ -202,26 +200,22 @@ endin // The arguments of function-like macros can be separated by # instead of '. // These two lines define the same macro. -#define OSCIL_MACRO(VOLUME'FREQUENCY'TABLE) #oscil $VOLUME, $FREQUENCY, $TABLE# -#define OSCIL_MACRO(VOLUME#FREQUENCY#TABLE) #oscil $VOLUME, $FREQUENCY, $TABLE# +#define OSCIL_MACRO(VOLUME' FREQUENCY' TABLE) #oscil $VOLUME, $FREQUENCY, $TABLE# +#define OSCIL_MACRO(VOLUME# FREQUENCY# TABLE) #oscil $VOLUME, $FREQUENCY, $TABLE# // Uses of macros can optionally be suffixed with a period. instr TestMacroPeriodSuffix - aSignal $OSCIL_MACRO.(1'$A_HZ'1) + aSignal $OSCIL_MACRO.(1' $A_HZ' 1) prints "The frequency of the oscillator is $A_HZ.Hz.\n" out aSignal endin // Csound has @ and @@ operator-like macros that, when followed by a literal // non-negative integer, expand to the next power of 2 and the next power of 2 -// plus 1: +// plus 1 : // @x = 2^(ceil(log2(x + 1))), x >= 0 // @@0 = 2 // @@x = 2^(ceil(log2(x))) + 1, x > 0 -// These macros are in -// (and -// ) -// and are described at . instr TestAt prints "%d %2d %2d\n", 0, @0, @@0 prints "%d %2d %2d\n", 1, @1, @@1 @@ -252,7 +246,7 @@ i "TestOscillator" 2 2 i "TestBitwiseNOT" 0 1 i "TestBitwiseXOR" 0 1 i "TestGoto" 0 1 -i "TestMacroPeriodSuffix" 4 1 +i "TestMacroPeriodSuffix" 0 1 i "TestAt" 0 1 i "MacroAbuse" 0 1 e diff --git a/tests/examplefiles/test.orc b/tests/examplefiles/test.orc index 36725342..36e35b34 100644 --- a/tests/examplefiles/test.orc +++ b/tests/examplefiles/test.orc @@ -11,6 +11,7 @@ nchnls_i = 1 sr = 44100 0dbfs = 1 ksmps = 10 +A4 = 440 // The control rate kr = sr / ksmps can be omitted when the number of audio // samples in a control period (ksmps) is set, but kr may appear in older @@ -78,9 +79,8 @@ instr TestOscillator outc(anOscillator(0dbfs, 110)) endin -// Python can be executed in Csound -// . So can Lua -// . +// You can execute Python and +// Lua in Csound. pyruni {{ import random @@ -104,13 +104,13 @@ def get_number_from_pool(n, p): #define A_HZ #440# // This is a function-like macro: -#define OSCIL_MACRO(VOLUME'FREQUENCY'TABLE) #oscil $VOLUME, $FREQUENCY, $TABLE# +#define OSCIL_MACRO(VOLUME' FREQUENCY' TABLE) #oscil $VOLUME, $FREQUENCY, $TABLE# // Bodies of macros are enclosed in # and can contain newlines. The arguments of // function-like macros are separated by single-quotes. Uses of macros are // prefixed with a dollar sign. instr TestMacro - aSignal $OSCIL_MACRO(1'$A_HZ'1) + aSignal $OSCIL_MACRO(1' $A_HZ' 1) // Not unlike PHP, macros expand in double-quoted strings. prints "The frequency of the oscillator is $A_HZ Hz.\n" out aSignal @@ -127,7 +127,7 @@ instr TestBitwiseNOT endin // Csound uses # for bitwise XOR, which the Csound manual calls bitwise -// non-equivalence . +// non-equivalence . instr TestBitwiseXOR print 0 # 0 print 0 # 1 @@ -176,10 +176,8 @@ loop_lt_label: od endin -// The prints and printks opcodes -// , arguably -// the primary methods of logging output, treat certain sequences of characters -// different from printf in C. +// The prints and printks opcodes, arguably the primary methods of logging +// output, treat certain sequences of characters different from printf in C. instr TestPrints // ^ prints an ESCAPE character (U+001B), not a CIRCUMFLEX ACCENT character // (U+005E). ^^ prints a CIRCUMFLEX ACCENT. @@ -200,26 +198,22 @@ endin // The arguments of function-like macros can be separated by # instead of '. // These two lines define the same macro. -#define OSCIL_MACRO(VOLUME'FREQUENCY'TABLE) #oscil $VOLUME, $FREQUENCY, $TABLE# -#define OSCIL_MACRO(VOLUME#FREQUENCY#TABLE) #oscil $VOLUME, $FREQUENCY, $TABLE# +#define OSCIL_MACRO(VOLUME' FREQUENCY' TABLE) #oscil $VOLUME, $FREQUENCY, $TABLE# +#define OSCIL_MACRO(VOLUME# FREQUENCY# TABLE) #oscil $VOLUME, $FREQUENCY, $TABLE# // Uses of macros can optionally be suffixed with a period. instr TestMacroPeriodSuffix - aSignal $OSCIL_MACRO.(1'$A_HZ'1) + aSignal $OSCIL_MACRO.(1' $A_HZ' 1) prints "The frequency of the oscillator is $A_HZ.Hz.\n" out aSignal endin // Csound has @ and @@ operator-like macros that, when followed by a literal // non-negative integer, expand to the next power of 2 and the next power of 2 -// plus 1: +// plus 1 : // @x = 2^(ceil(log2(x + 1))), x >= 0 // @@0 = 2 // @@x = 2^(ceil(log2(x))) + 1, x > 0 -// These macros are in -// (and -// ) -// and are described at . instr TestAt prints "%d %2d %2d\n", 0, @0, @@0 prints "%d %2d %2d\n", 1, @1, @@1 @@ -250,7 +244,7 @@ i "TestOscillator" 2 2 i "TestBitwiseNOT" 0 1 i "TestBitwiseXOR" 0 1 i "TestGoto" 0 1 -i "TestMacroPeriodSuffix" 4 1 +i "TestMacroPeriodSuffix" 0 1 i "TestAt" 0 1 i "MacroAbuse" 0 1 e diff --git a/tests/examplefiles/test.sco b/tests/examplefiles/test.sco index a0b39251..07818133 100644 --- a/tests/examplefiles/test.sco +++ b/tests/examplefiles/test.sco @@ -4,7 +4,7 @@ i "TestOscillator" 2 2 i "TestBitwiseNOT" 0 1 i "TestBitwiseXOR" 0 1 i "TestGoto" 0 1 -i "TestMacroPeriodSuffix" 4 1 +i "TestMacroPeriodSuffix" 0 1 i "TestAt" 0 1 i "MacroAbuse" 0 1 e -- cgit v1.2.1 From c666eec31bff48ee0e2850ee5075c6bbbf09b804 Mon Sep 17 00:00:00 2001 From: Nathan Whetsell Date: Sun, 11 Dec 2016 12:31:07 -0500 Subject: Use HTTPS URLs --- pygments/lexers/csound.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pygments/lexers/csound.py b/pygments/lexers/csound.py index 4d9ae398..892fd6d0 100644 --- a/pygments/lexers/csound.py +++ b/pygments/lexers/csound.py @@ -92,7 +92,7 @@ class CsoundLexer(RegexLexer): class CsoundScoreLexer(CsoundLexer): """ - For `Csound `_ scores. + For `Csound `_ scores. .. versionadded:: 2.1 """ @@ -135,7 +135,7 @@ class CsoundScoreLexer(CsoundLexer): class CsoundOrchestraLexer(CsoundLexer): """ - For `Csound `_ orchestras. + For `Csound `_ orchestras. .. versionadded:: 2.1 """ @@ -334,7 +334,7 @@ class CsoundOrchestraLexer(CsoundLexer): class CsoundDocumentLexer(RegexLexer): """ - For `Csound `_ documents. + For `Csound `_ documents. .. versionadded:: 2.1 """ -- cgit v1.2.1 From 552dc927f57684df1f96852ce702cf8c0a059586 Mon Sep 17 00:00:00 2001 From: Nathan Whetsell Date: Sun, 11 Dec 2016 12:32:27 -0500 Subject: Add .udo to Csound filenames --- pygments/lexers/csound.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pygments/lexers/csound.py b/pygments/lexers/csound.py index 892fd6d0..81a37395 100644 --- a/pygments/lexers/csound.py +++ b/pygments/lexers/csound.py @@ -142,7 +142,7 @@ class CsoundOrchestraLexer(CsoundLexer): name = 'Csound Orchestra' aliases = ['csound', 'csound-orc'] - filenames = ['*.orc'] + filenames = ['*.orc', '*.udo'] user_defined_opcodes = set() -- cgit v1.2.1 From 7fd626cad80ee0257ac4ac5b1365d5cbb52ac247 Mon Sep 17 00:00:00 2001 From: Nathan Reed Date: Thu, 29 Dec 2016 17:48:53 -0800 Subject: Add string literal parsing to HLSL lexer (copied from the one for C++). - Also added a snippet to the example file where a string shows up (uncommon in HLSL). --- pygments/lexers/graphics.py | 9 +++++++++ tests/examplefiles/example.hlsl | 11 +++++++++++ 2 files changed, 20 insertions(+) diff --git a/pygments/lexers/graphics.py b/pygments/lexers/graphics.py index a5da2375..b47e12b1 100644 --- a/pygments/lexers/graphics.py +++ b/pygments/lexers/graphics.py @@ -102,6 +102,7 @@ class HLSLShaderLexer(RegexLexer): (r'0[xX][0-9a-fA-F]*', Number.Hex), (r'0[0-7]*', Number.Oct), (r'[1-9][0-9]*', Number.Integer), + (r'"', String, 'string'), (words(( 'asm','asm_fragment','break','case','cbuffer','centroid','class', 'column_major','compile','compile_fragment','const','continue', @@ -217,6 +218,14 @@ class HLSLShaderLexer(RegexLexer): (r'\\$', Comment.Preproc), # backslash at end of line -- usually macro continuation (r'\s+', Text), ], + 'string': [ + (r'"', String, '#pop'), + (r'\\([\\abfnrtv"\']|x[a-fA-F0-9]{2,4}|' + r'u[a-fA-F0-9]{4}|U[a-fA-F0-9]{8}|[0-7]{1,3})', String.Escape), + (r'[^\\"\n]+', String), # all other characters + (r'\\\n', String), # line continuation + (r'\\', String), # stray backslash + ], } diff --git a/tests/examplefiles/example.hlsl b/tests/examplefiles/example.hlsl index 77f1fa49..21d0a672 100644 --- a/tests/examplefiles/example.hlsl +++ b/tests/examplefiles/example.hlsl @@ -155,3 +155,14 @@ void main( o_rgb = diffuseColor * diffuseLight; } + +[domain("quad")] +void ds( + in float edgeFactors[4] : SV_TessFactor, + in float insideFactors[2] : SV_InsideTessFactor, + in OutputPatch inp, + in float2 uv : SV_DomainLocation, + out float4 o_pos : SV_Position) +{ + o_pos = lerp(lerp(inp[0].pos, inp[1].pos, uv.x), lerp(inp[2].pos, inp[3].pos, uv.x), uv.y); +} -- cgit v1.2.1 From 5521b1c682a4e5078c5c0d7bb3122a2e18f81f30 Mon Sep 17 00:00:00 2001 From: Nathan Reed Date: Thu, 29 Dec 2016 17:51:30 -0800 Subject: Fix tabs --- pygments/lexers/graphics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pygments/lexers/graphics.py b/pygments/lexers/graphics.py index b47e12b1..d62b64ba 100644 --- a/pygments/lexers/graphics.py +++ b/pygments/lexers/graphics.py @@ -102,7 +102,7 @@ class HLSLShaderLexer(RegexLexer): (r'0[xX][0-9a-fA-F]*', Number.Hex), (r'0[0-7]*', Number.Oct), (r'[1-9][0-9]*', Number.Integer), - (r'"', String, 'string'), + (r'"', String, 'string'), (words(( 'asm','asm_fragment','break','case','cbuffer','centroid','class', 'column_major','compile','compile_fragment','const','continue', -- cgit v1.2.1 -- cgit v1.2.1 From 5730648f7ccd0fc07ee9aa3377a3156a74867c8c Mon Sep 17 00:00:00 2001 From: Nathan Whetsell Date: Thu, 19 Jan 2017 18:40:58 +0000 Subject: Add %f to format specifiers --- pygments/lexers/csound.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pygments/lexers/csound.py b/pygments/lexers/csound.py index 81a37395..75b48827 100644 --- a/pygments/lexers/csound.py +++ b/pygments/lexers/csound.py @@ -181,7 +181,7 @@ class CsoundOrchestraLexer(CsoundLexer): (r'\d+', Number.Integer), (r'"', String, 'single-line string'), (r'\{\{', String, 'multi-line string'), - (r'[+\-*/%^!=&|<>#~¬]', Operator), + (r'[+\-*/%^!=&|<>#~]', Operator), (r'[](),?:[]', Punctuation), (words(( # Keywords @@ -301,8 +301,8 @@ class CsoundOrchestraLexer(CsoundLexer): 'prints': [ include('macro use'), (r'"', String, '#pop'), - # https://github.com/csound/csound/search?q=sprints1+path%3AOpcodes+filename%3Afout.c - (r'%\d*(\.\d+)?[cdhilouxX]', String.Interpol), + # https://github.com/csound/csound/search?q=printksset_+path%3AOOps+filename%3Augrw1.c + (r'%\d*(\.\d+)?[cdfhilouxX]', String.Interpol), (r'%[!%nNrRtT]|[~^]|\\([\\aAbBnNrRtT"]|[0-7]{1,3})', String.Escape), (r'[^\\"~$%\^\n]+', String), (r'[\\"~$%\^\n]', String) -- cgit v1.2.1 From afcaa7d6ac6c676501446efa6f1a7de0d4eae86c Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Wed, 25 Jan 2017 07:58:50 +0100 Subject: PR#662: catch IOError from pkg_resources import --- pygments/plugin.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pygments/plugin.py b/pygments/plugin.py index 7987d646..08d9b5b4 100644 --- a/pygments/plugin.py +++ b/pygments/plugin.py @@ -40,14 +40,16 @@ FORMATTER_ENTRY_POINT = 'pygments.formatters' STYLE_ENTRY_POINT = 'pygments.styles' FILTER_ENTRY_POINT = 'pygments.filters' + def iter_entry_points(group_name): try: import pkg_resources - except ImportError: + except (ImportError, IOError): return [] return pkg_resources.iter_entry_points(group_name) + def find_plugin_lexers(): for entrypoint in iter_entry_points(LEXER_ENTRY_POINT): yield entrypoint.load() -- cgit v1.2.1 From 05f1ff72c4b5b665e1e39f6d09ac91aac27bd7ea Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Fri, 27 Jan 2017 07:22:40 +0100 Subject: Fixes #1320: restore Py2.6 compatibility --- pygments/lexers/_lua_builtins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pygments/lexers/_lua_builtins.py b/pygments/lexers/_lua_builtins.py index c60bf5a2..0561725d 100644 --- a/pygments/lexers/_lua_builtins.py +++ b/pygments/lexers/_lua_builtins.py @@ -288,7 +288,7 @@ if __name__ == '__main__': # pragma: no cover print('>> %s' % full_function_name) m = get_function_module(full_function_name) modules.setdefault(m, []).append(full_function_name) - modules = {k: tuple(v) for k, v in modules.iteritems()} + modules = dict((k, tuple(v)) for k, v in modules.iteritems()) regenerate(__file__, modules) -- cgit v1.2.1 From 43fcdda784228cf8c60287c4c64a575738653d94 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Fri, 27 Jan 2017 15:12:54 +0100 Subject: Add rs alias for Rust. --- pygments/lexers/_mapping.py | 4 ++-- pygments/lexers/rust.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py index ea54241c..3b747036 100644 --- a/pygments/lexers/_mapping.py +++ b/pygments/lexers/_mapping.py @@ -367,7 +367,7 @@ LEXERS = { 'RtsLexer': ('pygments.lexers.trafficscript', 'TrafficScript', ('rts', 'trafficscript'), ('*.rts',), ()), 'RubyConsoleLexer': ('pygments.lexers.ruby', 'Ruby irb session', ('rbcon', 'irb'), (), ('text/x-ruby-shellsession',)), 'RubyLexer': ('pygments.lexers.ruby', 'Ruby', ('rb', 'ruby', 'duby'), ('*.rb', '*.rbw', 'Rakefile', '*.rake', '*.gemspec', '*.rbx', '*.duby', 'Gemfile'), ('text/x-ruby', 'application/x-ruby')), - 'RustLexer': ('pygments.lexers.rust', 'Rust', ('rust',), ('*.rs', '*.rs.in'), ('text/rust',)), + 'RustLexer': ('pygments.lexers.rust', 'Rust', ('rust', 'rs'), ('*.rs', '*.rs.in'), ('text/rust',)), 'SASLexer': ('pygments.lexers.sas', 'SAS', ('sas',), ('*.SAS', '*.sas'), ('text/x-sas', 'text/sas', 'application/x-sas')), 'SLexer': ('pygments.lexers.r', 'S', ('splus', 's', 'r'), ('*.S', '*.R', '.Rhistory', '.Rprofile', '.Renviron'), ('text/S-plus', 'text/S', 'text/x-r-source', 'text/x-r', 'text/x-R', 'text/x-r-history', 'text/x-r-profile')), 'SMLLexer': ('pygments.lexers.ml', 'Standard ML', ('sml',), ('*.sml', '*.sig', '*.fun'), ('text/x-standardml', 'application/x-standardml')), @@ -417,7 +417,7 @@ LEXERS = { 'TurtleLexer': ('pygments.lexers.rdf', 'Turtle', ('turtle',), ('*.ttl',), ('text/turtle', 'application/x-turtle')), 'TwigHtmlLexer': ('pygments.lexers.templates', 'HTML+Twig', ('html+twig',), ('*.twig',), ('text/html+twig',)), 'TwigLexer': ('pygments.lexers.templates', 'Twig', ('twig',), (), ('application/x-twig',)), - 'TypeScriptLexer': ('pygments.lexers.javascript', 'TypeScript', ('ts', 'typescript'), ('*.ts',), ('text/x-typescript',)), + 'TypeScriptLexer': ('pygments.lexers.javascript', 'TypeScript', ('ts', 'typescript'), ('*.ts', '*.tsx'), ('text/x-typescript',)), 'TypoScriptCssDataLexer': ('pygments.lexers.typoscript', 'TypoScriptCssData', ('typoscriptcssdata',), (), ()), 'TypoScriptHtmlDataLexer': ('pygments.lexers.typoscript', 'TypoScriptHtmlData', ('typoscripthtmldata',), (), ()), 'TypoScriptLexer': ('pygments.lexers.typoscript', 'TypoScript', ('typoscript',), ('*.ts', '*.txt'), ('text/x-typoscript',)), diff --git a/pygments/lexers/rust.py b/pygments/lexers/rust.py index 6914f54d..10097fba 100644 --- a/pygments/lexers/rust.py +++ b/pygments/lexers/rust.py @@ -24,7 +24,7 @@ class RustLexer(RegexLexer): """ name = 'Rust' filenames = ['*.rs', '*.rs.in'] - aliases = ['rust'] + aliases = ['rust', 'rs'] mimetypes = ['text/rust'] keyword_types = ( -- cgit v1.2.1 From b527da9f8638a77fec2c5b6de395631736658b6f Mon Sep 17 00:00:00 2001 From: Nathan Whetsell Date: Fri, 27 Jan 2017 17:25:37 -0500 Subject: Update lexers --- pygments/lexers/_csound_builtins.py | 2678 +++++++++++++++++------------------ pygments/lexers/csound.py | 440 +++--- tests/examplefiles/test.csd | 258 +--- tests/examplefiles/test.orc | 300 +--- tests/examplefiles/test.sco | 32 +- 5 files changed, 1682 insertions(+), 2026 deletions(-) diff --git a/pygments/lexers/_csound_builtins.py b/pygments/lexers/_csound_builtins.py index 4ccac6b6..a0bca365 100644 --- a/pygments/lexers/_csound_builtins.py +++ b/pygments/lexers/_csound_builtins.py @@ -10,1358 +10,1336 @@ # Opcodes in Csound 6.08.0 at commit a1159a1 from # csound --list-opcodes3 # except for -# cggoto -# cigoto +# cggoto https://csound.github.io/docs/manual/cggoto.html +# cigoto https://csound.github.io/docs/manual/cigoto.html # cingoto (undocumented) -# ckgoto -# cngoto +# ckgoto https://csound.github.io/docs/manual/ckgoto.html +# cngoto https://csound.github.io/docs/manual/cngoto.html # cnkgoto (undocumented) -# endin -# igoto -# instr -# kgoto -# loop_ge -# loop_gt -# loop_le -# loop_lt -# opcode -# return -# rireturn -# rigoto -# tigoto -# timout +# endin https://csound.github.io/docs/manual/endin.html +# endop https://csound.github.io/docs/manual/endop.html +# goto https://csound.github.io/docs/manual/goto.html +# igoto https://csound.github.io/docs/manual/igoto.html +# instr https://csound.github.io/docs/manual/instr.html +# kgoto https://csound.github.io/docs/manual/kgoto.html +# loop_ge https://csound.github.io/docs/manual/loop_ge.html +# loop_gt https://csound.github.io/docs/manual/loop_gt.html +# loop_le https://csound.github.io/docs/manual/loop_le.html +# loop_lt https://csound.github.io/docs/manual/loop_lt.html +# opcode https://csound.github.io/docs/manual/opcode.html +# return https://csound.github.io/docs/manual/return.html +# rireturn https://csound.github.io/docs/manual/rireturn.html +# rigoto https://csound.github.io/docs/manual/rigoto.html +# tigoto https://csound.github.io/docs/manual/tigoto.html +# timout https://csound.github.io/docs/manual/timout.html # which are treated as keywords in csound.py. -OPCODES = set(( - 'ATSadd', - 'ATSaddnz', - 'ATSbufread', - 'ATScross', - 'ATSinfo', - 'ATSinterpread', - 'ATSpartialtap', - 'ATSread', - 'ATSreadnz', - 'ATSsinnoi', - 'FLbox', - 'FLbutBank', - 'FLbutton', - 'FLcloseButton', - 'FLcolor', - 'FLcolor2', - 'FLcount', - 'FLexecButton', - 'FLgetsnap', - 'FLgroup', - 'FLgroupEnd', - 'FLgroup_end', - 'FLhide', - 'FLhvsBox', - 'FLhvsBoxSetValue', - 'FLjoy', - 'FLkeyIn', - 'FLknob', - 'FLlabel', - 'FLloadsnap', - 'FLmouse', - 'FLpack', - 'FLpackEnd', - 'FLpack_end', - 'FLpanel', - 'FLpanelEnd', - 'FLpanel_end', - 'FLprintk', - 'FLprintk2', - 'FLroller', - 'FLrun', - 'FLsavesnap', - 'FLscroll', - 'FLscrollEnd', - 'FLscroll_end', - 'FLsetAlign', - 'FLsetBox', - 'FLsetColor', - 'FLsetColor2', - 'FLsetFont', - 'FLsetPosition', - 'FLsetSize', - 'FLsetSnapGroup', - 'FLsetText', - 'FLsetTextColor', - 'FLsetTextSize', - 'FLsetTextType', - 'FLsetVal', - 'FLsetVal_i', - 'FLsetVali', - 'FLsetsnap', - 'FLshow', - 'FLslidBnk', - 'FLslidBnk2', - 'FLslidBnk2Set', - 'FLslidBnk2Setk', - 'FLslidBnkGetHandle', - 'FLslidBnkSet', - 'FLslidBnkSetk', - 'FLslider', - 'FLtabs', - 'FLtabsEnd', - 'FLtabs_end', - 'FLtext', - 'FLupdate', - 'FLvalue', - 'FLvkeybd', - 'FLvslidBnk', - 'FLvslidBnk2', - 'FLxyin', - 'MixerClear', - 'MixerGetLevel', - 'MixerReceive', - 'MixerSend', - 'MixerSetLevel', - 'MixerSetLevel_i', - 'OSCinit', - 'OSCinitM', - 'OSClisten', - 'OSCsend', - 'S', - 'a', - 'abs', - 'active', - 'adsr', - 'adsyn', - 'adsynt', - 'adsynt2', - 'aftouch', - 'alpass', - 'alwayson', - 'ampdb', - 'ampdbfs', - 'ampmidi', - 'ampmidid', - 'areson', - 'aresonk', - 'atone', - 'atonek', - 'atonex', - 'babo', - 'balance', - 'bamboo', - 'barmodel', - 'bbcutm', - 'bbcuts', - 'betarand', - 'bexprnd', - 'bformdec1', - 'bformenc1', - 'binit', - 'biquad', - 'biquada', - 'birnd', - 'bqrez', - 'buchla', - 'butbp', - 'butbr', - 'buthp', - 'butlp', - 'butterbp', - 'butterbr', - 'butterhp', - 'butterlp', - 'button', - 'buzz', - 'c2r', - 'cabasa', - 'cauchy', - 'cauchyi', - 'ceil', - 'cell', - 'cent', - 'centroid', - 'ceps', - 'cepsinv', - #'cggoto', - 'chanctrl', - 'changed', - 'changed2', - 'chani', - 'chano', - 'chebyshevpoly', - 'checkbox', - 'chn_S', - 'chn_a', - 'chn_k', - 'chnclear', - 'chnexport', - 'chnget', - 'chnmix', - 'chnparams', - 'chnset', - 'chuap', - #'cigoto', - #'cingoto', - #'ckgoto', - 'clear', - 'clfilt', - 'clip', - 'clockoff', - 'clockon', - 'cmplxprod', - #'cngoto', - #'cnkgoto', - 'comb', - 'combinv', - 'compilecsd', - 'compileorc', - 'compilestr', - 'compress', - 'compress2', - 'connect', - 'control', - 'convle', - 'convolve', - 'copya2ftab', - 'copyf2array', - 'cos', - 'cosh', - 'cosinv', - 'cosseg', - 'cossegb', - 'cossegr', - 'cps2pch', - 'cpsmidi', - 'cpsmidib', - 'cpsmidinn', - 'cpsoct', - 'cpspch', - 'cpstmid', - 'cpstun', - 'cpstuni', - 'cpsxpch', - 'cpuprc', - 'cross2', - 'crossfm', - 'crossfmi', - 'crossfmpm', - 'crossfmpmi', - 'crosspm', - 'crosspmi', - 'crunch', - 'ctlchn', - 'ctrl14', - 'ctrl21', - 'ctrl7', - 'ctrlinit', - 'cuserrnd', - 'dam', - 'date', - 'dates', - 'db', - 'dbamp', - 'dbfsamp', - 'dcblock', - 'dcblock2', - 'dconv', - 'dct', - 'dctinv', - 'delay', - 'delay1', - 'delayk', - 'delayr', - 'delayw', - 'deltap', - 'deltap3', - 'deltapi', - 'deltapn', - 'deltapx', - 'deltapxw', - 'denorm', - 'diff', - 'directory', - 'diskgrain', - 'diskin', - 'diskin2', - 'dispfft', - 'display', - 'distort', - 'distort1', - 'divz', - 'doppler', - 'downsamp', - 'dripwater', - 'dumpk', - 'dumpk2', - 'dumpk3', - 'dumpk4', - 'duserrnd', - 'dust', - 'dust2', - #'endin', - #'endop', - 'envlpx', - 'envlpxr', - 'ephasor', - 'eqfil', - 'evalstr', - 'event', - 'event_i', - 'exciter', - 'exitnow', - 'exp', - 'expcurve', - 'expon', - 'exprand', - 'exprandi', - 'expseg', - 'expsega', - 'expsegb', - 'expsegba', - 'expsegr', - 'fareylen', - 'fareyleni', - 'faustaudio', - 'faustcompile', - 'faustctl', - 'faustgen', - 'fft', - 'fftinv', - 'ficlose', - 'filebit', - 'filelen', - 'filenchnls', - 'filepeak', - 'filescal', - 'filesr', - 'filevalid', - 'fillarray', - 'filter2', - 'fin', - 'fini', - 'fink', - 'fiopen', - 'flanger', - 'flashtxt', - 'flooper', - 'flooper2', - 'floor', - 'fluidAllOut', - 'fluidCCi', - 'fluidCCk', - 'fluidControl', - 'fluidEngine', - 'fluidLoad', - 'fluidNote', - 'fluidOut', - 'fluidProgramSelect', - 'fluidSetInterpMethod', - 'fmb3', - 'fmbell', - 'fmmetal', - 'fmpercfl', - 'fmrhode', - 'fmvoice', - 'fmwurlie', - 'fof', - 'fof2', - 'fofilter', - 'fog', - 'fold', - 'follow', - 'follow2', - 'foscil', - 'foscili', - 'fout', - 'fouti', - 'foutir', - 'foutk', - 'fprintks', - 'fprints', - 'frac', - 'fractalnoise', - 'framebuffer', - 'freeverb', - 'ftchnls', - 'ftconv', - 'ftcps', - 'ftfree', - 'ftgen', - 'ftgenonce', - 'ftgentmp', - 'ftlen', - 'ftload', - 'ftloadk', - 'ftlptim', - 'ftmorf', - 'ftresize', - 'ftresizei', - 'ftsamplebank', - 'ftsave', - 'ftsavek', - 'ftsr', - 'gain', - 'gainslider', - 'gauss', - 'gaussi', - 'gausstrig', - 'gbuzz', - 'genarray', - 'genarray_i', - 'gendy', - 'gendyc', - 'gendyx', - 'getcfg', - 'getcol', - 'getftargs', - 'getrow', - 'getseed', - 'gogobel', - #'goto', - 'grain', - 'grain2', - 'grain3', - 'granule', - 'guiro', - 'harmon', - 'harmon2', - 'harmon3', - 'harmon4', - 'hdf5read', - 'hdf5write', - 'hilbert', - 'hrtfearly', - 'hrtfmove', - 'hrtfmove2', - 'hrtfreverb', - 'hrtfstat', - 'hsboscil', - 'hvs1', - 'hvs2', - 'hvs3', - 'i', - #'igoto', - 'ihold', - 'imagecreate', - 'imagefree', - 'imagegetpixel', - 'imageload', - 'imagesave', - 'imagesetpixel', - 'imagesize', - 'in', - 'in32', - 'inch', - 'inh', - 'init', - 'initc14', - 'initc21', - 'initc7', - 'inleta', - 'inletf', - 'inletk', - 'inletkid', - 'inletv', - 'ino', - 'inq', - 'inrg', - 'ins', - 'insglobal', - 'insremot', - #'instr', - 'int', - 'integ', - 'interp', - 'invalue', - 'inx', - 'inz', - 'jitter', - 'jitter2', - 'jspline', - 'k', - #'kgoto', - 'lenarray', - 'lfo', - 'limit', - 'line', - 'linen', - 'linenr', - 'lineto', - 'linrand', - 'linseg', - 'linsegb', - 'linsegr', - 'locsend', - 'locsig', - 'log', - 'log10', - 'log2', - 'logbtwo', - 'logcurve', - #'loop_ge', - #'loop_gt', - #'loop_le', - #'loop_lt', - 'loopseg', - 'loopsegp', - 'looptseg', - 'loopxseg', - 'lorenz', - 'loscil', - 'loscil3', - 'loscilx', - 'lowpass2', - 'lowres', - 'lowresx', - 'lpf18', - 'lpform', - 'lpfreson', - 'lphasor', - 'lpinterp', - 'lposcil', - 'lposcil3', - 'lposcila', - 'lposcilsa', - 'lposcilsa2', - 'lpread', - 'lpreson', - 'lpshold', - 'lpsholdp', - 'lpslot', - 'lua_exec', - 'lua_ikopcall', - 'lua_opdef', - 'mac', - 'maca', - 'madsr', - 'mags', - 'mandel', - 'mandol', - 'maparray', - 'maparray_i', - 'marimba', - 'massign', - 'max', - 'max_k', - 'maxabs', - 'maxabsaccum', - 'maxaccum', - 'maxalloc', - 'maxarray', - 'mclock', - 'mdelay', - 'median', - 'mediank', - 'metro', - 'mfb', - 'midglobal', - 'midic14', - 'midic21', - 'midic7', - 'midichannelaftertouch', - 'midichn', - 'midicontrolchange', - 'midictrl', - 'mididefault', - 'midifilestatus', - 'midiin', - 'midinoteoff', - 'midinoteoncps', - 'midinoteonkey', - 'midinoteonoct', - 'midinoteonpch', - 'midion', - 'midion2', - 'midiout', - 'midipgm', - 'midipitchbend', - 'midipolyaftertouch', - 'midiprogramchange', - 'miditempo', - 'midremot', - 'min', - 'minabs', - 'minabsaccum', - 'minaccum', - 'minarray', - 'mincer', - 'mirror', - 'mode', - 'modmatrix', - 'monitor', - 'moog', - 'moogladder', - 'moogladder2', - 'moogvcf', - 'moogvcf2', - 'moscil', - 'mp3bitrate', - 'mp3in', - 'mp3len', - 'mp3nchnls', - 'mp3scal', - 'mp3sr', - 'mpulse', - 'mrtmsg', - 'multitap', - 'mute', - 'mvchpf', - 'mvclpf1', - 'mvclpf2', - 'mvclpf3', - 'mvclpf4', - 'mxadsr', - 'nchnls_hw', - 'nestedap', - 'nlalp', - 'nlfilt', - 'nlfilt2', - 'noise', - 'noteoff', - 'noteon', - 'noteondur', - 'noteondur2', - 'notnum', - 'nreverb', - 'nrpn', - 'nsamp', - 'nstance', - 'nstrnum', - 'ntrpol', - 'nxtpow2', - 'octave', - 'octcps', - 'octmidi', - 'octmidib', - 'octmidinn', - 'octpch', - 'olabuffer', - #'opcode', - 'oscbnk', - 'oscil', - 'oscil1', - 'oscil1i', - 'oscil3', - 'oscili', - 'oscilikt', - 'osciliktp', - 'oscilikts', - 'osciln', - 'oscils', - 'oscilx', - 'out', - 'out32', - 'outc', - 'outch', - 'outh', - 'outiat', - 'outic', - 'outic14', - 'outipat', - 'outipb', - 'outipc', - 'outkat', - 'outkc', - 'outkc14', - 'outkpat', - 'outkpb', - 'outkpc', - 'outleta', - 'outletf', - 'outletk', - 'outletkid', - 'outletv', - 'outo', - 'outq', - 'outq1', - 'outq2', - 'outq3', - 'outq4', - 'outrg', - 'outs', - 'outs1', - 'outs2', - 'outvalue', - 'outx', - 'outz', - 'p', - 'pan', - 'pan2', - 'pareq', - 'part2txt', - 'partials', - 'partikkel', - 'partikkelget', - 'partikkelset', - 'partikkelsync', - 'passign', - 'paulstretch', - 'pcauchy', - 'pchbend', - 'pchmidi', - 'pchmidib', - 'pchmidinn', - 'pchoct', - 'pconvolve', - 'pcount', - 'pdclip', - 'pdhalf', - 'pdhalfy', - 'peak', - 'pgmassign', - 'pgmchn', - 'phaser1', - 'phaser2', - 'phasor', - 'phasorbnk', - 'phs', - 'pindex', - 'pinker', - 'pinkish', - 'pitch', - 'pitchac', - 'pitchamdf', - 'planet', - 'platerev', - 'plltrack', - 'pluck', - 'poisson', - 'pol2rect', - 'polyaft', - 'polynomial', - 'port', - 'portk', - 'poscil', - 'poscil3', - 'pow', - 'powershape', - 'powoftwo', - 'pows', - 'prealloc', - 'prepiano', - 'print', - 'print_type', - 'printf', - 'printf_i', - 'printk', - 'printk2', - 'printks', - 'printks2', - 'prints', - 'product', - 'pset', - 'ptable', - 'ptable3', - 'ptablei', - 'ptableiw', - 'ptablew', - 'ptrack', - 'puts', - 'pvadd', - 'pvbufread', - 'pvcross', - 'pvinterp', - 'pvoc', - 'pvread', - 'pvs2array', - 'pvs2tab', - 'pvsadsyn', - 'pvsanal', - 'pvsarp', - 'pvsbandp', - 'pvsbandr', - 'pvsbin', - 'pvsblur', - 'pvsbuffer', - 'pvsbufread', - 'pvsbufread2', - 'pvscale', - 'pvscent', - 'pvsceps', - 'pvscross', - 'pvsdemix', - 'pvsdiskin', - 'pvsdisp', - 'pvsenvftw', - 'pvsfilter', - 'pvsfread', - 'pvsfreeze', - 'pvsfromarray', - 'pvsftr', - 'pvsftw', - 'pvsfwrite', - 'pvsgain', - 'pvsgendy', - 'pvshift', - 'pvsifd', - 'pvsin', - 'pvsinfo', - 'pvsinit', - 'pvslock', - 'pvsmaska', - 'pvsmix', - 'pvsmooth', - 'pvsmorph', - 'pvsosc', - 'pvsout', - 'pvspitch', - 'pvstanal', - 'pvstencil', - 'pvsvoc', - 'pvswarp', - 'pvsynth', - 'pwd', - 'pyassign', - 'pyassigni', - 'pyassignt', - 'pycall', - 'pycall1', - 'pycall1i', - 'pycall1t', - 'pycall2', - 'pycall2i', - 'pycall2t', - 'pycall3', - 'pycall3i', - 'pycall3t', - 'pycall4', - 'pycall4i', - 'pycall4t', - 'pycall5', - 'pycall5i', - 'pycall5t', - 'pycall6', - 'pycall6i', - 'pycall6t', - 'pycall7', - 'pycall7i', - 'pycall7t', - 'pycall8', - 'pycall8i', - 'pycall8t', - 'pycalli', - 'pycalln', - 'pycallni', - 'pycallt', - 'pyeval', - 'pyevali', - 'pyevalt', - 'pyexec', - 'pyexeci', - 'pyexect', - 'pyinit', - 'pylassign', - 'pylassigni', - 'pylassignt', - 'pylcall', - 'pylcall1', - 'pylcall1i', - 'pylcall1t', - 'pylcall2', - 'pylcall2i', - 'pylcall2t', - 'pylcall3', - 'pylcall3i', - 'pylcall3t', - 'pylcall4', - 'pylcall4i', - 'pylcall4t', - 'pylcall5', - 'pylcall5i', - 'pylcall5t', - 'pylcall6', - 'pylcall6i', - 'pylcall6t', - 'pylcall7', - 'pylcall7i', - 'pylcall7t', - 'pylcall8', - 'pylcall8i', - 'pylcall8t', - 'pylcalli', - 'pylcalln', - 'pylcallni', - 'pylcallt', - 'pyleval', - 'pylevali', - 'pylevalt', - 'pylexec', - 'pylexeci', - 'pylexect', - 'pylrun', - 'pylruni', - 'pylrunt', - 'pyrun', - 'pyruni', - 'pyrunt', - 'qinf', - 'qnan', - 'r2c', - 'rand', - 'randh', - 'randi', - 'random', - 'randomh', - 'randomi', - 'rbjeq', - 'readclock', - 'readf', - 'readfi', - 'readk', - 'readk2', - 'readk3', - 'readk4', - 'readks', - 'readscore', - 'readscratch', - 'rect2pol', - 'reinit', - 'release', - 'remoteport', - 'remove', - 'repluck', - 'reson', - 'resonk', - 'resonr', - 'resonx', - 'resonxk', - 'resony', - 'resonz', - 'resyn', - #'return', - 'reverb', - 'reverb2', - 'reverbsc', - 'rewindscore', - 'rezzy', - 'rfft', - 'rifft', - #'rigoto', - #'rireturn', - 'rms', - 'rnd', - 'rnd31', - 'round', - 'rspline', - 'rtclock', - 's16b14', - 's32b14', - 'samphold', - 'sandpaper', - 'scale', - 'scalearray', - 'scanhammer', - 'scans', - 'scantable', - 'scanu', - 'schedkwhen', - 'schedkwhennamed', - 'schedule', - 'schedwhen', - 'scoreline', - 'scoreline_i', - 'seed', - 'sekere', - 'semitone', - 'sense', - 'sensekey', - 'seqtime', - 'seqtime2', - 'serialBegin', - 'serialEnd', - 'serialFlush', - 'serialPrint', - 'serialRead', - 'serialWrite', - 'serialWrite_i', - 'setcol', - 'setctrl', - 'setksmps', - 'setrow', - 'setscorepos', - 'sfilist', - 'sfinstr', - 'sfinstr3', - 'sfinstr3m', - 'sfinstrm', - 'sfload', - 'sflooper', - 'sfpassign', - 'sfplay', - 'sfplay3', - 'sfplay3m', - 'sfplaym', - 'sfplist', - 'sfpreset', - 'shaker', - 'shiftin', - 'shiftout', - 'signalflowgraph', - 'signum', - 'sin', - 'sinh', - 'sininv', - 'sinsyn', - 'sleighbells', - 'slicearray', - 'slider16', - 'slider16f', - 'slider16table', - 'slider16tablef', - 'slider32', - 'slider32f', - 'slider32table', - 'slider32tablef', - 'slider64', - 'slider64f', - 'slider64table', - 'slider64tablef', - 'slider8', - 'slider8f', - 'slider8table', - 'slider8tablef', - 'sliderKawai', - 'sndloop', - 'sndwarp', - 'sndwarpst', - 'sockrecv', - 'sockrecvs', - 'socksend', - 'socksends', - 'soundin', - 'space', - 'spat3d', - 'spat3di', - 'spat3dt', - 'spdist', - 'splitrig', - 'sprintf', - 'sprintfk', - 'spsend', - 'sqrt', - 'statevar', - 'stix', - 'strcat', - 'strcatk', - 'strchar', - 'strchark', - 'strcmp', - 'strcmpk', - 'strcpy', - 'strcpyk', - 'strecv', - 'streson', - 'strfromurl', - 'strget', - 'strindex', - 'strindexk', - 'strlen', - 'strlenk', - 'strlower', - 'strlowerk', - 'strrindex', - 'strrindexk', - 'strset', - 'strsub', - 'strsubk', - 'strtod', - 'strtodk', - 'strtol', - 'strtolk', - 'strupper', - 'strupperk', - 'stsend', - 'subinstr', - 'subinstrinit', - 'sum', - 'sumarray', - 'svfilter', - 'syncgrain', - 'syncloop', - 'syncphasor', - 'system', - 'system_i', - 'tab', - 'tab2pvs', - 'tab_i', - 'tabifd', - 'table', - 'table3', - 'table3kt', - 'tablecopy', - 'tablefilter', - 'tablefilteri', - 'tablegpw', - 'tablei', - 'tableicopy', - 'tableigpw', - 'tableikt', - 'tableimix', - 'tableiw', - 'tablekt', - 'tablemix', - 'tableng', - 'tablera', - 'tableseg', - 'tableshuffle', - 'tableshufflei', - 'tablew', - 'tablewa', - 'tablewkt', - 'tablexkt', - 'tablexseg', - 'tabmorph', - 'tabmorpha', - 'tabmorphak', - 'tabmorphi', - 'tabplay', - 'tabrec', - 'tabsum', - 'tabw', - 'tabw_i', - 'tambourine', - 'tan', - 'tanh', - 'taninv', - 'taninv2', - 'tb0', - 'tb0_init', - 'tb1', - 'tb10', - 'tb10_init', - 'tb11', - 'tb11_init', - 'tb12', - 'tb12_init', - 'tb13', - 'tb13_init', - 'tb14', - 'tb14_init', - 'tb15', - 'tb15_init', - 'tb1_init', - 'tb2', - 'tb2_init', - 'tb3', - 'tb3_init', - 'tb4', - 'tb4_init', - 'tb5', - 'tb5_init', - 'tb6', - 'tb6_init', - 'tb7', - 'tb7_init', - 'tb8', - 'tb8_init', - 'tb9', - 'tb9_init', - 'tbvcf', - 'tempest', - 'tempo', - 'temposcal', - 'tempoval', - #'tigoto', - 'timedseq', - 'timeinstk', - 'timeinsts', - 'timek', - 'times', - #'timout', - 'tival', - 'tlineto', - 'tone', - 'tonek', - 'tonex', - 'tradsyn', - 'trandom', - 'transeg', - 'transegb', - 'transegr', - 'trcross', - 'trfilter', - 'trhighest', - 'trigger', - 'trigseq', - 'trirand', - 'trlowest', - 'trmix', - 'trscale', - 'trshift', - 'trsplit', - 'turnoff', - 'turnoff2', - 'turnon', - 'unirand', - 'unwrap', - 'upsamp', - 'urandom', - 'urd', - 'vactrol', - 'vadd', - 'vadd_i', - 'vaddv', - 'vaddv_i', - 'vaget', - 'valpass', - 'vaset', - 'vbap', - 'vbapg', - 'vbapgmove', - 'vbaplsinit', - 'vbapmove', - 'vbapz', - 'vbapzmove', - 'vcella', - 'vco', - 'vco2', - 'vco2ft', - 'vco2ift', - 'vco2init', - 'vcomb', - 'vcopy', - 'vcopy_i', - 'vdel_k', - 'vdelay', - 'vdelay3', - 'vdelayk', - 'vdelayx', - 'vdelayxq', - 'vdelayxs', - 'vdelayxw', - 'vdelayxwq', - 'vdelayxws', - 'vdivv', - 'vdivv_i', - 'vecdelay', - 'veloc', - 'vexp', - 'vexp_i', - 'vexpseg', - 'vexpv', - 'vexpv_i', - 'vibes', - 'vibr', - 'vibrato', - 'vincr', - 'vlimit', - 'vlinseg', - 'vlowres', - 'vmap', - 'vmirror', - 'vmult', - 'vmult_i', - 'vmultv', - 'vmultv_i', - 'voice', - 'vosim', - 'vphaseseg', - 'vport', - 'vpow', - 'vpow_i', - 'vpowv', - 'vpowv_i', - 'vpvoc', - 'vrandh', - 'vrandi', - 'vsubv', - 'vsubv_i', - 'vtaba', - 'vtabi', - 'vtabk', - 'vtable1k', - 'vtablea', - 'vtablei', - 'vtablek', - 'vtablewa', - 'vtablewi', - 'vtablewk', - 'vtabwa', - 'vtabwi', - 'vtabwk', - 'vwrap', - 'waveset', - 'weibull', - 'wgbow', - 'wgbowedbar', - 'wgbrass', - 'wgclar', - 'wgflute', - 'wgpluck', - 'wgpluck2', - 'wguide1', - 'wguide2', - 'wiiconnect', - 'wiidata', - 'wiirange', - 'wiisend', - 'window', - 'wrap', - 'writescratch', - 'wterrain', - 'xadsr', - 'xin', - 'xout', - 'xscanmap', - 'xscans', - 'xscansmap', - 'xscanu', - 'xtratim', - 'zacl', - 'zakinit', - 'zamod', - 'zar', - 'zarg', - 'zaw', - 'zawm', - 'zfilter2', - 'zir', - 'ziw', - 'ziwm', - 'zkcl', - 'zkmod', - 'zkr', - 'zkw', - 'zkwm' -)) +OPCODES = set(''' +ATSadd +ATSaddnz +ATSbufread +ATScross +ATSinfo +ATSinterpread +ATSpartialtap +ATSread +ATSreadnz +ATSsinnoi +FLbox +FLbutBank +FLbutton +FLcloseButton +FLcolor +FLcolor2 +FLcount +FLexecButton +FLgetsnap +FLgroup +FLgroupEnd +FLgroup_end +FLhide +FLhvsBox +FLhvsBoxSetValue +FLjoy +FLkeyIn +FLknob +FLlabel +FLloadsnap +FLmouse +FLpack +FLpackEnd +FLpack_end +FLpanel +FLpanelEnd +FLpanel_end +FLprintk +FLprintk2 +FLroller +FLrun +FLsavesnap +FLscroll +FLscrollEnd +FLscroll_end +FLsetAlign +FLsetBox +FLsetColor +FLsetColor2 +FLsetFont +FLsetPosition +FLsetSize +FLsetSnapGroup +FLsetText +FLsetTextColor +FLsetTextSize +FLsetTextType +FLsetVal +FLsetVal_i +FLsetVali +FLsetsnap +FLshow +FLslidBnk +FLslidBnk2 +FLslidBnk2Set +FLslidBnk2Setk +FLslidBnkGetHandle +FLslidBnkSet +FLslidBnkSetk +FLslider +FLtabs +FLtabsEnd +FLtabs_end +FLtext +FLupdate +FLvalue +FLvkeybd +FLvslidBnk +FLvslidBnk2 +FLxyin +MixerClear +MixerGetLevel +MixerReceive +MixerSend +MixerSetLevel +MixerSetLevel_i +OSCinit +OSCinitM +OSClisten +OSCsend +S +a +abs +active +adsr +adsyn +adsynt +adsynt2 +aftouch +alpass +alwayson +ampdb +ampdbfs +ampmidi +ampmidid +areson +aresonk +atone +atonek +atonex +babo +balance +bamboo +barmodel +bbcutm +bbcuts +betarand +bexprnd +bformdec1 +bformenc1 +binit +biquad +biquada +birnd +bqrez +buchla +butbp +butbr +buthp +butlp +butterbp +butterbr +butterhp +butterlp +button +buzz +c2r +cabasa +cauchy +cauchyi +ceil +cell +cent +centroid +ceps +cepsinv +chanctrl +changed +changed2 +chani +chano +chebyshevpoly +checkbox +chn_S +chn_a +chn_k +chnclear +chnexport +chnget +chnmix +chnparams +chnset +chuap +clear +clfilt +clip +clockoff +clockon +cmplxprod +comb +combinv +compilecsd +compileorc +compilestr +compress +compress2 +connect +control +convle +convolve +copya2ftab +copyf2array +cos +cosh +cosinv +cosseg +cossegb +cossegr +cps2pch +cpsmidi +cpsmidib +cpsmidinn +cpsoct +cpspch +cpstmid +cpstun +cpstuni +cpsxpch +cpuprc +cross2 +crossfm +crossfmi +crossfmpm +crossfmpmi +crosspm +crosspmi +crunch +ctlchn +ctrl14 +ctrl21 +ctrl7 +ctrlinit +cuserrnd +dam +date +dates +db +dbamp +dbfsamp +dcblock +dcblock2 +dconv +dct +dctinv +delay +delay1 +delayk +delayr +delayw +deltap +deltap3 +deltapi +deltapn +deltapx +deltapxw +denorm +diff +directory +diskgrain +diskin +diskin2 +dispfft +display +distort +distort1 +divz +doppler +downsamp +dripwater +dumpk +dumpk2 +dumpk3 +dumpk4 +duserrnd +dust +dust2 +envlpx +envlpxr +ephasor +eqfil +evalstr +event +event_i +exciter +exitnow +exp +expcurve +expon +exprand +exprandi +expseg +expsega +expsegb +expsegba +expsegr +fareylen +fareyleni +faustaudio +faustcompile +faustctl +faustgen +fft +fftinv +ficlose +filebit +filelen +filenchnls +filepeak +filescal +filesr +filevalid +fillarray +filter2 +fin +fini +fink +fiopen +flanger +flashtxt +flooper +flooper2 +floor +fluidAllOut +fluidCCi +fluidCCk +fluidControl +fluidEngine +fluidLoad +fluidNote +fluidOut +fluidProgramSelect +fluidSetInterpMethod +fmb3 +fmbell +fmmetal +fmpercfl +fmrhode +fmvoice +fmwurlie +fof +fof2 +fofilter +fog +fold +follow +follow2 +foscil +foscili +fout +fouti +foutir +foutk +fprintks +fprints +frac +fractalnoise +framebuffer +freeverb +ftchnls +ftconv +ftcps +ftfree +ftgen +ftgenonce +ftgentmp +ftlen +ftload +ftloadk +ftlptim +ftmorf +ftresize +ftresizei +ftsamplebank +ftsave +ftsavek +ftsr +gain +gainslider +gauss +gaussi +gausstrig +gbuzz +genarray +genarray_i +gendy +gendyc +gendyx +getcfg +getcol +getftargs +getrow +getseed +gogobel +grain +grain2 +grain3 +granule +guiro +harmon +harmon2 +harmon3 +harmon4 +hdf5read +hdf5write +hilbert +hrtfearly +hrtfmove +hrtfmove2 +hrtfreverb +hrtfstat +hsboscil +hvs1 +hvs2 +hvs3 +i +ihold +imagecreate +imagefree +imagegetpixel +imageload +imagesave +imagesetpixel +imagesize +in +in32 +inch +inh +init +initc14 +initc21 +initc7 +inleta +inletf +inletk +inletkid +inletv +ino +inq +inrg +ins +insglobal +insremot +int +integ +interp +invalue +inx +inz +jitter +jitter2 +jspline +k +lenarray +lfo +limit +line +linen +linenr +lineto +linrand +linseg +linsegb +linsegr +locsend +locsig +log +log10 +log2 +logbtwo +logcurve +loopseg +loopsegp +looptseg +loopxseg +lorenz +loscil +loscil3 +loscilx +lowpass2 +lowres +lowresx +lpf18 +lpform +lpfreson +lphasor +lpinterp +lposcil +lposcil3 +lposcila +lposcilsa +lposcilsa2 +lpread +lpreson +lpshold +lpsholdp +lpslot +lua_exec +lua_ikopcall +lua_opdef +mac +maca +madsr +mags +mandel +mandol +maparray +maparray_i +marimba +massign +max +max_k +maxabs +maxabsaccum +maxaccum +maxalloc +maxarray +mclock +mdelay +median +mediank +metro +mfb +midglobal +midic14 +midic21 +midic7 +midichannelaftertouch +midichn +midicontrolchange +midictrl +mididefault +midifilestatus +midiin +midinoteoff +midinoteoncps +midinoteonkey +midinoteonoct +midinoteonpch +midion +midion2 +midiout +midipgm +midipitchbend +midipolyaftertouch +midiprogramchange +miditempo +midremot +min +minabs +minabsaccum +minaccum +minarray +mincer +mirror +mode +modmatrix +monitor +moog +moogladder +moogladder2 +moogvcf +moogvcf2 +moscil +mp3bitrate +mp3in +mp3len +mp3nchnls +mp3scal +mp3sr +mpulse +mrtmsg +multitap +mute +mvchpf +mvclpf1 +mvclpf2 +mvclpf3 +mvclpf4 +mxadsr +nchnls_hw +nestedap +nlalp +nlfilt +nlfilt2 +noise +noteoff +noteon +noteondur +noteondur2 +notnum +nreverb +nrpn +nsamp +nstance +nstrnum +ntrpol +nxtpow2 +octave +octcps +octmidi +octmidib +octmidinn +octpch +olabuffer +oscbnk +oscil +oscil1 +oscil1i +oscil3 +oscili +oscilikt +osciliktp +oscilikts +osciln +oscils +oscilx +out +out32 +outc +outch +outh +outiat +outic +outic14 +outipat +outipb +outipc +outkat +outkc +outkc14 +outkpat +outkpb +outkpc +outleta +outletf +outletk +outletkid +outletv +outo +outq +outq1 +outq2 +outq3 +outq4 +outrg +outs +outs1 +outs2 +outvalue +outx +outz +p +pan +pan2 +pareq +part2txt +partials +partikkel +partikkelget +partikkelset +partikkelsync +passign +paulstretch +pcauchy +pchbend +pchmidi +pchmidib +pchmidinn +pchoct +pconvolve +pcount +pdclip +pdhalf +pdhalfy +peak +pgmassign +pgmchn +phaser1 +phaser2 +phasor +phasorbnk +phs +pindex +pinker +pinkish +pitch +pitchac +pitchamdf +planet +platerev +plltrack +pluck +poisson +pol2rect +polyaft +polynomial +port +portk +poscil +poscil3 +pow +powershape +powoftwo +pows +prealloc +prepiano +print +print_type +printf +printf_i +printk +printk2 +printks +printks2 +prints +product +pset +ptable +ptable3 +ptablei +ptableiw +ptablew +ptrack +puts +pvadd +pvbufread +pvcross +pvinterp +pvoc +pvread +pvs2array +pvs2tab +pvsadsyn +pvsanal +pvsarp +pvsbandp +pvsbandr +pvsbin +pvsblur +pvsbuffer +pvsbufread +pvsbufread2 +pvscale +pvscent +pvsceps +pvscross +pvsdemix +pvsdiskin +pvsdisp +pvsenvftw +pvsfilter +pvsfread +pvsfreeze +pvsfromarray +pvsftr +pvsftw +pvsfwrite +pvsgain +pvsgendy +pvshift +pvsifd +pvsin +pvsinfo +pvsinit +pvslock +pvsmaska +pvsmix +pvsmooth +pvsmorph +pvsosc +pvsout +pvspitch +pvstanal +pvstencil +pvsvoc +pvswarp +pvsynth +pwd +pyassign +pyassigni +pyassignt +pycall +pycall1 +pycall1i +pycall1t +pycall2 +pycall2i +pycall2t +pycall3 +pycall3i +pycall3t +pycall4 +pycall4i +pycall4t +pycall5 +pycall5i +pycall5t +pycall6 +pycall6i +pycall6t +pycall7 +pycall7i +pycall7t +pycall8 +pycall8i +pycall8t +pycalli +pycalln +pycallni +pycallt +pyeval +pyevali +pyevalt +pyexec +pyexeci +pyexect +pyinit +pylassign +pylassigni +pylassignt +pylcall +pylcall1 +pylcall1i +pylcall1t +pylcall2 +pylcall2i +pylcall2t +pylcall3 +pylcall3i +pylcall3t +pylcall4 +pylcall4i +pylcall4t +pylcall5 +pylcall5i +pylcall5t +pylcall6 +pylcall6i +pylcall6t +pylcall7 +pylcall7i +pylcall7t +pylcall8 +pylcall8i +pylcall8t +pylcalli +pylcalln +pylcallni +pylcallt +pyleval +pylevali +pylevalt +pylexec +pylexeci +pylexect +pylrun +pylruni +pylrunt +pyrun +pyruni +pyrunt +qinf +qnan +r2c +rand +randh +randi +random +randomh +randomi +rbjeq +readclock +readf +readfi +readk +readk2 +readk3 +readk4 +readks +readscore +readscratch +rect2pol +reinit +release +remoteport +remove +repluck +reson +resonk +resonr +resonx +resonxk +resony +resonz +resyn +reverb +reverb2 +reverbsc +rewindscore +rezzy +rfft +rifft +rms +rnd +rnd31 +round +rspline +rtclock +s16b14 +s32b14 +samphold +sandpaper +scale +scalearray +scanhammer +scans +scantable +scanu +schedkwhen +schedkwhennamed +schedule +schedwhen +scoreline +scoreline_i +seed +sekere +semitone +sense +sensekey +seqtime +seqtime2 +serialBegin +serialEnd +serialFlush +serialPrint +serialRead +serialWrite +serialWrite_i +setcol +setctrl +setksmps +setrow +setscorepos +sfilist +sfinstr +sfinstr3 +sfinstr3m +sfinstrm +sfload +sflooper +sfpassign +sfplay +sfplay3 +sfplay3m +sfplaym +sfplist +sfpreset +shaker +shiftin +shiftout +signalflowgraph +signum +sin +sinh +sininv +sinsyn +sleighbells +slicearray +slider16 +slider16f +slider16table +slider16tablef +slider32 +slider32f +slider32table +slider32tablef +slider64 +slider64f +slider64table +slider64tablef +slider8 +slider8f +slider8table +slider8tablef +sliderKawai +sndloop +sndwarp +sndwarpst +sockrecv +sockrecvs +socksend +socksends +soundin +space +spat3d +spat3di +spat3dt +spdist +splitrig +sprintf +sprintfk +spsend +sqrt +statevar +stix +strcat +strcatk +strchar +strchark +strcmp +strcmpk +strcpy +strcpyk +strecv +streson +strfromurl +strget +strindex +strindexk +strlen +strlenk +strlower +strlowerk +strrindex +strrindexk +strset +strsub +strsubk +strtod +strtodk +strtol +strtolk +strupper +strupperk +stsend +subinstr +subinstrinit +sum +sumarray +svfilter +syncgrain +syncloop +syncphasor +system +system_i +tab +tab2pvs +tab_i +tabifd +table +table3 +table3kt +tablecopy +tablefilter +tablefilteri +tablegpw +tablei +tableicopy +tableigpw +tableikt +tableimix +tableiw +tablekt +tablemix +tableng +tablera +tableseg +tableshuffle +tableshufflei +tablew +tablewa +tablewkt +tablexkt +tablexseg +tabmorph +tabmorpha +tabmorphak +tabmorphi +tabplay +tabrec +tabsum +tabw +tabw_i +tambourine +tan +tanh +taninv +taninv2 +tb0 +tb0_init +tb1 +tb10 +tb10_init +tb11 +tb11_init +tb12 +tb12_init +tb13 +tb13_init +tb14 +tb14_init +tb15 +tb15_init +tb1_init +tb2 +tb2_init +tb3 +tb3_init +tb4 +tb4_init +tb5 +tb5_init +tb6 +tb6_init +tb7 +tb7_init +tb8 +tb8_init +tb9 +tb9_init +tbvcf +tempest +tempo +temposcal +tempoval +timedseq +timeinstk +timeinsts +timek +times +tival +tlineto +tone +tonek +tonex +tradsyn +trandom +transeg +transegb +transegr +trcross +trfilter +trhighest +trigger +trigseq +trirand +trlowest +trmix +trscale +trshift +trsplit +turnoff +turnoff2 +turnon +unirand +unwrap +upsamp +urandom +urd +vactrol +vadd +vadd_i +vaddv +vaddv_i +vaget +valpass +vaset +vbap +vbapg +vbapgmove +vbaplsinit +vbapmove +vbapz +vbapzmove +vcella +vco +vco2 +vco2ft +vco2ift +vco2init +vcomb +vcopy +vcopy_i +vdel_k +vdelay +vdelay3 +vdelayk +vdelayx +vdelayxq +vdelayxs +vdelayxw +vdelayxwq +vdelayxws +vdivv +vdivv_i +vecdelay +veloc +vexp +vexp_i +vexpseg +vexpv +vexpv_i +vibes +vibr +vibrato +vincr +vlimit +vlinseg +vlowres +vmap +vmirror +vmult +vmult_i +vmultv +vmultv_i +voice +vosim +vphaseseg +vport +vpow +vpow_i +vpowv +vpowv_i +vpvoc +vrandh +vrandi +vsubv +vsubv_i +vtaba +vtabi +vtabk +vtable1k +vtablea +vtablei +vtablek +vtablewa +vtablewi +vtablewk +vtabwa +vtabwi +vtabwk +vwrap +waveset +weibull +wgbow +wgbowedbar +wgbrass +wgclar +wgflute +wgpluck +wgpluck2 +wguide1 +wguide2 +wiiconnect +wiidata +wiirange +wiisend +window +wrap +writescratch +wterrain +xadsr +xin +xout +xscanmap +xscans +xscansmap +xscanu +xtratim +zacl +zakinit +zamod +zar +zarg +zaw +zawm +zfilter2 +zir +ziw +ziwm +zkcl +zkmod +zkr +zkw +zkwm +'''.split()) -DEPRECATED_OPCODES = set(( - 'array', - 'bformdec', - 'bformenc', - 'copy2ftab', - 'copy2ttab', - 'hrtfer', - 'ktableseg', - 'lentab', - 'maxtab', - 'mintab', - 'pop', - 'pop_f', - 'push', - 'push_f', - 'scalet', - 'sndload', - 'soundout', - 'soundouts', - 'specaddm', - 'specdiff', - 'specdisp', - 'specfilt', - 'spechist', - 'specptrk', - 'specscal', - 'specsum', - 'spectrum', - 'stack', - 'sumtab', - 'tabgen', - 'tabmap', - 'tabmap_i', - 'tabslice', - 'vbap16', - 'vbap4', - 'vbap4move', - 'vbap8', - 'vbap8move', - 'xyin' -)) +DEPRECATED_OPCODES = set(''' +array +bformdec +bformenc +copy2ftab +copy2ttab +hrtfer +ktableseg +lentab +maxtab +mintab +pop +pop_f +push +push_f +scalet +sndload +soundout +soundouts +specaddm +specdiff +specdisp +specfilt +spechist +specptrk +specscal +specsum +spectrum +stack +sumtab +tabgen +tabmap +tabmap_i +tabslice +vbap16 +vbap4 +vbap4move +vbap8 +vbap8move +xyin +'''.split()) diff --git a/pygments/lexers/csound.py b/pygments/lexers/csound.py index 677bfa89..b492ea9d 100644 --- a/pygments/lexers/csound.py +++ b/pygments/lexers/csound.py @@ -12,8 +12,8 @@ import re from pygments.lexer import RegexLexer, bygroups, default, include, using, words -from pygments.token import Comment, Keyword, Name, Number, Operator, Punctuation, \ - String, Text +from pygments.token import Comment, Error, Keyword, Name, Number, Operator, Punctuation, \ + String, Text, Whitespace from pygments.lexers._csound_builtins import OPCODES, DEPRECATED_OPCODES from pygments.lexers.html import HtmlLexer from pygments.lexers.python import PythonLexer @@ -28,64 +28,103 @@ class CsoundLexer(RegexLexer): tokens = { 'whitespace': [ (r'[ \t]+', Text), - (r'\\\n', Text), - (r'/[*](.|\n)*?[*]/', Comment.Multiline) - ], - - 'macro use': [ - (r'(\$[A-Z_a-z]\w*\.?)(\()', bygroups(Comment.Preproc, Punctuation), - 'function macro use'), - (r'\$[A-Z_a-z]\w*(\.|\b)', Comment.Preproc) - ], - 'function macro use': [ - (r"((?:\\['\)]|[^'\)])+)(')", bygroups(Comment.Preproc, Punctuation)), - (r"([^'\)]+)(\))", bygroups(Comment.Preproc, Punctuation), '#pop') - ], - - 'whitespace or macro use': [ - include('whitespace'), - include('macro use') + (r'/[*](?:.|\n)*?[*]/', Comment.Multiline), + (r'(?:;|//).*$', Comment.Single), + (r'(\\)(\n)', bygroups(Whitespace, Text)) ], 'preprocessor directives': [ - (r'#(e(nd(if)?|lse)|ifn?def|undef)\b|##', Comment.Preproc), - (r'#include', Comment.Preproc, 'include'), - (r'#[ \t]*define', Comment.Preproc, 'macro name'), - (r'@+[ \t]*\d*', Comment.Preproc) + (r'#(?:e(?:nd(?:if)?|lse)\b|##)|@@?[ \t]*\d+', Comment.Preproc), + (r'#include', Comment.Preproc, 'include directive'), + (r'#[ \t]*define', Comment.Preproc, 'define directive'), + (r'#(?:ifn?def|undef)\b', Comment.Preproc, 'macro directive') ], - 'include': [ + 'include directive': [ include('whitespace'), (r'([^ \t]).*?\1', String, '#pop') ], - 'macro name': [ + 'define directive': [ + (r'\n', Text), include('whitespace'), (r'([A-Z_a-z]\w*)(\()', bygroups(Comment.Preproc, Punctuation), - 'function macro parameter list'), - (r'[A-Z_a-z]\w*', Comment.Preproc, 'object macro definition after name') + ('#pop', 'macro parameter name list')), + (r'[A-Z_a-z]\w*', Comment.Preproc, ('#pop', 'before macro body')) + ], + 'macro parameter name list': [ + include('whitespace'), + (r'[A-Z_a-z]\w*', Comment.Preproc), + (r"['#]", Punctuation), + (r'\)', Punctuation, ('#pop', 'before macro body')) ], - 'object macro definition after name': [ + 'before macro body': [ + (r'\n', Text), include('whitespace'), - (r'#', Punctuation, 'object macro replacement text') + (r'#', Punctuation, ('#pop', 'macro body')) ], - 'object macro replacement text': [ - (r'(\\#|[^#])+', Comment.Preproc), - (r'#', Punctuation, '#pop:3') + 'macro body': [ + (r'(?:\\(?!#)|[^#\\]|\n)+', Comment.Preproc), # TODO + (r'\\#', Comment.Preproc), + (r'(?. For example, + # $MACRO((value)) + # passes (value, not (value), to MACRO. The workaround for this implemented in + # https://github.com/csound/csound/commit/3e0b441b55fd8e07d70b0908da8165b889feb883 + # is to require escaping right parentheses. Because this is not required by + # the C preprocessor, it is likely to be unexpected, so use a different token + # type for parentheses in macro parameter values. + (r'"', String, 'macro parameter value quoted string'), + (r'\{\{', String, 'macro parameter value braced string'), + (r'\(', Comment.Preproc, 'macro parameter value parenthetical'), + (r'\)', Punctuation, '#pop') + ], + 'macro parameter value quoted string': [ + (r'\\\)', Comment.Preproc), + (r'\)', Error), + include('quoted string') + ], + 'macro parameter value braced string': [ + (r'\\\)', Comment.Preproc), + (r'\)', Error), + include('braced string') + ], + 'macro parameter value parenthetical': [ + (r'[^\\()]+', Comment.Preproc), # TODO + (r'\(', Comment.Preproc, '#push'), + (r'\\\)', Comment.Preproc, '#pop'), + (r'\)', Error, '#pop') + ], + + 'whitespace and macro uses': [ + include('whitespace'), + include('macro uses') + ], + + 'numbers': [ + (r'\d+[Ee][+-]?\d+|(\d+\.\d*|\d*\.\d+)([Ee][+-]?\d+)?', Number.Float), + (r'0[Xx][0-9A-Fa-f]+', Number.Hex), + (r'\d+', Number.Integer) ], - 'function macro replacement text': [ - (r'(\\#|[^#])+', Comment.Preproc), - (r'#', Punctuation, '#pop:4') + + 'braced string': [ + # Do nothing. This must be defined in subclasses. ] } @@ -102,33 +141,63 @@ class CsoundScoreLexer(CsoundLexer): filenames = ['*.sco'] tokens = { - 'partial statement': [ + 'root': [ + (r'\n', Text), + include('whitespace and macro uses'), include('preprocessor directives'), - (r'\d+[Ee][+-]?\d+|(\d+\.\d*|\d*\.\d+)([Ee][+-]?\d+)?', Number.Float), - (r'0[Xx][0-9A-Fa-f]+', Number.Hex), - (r'\d+', Number.Integer), - (r'"', String, 'single-line string'), - (r'[+\-*/%^!=<>|&#~.]', Operator), - (r'[]()[]', Punctuation), - (r'\w+', Comment.Preproc) - ], - 'statement': [ - include('whitespace or macro use'), - newline + ('#pop',), - include('partial statement') + (r'[abCefiqstvxy]', Keyword), + # There is also a w statement that is generated internally and should not be + # used; see https://github.com/csound/csound/issues/750. + + (r'z', Keyword.Constant), + # z is a constant equal to 800,000,000,000. 800 billion seconds is about + # 25,367.8 years. See also + # https://csound.github.io/docs/manual/ScoreTop.html and + # https://github.com/csound/csound/search?q=stof+path%3AEngine+filename%3Asread.c. + + (r'([nNpP][pP])(\d+)', bygroups(Keyword, Number.Integer)), + + (r'[mn]', Keyword, 'mark statement'), + + include('numbers'), + (r'[!+\-*/^%&|<>#~.]', Operator), + (r'[()\[\]]', Punctuation), + (r'"', String, 'quoted string'), + (r'\{', Comment.Preproc, 'loop after left brace'), ], - 'root': [ - newline, - include('whitespace or macro use'), - (r'[{}]', Punctuation, 'statement'), - (r'[abefimq-tv-z]|[nN][pP]?', Keyword, 'statement') + 'mark statement': [ + include('whitespace and macro uses'), + (r'[A-Z_a-z]\w*', Name.Label), + (r'\n', Text, '#pop') ], - 'single-line string': [ + 'quoted string': [ (r'"', String, '#pop'), - (r'[^\\"]+', String) + (r'[^"$]+', String), + include('macro uses'), + (r'[$]', String) + ], + + 'loop after left brace': [ + include('whitespace and macro uses'), + (r'\d+', Number.Integer, ('#pop', 'loop after repeat count')), + ], + 'loop after repeat count': [ + include('whitespace and macro uses'), + (r'[A-Z_a-z]\w*', Comment.Preproc, ('#pop', 'loop')) + ], + 'loop': [ + (r'\}', Comment.Preproc, '#pop'), + include('root') + ], + + # Braced strings are not allowed in Csound scores, but this is needed + # because the superclass includes it. + 'braced string': [ + (r'\}\}', String, '#pop'), + (r'[^}]|\}(?!\})', String) ] } @@ -169,163 +238,162 @@ class CsoundOrchestraLexer(CsoundLexer): yield match.start(), Name, name tokens = { - 'label': [ - (r'^([ \t]*)(\w+)(:)', bygroups(Text, Name.Label, Punctuation)) - ], + 'root': [ + (r'\n', Text), - 'partial expression': [ + (r'^([ \t]*)(\w+)(:)(?:[ \t]+|$)', bygroups(Text, Name.Label, Punctuation)), + + include('whitespace and macro uses'), include('preprocessor directives'), - (r'\b(0dbfs|A4|k(r|smps)|nchnls(_i)?|sr)\b', Name.Variable.Global), - (r'\d+[Ee][+-]?\d+|(\d+\.\d*|\d*\.\d+)([Ee][+-]?\d+)?', Number.Float), - (r'0[Xx][0-9A-Fa-f]+', Number.Hex), - (r'\d+', Number.Integer), - (r'"', String, 'single-line string'), - (r'\{\{', String, 'multi-line string'), - (r'[+\-*/%^!=&|<>#~¬]', Operator), - (r'[](),?:[]', Punctuation), + + (r'\binstr\b', Keyword.Declaration, 'instrument numbers and identifiers'), + (r'\bopcode\b', Keyword.Declaration, 'after opcode keyword'), + (r'\b(?:end(?:in|op))\b', Keyword.Declaration), + + include('partial statements') + ], + + 'partial statements': [ + (r'\b(?:0dbfs|A4|k(?:r|smps)|nchnls(?:_i)?|sr)\b', Name.Variable.Global), + + include('numbers'), + + (r'\+=|-=|\*=|/=|<<|>>|<=|>=|==|!=|&&|\|\||[~¬]|[=!+\-*/^%&|<>#?:]', Operator), + (r'[(),\[\]]', Punctuation), + + (r'"', String, 'quoted string'), + (r'\{\{', String, 'braced string'), + (words(( - # Keywords 'do', 'else', 'elseif', 'endif', 'enduntil', 'fi', 'if', 'ithen', 'kthen', 'od', 'then', 'until', 'while', - # Opcodes that act as control structures - 'return', 'rireturn' ), prefix=r'\b', suffix=r'\b'), Keyword), - (words(('goto', 'igoto', 'kgoto', 'reinit', 'rigoto', 'tigoto'), - prefix=r'\b', suffix=r'\b'), Keyword, 'goto label'), - (words(('cggoto', 'cigoto', 'cingoto', 'ckgoto', 'cngoto', 'cnkgoto'), - prefix=r'\b', suffix=r'\b'), Keyword, - ('goto label', 'goto expression')), - (r'\btimout\b', Keyword, - ('goto label', 'goto expression', 'goto expression')), - (words(('loop_ge', 'loop_gt', 'loop_le', 'loop_lt'), - prefix=r'\b', suffix=r'\b'), Keyword, - ('goto label', 'goto expression', 'goto expression', 'goto expression')), - (r'\bscoreline(_i)?\b', Name.Builtin, 'scoreline opcode'), + (words(('return', 'rireturn'), prefix=r'\b', suffix=r'\b'), Keyword.Pseudo), + + (r'\b[ik]?goto\b', Keyword, 'goto label'), + (r'\b(r(?:einit|igoto)|tigoto)(\(|\b)', bygroups(Keyword.Pseudo, Punctuation), + 'goto label'), + (r'\b(c(?:g|in?|k|nk?)goto)(\(|\b)', bygroups(Keyword.Pseudo, Punctuation), + ('goto label', 'goto argument')), + (r'\b(timout)(\(|\b)', bygroups(Keyword.Pseudo, Punctuation), + ('goto label', 'goto argument', 'goto argument')), + (r'\b(loop_[gl][et])(\(|\b)', bygroups(Keyword.Pseudo, Punctuation), + ('goto label', 'goto argument', 'goto argument', 'goto argument')), + (r'\bprintk?s\b', Name.Builtin, 'prints opcode'), - (r'\bpyl?run[it]?\b', Name.Builtin, 'python opcode'), - (r'\blua_(exec|opdef)\b', Name.Builtin, 'lua opcode'), - (r'\bp\d+\b', Name.Builtin), + (r'\b(?:readscore|scoreline(?:_i)?)\b', Name.Builtin, 'Csound score opcode'), + (r'\bpyl?run[it]?\b', Name.Builtin, 'Python opcode'), + (r'\blua_(?:exec|opdef)\b', Name.Builtin, 'Lua opcode'), + (r'\bp\d+\b', Name.Variable.Instance), (r'\b([A-Z_a-z]\w*)(?:(:)([A-Za-z]))?\b', name_callback) ], - 'expression': [ - include('whitespace or macro use'), - newline + ('#pop',), - include('partial expression') - ], - - 'root': [ - newline, - include('whitespace or macro use'), - (r'\binstr\b', Keyword, ('instrument block', 'instrument name list')), - (r'\bopcode\b', Keyword, ('opcode block', 'opcode parameter list', - 'opcode types', 'opcode types', 'opcode name')), - include('label'), - default('expression') - ], - - 'instrument name list': [ - include('whitespace or macro use'), + 'instrument numbers and identifiers': [ + include('whitespace and macro uses'), (r'\d+|[A-Z_a-z]\w*', Name.Function), (r'[+,]', Punctuation), - newline + ('#pop',) - ], - 'instrument block': [ - newline, - include('whitespace or macro use'), - (r'\bendin\b', Keyword, '#pop'), - include('label'), - default('expression') + (r'\n', Text, '#pop') ], - 'opcode name': [ - include('whitespace or macro use'), - (r'[A-Z_a-z]\w*', opcode_name_callback, '#pop') + 'after opcode keyword': [ + include('whitespace and macro uses'), + (r'[A-Z_a-z]\w*', opcode_name_callback, ('#pop', 'opcode type signatures')), + (r'\n', Text, '#pop') ], - 'opcode types': [ - include('whitespace or macro use'), - (r'0|[]afijkKoOpPStV[]+', Keyword.Type, '#pop'), - (r',', Punctuation) + 'opcode type signatures': [ + include('whitespace and macro uses'), + (r'\b(?:0|[afijkKoOpPStV\[\]]+)\b', Keyword.Type), + (r',', Punctuation), + (r'\n', Text, '#pop') + ], + + 'quoted string': [ + (r'"', String, '#pop'), + (r'[^\\"$%]', String), + include('macro uses'), + include('escape sequences'), + include('format specifiers'), + (r'[\\$%]', String) ], - 'opcode parameter list': [ - include('whitespace or macro use'), - newline + ('#pop',) + 'braced string': [ + (r'\}\}', String, '#pop'), + (r'[^\\}]|\}(?!\})', String), + include('escape sequences'), + include('format specifiers'), + (r'[\\%]', String) ], - 'opcode block': [ - newline, - include('whitespace or macro use'), - (r'\bendop\b', Keyword, '#pop'), - include('label'), - default('expression') + 'escape sequences': [ + # https://github.com/csound/csound/search?q=unquote_string+path%3AEngine+filename%3Acsound_orc_compile.c + (r'\\(?:[\\abnrt"]|[0-7]{1,3})', String.Escape) + ], + # Format specifiers are highlighted in all strings, even though only + # fprintks https://csound.github.io/docs/manual/fprintks.html + # fprints https://csound.github.io/docs/manual/fprints.html + # printf/printf_i https://csound.github.io/docs/manual/printf.html + # printks https://csound.github.io/docs/manual/printks.html + # prints https://csound.github.io/docs/manual/prints.html + # sprintf https://csound.github.io/docs/manual/sprintf.html + # sprintfk https://csound.github.io/docs/manual/sprintfk.html + # work with strings that contain format specifiers. In addition, these + # opcodes’ handling of format specifiers is inconsistent: + # - fprintks, fprints, printks, and prints do accept %a and %A + # specifiers, but can’t accept %s specifiers. + # - printf, printf_i, sprintf, and sprintfk don’t accept %a and %A + # specifiers, but can accept %s specifiers. + # See https://github.com/csound/csound/issues/747 for more information. + 'format specifiers': [ + (r'%[#0\- +]*\d*(?:\.\d+)?[diuoxXfFeEgGaAcs]', String.Interpol), + (r'%%', String.Escape) + ], + + 'goto argument': [ + include('whitespace and macro uses'), + (r',', Punctuation, '#pop'), + include('partial statements') ], - 'goto label': [ - include('whitespace or macro use'), + include('whitespace and macro uses'), (r'\w+', Name.Label, '#pop'), default('#pop') ], - 'goto expression': [ - include('whitespace or macro use'), - (r',', Punctuation, '#pop'), - include('partial expression') - ], - 'single-line string': [ - include('macro use'), - (r'"', String, '#pop'), - # https://github.com/csound/csound/search?q=unquote_string+path%3AEngine+filename%3Acsound_orc_compile.c - (r'\\([\\abnrt"]|[0-7]{1,3})', String.Escape), - (r'[^\\"$\n]+', String), - (r'[\\"$\n]', String) + 'prints opcode': [ + include('whitespace and macro uses'), + (r'"', String, 'prints quoted string'), + default('#pop') ], - 'multi-line string': [ - (r'\}\}', String, '#pop'), - (r'[^}]+|\}(?!\})', String) + 'prints quoted string': [ + (r'\\\\[aAbBnNrRtT]', String.Escape), + (r'%[!nNrRtT]|[~^]{1,2}', String.Escape), + include('quoted string') ], - 'scoreline opcode': [ - include('whitespace or macro use'), - (r'\{\{', String, 'scoreline'), - default('#pop') + 'Csound score opcode': [ + include('whitespace and macro uses'), + (r'\{\{', String, 'Csound score'), + (r'\n', Text, '#pop') ], - 'scoreline': [ + 'Csound score': [ (r'\}\}', String, '#pop'), (r'([^}]+)|\}(?!\})', using(CsoundScoreLexer)) ], - 'prints opcode': [ - include('whitespace or macro use'), - (r'"', String, 'prints'), - default('#pop') + 'Python opcode': [ + include('whitespace and macro uses'), + (r'\{\{', String, 'Python'), + (r'\n', Text, '#pop') ], - 'prints': [ - include('macro use'), - (r'"', String, '#pop'), - # https://github.com/csound/csound/search?q=printksset_+path%3AOOps+filename%3Augrw1.c - (r'%\d*(\.\d+)?[cdfhilouxX]', String.Interpol), - (r'%[!%nNrRtT]|[~^]|\\([\\aAbBnNrRtT"]|[0-7]{1,3})', String.Escape), - (r'[^\\"~$%\^\n]+', String), - (r'[\\"~$%\^\n]', String) - ], - - 'python opcode': [ - include('whitespace or macro use'), - (r'\{\{', String, 'python'), - default('#pop') - ], - 'python': [ + 'Python': [ (r'\}\}', String, '#pop'), (r'([^}]+)|\}(?!\})', using(PythonLexer)) ], - 'lua opcode': [ - include('whitespace or macro use'), - (r'"', String, 'single-line string'), - (r'\{\{', String, 'lua'), - (r',', Punctuation), - default('#pop') + 'Lua opcode': [ + include('whitespace and macro uses'), + (r'\{\{', String, 'Lua'), + (r'\n', Text, '#pop') ], - 'lua': [ + 'Lua': [ (r'\}\}', String, '#pop'), (r'([^}]+)|\}(?!\})', using(LuaLexer)) ] @@ -352,15 +420,18 @@ class CsoundDocumentLexer(RegexLexer): # be XML files. tokens = { 'root': [ - newline, (r'/[*](.|\n)*?[*]/', Comment.Multiline), - (r'[^<&;/]+', Text), + (r'(?:;|//).*$', Comment.Single), + (r'[^/;<]+|/(?!/)', Text), + (r'<\s*CsInstruments', Name.Tag, ('orchestra', 'tag')), (r'<\s*CsScore', Name.Tag, ('score', 'tag')), (r'<\s*[hH][tT][mM][lL]', Name.Tag, ('HTML', 'tag')), + (r'<\s*[\w:.-]+', Name.Tag, 'tag'), (r'<\s*/\s*[\w:.-]+\s*>', Name.Tag) ], + 'orchestra': [ (r'<\s*/\s*CsInstruments\s*>', Name.Tag, '#pop'), (r'(.|\n)+?(?=<\s*/\s*CsInstruments\s*>)', using(CsoundOrchestraLexer)) @@ -373,6 +444,7 @@ class CsoundDocumentLexer(RegexLexer): (r'<\s*/\s*[hH][tT][mM][lL]\s*>', Name.Tag, '#pop'), (r'(.|\n)+?(?=<\s*/\s*[hH][tT][mM][lL]\s*>)', using(HtmlLexer)) ], + 'tag': [ (r'\s+', Text), (r'[\w.:-]+\s*=', Name.Attribute, 'attr'), diff --git a/tests/examplefiles/test.csd b/tests/examplefiles/test.csd index 3f43bede..6512d99e 100644 --- a/tests/examplefiles/test.csd +++ b/tests/examplefiles/test.csd @@ -1,254 +1,18 @@ +/* + * comment + */ +; comment +// comment +/ -// This is a Csound orchestra file for testing a Pygments -// lexer. Csound single-line comments can be preceded by a pair of forward -// slashes... -; ...or a semicolon. - -/* Block comments begin with /* and end with */ - -// Orchestras begin with a header of audio parameters. -nchnls = 1 -nchnls_i = 1 -sr = 44100 0dbfs = 1 -ksmps = 10 -A4 = 440 - -// The control rate kr = sr / ksmps can be omitted when the number of audio -// samples in a control period (ksmps) is set, but kr may appear in older -// orchestras. -kr = 4410 - -// Orchestras contain instruments. These begin with the keyword instr followed -// by a comma-separated list of numbers or names of the instrument. Instruments -// end at the endin keyword and cannot be nested. -instr 1, N_a_M_e_, +Name - // Instruments contain statements. Here is a typical statement: - aSignal oscil 0dbfs, 440, 1 - // Statements are terminated with a newline (possibly preceded by a comment). - // To write a statement on several lines, precede the newline with a - // backslash. - prints \ - "hello, world\n";comment - - // Csound 6 introduced function syntax for opcodes with one or zero outputs. - // The oscil statement above is the same as - aSignal = oscil(0dbfs, 440, 1) - - // Instruments can contain control structures. - kNote = p3 - if (kNote == 0) then - kFrequency = 220 - elseif kNote == 1 then // Parentheses around binary expressions are optional. - kFrequency = 440 - endif - - // Csound 6 introduced looping structures. - iIndex = 0 - while iIndex < 5 do - print iIndex - iIndex += 1 - od - iIndex = 0 - until iIndex >= 5 do - print iIndex - iIndex += 1 - enduntil - // Both kinds of loops can be terminated by either od or enduntil. - - // Single-line strings are enclosed in double-quotes. - prints "string\\\r\n\t\"" - // Multi-line strings are enclosed in pairs of curly braces. - prints {{ - hello, - - world - }} - - // Instruments often end with a statement containing an output opcode. - outc aSignal -endin - -// Orchestras can also contain user-defined opcodes (UDOs). Here is an -// oscillator with one audio-rate output and two control-rate inputs: -opcode anOscillator, a, kk - kAmplitude, kFrequency xin - aSignal vco2 kAmplitude, kFrequency - xout aSignal -endop -instr TestOscillator - outc(anOscillator(0dbfs, 110)) -endin - -// You can execute Python and -// Lua in Csound. -pyruni {{ -import random - -pool = [(1 + i / 10.0) ** 1.2 for i in range(100)] - -def get_number_from_pool(n, p): - if random.random() < p: - i = int(random.random() * len(pool)) - pool[i] = n - return random.choice(pool) -}} - -// The Csound preprocessor supports conditional compilation and including files. -#ifdef DEBUG -#undef DEBUG -#include "filename.orc" -#endif - -// The preprocessor also supports object- and function-like macros. This is an -// object-like macro that defines a number: -#define A_HZ #440# - -// This is a function-like macro: -#define OSCIL_MACRO(VOLUME' FREQUENCY' TABLE) #oscil $VOLUME, $FREQUENCY, $TABLE# - -// Bodies of macros are enclosed in # and can contain newlines. The arguments of -// function-like macros are separated by single-quotes. Uses of macros are -// prefixed with a dollar sign. -instr TestMacro - aSignal $OSCIL_MACRO(1' $A_HZ' 1) - // Not unlike PHP, macros expand in double-quoted strings. - prints "The frequency of the oscillator is $A_HZ Hz.\n" - out aSignal -endin - -// Here are other things to note about Csound. - -// There are two bitwise NOT operators, ~ and ¬ (U+00AC). The latter is common -// on keyboards in the United Kingdom -// . -instr TestBitwiseNOT - print ~42 - print ¬42 -endin - -// Csound uses # for bitwise XOR, which the Csound manual calls bitwise -// non-equivalence . -instr TestBitwiseXOR - print 0 # 0 - print 0 # 1 - print 1 # 0 - print 1 # 1 -endin - -// Loops and if-then statements are relatively recent additions to Csound. There -// are many flow-control opcodes that involve goto and labels. -instr TestGoto - // This... - if p3 > 0 goto if_label - goto else_label -if_label: - prints "if branch\n" - goto endif_label -else_label: - prints "else branch\n" -endif_label: - - // ...is the same as this. - if p3 > 0 then - prints "if branch\n" - else - prints "else branch\n" - endif - - // This... - iIndex = 0 -loop_label: - print iIndex - iIndex += 1 - if iIndex < 10 goto loop_label - - // ...is the same as this... - iIndex = 0 -loop_lt_label: - print iIndex - loop_lt iIndex, 1, 10, loop_lt_label - - // ...and this. - iIndex = 0 - while iIndex < 10 do - print iIndex - iIndex += 1 - od -endin - -// The prints and printks opcodes, arguably the primary methods of logging -// output, treat certain sequences of characters different from printf in C. -instr TestPrints - // ^ prints an ESCAPE character (U+001B), not a CIRCUMFLEX ACCENT character - // (U+005E). ^^ prints a CIRCUMFLEX ACCENT. - prints "^^\n" - // ~ prints an ESCAPE character (U+001B) followed by a [, not a TILDE - // character (U+007E). ~~ prints a TILDE. - prints "~~\n" - // \A, \B, \N, \R, and \T correspond to the escaped lowercase characters (that - // is, BELL (U+0007), BACKSPACE (U+0008), new line (U+000A), CARRIAGE RETURN - // (U+000D), and tab (U+0009)). - prints "\T\R\N" - // %n, %r, and %t are the same as \n, \r, and \t, as are %N, %R, and %T. - prints "%t%r%n" - // %! prints a semicolon. This is a hold-over from old versions of Csound that - // allowed comments to begin in strings. - prints "; %!\n" -endin - -// The arguments of function-like macros can be separated by # instead of '. -// These two lines define the same macro. -#define OSCIL_MACRO(VOLUME' FREQUENCY' TABLE) #oscil $VOLUME, $FREQUENCY, $TABLE# -#define OSCIL_MACRO(VOLUME# FREQUENCY# TABLE) #oscil $VOLUME, $FREQUENCY, $TABLE# - -// Uses of macros can optionally be suffixed with a period. -instr TestMacroPeriodSuffix - aSignal $OSCIL_MACRO.(1' $A_HZ' 1) - prints "The frequency of the oscillator is $A_HZ.Hz.\n" - out aSignal -endin - -// Csound has @ and @@ operator-like macros that, when followed by a literal -// non-negative integer, expand to the next power of 2 and the next power of 2 -// plus 1 : -// @x = 2^(ceil(log2(x + 1))), x >= 0 -// @@0 = 2 -// @@x = 2^(ceil(log2(x))) + 1, x > 0 -instr TestAt - prints "%d %2d %2d\n", 0, @0, @@0 - prints "%d %2d %2d\n", 1, @1, @@1 - prints "%d %2d %2d\n", 2, @2, @@2 - prints "%d %2d %2d\n", 3, @3, @@3 - prints "%d %2d %2d\n", 4, @4, @@4 - prints "%d %2d %2d\n", 5, @5, @@5 - prints "%d %2d %2d\n", 6, @6, @@6 - prints "%d %2d %2d\n", 7, @7, @@7 - prints "%d %2d %2d\n", 8, @8, @@8 - prints "%d %2d %2d\n", 9, @9, @@9 -endin - -// Including newlines in macros can lead to confusing code, but it tests the -// lexer. -instr MacroAbuse - if 1 == 1 then - prints "on\n" -#define FOO# -BAR -#endif // This ends the if statement. It is not a preprocessor directive. -endin +prints "hello, world\n" -f 1 0 16384 10 1 -i "N_a_M_e_" 0 2 -i "TestOscillator" 2 2 -i "TestBitwiseNOT" 0 1 -i "TestBitwiseXOR" 0 1 -i "TestGoto" 0 1 -i "TestMacroPeriodSuffix" 0 1 -i "TestAt" 0 1 -i "MacroAbuse" 0 1 -e +i 1 0 0 + + + diff --git a/tests/examplefiles/test.orc b/tests/examplefiles/test.orc index 36e35b34..59bf7f66 100644 --- a/tests/examplefiles/test.orc +++ b/tests/examplefiles/test.orc @@ -1,251 +1,81 @@ -// This is a Csound orchestra file for testing a Pygments -// lexer. Csound single-line comments can be preceded by a pair of forward -// slashes... -; ...or a semicolon. - -/* Block comments begin with /* and end with */ - -// Orchestras begin with a header of audio parameters. -nchnls = 1 -nchnls_i = 1 -sr = 44100 -0dbfs = 1 -ksmps = 10 -A4 = 440 - -// The control rate kr = sr / ksmps can be omitted when the number of audio -// samples in a control period (ksmps) is set, but kr may appear in older -// orchestras. -kr = 4410 - -// Orchestras contain instruments. These begin with the keyword instr followed -// by a comma-separated list of numbers or names of the instrument. Instruments -// end at the endin keyword and cannot be nested. -instr 1, N_a_M_e_, +Name - // Instruments contain statements. Here is a typical statement: - aSignal oscil 0dbfs, 440, 1 - // Statements are terminated with a newline (possibly preceded by a comment). - // To write a statement on several lines, precede the newline with a - // backslash. - prints \ - "hello, world\n";comment - - // Csound 6 introduced function syntax for opcodes with one or zero outputs. - // The oscil statement above is the same as - aSignal = oscil(0dbfs, 440, 1) - - // Instruments can contain control structures. - kNote = p3 - if (kNote == 0) then - kFrequency = 220 - elseif kNote == 1 then // Parentheses around binary expressions are optional. - kFrequency = 440 - endif - - // Csound 6 introduced looping structures. - iIndex = 0 - while iIndex < 5 do - print iIndex - iIndex += 1 - od - iIndex = 0 - until iIndex >= 5 do - print iIndex - iIndex += 1 - enduntil - // Both kinds of loops can be terminated by either od or enduntil. - - // Single-line strings are enclosed in double-quotes. - prints "string\\\r\n\t\"" - // Multi-line strings are enclosed in pairs of curly braces. - prints {{ - hello, - - world - }} - - // Instruments often end with a statement containing an output opcode. - outc aSignal +/* + * comment + */ +; comment +// comment + +instr/**/1,/**/N_a_M_e_,/**/+Name/**/// + iDuration = p3 + outc:a(aSignal) endin -// Orchestras can also contain user-defined opcodes (UDOs). Here is an -// oscillator with one audio-rate output and two control-rate inputs: -opcode anOscillator, a, kk - kAmplitude, kFrequency xin - aSignal vco2 kAmplitude, kFrequency - xout aSignal +opcode/**/aUDO,/**/0,/**/aik// + aUDO endop -instr TestOscillator - outc(anOscillator(0dbfs, 110)) -endin -// You can execute Python and -// Lua in Csound. -pyruni {{ -import random +123 0123456789 +0xabcdef0123456789 0XABCDEF +1e2 3e+4 5e-6 7E8 9E+0 1E-2 3. 4.56 .789 -pool = [(1 + i / 10.0) ** 1.2 for i in range(100)] +"characters$MACRO." +"\\\a\b\n\r\t\012\345\67\"" -def get_number_from_pool(n, p): - if random.random() < p: - i = int(random.random() * len(pool)) - pool[i] = n - return random.choice(pool) +{{ +characters$MACRO. }} +{{\\\a\b\n\r\t\"\012\345\67}} -// The Csound preprocessor supports conditional compilation and including files. -#ifdef DEBUG -#undef DEBUG -#include "filename.orc" -#endif ++ - ~ ¬ ! * / ^ % << >> < > <= >= == != & # | && || ? : += -= *= /= -// The preprocessor also supports object- and function-like macros. This is an -// object-like macro that defines a number: -#define A_HZ #440# - -// This is a function-like macro: -#define OSCIL_MACRO(VOLUME' FREQUENCY' TABLE) #oscil $VOLUME, $FREQUENCY, $TABLE# - -// Bodies of macros are enclosed in # and can contain newlines. The arguments of -// function-like macros are separated by single-quotes. Uses of macros are -// prefixed with a dollar sign. -instr TestMacro - aSignal $OSCIL_MACRO(1' $A_HZ' 1) - // Not unlike PHP, macros expand in double-quoted strings. - prints "The frequency of the oscillator is $A_HZ Hz.\n" - out aSignal -endin +0dbfs A4 kr ksmps nchnls nchnls_i sr -// Here are other things to note about Csound. - -// There are two bitwise NOT operators, ~ and ¬ (U+00AC). The latter is common -// on keyboards in the United Kingdom -// . -instr TestBitwiseNOT - print ~42 - print ¬42 -endin - -// Csound uses # for bitwise XOR, which the Csound manual calls bitwise -// non-equivalence . -instr TestBitwiseXOR - print 0 # 0 - print 0 # 1 - print 1 # 0 - print 1 # 1 -endin - -// Loops and if-then statements are relatively recent additions to Csound. There -// are many flow-control opcodes that involve goto and labels. -instr TestGoto - // This... - if p3 > 0 goto if_label - goto else_label -if_label: - prints "if branch\n" - goto endif_label -else_label: - prints "else branch\n" -endif_label: - - // ...is the same as this. - if p3 > 0 then - prints "if branch\n" - else - prints "else branch\n" - endif - - // This... - iIndex = 0 -loop_label: - print iIndex - iIndex += 1 - if iIndex < 10 goto loop_label - - // ...is the same as this... - iIndex = 0 -loop_lt_label: - print iIndex - loop_lt iIndex, 1, 10, loop_lt_label - - // ...and this. - iIndex = 0 - while iIndex < 10 do - print iIndex - iIndex += 1 - od -endin +do else elseif endif enduntil fi if ithen kthen od then until while +return rireturn -// The prints and printks opcodes, arguably the primary methods of logging -// output, treat certain sequences of characters different from printf in C. -instr TestPrints - // ^ prints an ESCAPE character (U+001B), not a CIRCUMFLEX ACCENT character - // (U+005E). ^^ prints a CIRCUMFLEX ACCENT. - prints "^^\n" - // ~ prints an ESCAPE character (U+001B) followed by a [, not a TILDE - // character (U+007E). ~~ prints a TILDE. - prints "~~\n" - // \A, \B, \N, \R, and \T correspond to the escaped lowercase characters (that - // is, BELL (U+0007), BACKSPACE (U+0008), new line (U+000A), CARRIAGE RETURN - // (U+000D), and tab (U+0009)). - prints "\T\R\N" - // %n, %r, and %t are the same as \n, \r, and \t, as are %N, %R, and %T. - prints "%t%r%n" - // %! prints a semicolon. This is a hold-over from old versions of Csound that - // allowed comments to begin in strings. - prints "; %!\n" -endin +aLabel: + label2: -// The arguments of function-like macros can be separated by # instead of '. -// These two lines define the same macro. -#define OSCIL_MACRO(VOLUME' FREQUENCY' TABLE) #oscil $VOLUME, $FREQUENCY, $TABLE# -#define OSCIL_MACRO(VOLUME# FREQUENCY# TABLE) #oscil $VOLUME, $FREQUENCY, $TABLE# +goto aLabel +reinit aLabel +cggoto 1==0, aLabel +timout 0, 0, aLabel +loop_ge 0, 0, 0, aLabel -// Uses of macros can optionally be suffixed with a period. -instr TestMacroPeriodSuffix - aSignal $OSCIL_MACRO.(1' $A_HZ' 1) - prints "The frequency of the oscillator is $A_HZ.Hz.\n" - out aSignal -endin +prints "%! %% %n%N %r%R %t%T \\a\\A \\b\\B \\n\\N \\r\\R \\t\\T" +prints Soutput -// Csound has @ and @@ operator-like macros that, when followed by a literal -// non-negative integer, expand to the next power of 2 and the next power of 2 -// plus 1 : -// @x = 2^(ceil(log2(x + 1))), x >= 0 -// @@0 = 2 -// @@x = 2^(ceil(log2(x))) + 1, x > 0 -instr TestAt - prints "%d %2d %2d\n", 0, @0, @@0 - prints "%d %2d %2d\n", 1, @1, @@1 - prints "%d %2d %2d\n", 2, @2, @@2 - prints "%d %2d %2d\n", 3, @3, @@3 - prints "%d %2d %2d\n", 4, @4, @@4 - prints "%d %2d %2d\n", 5, @5, @@5 - prints "%d %2d %2d\n", 6, @6, @@6 - prints "%d %2d %2d\n", 7, @7, @@7 - prints "%d %2d %2d\n", 8, @8, @@8 - prints "%d %2d %2d\n", 9, @9, @@9 -endin +readscore {{ +i 1 0 0 +}} +pyrun {{ +# Python +}} +lua_exec {{ +-- Lua +}} -// Including newlines in macros can lead to confusing code, but it tests the -// lexer. -instr MacroAbuse - if 1 == 1 then - prints "on\n" -#define FOO# -BAR -#endif // This ends the if statement. It is not a preprocessor directive. -endin +#include/**/"file.udo" +#include/**/|file.udo| -scoreline_i {{ -f 1 0 16384 10 1 -i "N_a_M_e_" 0 2 -i "TestOscillator" 2 2 -i "TestBitwiseNOT" 0 1 -i "TestBitwiseXOR" 0 1 -i "TestGoto" 0 1 -i "TestMacroPeriodSuffix" 0 1 -i "TestAt" 0 1 -i "MacroAbuse" 0 1 -e -}} +#ifdef MACRO +#else +#ifndef MACRO +#endif +#undef MACRO + +# define MACRO#macro_body# +#define/**/ +MACRO/**/ +#\#macro +body\## + +#define MACRO(ARG1#ARG2) #macro_body# +#define/**/ +MACRO(ARG1'ARG2'ARG3)/**/ +#\#macro +body\## + +$MACRO $MACRO. +$MACRO(x) +@0 +@@ 1 diff --git a/tests/examplefiles/test.sco b/tests/examplefiles/test.sco index 07818133..e855e563 100644 --- a/tests/examplefiles/test.sco +++ b/tests/examplefiles/test.sco @@ -1,10 +1,22 @@ -f 1 0 16384 10 1 -i "N_a_M_e_" 0 2 -i "TestOscillator" 2 2 -i "TestBitwiseNOT" 0 1 -i "TestBitwiseXOR" 0 1 -i "TestGoto" 0 1 -i "TestMacroPeriodSuffix" 0 1 -i "TestAt" 0 1 -i "MacroAbuse" 0 1 -e +/* + * comment + */ +; comment +// comment +a b C e f i q s t v x y +z +np0 nP1 Np2 NP3 +m/**/label; +n label +123 0123456789 +0xabcdef0123456789 0XABCDEF +1e2 3e+4 5e-6 7E8 9E+0 1E-2 3. 4.56 .789 +"characters$MACRO." +{ 1 I + { 2 J + { 3 K + $I $J $K + } + } +} +#include "score.sco" -- cgit v1.2.1 From 1ce702fcde424a8cb16179d0d7a9e32c20636003 Mon Sep 17 00:00:00 2001 From: Nathan Whetsell Date: Sat, 28 Jan 2017 12:15:10 -0500 Subject: Add Csound lexer tests --- pygments/lexers/csound.py | 16 +- tests/test_csound.py | 471 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 479 insertions(+), 8 deletions(-) create mode 100644 tests/test_csound.py diff --git a/pygments/lexers/csound.py b/pygments/lexers/csound.py index b492ea9d..462307d9 100644 --- a/pygments/lexers/csound.py +++ b/pygments/lexers/csound.py @@ -64,7 +64,7 @@ class CsoundLexer(RegexLexer): (r'#', Punctuation, ('#pop', 'macro body')) ], 'macro body': [ - (r'(?:\\(?!#)|[^#\\]|\n)+', Comment.Preproc), # TODO + (r'(?:\\(?!#)|[^#\\]|\n)+', Comment.Preproc), (r'\\#', Comment.Preproc), (r'(?. For example, # $MACRO((value)) @@ -106,7 +106,7 @@ class CsoundLexer(RegexLexer): include('braced string') ], 'macro parameter value parenthetical': [ - (r'[^\\()]+', Comment.Preproc), # TODO + (r'[^\\()]+', Comment.Preproc), (r'\(', Comment.Preproc, '#push'), (r'\\\)', Comment.Preproc, '#pop'), (r'\)', Error, '#pop') @@ -309,18 +309,18 @@ class CsoundOrchestraLexer(CsoundLexer): 'quoted string': [ (r'"', String, '#pop'), - (r'[^\\"$%]', String), + (r'[^\\"$%)]+', String), include('macro uses'), include('escape sequences'), include('format specifiers'), - (r'[\\$%]', String) + (r'[\\$%)]', String) ], 'braced string': [ (r'\}\}', String, '#pop'), - (r'[^\\}]|\}(?!\})', String), + (r'(?:[^\\%)}]|\}(?!\}))+', String), include('escape sequences'), include('format specifiers'), - (r'[\\%]', String) + (r'[\\%)]', String) ], 'escape sequences': [ # https://github.com/csound/csound/search?q=unquote_string+path%3AEngine+filename%3Acsound_orc_compile.c diff --git a/tests/test_csound.py b/tests/test_csound.py new file mode 100644 index 00000000..610dec1a --- /dev/null +++ b/tests/test_csound.py @@ -0,0 +1,471 @@ +# -*- coding: utf-8 -*- +""" + Csound lexer tests + ~~~~~~~~~~~~~~~~~~~~ + + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import unittest +from textwrap import dedent + +from pygments.token import Comment, Error, Keyword, Name, Number, Operator, Punctuation, \ + String, Text +from pygments.lexers import CsoundOrchestraLexer + + +class CsoundOrchestraTest(unittest.TestCase): + + def setUp(self): + self.lexer = CsoundOrchestraLexer() + self.maxDiff = None + + def testComments(self): + fragment = dedent('''\ + /* + * comment + */ + ; comment + // comment + ''') + tokens = [ + (Comment.Multiline, u'/*\n * comment\n */'), + (Text, u'\n'), + (Comment.Single, u'; comment'), + (Text, u'\n'), + (Comment.Single, u'// comment'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testInstrumentBlocks(self): + fragment = dedent('''\ + instr/**/1,/**/N_a_M_e_,/**/+Name/**/// + iDuration = p3 + outc:a(aSignal) + endin + ''') + tokens = [ + (Keyword.Declaration, u'instr'), + (Comment.Multiline, u'/**/'), + (Name.Function, u'1'), + (Punctuation, u','), + (Comment.Multiline, u'/**/'), + (Name.Function, u'N_a_M_e_'), + (Punctuation, u','), + (Comment.Multiline, u'/**/'), + (Punctuation, u'+'), + (Name.Function, u'Name'), + (Comment.Multiline, u'/**/'), + (Comment.Single, u'//'), + (Text, u'\n'), + (Text, u' '), + (Keyword.Type, u'i'), + (Name, u'Duration'), + (Text, u' '), + (Operator, u'='), + (Text, u' '), + (Name.Variable.Instance, u'p3'), + (Text, u'\n'), + (Text, u' '), + (Name.Builtin, u'outc'), + (Punctuation, u':'), + (Keyword.Type, u'a'), + (Punctuation, u'('), + (Keyword.Type, u'a'), + (Name, u'Signal'), + (Punctuation, u')'), + (Text, u'\n'), + (Keyword.Declaration, u'endin'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testUserDefinedOpcodes(self): + fragment = dedent('''\ + opcode/**/aUDO,/**/0,/**/aik// + aUDO + endop + ''') + tokens = [ + (Keyword.Declaration, u'opcode'), + (Comment.Multiline, u'/**/'), + (Name.Function, u'aUDO'), + (Punctuation, u','), + (Comment.Multiline, u'/**/'), + (Keyword.Type, u'0'), + (Punctuation, u','), + (Comment.Multiline, u'/**/'), + (Keyword.Type, u'aik'), + (Comment.Single, u'//'), + (Text, u'\n'), + (Text, u' '), + (Name.Function, u'aUDO'), + (Text, u'\n'), + (Keyword.Declaration, u'endop'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testNumbers(self): + fragment = '123 0123456789' + tokens = [ + (Number.Integer, u'123'), + (Text, u' '), + (Number.Integer, u'0123456789'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + fragment = '0xabcdef0123456789 0XABCDEF' + tokens = [ + (Number.Hex, u'0xabcdef0123456789'), + (Text, u' '), + (Number.Hex, u'0XABCDEF'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + fragments = ['1e2', '3e+4', '5e-6', '7E8', '9E+0', '1E-2', '3.', '4.56', '.789'] + for fragment in fragments: + tokens = [ + (Number.Float, fragment), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testQuotedStrings(self): + fragment = '"characters$MACRO."' + tokens = [ + (String, u'"'), + (String, u'characters'), + (Comment.Preproc, u'$MACRO.'), + (String, u'"'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testBracedStrings(self): + fragment = dedent('''\ + {{ + characters$MACRO. + }} + ''') + tokens = [ + (String, u'{{'), + (String, u'\ncharacters$MACRO.\n'), + (String, u'}}'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testEscapeSequences(self): + for character in ['\\', 'a', 'b', 'n', 'r', 't', '"', '012', '345', '67']: + escapedCharacter = '\\' + character + fragment = '"' + escapedCharacter + '"' + tokens = [ + (String, u'"'), + (String.Escape, escapedCharacter), + (String, u'"'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + fragment = '{{' + escapedCharacter + '}}' + tokens = [ + (String, u'{{'), + (String.Escape, escapedCharacter), + (String, u'}}'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testOperators(self): + fragments = ['+', '-', '~', u'¬', '!', '*', '/', '^', '%', '<<', '>>', '<', '>', + '<=', '>=', '==', '!=', '&', '#', '|', '&&', '||', '?', ':', '+=', + '-=', '*=', '/='] + for fragment in fragments: + tokens = [ + (Operator, fragment), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testGlobalValueIdentifiers(self): + for fragment in ['0dbfs', 'A4', 'kr', 'ksmps', 'nchnls', 'nchnls_i', 'sr']: + tokens = [ + (Name.Variable.Global, fragment), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testKeywords(self): + fragments = ['do', 'else', 'elseif', 'endif', 'enduntil', 'fi', 'if', 'ithen', + 'kthen', 'od', 'then', 'until', 'while'] + for fragment in fragments: + tokens = [ + (Keyword, fragment), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + for fragment in ['return', 'rireturn']: + tokens = [ + (Keyword.Pseudo, fragment), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testLabels(self): + fragment = dedent('''\ + aLabel: + label2: + ''') + tokens = [ + (Name.Label, u'aLabel'), + (Punctuation, u':'), + (Text, u'\n'), + (Text, u' '), + (Name.Label, u'label2'), + (Punctuation, u':'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testPrintksAndPrintsEscapeSequences(self): + escapedCharacters = ['%!', '%%', '%n', '%N', '%r', '%R', '%t', '%T', '\\\\a', + '\\\\A', '\\\\b', '\\\\B', '\\\\n', '\\\\N', '\\\\r', + '\\\\R', '\\\\t', '\\\\T'] + for opcode in ['printks', 'prints']: + for escapedCharacter in escapedCharacters: + fragment = opcode + ' "' + escapedCharacter + '"' + tokens = [ + (Name.Builtin, opcode), + (Text, u' '), + (String, u'"'), + (String.Escape, escapedCharacter), + (String, u'"'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testGotoStatements(self): + for keyword in ['goto', 'igoto', 'kgoto']: + fragment = keyword + ' aLabel' + tokens = [ + (Keyword, keyword), + (Text, u' '), + (Name.Label, u'aLabel'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + for opcode in ['reinit', 'rigoto', 'tigoto']: + fragment = opcode + ' aLabel' + tokens = [ + (Keyword.Pseudo, opcode), + (Text, u' '), + (Name.Label, u'aLabel'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + for opcode in ['cggoto', 'cigoto', 'cingoto', 'ckgoto', 'cngoto', 'cnkgoto']: + fragment = opcode + ' 1==0, aLabel' + tokens = [ + (Keyword.Pseudo, opcode), + (Text, u' '), + (Number.Integer, u'1'), + (Operator, u'=='), + (Number.Integer, u'0'), + (Punctuation, u','), + (Text, u' '), + (Name.Label, u'aLabel'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + fragment = 'timout 0, 0, aLabel' + tokens = [ + (Keyword.Pseudo, 'timout'), + (Text, u' '), + (Number.Integer, u'0'), + (Punctuation, u','), + (Text, u' '), + (Number.Integer, u'0'), + (Punctuation, u','), + (Text, u' '), + (Name.Label, u'aLabel'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + for opcode in ['loop_ge', 'loop_gt', 'loop_le', 'loop_lt']: + fragment = opcode + ' 0, 0, 0, aLabel' + tokens = [ + (Keyword.Pseudo, opcode), + (Text, u' '), + (Number.Integer, u'0'), + (Punctuation, u','), + (Text, u' '), + (Number.Integer, u'0'), + (Punctuation, u','), + (Text, u' '), + (Number.Integer, u'0'), + (Punctuation, u','), + (Text, u' '), + (Name.Label, u'aLabel'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testIncludeDirectives(self): + for character in ['"', '|']: + fragment = '#include/**/' + character + 'file.udo' + character + tokens = [ + (Comment.Preproc, u'#include'), + (Comment.Multiline, u'/**/'), + (String, character + u'file.udo' + character), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testObjectLikeMacroDefinitions(self): + fragment = dedent('''\ + # \tdefine MACRO#macro_body# + #define/**/ + MACRO/**/ + #\\#macro + body\\## + ''') + tokens = [ + (Comment.Preproc, u'# \tdefine'), + (Text, u' '), + (Comment.Preproc, u'MACRO'), + (Punctuation, u'#'), + (Comment.Preproc, u'macro_body'), + (Punctuation, u'#'), + (Text, u'\n'), + (Comment.Preproc, u'#define'), + (Comment.Multiline, u'/**/'), + (Text, u'\n'), + (Comment.Preproc, u'MACRO'), + (Comment.Multiline, u'/**/'), + (Text, u'\n'), + (Punctuation, u'#'), + (Comment.Preproc, u'\\#'), + (Comment.Preproc, u'macro\nbody'), + (Comment.Preproc, u'\\#'), + (Punctuation, u'#'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testFunctionLikeMacroDefinitions(self): + fragment = dedent('''\ + #define MACRO(ARG1#ARG2) #macro_body# + #define/**/ + MACRO(ARG1'ARG2' ARG3)/**/ + #\\#macro + body\\## + ''') + tokens = [ + (Comment.Preproc, u'#define'), + (Text, u' '), + (Comment.Preproc, u'MACRO'), + (Punctuation, u'('), + (Comment.Preproc, u'ARG1'), + (Punctuation, u'#'), + (Comment.Preproc, u'ARG2'), + (Punctuation, u')'), + (Text, u' '), + (Punctuation, u'#'), + (Comment.Preproc, u'macro_body'), + (Punctuation, u'#'), + (Text, u'\n'), + (Comment.Preproc, u'#define'), + (Comment.Multiline, u'/**/'), + (Text, u'\n'), + (Comment.Preproc, u'MACRO'), + (Punctuation, u'('), + (Comment.Preproc, u'ARG1'), + (Punctuation, u"'"), + (Comment.Preproc, u'ARG2'), + (Punctuation, u"'"), + (Text, u' '), + (Comment.Preproc, u'ARG3'), + (Punctuation, u')'), + (Comment.Multiline, u'/**/'), + (Text, u'\n'), + (Punctuation, u'#'), + (Comment.Preproc, u'\\#'), + (Comment.Preproc, u'macro\nbody'), + (Comment.Preproc, u'\\#'), + (Punctuation, u'#'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testMacroPreprocessorDirectives(self): + for directive in ['#ifdef', '#ifndef', '#undef']: + fragment = directive + ' MACRO' + tokens = [ + (Comment.Preproc, directive), + (Text, u' '), + (Comment.Preproc, u'MACRO'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testOtherPreprocessorDirectives(self): + fragment = dedent('''\ + #else + #end + #endif + ### + @ \t12345 + @@ \t67890 + ''') + tokens = [ + (Comment.Preproc, u'#else'), + (Text, u'\n'), + (Comment.Preproc, u'#end'), + (Text, u'\n'), + (Comment.Preproc, u'#endif'), + (Text, u'\n'), + (Comment.Preproc, u'###'), + (Text, u'\n'), + (Comment.Preproc, u'@ \t12345'), + (Text, u'\n'), + (Comment.Preproc, u'@@ \t67890'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testFunctionLikeMacros(self): + fragment = '$MACRO.(((x\\))\' "x)\\)x\\))"# {{x\\))x)\\)}})' + tokens = [ + (Comment.Preproc, u'$MACRO.'), + (Punctuation, u'('), + (Comment.Preproc, u'('), + (Comment.Preproc, u'('), + (Comment.Preproc, u'x'), + (Comment.Preproc, u'\\)'), + (Error, u')'), + (Punctuation, u"'"), + (Comment.Preproc, u' '), + (String, u'"'), + (String, u'x'), + (Error, u')'), + (Comment.Preproc, u'\\)'), + (String, u'x'), + (Comment.Preproc, u'\\)'), + (Error, u')'), + (String, u'"'), + (Punctuation, u'#'), + (Comment.Preproc, u' '), + (String, u'{{'), + (String, u'x'), + (Comment.Preproc, u'\\)'), + (Error, u')'), + (String, u'x'), + (Error, u')'), + (Comment.Preproc, u'\\)'), + (String, u'}}'), + (Punctuation, u')'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) -- cgit v1.2.1 From 76bf6c18c554bda6b77199ef3fb0570a9017453f Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 29 Jan 2017 08:32:20 +0100 Subject: Clojure: extend valid_name to contain "|". --- pygments/lexers/jvm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pygments/lexers/jvm.py b/pygments/lexers/jvm.py index f4392839..ccff41c1 100644 --- a/pygments/lexers/jvm.py +++ b/pygments/lexers/jvm.py @@ -801,7 +801,7 @@ class ClojureLexer(RegexLexer): # TODO / should divide keywords/symbols into namespace/rest # but that's hard, so just pretend / is part of the name - valid_name = r'(?!#)[\w!$%*+<=>?/.#-]+' + valid_name = r'(?!#)[\w!$%*+<=>?/.#|-]+' tokens = { 'root': [ -- cgit v1.2.1 From 39a6495a2305444f3e4219635398c345cd106f49 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 29 Jan 2017 08:35:40 +0100 Subject: SCSS: fix wrong token type for single-quote strings (fixes #1322). --- pygments/lexers/css.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/pygments/lexers/css.py b/pygments/lexers/css.py index 29d83707..ce97730e 100644 --- a/pygments/lexers/css.py +++ b/pygments/lexers/css.py @@ -125,7 +125,7 @@ _css_properties = ( 'wrap-flow', 'wrap-inside', 'wrap-through', 'writing-mode', 'z-index', ) -# List of keyword values obtained from: +# List of keyword values obtained from: # http://cssvalues.com/ _keyword_values = ( 'absolute', 'alias', 'all', 'all-petite-caps', 'all-scroll', @@ -263,7 +263,7 @@ _time_units = ( 's', 'ms', ) _all_units = _angle_units + _frequency_units + _length_units + \ - _resolution_units + _time_units + _resolution_units + _time_units class CssLexer(RegexLexer): @@ -322,16 +322,18 @@ class CssLexer(RegexLexer): include('urls'), (r'('+r'|'.join(_functional_notation_keyword_values)+r')(\()', bygroups(Name.Builtin, Punctuation), 'function-start'), - (r'([a-zA-Z_][\w-]+)(\()', bygroups(Name.Function, Punctuation), 'function-start'), + (r'([a-zA-Z_][\w-]+)(\()', + bygroups(Name.Function, Punctuation), 'function-start'), (words(_keyword_values, suffix=r'\b'), Keyword.Constant), (words(_other_keyword_values, suffix=r'\b'), Keyword.Constant), (words(_color_keywords, suffix=r'\b'), Keyword.Constant), - (words(_css_properties, suffix=r'\b'), Keyword), # for transition-property etc. + # for transition-property etc. + (words(_css_properties, suffix=r'\b'), Keyword), (r'\!important', Comment.Preproc), (r'/\*(?:.|\n)*?\*/', Comment), include('numeric-values'), - + (r'[~^*!%&<>|+=@:./?-]+', Operator), (r'[\[\](),]+', Punctuation), (r'"(\\\\|\\"|[^"])*"', String.Double), @@ -351,7 +353,8 @@ class CssLexer(RegexLexer): # function-start may be entered recursively (r'(' + r'|'.join(_functional_notation_keyword_values) + r')(\()', bygroups(Name.Builtin, Punctuation), 'function-start'), - (r'([a-zA-Z_][\w-]+)(\()', bygroups(Name.Function, Punctuation), 'function-start'), + (r'([a-zA-Z_][\w-]+)(\()', + bygroups(Name.Function, Punctuation), 'function-start'), (r'/\*(?:.|\n)*?\*/', Comment), include('numeric-values'), @@ -373,8 +376,8 @@ class CssLexer(RegexLexer): 'numeric-values': [ (r'\#[a-zA-Z0-9]{1,6}', Number.Hex), (r'[+\-]?[0-9]*[.][0-9]+', Number.Float, 'numeric-end'), - (r'[+\-]?[0-9]+', Number.Integer, 'numeric-end'), - ], + (r'[+\-]?[0-9]+', Number.Integer, 'numeric-end'), + ], 'numeric-end': [ (words(_all_units, suffix=r'\b'), Keyword.Type), (r'%', Keyword.Type), @@ -466,9 +469,9 @@ common_sass_tokens = { ], 'string-single': [ - (r"(\\.|#(?=[^\n{])|[^\n'#])+", String.Double), + (r"(\\.|#(?=[^\n{])|[^\n'#])+", String.Single), (r'#\{', String.Interpol, 'interpolation'), - (r"'", String.Double, '#pop'), + (r"'", String.Single, '#pop'), ], 'string-url': [ -- cgit v1.2.1 From 968b078eb424095305a53470760a68d94cfb629b Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 5 Feb 2017 17:17:05 +0100 Subject: AMPL: fix markup --- pygments/lexers/ampl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pygments/lexers/ampl.py b/pygments/lexers/ampl.py index d439cb19..638d025d 100644 --- a/pygments/lexers/ampl.py +++ b/pygments/lexers/ampl.py @@ -3,7 +3,7 @@ pygments.lexers.ampl ~~~~~~~~~~~~~~~~~~~~ - Lexers for the ampl language. + Lexers for the AMPL language. :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. @@ -18,7 +18,7 @@ __all__ = ['AmplLexer'] class AmplLexer(RegexLexer): """ - For AMPL source code. + For `AMPL `_ source code. .. versionadded:: 2.2 """ -- cgit v1.2.1 From 4f5f39dcfe35c329aa2d1be65724782092dd783e Mon Sep 17 00:00:00 2001 From: Erkki Mattila Date: Sun, 5 Feb 2017 17:52:37 +0200 Subject: Change instances of Comment.Singleline to Comment.Single --- pygments/lexers/asm.py | 2 +- pygments/lexers/automation.py | 4 ++-- pygments/lexers/c_like.py | 2 +- pygments/lexers/idl.py | 2 +- pygments/lexers/perl.py | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pygments/lexers/asm.py b/pygments/lexers/asm.py index 9c58478e..5b8cab80 100644 --- a/pygments/lexers/asm.py +++ b/pygments/lexers/asm.py @@ -265,7 +265,7 @@ class HsailLexer(RegexLexer): ], 'comments': [ (r'/\*.*?\*/', Comment.Multiline), - (r'//.*?\n', Comment.Singleline), + (r'//.*?\n', Comment.Single), ], 'keyword': [ # Types diff --git a/pygments/lexers/automation.py b/pygments/lexers/automation.py index be1ec129..3ef42e48 100644 --- a/pygments/lexers/automation.py +++ b/pygments/lexers/automation.py @@ -31,8 +31,8 @@ class AutohotkeyLexer(RegexLexer): 'root': [ (r'^(\s*)(/\*)', bygroups(Text, Comment.Multiline), 'incomment'), (r'^(\s*)(\()', bygroups(Text, Generic), 'incontinuation'), - (r'\s+;.*?$', Comment.Singleline), - (r'^;.*?$', Comment.Singleline), + (r'\s+;.*?$', Comment.Single), + (r'^;.*?$', Comment.Single), (r'[]{}(),;[]', Punctuation), (r'(in|is|and|or|not)\b', Operator.Word), (r'\%[a-zA-Z_#@$][\w#@$]*\%', Name.Variable), diff --git a/pygments/lexers/c_like.py b/pygments/lexers/c_like.py index f7ba7e8f..38827219 100644 --- a/pygments/lexers/c_like.py +++ b/pygments/lexers/c_like.py @@ -105,7 +105,7 @@ class ClayLexer(RegexLexer): tokens = { 'root': [ (r'\s', Text), - (r'//.*?$', Comment.Singleline), + (r'//.*?$', Comment.Single), (r'/(\\\n)?[*](.|\n)*?[*](\\\n)?/', Comment.Multiline), (r'\b(public|private|import|as|record|variant|instance' r'|define|overload|default|external|alias' diff --git a/pygments/lexers/idl.py b/pygments/lexers/idl.py index 99078970..2fc39318 100644 --- a/pygments/lexers/idl.py +++ b/pygments/lexers/idl.py @@ -249,7 +249,7 @@ class IDLLexer(RegexLexer): tokens = { 'root': [ - (r'^\s*;.*?\n', Comment.Singleline), + (r'^\s*;.*?\n', Comment.Single), (words(_RESERVED, prefix=r'\b', suffix=r'\b'), Keyword), (words(_BUILTIN_LIB, prefix=r'\b', suffix=r'\b'), Name.Builtin), (r'\+=|-=|\^=|\*=|/=|#=|##=|<=|>=|=', Operator), diff --git a/pygments/lexers/perl.py b/pygments/lexers/perl.py index 4d5ab3b3..db5a9361 100644 --- a/pygments/lexers/perl.py +++ b/pygments/lexers/perl.py @@ -489,7 +489,7 @@ class Perl6Lexer(ExtendedRegexLexer): 'common': [ (r'#[`|=](?P(?P[' + ''.join(PERL6_BRACKETS) + r'])(?P=first_char)*)', brackets_callback(Comment.Multiline)), - (r'#[^\n]*$', Comment.Singleline), + (r'#[^\n]*$', Comment.Single), (r'^(\s*)=begin\s+(\w+)\b.*?^\1=end\s+\2', Comment.Multiline), (r'^(\s*)=for.*?\n\s*?\n', Comment.Multiline), (r'^=.*?\n\s*?\n', Comment.Multiline), @@ -558,7 +558,7 @@ class Perl6Lexer(ExtendedRegexLexer): # make sure that '#' characters in quotes aren't treated as comments (r"(? Date: Sat, 11 Feb 2017 15:28:03 -0700 Subject: Add built-in aliases to PowerShellLexer --- pygments/lexers/shell.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pygments/lexers/shell.py b/pygments/lexers/shell.py index ceb6f14d..2168583a 100644 --- a/pygments/lexers/shell.py +++ b/pygments/lexers/shell.py @@ -646,6 +646,18 @@ class PowerShellLexer(RegexLexer): 'disable debug cxnew copy convertto convertfrom convert connect ' 'complete compare clear checkpoint aggregate add').split() + aliases = ( + 'ac asnp cat cd cfs chdir clc clear clhy cli clp cls clv cnsn ' + 'compare copy cp cpi cpp curl cvpa dbp del diff dir dnsn ebp echo epal ' + 'epcsv epsn erase etsn exsn fc fhx fl foreach ft fw gal gbp gc gci gcm ' + 'gcs gdr ghy gi gjb gl gm gmo gp gps gpv group gsn gsnp gsv gu gv gwmi ' + 'h history icm iex ihy ii ipal ipcsv ipmo ipsn irm ise iwmi iwr kill lp ' + 'ls man md measure mi mount move mp mv nal ndr ni nmo npssc nsn nv ogv ' + 'oh popd ps pushd pwd r rbp rcjb rcsn rd rdr ren ri rjb rm rmdir rmo ' + 'rni rnp rp rsn rsnp rujb rv rvpa rwmi sajb sal saps sasv sbp sc select ' + 'set shcm si sl sleep sls sort sp spjb spps spsv start sujb sv swmi tee ' + 'trcm type wget where wjb write').split() + commenthelp = ( 'component description example externalhelp forwardhelpcategory ' 'forwardhelptargetname functionality inputs link ' @@ -672,6 +684,7 @@ class PowerShellLexer(RegexLexer): (r'(%s)\b' % '|'.join(keywords), Keyword), (r'-(%s)\b' % '|'.join(operators), Operator), (r'(%s)-[a-z_]\w*\b' % '|'.join(verbs), Name.Builtin), + (r'(%s)\s' % '|'.join(aliases), Name.Builtin), (r'\[[a-z_\[][\w. `,\[\]]*\]', Name.Constant), # .net [type]s (r'-[a-z_]\w*', Name), (r'\w+', Name), -- cgit v1.2.1 From 2ec0231e23dfac5933e43c0e7c92252c2daaaaf2 Mon Sep 17 00:00:00 2001 From: Chris Duck Date: Sat, 11 Feb 2017 15:57:33 -0700 Subject: Add verbs from PS v5.1 Adds the following verbs: approve, assert, backup, block, close, compress, confirm, deny, dismount, edit, find, grant, hide, initialize, install, lock, merge, mount, open, optimize, protect, publish, redo, repair, request, revoke, save, search, step, submit, switch, sync, unblock, uninstall, unlock, unprotect, unpublish, watch --- pygments/lexers/shell.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/pygments/lexers/shell.py b/pygments/lexers/shell.py index 2168583a..68150811 100644 --- a/pygments/lexers/shell.py +++ b/pygments/lexers/shell.py @@ -638,13 +638,17 @@ class PowerShellLexer(RegexLexer): 'wildcard').split() verbs = ( - 'write where wait use update unregister undo trace test tee take ' - 'suspend stop start split sort skip show set send select scroll resume ' - 'restore restart resolve resize reset rename remove register receive ' - 'read push pop ping out new move measure limit join invoke import ' - 'group get format foreach export expand exit enter enable disconnect ' - 'disable debug cxnew copy convertto convertfrom convert connect ' - 'complete compare clear checkpoint aggregate add').split() + 'write where watch wait use update unregister unpublish unprotect ' + 'unlock uninstall undo unblock trace test tee take sync switch ' + 'suspend submit stop step start split sort skip show set send select ' + 'search scroll save revoke resume restore restart resolve resize ' + 'reset request repair rename remove register redo receive read push ' + 'publish protect pop ping out optimize open new move mount merge ' + 'measure lock limit join invoke install initialize import hide group ' + 'grant get format foreach find export expand exit enter enable edit ' + 'dismount disconnect disable deny debug cxnew copy convertto ' + 'convertfrom convert connect confirm compress complete compare close ' + 'clear checkpoint block backup assert approve aggregate add').split() aliases = ( 'ac asnp cat cd cfs chdir clc clear clhy cli clp cls clv cnsn ' -- cgit v1.2.1 From 4452f14d50f695c4cfaa2d69038dde09a9754b71 Mon Sep 17 00:00:00 2001 From: witchard Date: Sat, 18 Feb 2017 20:18:44 +0000 Subject: Added support for text/x-shellscript mime type. Fixes #1332. --- pygments/lexers/shell.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pygments/lexers/shell.py b/pygments/lexers/shell.py index ceb6f14d..dd869ef5 100644 --- a/pygments/lexers/shell.py +++ b/pygments/lexers/shell.py @@ -38,7 +38,7 @@ class BashLexer(RegexLexer): '*.exheres-0', '*.exlib', '*.zsh', '.bashrc', 'bashrc', '.bash_*', 'bash_*', 'zshrc', '.zshrc', 'PKGBUILD'] - mimetypes = ['application/x-sh', 'application/x-shellscript'] + mimetypes = ['application/x-sh', 'application/x-shellscript', 'text/x-shellscript'] tokens = { 'root': [ -- cgit v1.2.1 From ba01dca635a1d141f26aa4f1d11cc2326c988769 Mon Sep 17 00:00:00 2001 From: Jakub Wilk Date: Mon, 20 Feb 2017 12:45:58 +0100 Subject: Fix syntax error in a regexp. --- pygments/lexers/rebol.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pygments/lexers/rebol.py b/pygments/lexers/rebol.py index f3d00200..eff3fce0 100644 --- a/pygments/lexers/rebol.py +++ b/pygments/lexers/rebol.py @@ -239,7 +239,7 @@ class RebolLexer(RegexLexer): if re.match(r'^\s*REBOL\s*\[', text, re.IGNORECASE): # The code starts with REBOL header return 1.0 - elif re.search(r'\s*REBOL\s*[', text, re.IGNORECASE): + elif re.search(r'\s*REBOL\s*\[', text, re.IGNORECASE): # The code contains REBOL header but also some text before it return 0.5 -- cgit v1.2.1 From 8432a3525bd911e3548072ec9bb727f217a2d097 Mon Sep 17 00:00:00 2001 From: Camil Staps Date: Tue, 28 Feb 2017 14:33:06 +0100 Subject: Adds a solarized style --- AUTHORS | 2 +- pygments/styles/__init__.py | 2 + pygments/styles/solarized.py | 129 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 132 insertions(+), 1 deletion(-) create mode 100644 pygments/styles/solarized.py diff --git a/AUTHORS b/AUTHORS index f9ba2675..099fde55 100644 --- a/AUTHORS +++ b/AUTHORS @@ -185,7 +185,7 @@ Other contributors, listed alphabetically, are: * Alexander Smishlajev -- Visual FoxPro lexer * Steve Spigarelli -- XQuery lexer * Jerome St-Louis -- eC lexer -* Camil Staps -- Clean and NuSMV lexers +* Camil Staps -- Clean and NuSMV lexers; Solarized style * James Strachan -- Kotlin lexer * Tom Stuart -- Treetop lexer * Colin Sullivan -- SuperCollider lexer diff --git a/pygments/styles/__init__.py b/pygments/styles/__init__.py index 839a9b78..1f39c692 100644 --- a/pygments/styles/__init__.py +++ b/pygments/styles/__init__.py @@ -44,6 +44,8 @@ STYLE_MAP = { 'arduino': 'arduino::ArduinoStyle', 'rainbow_dash': 'rainbow_dash::RainbowDashStyle', 'abap': 'abap::AbapStyle', + 'solarized-dark': 'solarized::SolarizedDarkStyle', + 'solarized-light': 'solarized::SolarizedLightStyle', } diff --git a/pygments/styles/solarized.py b/pygments/styles/solarized.py new file mode 100644 index 00000000..a1790b08 --- /dev/null +++ b/pygments/styles/solarized.py @@ -0,0 +1,129 @@ +# -*- coding: utf-8 -*- +""" + pygments.styles.solarized + ~~~~~~~~~~~~~~~~~~~~~~~~ + + Solarized by Camil Staps + + A Pygments style for the Solarized themes (licensed under MIT). + See: https://github.com/altercation/solarized + + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.style import Style +from pygments.token import Comment, Error, Generic, Keyword, Name, Number, \ + Operator, String, Token + + +def make_style(colors): + return { + Token: colors['base0'], + + Comment: 'italic ' + colors['base01'], + Comment.Hashbang: colors['base01'], + Comment.Multiline: colors['base01'], + Comment.Preproc: 'noitalic ' + colors['magenta'], + Comment.PreprocFile: 'noitalic ' + colors['base01'], + + Keyword: colors['green'], + Keyword.Constant: colors['cyan'], + Keyword.Declaration: colors['cyan'], + Keyword.Namespace: colors['orange'], + Keyword.Type: colors['yellow'], + + Operator: colors['base01'], + Operator.Word: colors['green'], + + Name.Builtin: colors['blue'], + Name.Builtin.Pseudo: colors['blue'], + Name.Class: colors['blue'], + Name.Constant: colors['blue'], + Name.Decorator: colors['blue'], + Name.Entity: colors['blue'], + Name.Exception: colors['blue'], + Name.Function: colors['blue'], + Name.Function.Magic: colors['blue'], + Name.Label: colors['blue'], + Name.Namespace: colors['blue'], + Name.Tag: colors['blue'], + Name.Variable: colors['blue'], + Name.Variable.Global:colors['blue'], + Name.Variable.Magic: colors['blue'], + + String: colors['cyan'], + String.Doc: colors['base01'], + String.Regex: colors['orange'], + + Number: colors['cyan'], + + Generic.Deleted: colors['red'], + Generic.Emph: 'italic', + Generic.Error: colors['red'], + Generic.Heading: 'bold', + Generic.Subheading: 'underline', + Generic.Inserted: colors['green'], + Generic.Strong: 'bold', + Generic.Traceback: colors['blue'], + + Error: 'bg:' + colors['red'], + } + + +DARK_COLORS = { + 'base03': '#002b36', + 'base02': '#073642', + 'base01': '#586e75', + 'base00': '#657b83', + 'base0': '#839496', + 'base1': '#93a1a1', + 'base2': '#eee8d5', + 'base3': '#fdf6e3', + 'yellow': '#b58900', + 'orange': '#cb4b16', + 'red': '#dc322f', + 'magenta': '#d33682', + 'violet': '#6c71c4', + 'blue': '#268bd2', + 'cyan': '#2aa198', + 'green': '#859900', +} + +LIGHT_COLORS = { + 'base3': '#002b36', + 'base2': '#073642', + 'base1': '#586e75', + 'base0': '#657b83', + 'base00': '#839496', + 'base01': '#93a1a1', + 'base02': '#eee8d5', + 'base03': '#fdf6e3', + 'yellow': '#b58900', + 'orange': '#cb4b16', + 'red': '#dc322f', + 'magenta': '#d33682', + 'violet': '#6c71c4', + 'blue': '#268bd2', + 'cyan': '#2aa198', + 'green': '#859900', +} + + +class SolarizedDarkStyle(Style): + """ + The solarized style, dark. + """ + + styles = make_style(DARK_COLORS) + background_color = DARK_COLORS['base03'] + highlight_color = DARK_COLORS['base02'] + +class SolarizedLightStyle(SolarizedDarkStyle): + """ + The solarized style, light. + """ + + styles = make_style(LIGHT_COLORS) + background_color = LIGHT_COLORS['base03'] + highlight_color = LIGHT_COLORS['base02'] -- cgit v1.2.1 From 4cd7caf75364f579600b1a526b92a1b62614177e Mon Sep 17 00:00:00 2001 From: Camil Staps Date: Mon, 6 Mar 2017 18:37:22 +0100 Subject: Fix strings & add quantified identifiers --- pygments/lexers/clean.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/pygments/lexers/clean.py b/pygments/lexers/clean.py index c41fa400..c0fb63c3 100644 --- a/pygments/lexers/clean.py +++ b/pygments/lexers/clean.py @@ -71,10 +71,8 @@ class CleanLexer(ExtendedRegexLexer): (words(keywords, prefix=r'\b', suffix=r'\b'), Keyword), ], 'module': [ - include('comments'), (words(modulewords, prefix=r'\b', suffix=r'\b'), Keyword.Namespace), (r'\bmodule\b', Keyword.Namespace, 'module.name'), - include('whitespace'), ], 'module.name': [ include('whitespace'), @@ -92,10 +90,9 @@ class CleanLexer(ExtendedRegexLexer): (r'[+~-]?0x[\da-fA-F]+\b', Number.Hex), (r'True|False', Literal), (r'"', String.Double, 'literals.stringd'), - #(r'\[\'', Literal. ], 'literals.stringd': [ - (r'[^\\"$\n]+', String.Double), + (r'[^\\"\n]+', String.Double), (r'"', String.Double, '#pop'), (r'\\.', String.Double), (r'[$\n]', Error, '#pop'), @@ -105,7 +102,8 @@ class CleanLexer(ExtendedRegexLexer): (r'\b_+\b', Operator), ], 'delimiters': [ - (r'[,;(){}\[\]]', Punctuation) + (r'[,;(){}\[\]]', Punctuation), + (r'\'[\w`.]+\'', Name.Class), ], 'names': [ (r'\b' + lowerId, Name), -- cgit v1.2.1 From 3793cbc815769924fa6a942446ea9297fe204303 Mon Sep 17 00:00:00 2001 From: Camil Staps Date: Mon, 6 Mar 2017 18:43:12 +0100 Subject: Better quantified identifiers --- pygments/lexers/clean.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pygments/lexers/clean.py b/pygments/lexers/clean.py index c0fb63c3..b87f06ec 100644 --- a/pygments/lexers/clean.py +++ b/pygments/lexers/clean.py @@ -9,7 +9,7 @@ :license: BSD, see LICENSE for details. """ -from pygments.lexer import ExtendedRegexLexer, words, include +from pygments.lexer import ExtendedRegexLexer, words, include, bygroups from pygments.token import Comment, Error, Keyword, Literal, Name, Number, \ Operator, Punctuation, String, Whitespace @@ -103,7 +103,8 @@ class CleanLexer(ExtendedRegexLexer): ], 'delimiters': [ (r'[,;(){}\[\]]', Punctuation), - (r'\'[\w`.]+\'', Name.Class), + (r'(\')([\w`.]+)(\')', + bygroups(Punctuation, Name.Class, Punctuation)), ], 'names': [ (r'\b' + lowerId, Name), -- cgit v1.2.1 From 8d6dced7b7bf0854089c185679420861888abaf2 Mon Sep 17 00:00:00 2001 From: Maxime Vidori Date: Tue, 28 Mar 2017 11:21:29 +0200 Subject: Update Dockerfile lexer Add Dockerfiles new keyword in the lexer Improve lexer to parse json arrays on specific commands Handle line breaks as a bash syntax --- pygments/lexers/configs.py | 22 ++++++++++++++-------- tests/examplefiles/docker.docker | 33 +++++++++++++++++++++++++++++++-- 2 files changed, 45 insertions(+), 10 deletions(-) diff --git a/pygments/lexers/configs.py b/pygments/lexers/configs.py index c39b1a52..4af2adb6 100644 --- a/pygments/lexers/configs.py +++ b/pygments/lexers/configs.py @@ -15,6 +15,7 @@ from pygments.lexer import RegexLexer, default, words, bygroups, include, using from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ Number, Punctuation, Whitespace, Literal from pygments.lexers.shell import BashLexer +from pygments.lexers.data import JsonLexer __all__ = ['IniLexer', 'RegeditLexer', 'PropertiesLexer', 'KconfigLexer', 'Cfengine3Lexer', 'ApacheConfLexer', 'SquidConfLexer', @@ -539,20 +540,25 @@ class DockerLexer(RegexLexer): filenames = ['Dockerfile', '*.docker'] mimetypes = ['text/x-dockerfile-config'] - _keywords = (r'(?:FROM|MAINTAINER|CMD|EXPOSE|ENV|ADD|ENTRYPOINT|' - r'VOLUME|WORKDIR)') - + _keywords = (r'(?:FROM|MAINTAINER|EXPOSE|WORKDIR|USER|STOPSIGNAL)') + _bash_keywords = (r'(?:RUN|CMD|ENTRYPOINT|ENV|ARG|LABEL|ADD|COPY)') + _lb = r'(?:\s*\\?\s*)' # dockerfile line break regex flags = re.IGNORECASE | re.MULTILINE tokens = { 'root': [ - (r'^(ONBUILD)(\s+)(%s)\b' % (_keywords,), - bygroups(Name.Keyword, Whitespace, Keyword)), - (r'^(%s)\b(.*)' % (_keywords,), bygroups(Keyword, String)), (r'#.*', Comment), - (r'RUN', Keyword), # Rest of line falls through + (r'(ONBUILD)(%s)' % (_lb,), bygroups(Keyword, using(BashLexer))), + (r'(HEALTHCHECK)((%s--\w+=\w+%s)*)' % (_lb, _lb), + bygroups(Keyword, using(BashLexer))), + (r'(VOLUME|ENTRYPOINT|CMD|SHELL)(%s)(\[.*?\])' % (_lb,), + bygroups(Keyword, using(BashLexer), using(JsonLexer))), + (r'(LABEL|ENV|ARG)((%s\w+=\w+%s)*)' % (_lb, _lb), + bygroups(Keyword, using(BashLexer))), + (r'(%s|VOLUME)\b(.*)' % (_keywords), bygroups(Keyword, String)), + (r'(%s)' % (_bash_keywords,), Keyword), (r'(.*\\\n)*.+', using(BashLexer)), - ], + ] } diff --git a/tests/examplefiles/docker.docker b/tests/examplefiles/docker.docker index d65385b6..1ae3c3a1 100644 --- a/tests/examplefiles/docker.docker +++ b/tests/examplefiles/docker.docker @@ -1,5 +1,34 @@ -maintainer First O'Last +FROM alpine:3.5 +MAINTAINER First O'Last +# comment run echo \ 123 $bar -# comment +RUN apk --update add rsync dumb-init + +# Test env with both syntax +ENV FOO = "BAR" +ENV FOO \ + "BAR" + +COPY foo "bar" +COPY foo \ + "bar" + +HEALTHCHECK \ + --interval=5m --timeout=3s \ + CMD curl -f http://localhost/ || exit 1 + +# ONBUILD keyword, then with linebreak +ONBUILD ADD . /app/src +ONBUILD \ + RUN echo 123 $bar + +# Potential JSON array parsing, mixed with linebreaks +VOLUME \ + /foo +VOLUME \ + ["/bar"] +VOLUME ["/bar"] +VOLUME /foo +CMD ["foo", "bar"] -- cgit v1.2.1 From b3b591c8fd850469f6dc7afc5232176360bcc3f2 Mon Sep 17 00:00:00 2001 From: cocoatomo Date: Wed, 29 Mar 2017 01:45:36 +0000 Subject: Insert exception names added in Python version 3.5 and 3.6. --- pygments/lexers/python.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pygments/lexers/python.py b/pygments/lexers/python.py index 390eafe8..1ac176cd 100644 --- a/pygments/lexers/python.py +++ b/pygments/lexers/python.py @@ -124,10 +124,10 @@ class PythonLexer(RegexLexer): 'Exception', 'FloatingPointError', 'FutureWarning', 'GeneratorExit', 'IOError', 'ImportError', 'ImportWarning', 'IndentationError', 'IndexError', 'KeyError', 'KeyboardInterrupt', 'LookupError', - 'MemoryError', 'NameError', 'NotImplemented', 'NotImplementedError', + 'MemoryError', 'ModuleNotFoundError', 'NameError', 'NotImplemented', 'NotImplementedError', 'OSError', 'OverflowError', 'OverflowWarning', 'PendingDeprecationWarning', - 'ReferenceError', 'RuntimeError', 'RuntimeWarning', 'StandardError', - 'StopIteration', 'SyntaxError', 'SyntaxWarning', 'SystemError', + 'RecursionError', 'ReferenceError', 'RuntimeError', 'RuntimeWarning', 'StandardError', + 'StopIteration', 'StopAsyncIteration', 'SyntaxError', 'SyntaxWarning', 'SystemError', 'SystemExit', 'TabError', 'TypeError', 'UnboundLocalError', 'UnicodeDecodeError', 'UnicodeEncodeError', 'UnicodeError', 'UnicodeTranslateError', 'UnicodeWarning', 'UserWarning', -- cgit v1.2.1 From 8bbe382d011e110f676e0731725e205bde48ad33 Mon Sep 17 00:00:00 2001 From: St?phane GALLAND Date: Tue, 4 Apr 2017 17:27:11 +0000 Subject: Add the lexer for the SARL language. --- pygments/lexers/jvm.py | 56 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/pygments/lexers/jvm.py b/pygments/lexers/jvm.py index f4392839..d81ebbf4 100644 --- a/pygments/lexers/jvm.py +++ b/pygments/lexers/jvm.py @@ -21,7 +21,7 @@ from pygments import unistring as uni __all__ = ['JavaLexer', 'ScalaLexer', 'GosuLexer', 'GosuTemplateLexer', 'GroovyLexer', 'IokeLexer', 'ClojureLexer', 'ClojureScriptLexer', 'KotlinLexer', 'XtendLexer', 'AspectJLexer', 'CeylonLexer', - 'PigLexer', 'GoloLexer', 'JasminLexer'] + 'PigLexer', 'GoloLexer', 'JasminLexer', 'SarlLexer'] class JavaLexer(RegexLexer): @@ -1571,3 +1571,57 @@ class JasminLexer(RegexLexer): re.MULTILINE): score += 0.6 return score + + +class SarlLexer(RegexLexer): + """ + For `SARL `_ source code. + + .. versionadded:: 0.6 + """ + + name = 'SARL' + aliases = ['sarl'] + filenames = ['*.sarl'] + mimetypes = ['text/x-sarl'] + + flags = re.MULTILINE | re.DOTALL + + tokens = { + 'root': [ + # method names + (r'^(\s*(?:[a-zA-Z_][\w.\[\]]*\s+)+?)' # return arguments + r'([a-zA-Z_$][\w$]*)' # method name + r'(\s*)(\()', # signature start + bygroups(using(this), Name.Function, Text, Operator)), + (r'[^\S\n]+', Text), + (r'//.*?\n', Comment.Single), + (r'/\*.*?\*/', Comment.Multiline), + (r'@[a-zA-Z_][\w.]*', Name.Decorator), + (r'(as|break|case|catch|default|do|else|extends|extension|finally|fires|for|if|implements|instanceof|new|on|requires|return|super|switch|throw|throws|try|typeof|uses|while|with)\b', + Keyword), + (r'(abstract|def|dispatch|final|native|override|private|protected|public|static|strictfp|synchronized|transient|val|var|volatile)\b', Keyword.Declaration), + (r'(boolean|byte|char|double|float|int|long|short|void)\b', + Keyword.Type), + (r'(package)(\s+)', bygroups(Keyword.Namespace, Text)), + (r'(false|it|null|occurrence|this|true|void)\b', Keyword.Constant), + (r'(agent|annotation|artifact|behavior|capacity|class|enum|event|interface|skill|space)(\s+)', bygroups(Keyword.Declaration, Text), + 'class'), + (r'(import)(\s+)', bygroups(Keyword.Namespace, Text), 'import'), + (r'"(\\\\|\\"|[^"])*"', String), + (r"'(\\\\|\\'|[^'])*'", String), + (r'[a-zA-Z_]\w*:', Name.Label), + (r'[a-zA-Z_$]\w*', Name), + (r'[~^*!%&\[\](){}<>\|+=:;,./?-]', Operator), + (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float), + (r'0x[0-9a-fA-F]+', Number.Hex), + (r'[0-9]+L?', Number.Integer), + (r'\n', Text) + ], + 'class': [ + (r'[a-zA-Z_]\w*', Name.Class, '#pop') + ], + 'import': [ + (r'[\w.]+\*?', Name.Namespace, '#pop') + ], + } -- cgit v1.2.1 From b28b24c5ffe0274a2ac45398766213e83bf6b2d7 Mon Sep 17 00:00:00 2001 From: Kevin Stone Date: Tue, 2 May 2017 10:03:14 -0700 Subject: Added pep 515 support to the python lexer Fixes #1299 --- pygments/lexers/python.py | 9 +++++++++ tests/test_python.py | 19 +++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/pygments/lexers/python.py b/pygments/lexers/python.py index 390eafe8..c9f024e1 100644 --- a/pygments/lexers/python.py +++ b/pygments/lexers/python.py @@ -395,6 +395,15 @@ class Python3Lexer(RegexLexer): ] tokens['strings-single'] = innerstring_rules(String.Single) tokens['strings-double'] = innerstring_rules(String.Double) + tokens['numbers'] = [ + (r'(\d(?:_?\d)*\.(?:\d(?:_?\d)*)?|(?:\d(?:_?\d)*)?\.\d(?:_?\d)*)([eE][+-]?\d(?:_?\d)*)?', Number.Float), + (r'\d+[eE][+-]?[0-9]+j?', Number.Float), + (r'0[oO](?:_?[0-7])+', Number.Oct), + (r'0[bB](?:_?[01])+', Number.Bin), + (r'0[xX](?:_?[a-fA-F0-9])+', Number.Hex), + (r'\d(?:_?\d)*', Number.Integer) + ] + def analyse_text(text): return shebang_matches(text, r'pythonw?3(\.\d)?') diff --git a/tests/test_python.py b/tests/test_python.py index e99687a6..6ef8169d 100644 --- a/tests/test_python.py +++ b/tests/test_python.py @@ -111,3 +111,22 @@ class Python3Test(unittest.TestCase): (Token.Text, u'\n'), ] self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def test_pep_515(self): + """ + Tests that the lexer can parse numeric literals with underscores + """ + fragments = ( + (Token.Literal.Number.Integer, u'1_000_000'), + (Token.Literal.Number.Float, u'1_000.000_001'), + (Token.Literal.Number.Hex, u'0xCAFE_F00D'), + (Token.Literal.Number.Bin, u'0b_0011_1111_0100_1110'), + (Token.Literal.Number.Oct, u'0o_777_123'), + ) + + for token, fragment in fragments: + tokens = [ + (token, fragment), + (Token.Text, u'\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) -- cgit v1.2.1 From ed3bd801548a301071823a38ce6e19aa18e8215b Mon Sep 17 00:00:00 2001 From: Kevin Stone Date: Tue, 2 May 2017 22:46:05 -0700 Subject: Fixed missing complex float case --- pygments/lexers/python.py | 2 +- tests/test_python.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pygments/lexers/python.py b/pygments/lexers/python.py index c9f024e1..9e861aab 100644 --- a/pygments/lexers/python.py +++ b/pygments/lexers/python.py @@ -397,7 +397,7 @@ class Python3Lexer(RegexLexer): tokens['strings-double'] = innerstring_rules(String.Double) tokens['numbers'] = [ (r'(\d(?:_?\d)*\.(?:\d(?:_?\d)*)?|(?:\d(?:_?\d)*)?\.\d(?:_?\d)*)([eE][+-]?\d(?:_?\d)*)?', Number.Float), - (r'\d+[eE][+-]?[0-9]+j?', Number.Float), + (r'\d(?:_?\d)*[eE][+-]?\d(?:_?\d)*j?', Number.Float), (r'0[oO](?:_?[0-7])+', Number.Oct), (r'0[bB](?:_?[01])+', Number.Bin), (r'0[xX](?:_?[a-fA-F0-9])+', Number.Hex), diff --git a/tests/test_python.py b/tests/test_python.py index 6ef8169d..6445022c 100644 --- a/tests/test_python.py +++ b/tests/test_python.py @@ -119,6 +119,7 @@ class Python3Test(unittest.TestCase): fragments = ( (Token.Literal.Number.Integer, u'1_000_000'), (Token.Literal.Number.Float, u'1_000.000_001'), + (Token.Literal.Number.Float, u'1_000e1_000j'), (Token.Literal.Number.Hex, u'0xCAFE_F00D'), (Token.Literal.Number.Bin, u'0b_0011_1111_0100_1110'), (Token.Literal.Number.Oct, u'0o_777_123'), -- cgit v1.2.1 From 2235e4a7cd8b91cbd277643ad523610fcee84f98 Mon Sep 17 00:00:00 2001 From: Nathan Whetsell Date: Fri, 26 May 2017 17:29:45 -0400 Subject: Fix issue with opcode types --- pygments/lexers/csound.py | 5 ++++- tests/examplefiles/test.orc | 2 +- tests/test_csound.py | 4 ++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/pygments/lexers/csound.py b/pygments/lexers/csound.py index 462307d9..110d73aa 100644 --- a/pygments/lexers/csound.py +++ b/pygments/lexers/csound.py @@ -302,7 +302,10 @@ class CsoundOrchestraLexer(CsoundLexer): ], 'opcode type signatures': [ include('whitespace and macro uses'), - (r'\b(?:0|[afijkKoOpPStV\[\]]+)\b', Keyword.Type), + + # https://github.com/csound/csound/search?q=XIDENT+path%3AEngine+filename%3Acsound_orc.lex + (r'0|[afijkKoOpPStV\[\]]+', Keyword.Type), + (r',', Punctuation), (r'\n', Text, '#pop') ], diff --git a/tests/examplefiles/test.orc b/tests/examplefiles/test.orc index 59bf7f66..d113303e 100644 --- a/tests/examplefiles/test.orc +++ b/tests/examplefiles/test.orc @@ -9,7 +9,7 @@ instr/**/1,/**/N_a_M_e_,/**/+Name/**/// outc:a(aSignal) endin -opcode/**/aUDO,/**/0,/**/aik// +opcode/**/aUDO,/**/i[],/**/aik// aUDO endop diff --git a/tests/test_csound.py b/tests/test_csound.py index 610dec1a..b5094fc6 100644 --- a/tests/test_csound.py +++ b/tests/test_csound.py @@ -84,7 +84,7 @@ class CsoundOrchestraTest(unittest.TestCase): def testUserDefinedOpcodes(self): fragment = dedent('''\ - opcode/**/aUDO,/**/0,/**/aik// + opcode/**/aUDO,/**/i[],/**/aik// aUDO endop ''') @@ -94,7 +94,7 @@ class CsoundOrchestraTest(unittest.TestCase): (Name.Function, u'aUDO'), (Punctuation, u','), (Comment.Multiline, u'/**/'), - (Keyword.Type, u'0'), + (Keyword.Type, u'i[]'), (Punctuation, u','), (Comment.Multiline, u'/**/'), (Keyword.Type, u'aik'), -- cgit v1.2.1 From 1d3874f78836393858c0f3a287cbe40ce18e8fc4 Mon Sep 17 00:00:00 2001 From: Nathan Whetsell Date: Fri, 26 May 2017 17:30:48 -0400 Subject: Update for Csound 6.09.0 --- pygments/lexers/_csound_builtins.py | 293 +++++++++++++++++++++++++++++++++++- 1 file changed, 292 insertions(+), 1 deletion(-) diff --git a/pygments/lexers/_csound_builtins.py b/pygments/lexers/_csound_builtins.py index a0bca365..a41c1aea 100644 --- a/pygments/lexers/_csound_builtins.py +++ b/pygments/lexers/_csound_builtins.py @@ -7,7 +7,7 @@ :license: BSD, see LICENSE for details. """ -# Opcodes in Csound 6.08.0 at commit a1159a1 from +# Opcodes in Csound 6.09.0 at commit 73a8df5 from # csound --list-opcodes3 # except for # cggoto https://csound.github.io/docs/manual/cggoto.html @@ -124,8 +124,53 @@ MixerSetLevel_i OSCinit OSCinitM OSClisten +OSCraw OSCsend +OSCsendA +OSCsend_lo +JackoAudioIn +JackoAudioInConnect +JackoAudioOut +JackoAudioOutConnect +JackoFreewheel +JackoInfo +JackoInit +JackoMidiInConnect +JackoMidiOut +JackoMidiOutConnect +JackoNoteOut +JackoOn +JackoTransport +K35_hpf +K35_lpf S +STKBandedWG +STKBeeThree +STKBlowBotl +STKBlowHole +STKBowed +STKBrass +STKClarinet +STKDrummer +STKFlute +STKFMVoices +STKHevyMetl +STKMandolin +STKModalBar +STKMoog +STKPercFlut +STKPlucked +STKResonate +STKRhodey +STKSaxofony +STKShakers +STKSimple +STKSitar +STKStifKarp +STKTubeBell +STKVoicForm +STKWhistle +STKWurley a abs active @@ -159,6 +204,7 @@ binit biquad biquada birnd +bpf bqrez buchla butbp @@ -175,6 +221,7 @@ c2r cabasa cauchy cauchyi +cbrt ceil cell cent @@ -203,6 +250,7 @@ clfilt clip clockoff clockon +cmp cmplxprod comb combinv @@ -233,6 +281,7 @@ cpstmid cpstun cpstuni cpsxpch +cpumeter cpuprc cross2 crossfm @@ -272,6 +321,7 @@ deltapx deltapxw denorm diff +diode_ladder directory diskgrain diskin @@ -282,8 +332,14 @@ distort distort1 divz doppler +dot downsamp dripwater +dssiactivate +dssiaudio +dssictls +dssiinit +dssilist dumpk dumpk2 dumpk3 @@ -347,6 +403,7 @@ fluidNote fluidOut fluidProgramSelect fluidSetInterpMethod +fmanal fmb3 fmbell fmmetal @@ -385,6 +442,7 @@ ftload ftloadk ftlptim ftmorf +ftom ftresize ftresizei ftsamplebank @@ -420,6 +478,7 @@ harmon4 hdf5read hdf5write hilbert +hilbert2 hrtfearly hrtfmove hrtfmove2 @@ -463,21 +522,216 @@ interp invalue inx inz +jacktransport jitter jitter2 +joystick jspline k +la_i_add_mc +la_i_add_mr +la_i_add_vc +la_i_add_vr +la_i_assign_mc +la_i_assign_mr +la_i_assign_t +la_i_assign_vc +la_i_assign_vr +la_i_conjugate_mc +la_i_conjugate_mr +la_i_conjugate_vc +la_i_conjugate_vr +la_i_distance_vc +la_i_distance_vr +la_i_divide_mc +la_i_divide_mr +la_i_divide_vc +la_i_divide_vr +la_i_dot_mc +la_i_dot_mc_vc +la_i_dot_mr +la_i_dot_mr_vr +la_i_dot_vc +la_i_dot_vr +la_i_get_mc +la_i_get_mr +la_i_get_vc +la_i_get_vr +la_i_invert_mc +la_i_invert_mr +la_i_lower_solve_mc +la_i_lower_solve_mr +la_i_lu_det_mc +la_i_lu_det_mr +la_i_lu_factor_mc +la_i_lu_factor_mr +la_i_lu_solve_mc +la_i_lu_solve_mr +la_i_mc_create +la_i_mc_set +la_i_mr_create +la_i_mr_set +la_i_multiply_mc +la_i_multiply_mr +la_i_multiply_vc +la_i_multiply_vr +la_i_norm_euclid_mc +la_i_norm_euclid_mr +la_i_norm_euclid_vc +la_i_norm_euclid_vr +la_i_norm_inf_mc +la_i_norm_inf_mr +la_i_norm_inf_vc +la_i_norm_inf_vr +la_i_norm_max_mc +la_i_norm_max_mr +la_i_norm1_mc +la_i_norm1_mr +la_i_norm1_vc +la_i_norm1_vr +la_i_print_mc +la_i_print_mr +la_i_print_vc +la_i_print_vr +la_i_qr_eigen_mc +la_i_qr_eigen_mr +la_i_qr_factor_mc +la_i_qr_factor_mr +la_i_qr_sym_eigen_mc +la_i_qr_sym_eigen_mr +la_i_random_mc +la_i_random_mr +la_i_random_vc +la_i_random_vr +la_i_size_mc +la_i_size_mr +la_i_size_vc +la_i_size_vr +la_i_subtract_mc +la_i_subtract_mr +la_i_subtract_vc +la_i_subtract_vr +la_i_t_assign +la_i_trace_mc +la_i_trace_mr +la_i_transpose_mc +la_i_transpose_mr +la_i_upper_solve_mc +la_i_upper_solve_mr +la_i_vc_create +la_i_vc_set +la_i_vr_create +la_i_vr_set +la_k_a_assign +la_k_add_mc +la_k_add_mr +la_k_add_vc +la_k_add_vr +la_k_assign_a +la_k_assign_f +la_k_assign_mc +la_k_assign_mr +la_k_assign_t +la_k_assign_vc +la_k_assign_vr +la_k_conjugate_mc +la_k_conjugate_mr +la_k_conjugate_vc +la_k_conjugate_vr +la_k_current_f +la_k_current_vr +la_k_distance_vc +la_k_distance_vr +la_k_divide_mc +la_k_divide_mr +la_k_divide_vc +la_k_divide_vr +la_k_dot_mc +la_k_dot_mc_vc +la_k_dot_mr +la_k_dot_mr_vr +la_k_dot_vc +la_k_dot_vr +la_k_f_assign +la_k_get_mc +la_k_get_mr +la_k_get_vc +la_k_get_vr +la_k_invert_mc +la_k_invert_mr +la_k_lower_solve_mc +la_k_lower_solve_mr +la_k_lu_det_mc +la_k_lu_det_mr +la_k_lu_factor_mc +la_k_lu_factor_mr +la_k_lu_solve_mc +la_k_lu_solve_mr +la_k_mc_set +la_k_mr_set +la_k_multiply_mc +la_k_multiply_mr +la_k_multiply_vc +la_k_multiply_vr +la_k_norm_euclid_mc +la_k_norm_euclid_mr +la_k_norm_euclid_vc +la_k_norm_euclid_vr +la_k_norm_inf_mc +la_k_norm_inf_mr +la_k_norm_inf_vc +la_k_norm_inf_vr +la_k_norm_max_mc +la_k_norm_max_mr +la_k_norm1_mc +la_k_norm1_mr +la_k_norm1_vc +la_k_norm1_vr +la_k_qr_eigen_mc +la_k_qr_eigen_mr +la_k_qr_factor_mc +la_k_qr_factor_mr +la_k_qr_sym_eigen_mc +la_k_qr_sym_eigen_mr +la_k_random_mc +la_k_random_mr +la_k_random_vc +la_k_random_vr +la_k_subtract_mc +la_k_subtract_mr +la_k_subtract_vc +la_k_subtract_vr +la_k_t_assign +la_k_trace_mc +la_k_trace_mr +la_k_upper_solve_mc +la_k_upper_solve_mr +la_k_vc_set +la_k_vr_set lenarray lfo limit +limit1 line linen linenr lineto +link_beat_force +link_beat_get +link_beat_request +link_create +link_enable +link_is_enabled +link_metro +link_peers +link_tempo_get +link_tempo_set +linlin linrand linseg linsegb linsegr +liveconv locsend locsig log @@ -512,7 +766,12 @@ lpshold lpsholdp lpslot lua_exec +lua_iaopcall +lua_iaopcall_off lua_ikopcall +lua_ikopcall_off +lua_iopcall +lua_iopcall_off lua_opdef mac maca @@ -538,6 +797,7 @@ mediank metro mfb midglobal +midiarp midic14 midic21 midic7 @@ -583,9 +843,16 @@ mp3in mp3len mp3nchnls mp3scal +mp3scal_check +mp3scal_load +mp3scal_load2 +mp3scal_play +mp3scal_play2 mp3sr mpulse mrtmsg +mtof +mton multitap mute mvchpf @@ -610,6 +877,7 @@ nrpn nsamp nstance nstrnum +ntom ntrpol nxtpow2 octave @@ -667,6 +935,8 @@ outvalue outx outz p +p5gconnect +p5gdata pan pan2 pareq @@ -684,6 +954,7 @@ pchmidi pchmidib pchmidinn pchoct +pchtom pconvolve pcount pdclip @@ -789,6 +1060,7 @@ pvsout pvspitch pvstanal pvstencil +pvstrace pvsvoc pvswarp pvsynth @@ -927,6 +1199,10 @@ s16b14 s32b14 samphold sandpaper +sc_lag +sc_lagud +sc_phasor +sc_trig scale scalearray scanhammer @@ -941,6 +1217,7 @@ scoreline scoreline_i seed sekere +select semitone sense sensekey @@ -1006,7 +1283,10 @@ sndwarpst sockrecv sockrecvs socksend +socksend_k socksends +sorta +sortd soundin space spat3d @@ -1053,6 +1333,7 @@ stsend subinstr subinstrinit sum +sumTableFilter sumarray svfilter syncgrain @@ -1060,6 +1341,7 @@ syncloop syncphasor system system_i +systime tab tab2pvs tab_i @@ -1169,6 +1451,7 @@ trsplit turnoff turnoff2 turnon +tvconv unirand unwrap upsamp @@ -1183,6 +1466,7 @@ vaget valpass vaset vbap +vbap1move vbapg vbapgmove vbaplsinit @@ -1258,6 +1542,7 @@ vtabwi vtabwk vwrap waveset +websocket weibull wgbow wgbowedbar @@ -1284,6 +1569,7 @@ xscans xscansmap xscanu xtratim +xyscale zacl zakinit zamod @@ -1291,6 +1577,11 @@ zar zarg zaw zawm +zdf_1pole +zdf_1pole_mode +zdf_2pole +zdf_2pole_mode +zdf_ladder zfilter2 zir ziw -- cgit v1.2.1 From a20d1aa299b7fadb347a1d5ed482696879e4e9b1 Mon Sep 17 00:00:00 2001 From: Nathan Whetsell Date: Sat, 27 May 2017 08:17:33 -0400 Subject: Add Csound Score d statement --- pygments/lexers/csound.py | 2 +- tests/examplefiles/test.sco | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pygments/lexers/csound.py b/pygments/lexers/csound.py index 110d73aa..d26ba156 100644 --- a/pygments/lexers/csound.py +++ b/pygments/lexers/csound.py @@ -146,7 +146,7 @@ class CsoundScoreLexer(CsoundLexer): include('whitespace and macro uses'), include('preprocessor directives'), - (r'[abCefiqstvxy]', Keyword), + (r'[abCdefiqstvxy]', Keyword), # There is also a w statement that is generated internally and should not be # used; see https://github.com/csound/csound/issues/750. diff --git a/tests/examplefiles/test.sco b/tests/examplefiles/test.sco index e855e563..d997c1b3 100644 --- a/tests/examplefiles/test.sco +++ b/tests/examplefiles/test.sco @@ -3,7 +3,7 @@ */ ; comment // comment -a b C e f i q s t v x y +a b C d e f i q s t v x y z np0 nP1 Np2 NP3 m/**/label; -- cgit v1.2.1 From 213f64436cf28d67c3e3dc47c681dab4a296b34d Mon Sep 17 00:00:00 2001 From: Rene? Schwaiger Date: Fri, 16 Jun 2017 21:25:18 +0200 Subject: YAML: Fix minor spelling mistakes --- pygments/lexers/data.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pygments/lexers/data.py b/pygments/lexers/data.py index 296366c2..acf75131 100644 --- a/pygments/lexers/data.py +++ b/pygments/lexers/data.py @@ -205,7 +205,7 @@ class YamlLexer(ExtendedRegexLexer): bygroups(Text, Number), 'ignored-line'), ], - # the %YAG directive + # the %TAG directive 'tag-directive': [ # a tag handle and the corresponding prefix (r'([ ]+)(!|![\w-]*!)' @@ -218,7 +218,7 @@ class YamlLexer(ExtendedRegexLexer): 'indentation': [ # trailing whitespaces are ignored (r'[ ]*$', something(Text), '#pop:2'), - # whitespaces preceeding block collection indicators + # whitespaces preceding block collection indicators (r'[ ]+(?=[?:-](?:[ ]|$))', save_indent(Text)), # block collection indicators (r'[?:-](?=[ ]|$)', set_indent(Punctuation.Indicator)), -- cgit v1.2.1 From f333e139fb4b2217df2f4d085e1f48fc7c0480ef Mon Sep 17 00:00:00 2001 From: Rene? Schwaiger Date: Fri, 16 Jun 2017 21:26:54 +0200 Subject: YAML: Recognize non-specific tag `!` Before this change Pygments would incorrectly mark the non-specific tag as error token. --- pygments/lexers/data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pygments/lexers/data.py b/pygments/lexers/data.py index acf75131..a795a7b0 100644 --- a/pygments/lexers/data.py +++ b/pygments/lexers/data.py @@ -250,7 +250,7 @@ class YamlLexer(ExtendedRegexLexer): (r'!<[\w#;/?:@&=+$,.!~*\'()\[\]%-]+>', Keyword.Type), # a tag in the form '!', '!suffix' or '!handle!suffix' (r'!(?:[\w-]+!)?' - r'[\w#;/?:@&=+$,.!~*\'()\[\]%-]+', Keyword.Type), + r'[\w#;/?:@&=+$,.!~*\'()\[\]%-]*', Keyword.Type), # an anchor (r'&[\w-]+', Name.Label), # an alias -- cgit v1.2.1 From 804580a406c4c26ee6721c6ccbea235f3e41e57f Mon Sep 17 00:00:00 2001 From: PurpleMyst Date: Sun, 23 Jul 2017 19:23:30 +0200 Subject: Highlight %a in Python3Lexer --- pygments/lexers/python.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pygments/lexers/python.py b/pygments/lexers/python.py index 390eafe8..435dfe9c 100644 --- a/pygments/lexers/python.py +++ b/pygments/lexers/python.py @@ -262,7 +262,7 @@ class Python3Lexer(RegexLexer): return [ # the old style '%s' % (...) string formatting (still valid in Py3) (r'%(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?' - '[hlL]?[E-GXc-giorsux%]', String.Interpol), + '[hlL]?[E-GXc-giorsaux%]', String.Interpol), # the new style '{}'.format(...) string formatting (r'\{' '((\w+)((\.\w+)|(\[[^\]]+\]))*)?' # field name -- cgit v1.2.1 From 64047332da7a0a5d574824b070caa90b39518047 Mon Sep 17 00:00:00 2001 From: natashawatkins Date: Thu, 27 Jul 2017 09:40:57 +0000 Subject: Add julia 0.6 language features to julia.py --- pygments/lexers/julia.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pygments/lexers/julia.py b/pygments/lexers/julia.py index 67453aba..ab6d76a8 100644 --- a/pygments/lexers/julia.py +++ b/pygments/lexers/julia.py @@ -48,6 +48,7 @@ class JuliaLexer(RegexLexer): # keywords (r'in\b', Keyword.Pseudo), + (r'isa\b', Keyword.Pseudo), (r'(true|false)\b', Keyword.Constant), (r'(local|global|const)\b', Keyword.Declaration), (words([ @@ -55,7 +56,8 @@ class JuliaLexer(RegexLexer): 'baremodule', 'begin', 'bitstype', 'break', 'catch', 'ccall', 'continue', 'do', 'else', 'elseif', 'end', 'export', 'finally', 'for', 'if', 'import', 'importall', 'let', 'macro', 'module', - 'quote', 'return', 'try', 'using', 'while'], + 'mutable', 'primitive', 'quote', 'return', 'struct', 'try', + 'using', 'while'], suffix=r'\b'), Keyword), # NOTE -- cgit v1.2.1 From 87fdfbab1a8bd27420d143816eb3645b22a6b4e0 Mon Sep 17 00:00:00 2001 From: Nathan Whetsell Date: Sun, 6 Aug 2017 13:17:38 -0400 Subject: Update for Csound 6.09.1 --- pygments/lexers/_csound_builtins.py | 35 ++++++++++++++++++----------------- pygments/lexers/csound.py | 25 ++++++++----------------- tests/test_csound.py | 21 +++++++++++++++------ 3 files changed, 41 insertions(+), 40 deletions(-) diff --git a/pygments/lexers/_csound_builtins.py b/pygments/lexers/_csound_builtins.py index a41c1aea..f51b37b7 100644 --- a/pygments/lexers/_csound_builtins.py +++ b/pygments/lexers/_csound_builtins.py @@ -7,7 +7,7 @@ :license: BSD, see LICENSE for details. """ -# Opcodes in Csound 6.09.0 at commit 73a8df5 from +# Opcodes in Csound 6.09.1 at commit 0815e64 from # csound --list-opcodes3 # except for # cggoto https://csound.github.io/docs/manual/cggoto.html @@ -115,19 +115,6 @@ FLvkeybd FLvslidBnk FLvslidBnk2 FLxyin -MixerClear -MixerGetLevel -MixerReceive -MixerSend -MixerSetLevel -MixerSetLevel_i -OSCinit -OSCinitM -OSClisten -OSCraw -OSCsend -OSCsendA -OSCsend_lo JackoAudioIn JackoAudioInConnect JackoAudioOut @@ -143,6 +130,19 @@ JackoOn JackoTransport K35_hpf K35_lpf +MixerClear +MixerGetLevel +MixerReceive +MixerSend +MixerSetLevel +MixerSetLevel_i +OSCinit +OSCinitM +OSClisten +OSCraw +OSCsend +OSCsendA +OSCsend_lo S STKBandedWG STKBeeThree @@ -404,9 +404,12 @@ fluidOut fluidProgramSelect fluidSetInterpMethod fmanal +fmax fmb3 fmbell +fmin fmmetal +fmod fmpercfl fmrhode fmvoice @@ -488,6 +491,7 @@ hsboscil hvs1 hvs2 hvs3 +hypot i ihold imagecreate @@ -1333,7 +1337,6 @@ stsend subinstr subinstrinit sum -sumTableFilter sumarray svfilter syncgrain @@ -1341,7 +1344,6 @@ syncloop syncphasor system system_i -systime tab tab2pvs tab_i @@ -1466,7 +1468,6 @@ vaget valpass vaset vbap -vbap1move vbapg vbapgmove vbaplsinit diff --git a/pygments/lexers/csound.py b/pygments/lexers/csound.py index d26ba156..3294d2d3 100644 --- a/pygments/lexers/csound.py +++ b/pygments/lexers/csound.py @@ -82,34 +82,25 @@ class CsoundLexer(RegexLexer): 'macro parameter value list': [ (r'(?:[^\'#"{()]|\{(?!\{))+', Comment.Preproc), (r"['#]", Punctuation), - # Csound’s preprocessor can’t handle right parentheses in parameter values - # . For example, - # $MACRO((value)) - # passes (value, not (value), to MACRO. The workaround for this implemented in - # https://github.com/csound/csound/commit/3e0b441b55fd8e07d70b0908da8165b889feb883 - # is to require escaping right parentheses. Because this is not required by - # the C preprocessor, it is likely to be unexpected, so use a different token - # type for parentheses in macro parameter values. (r'"', String, 'macro parameter value quoted string'), (r'\{\{', String, 'macro parameter value braced string'), (r'\(', Comment.Preproc, 'macro parameter value parenthetical'), (r'\)', Punctuation, '#pop') ], 'macro parameter value quoted string': [ - (r'\\\)', Comment.Preproc), - (r'\)', Error), + (r"\\[#'()]", Comment.Preproc), + (r"[#'()]", Error), include('quoted string') ], 'macro parameter value braced string': [ - (r'\\\)', Comment.Preproc), - (r'\)', Error), + (r"\\[#'()]", Comment.Preproc), + (r"[#'()]", Error), include('braced string') ], 'macro parameter value parenthetical': [ - (r'[^\\()]+', Comment.Preproc), + (r'(?:[^\\()]|\\\))+', Comment.Preproc), (r'\(', Comment.Preproc, '#push'), - (r'\\\)', Comment.Preproc, '#pop'), - (r'\)', Error, '#pop') + (r'\)', Comment.Preproc, '#pop') ], 'whitespace and macro uses': [ @@ -119,7 +110,7 @@ class CsoundLexer(RegexLexer): 'numbers': [ (r'\d+[Ee][+-]?\d+|(\d+\.\d*|\d*\.\d+)([Ee][+-]?\d+)?', Number.Float), - (r'0[Xx][0-9A-Fa-f]+', Number.Hex), + (r'(0[Xx])([0-9A-Fa-f]+)', bygroups(Keyword.Type, Number.Hex)), (r'\d+', Number.Integer) ], @@ -230,7 +221,7 @@ class CsoundOrchestraLexer(CsoundLexer): elif name in lexer.user_defined_opcodes: yield match.start(), Name.Function, name else: - nameMatch = re.search(r'^(g?[aikSw])(\w+)', name) + nameMatch = re.search(r'^(g?[afikSw])(\w+)', name) if nameMatch: yield nameMatch.start(1), Keyword.Type, nameMatch.group(1) yield nameMatch.start(2), Name, nameMatch.group(2) diff --git a/tests/test_csound.py b/tests/test_csound.py index b5094fc6..4d10c267 100644 --- a/tests/test_csound.py +++ b/tests/test_csound.py @@ -119,9 +119,11 @@ class CsoundOrchestraTest(unittest.TestCase): self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) fragment = '0xabcdef0123456789 0XABCDEF' tokens = [ - (Number.Hex, u'0xabcdef0123456789'), + (Keyword.Type, u'0x'), + (Number.Hex, u'abcdef0123456789'), (Text, u' '), - (Number.Hex, u'0XABCDEF'), + (Keyword.Type, u'0X'), + (Number.Hex, u'ABCDEF'), (Text, u'\n') ] self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) @@ -436,18 +438,21 @@ class CsoundOrchestraTest(unittest.TestCase): self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) def testFunctionLikeMacros(self): - fragment = '$MACRO.(((x\\))\' "x)\\)x\\))"# {{x\\))x)\\)}})' + fragment = "$MACRO.(((x#y\\)))' \"(#'x)\\)x\\))\"# {{x\\))x)\\)(#'}});" tokens = [ (Comment.Preproc, u'$MACRO.'), (Punctuation, u'('), (Comment.Preproc, u'('), (Comment.Preproc, u'('), - (Comment.Preproc, u'x'), - (Comment.Preproc, u'\\)'), - (Error, u')'), + (Comment.Preproc, u'x#y\\)'), + (Comment.Preproc, u')'), + (Comment.Preproc, u')'), (Punctuation, u"'"), (Comment.Preproc, u' '), (String, u'"'), + (Error, u'('), + (Error, u'#'), + (Error, u"'"), (String, u'x'), (Error, u')'), (Comment.Preproc, u'\\)'), @@ -464,8 +469,12 @@ class CsoundOrchestraTest(unittest.TestCase): (String, u'x'), (Error, u')'), (Comment.Preproc, u'\\)'), + (Error, u'('), + (Error, u'#'), + (Error, u"'"), (String, u'}}'), (Punctuation, u')'), + (Comment.Single, u';'), (Text, u'\n') ] self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) -- cgit v1.2.1 From c140390a275ad745231f39628c0bc888dc093d7e Mon Sep 17 00:00:00 2001 From: Nathan Whetsell Date: Wed, 30 Aug 2017 15:22:42 -0400 Subject: Style nit --- pygments/lexers/csound.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pygments/lexers/csound.py b/pygments/lexers/csound.py index 3294d2d3..b86de091 100644 --- a/pygments/lexers/csound.py +++ b/pygments/lexers/csound.py @@ -420,7 +420,7 @@ class CsoundDocumentLexer(RegexLexer): (r'<\s*CsInstruments', Name.Tag, ('orchestra', 'tag')), (r'<\s*CsScore', Name.Tag, ('score', 'tag')), - (r'<\s*[hH][tT][mM][lL]', Name.Tag, ('HTML', 'tag')), + (r'<\s*[Hh][Tt][Mm][Ll]', Name.Tag, ('HTML', 'tag')), (r'<\s*[\w:.-]+', Name.Tag, 'tag'), (r'<\s*/\s*[\w:.-]+\s*>', Name.Tag) @@ -435,8 +435,8 @@ class CsoundDocumentLexer(RegexLexer): (r'(.|\n)+?(?=<\s*/\s*CsScore\s*>)', using(CsoundScoreLexer)) ], 'HTML': [ - (r'<\s*/\s*[hH][tT][mM][lL]\s*>', Name.Tag, '#pop'), - (r'(.|\n)+?(?=<\s*/\s*[hH][tT][mM][lL]\s*>)', using(HtmlLexer)) + (r'<\s*/\s*[Hh][Tt][Mm][Ll]\s*>', Name.Tag, '#pop'), + (r'(.|\n)+?(?=<\s*/\s*[Hh][Tt][Mm][Ll]\s*>)', using(HtmlLexer)) ], 'tag': [ -- cgit v1.2.1 From 493703060cc12df987148b0a55a3c201f5eed9dc Mon Sep 17 00:00:00 2001 From: Nathan Whetsell Date: Wed, 30 Aug 2017 15:22:58 -0400 Subject: Remove reinit from list of opcodes --- pygments/lexers/_csound_builtins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pygments/lexers/_csound_builtins.py b/pygments/lexers/_csound_builtins.py index f51b37b7..84c06956 100644 --- a/pygments/lexers/_csound_builtins.py +++ b/pygments/lexers/_csound_builtins.py @@ -27,6 +27,7 @@ # loop_le https://csound.github.io/docs/manual/loop_le.html # loop_lt https://csound.github.io/docs/manual/loop_lt.html # opcode https://csound.github.io/docs/manual/opcode.html +# reinit https://csound.github.io/docs/manual/reinit.html # return https://csound.github.io/docs/manual/return.html # rireturn https://csound.github.io/docs/manual/rireturn.html # rigoto https://csound.github.io/docs/manual/rigoto.html @@ -1173,7 +1174,6 @@ readks readscore readscratch rect2pol -reinit release remoteport remove -- cgit v1.2.1 From 545bef0ea5ab41111858c801efea2f285e3d042a Mon Sep 17 00:00:00 2001 From: Brian Tiffin Date: Thu, 31 Aug 2017 04:00:50 +0000 Subject: _mapping.py edited online with Bitbucket --- pygments/lexers/_mapping.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py index b48ee1d1..e0ec34af 100644 --- a/pygments/lexers/_mapping.py +++ b/pygments/lexers/_mapping.py @@ -190,6 +190,7 @@ LEXERS = { 'HxmlLexer': ('pygments.lexers.haxe', 'Hxml', ('haxeml', 'hxml'), ('*.hxml',), ()), 'HyLexer': ('pygments.lexers.lisp', 'Hy', ('hylang',), ('*.hy',), ('text/x-hy', 'application/x-hy')), 'HybrisLexer': ('pygments.lexers.scripting', 'Hybris', ('hybris', 'hy'), ('*.hy', '*.hyb'), ('text/x-hybris', 'application/x-hybris')), + 'IconLexer': ('pygments.lexers.unicon', 'Icon', ('icon',), ('*.icon', '*.ICON'), ()), 'IDLLexer': ('pygments.lexers.idl', 'IDL', ('idl',), ('*.pro',), ('text/idl',)), 'IdrisLexer': ('pygments.lexers.haskell', 'Idris', ('idris', 'idr'), ('*.idr',), ('text/x-idris',)), 'IgorLexer': ('pygments.lexers.igor', 'Igor', ('igor', 'igorpro'), ('*.ipf',), ('text/ipf',)), @@ -421,6 +422,8 @@ LEXERS = { 'TypoScriptCssDataLexer': ('pygments.lexers.typoscript', 'TypoScriptCssData', ('typoscriptcssdata',), (), ()), 'TypoScriptHtmlDataLexer': ('pygments.lexers.typoscript', 'TypoScriptHtmlData', ('typoscripthtmldata',), (), ()), 'TypoScriptLexer': ('pygments.lexers.typoscript', 'TypoScript', ('typoscript',), ('*.ts', '*.txt'), ('text/x-typoscript',)), + 'UcodeLexer': ('pygments.lexers.unicon', 'ucode', ('ucode',), ('*.u', '*.u1', '*.u2'), ()), + 'UniconLexer': ('pygments.lexers.unicon', 'Unicon', ('unicon',), ('*.icn',), ('text/unicon',)), 'UrbiscriptLexer': ('pygments.lexers.urbi', 'UrbiScript', ('urbiscript',), ('*.u',), ('application/x-urbiscript',)), 'VCLLexer': ('pygments.lexers.varnish', 'VCL', ('vcl',), ('*.vcl',), ('text/x-vclsrc',)), 'VCLSnippetLexer': ('pygments.lexers.varnish', 'VCLSnippets', ('vclsnippets', 'vclsnippet'), (), ('text/x-vclsnippet',)), -- cgit v1.2.1 From 48fb55767e117743a4ea4a271810e5026c19727d Mon Sep 17 00:00:00 2001 From: Brian Tiffin Date: Thu, 31 Aug 2017 04:05:20 +0000 Subject: unicon.py created online with Bitbucket --- pygments/lexers/unicon.py | 390 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 390 insertions(+) create mode 100644 pygments/lexers/unicon.py diff --git a/pygments/lexers/unicon.py b/pygments/lexers/unicon.py new file mode 100644 index 00000000..ae825150 --- /dev/null +++ b/pygments/lexers/unicon.py @@ -0,0 +1,390 @@ +# -*- coding: utf-8 -*- +""" + pygments.lexers.icon + ~~~~~~~~~~~~~~~~~~~~ + + Lexers for the Icon and Unicon languages, including ucode VM. + + :copyright: Copyright 2006-2016 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import re + +from pygments.lexer import Lexer, RegexLexer, include, bygroups, words, \ + using, this, default +from pygments.util import get_bool_opt, get_list_opt +from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ + Number, Punctuation, Error +from pygments.scanner import Scanner + +__all__ = ['IconLexer', 'UcodeLexer', 'UniconLexer'] + +class UniconLexer(RegexLexer): + """ + For Unicon source code. + + .. versionadded:: 2.? + """ + + name = 'Unicon' + aliases = ['unicon'] + filenames = ['*.icn'] + mimetypes = ['text/unicon'] + + flags = re.MULTILINE + + tokens = { + 'root': [ + (r'[^\S\n]+', Text), + (r'#.*?\n', Comment.Single), + (r'[^\S\n]+', Text), + (r'class|method|procedure', Keyword.Declaration, 'subprogram'), + (r'(record)(\s+)(\w+)', + bygroups(Keyword.Declaration, Text, Keyword.Type), 'type_def'), + (r'(#line|\$C|\$Cend|\$define|\$else|\$endif|\$error|\$ifdef|' + r'\$ifndef|\$include|\$line|\$undef)\b', Keyword.PreProc), + (r'(&null|&fail)\b', Keyword.Constant), + (r'&allocated|&ascii|&clock|&collections|&column|&col|&control|' + r'&cset|¤t|&dateline|&date|&digits|&dump|' + r'&errno|&errornumber|&errortext|&errorvalue|&error|&errout|' + r'&eventcode|&eventvalue|&eventsource|&e|' + r'&features|&file|&host|&input|&interval|&lcase|&letters|' + r'&level|&line|&ldrag|&lpress|&lrelease|' + r'&main|&mdrag|&meta|&mpress|&mrelease|&now|&output|' + r'&phi|&pick|&pi|&pos|&progname|' + r'&random|&rdrag|®ions|&resize|&row|&rpress|&rrelease|' + r'&shift|&source|&storage|&subject|' + r'&time|&trace|&ucase|&version|' + r'&window|&x|&y', Keyword.Reserved), + (r'(by|of|not|to)\b', Keyword.Reserved), + (r'(global|local|static|abstract)\b', Keyword.Reserved), + (r'package|link|import', Keyword.Declaration), + (words(( + 'break', 'case', 'create', 'critical', 'default', 'end', 'all', + 'do', 'else', 'every', 'fail', 'if', 'import', 'initial', + 'initially', 'invocable', 'next', + 'repeat', 'return', 'suspend', + 'then', 'thread', 'until', 'while'), prefix=r'\b', suffix=r'\b'), + Keyword.Reserved), + (words(( + 'Abort', 'abs', 'acos', 'Active', 'Alert', 'any', 'Any', 'Arb', + 'Arbno', 'args', 'array', 'asin', 'atan', 'atanh', 'Attrib', + 'Bal', 'bal', 'Bg', 'Break', 'Breakx', + 'callout', 'center', 'char', 'chdir', 'chmod', 'chown', 'chroot', + 'classname', 'Clip', 'Clone', 'close', 'cofail', 'collect', + 'Color', 'ColorValue', 'condvar', 'constructor', 'copy', + 'CopyArea', 'cos', 'Couple', 'crypt', 'cset', 'ctime', + 'dbcolumns', 'dbdriver', 'dbkeys', 'dblimits', 'dbproduct', + 'dbtables', 'delay', 'delete', 'detab', 'display', 'DrawArc', + 'DrawCircle', 'DrawCube', 'DrawCurve', 'DrawCylinder', + 'DrawDisk', 'DrawImage', 'DrawLine', 'DrawPoint', 'DrawPolygon', + 'DrawRectangle', 'DrawSegment', 'DrawSphere', 'DrawString', + 'DrawTorus', 'dtor', + 'entab', 'EraseArea', 'errorclear', 'Event', 'eventmask', + 'EvGet', 'EvSend', 'exec', 'exit', 'exp', 'Eye', + 'Fail', 'fcntl', 'fdup', 'Fence', 'fetch', 'Fg', 'fieldnames', + 'filepair', 'FillArc', 'FillCircle', 'FillPolygon', + 'FillRectangle', 'find', 'flock', 'flush', 'Font', 'fork', + 'FreeColor', 'FreeSpace', 'function', + 'get', 'getch', 'getche', 'getegid', 'getenv', 'geteuid', + 'getgid', 'getgr', 'gethost', 'getpgrp', 'getpid', 'getppid', + 'getpw', 'getrusage', 'getserv', 'GetSpace', 'gettimeofday', + 'getuid', 'globalnames', 'GotoRC', 'GotoXY', 'gtime', 'hardlink', + 'iand', 'icom', 'IdentityMatrix', 'image', 'InPort', 'insert', + 'Int86', 'integer', 'ioctl', 'ior', 'ishift', 'istate', 'ixor', + 'kbhit', 'key', 'keyword', 'kill', + 'left', 'Len', 'list', 'load', 'loadfunc', 'localnames', + 'lock', 'log', 'Lower', 'lstat', + 'many', 'map', 'match', 'MatrixMode', 'max', 'member', + 'membernames', 'methodnames', 'methods', 'min', 'mkdir', 'move', + 'MultMatrix', 'mutex', + 'name', 'NewColor', 'Normals', 'NotAny', 'numeric', + 'open', 'opencl', 'oprec', 'ord', 'OutPort', + 'PaletteChars', 'PaletteColor', 'PaletteKey', 'paramnames', + 'parent', 'Pattern', 'Peek', 'Pending', 'pipe', 'Pixel', + 'PlayAudio', 'Poke', 'pop', 'PopMatrix', 'Pos', 'pos', + 'proc', 'pull', 'push', 'PushMatrix', 'PushRotate', 'PushScale', + 'PushTranslate', 'put', + 'QueryPointer', + 'Raise', 'read', 'ReadImage', 'readlink', 'reads', 'ready', + 'real', 'receive', 'Refresh', 'Rem', 'remove', 'rename', + 'repl', 'reverse', 'right', 'rmdir', 'Rotate', 'Rpos', + 'Rtab', 'rtod', 'runerr', + 'save', 'Scale', 'seek', 'select', 'send', 'seq', + 'serial', 'set', 'setenv', 'setgid', 'setgrent', + 'sethostent', 'setpgrp', 'setpwent', 'setservent', + 'setuid', 'signal', 'sin', 'sort', 'sortf', 'Span', + 'spawn', 'sql', 'sqrt', 'stat', 'staticnames', 'stop', + 'StopAudio', 'string', 'structure', 'Succeed', 'Swi', + 'symlink', 'sys_errstr', 'system', 'syswrite', + 'Tab', 'tab', 'table', 'tan', + 'Texcoord', 'Texture', 'TextWidth', 'Translate', + 'trap', 'trim', 'truncate', 'trylock', 'type', + 'umask', 'Uncouple', 'unlock', 'upto', 'utime', + 'variable', 'VAttrib', + 'wait', 'WAttrib', 'WDefault', 'WFlush', 'where', + 'WinAssociate', 'WinButton', 'WinColorDialog', 'WindowContents', + 'WinEditRegion', 'WinFontDialog', 'WinMenuBar', 'WinOpenDialog', + 'WinPlayMedia', 'WinSaveDialog', 'WinScrollBar', 'WinSelectDialog', + 'write', 'WriteImage', 'writes', 'WSection', + 'WSync'), prefix=r'\b', suffix=r'\b'), + Name.Function), + include('numbers'), + (r'<@|<<@|>@|>>@|\.>|\->', Operator), + (r'\*\*|\+\+|\-\-|\.|\=|\~\=|<\=|>\=|\=\=|\~\=\=|<<|<<\=|>>|>>\=', Operator), + (r':\=|:\=:|\->|<\->|\+:\=|\|', Operator), + (r'\=\=\=|\~\=\=\=', Operator), + (r'"(?:[^\\"]|\\.)*"', String), + (r"'(?:[^\\']|\\.)*'", String.Character), + (r'[*<>+=/&!?@~\\-]', Operator), + (r'\^', Operator), + (r'(\w+)(\s*|[(,])', bygroups(Name, using(this))), + (r"([[\]])", Punctuation), + (r"(<>|=>|[()|:;,.'`]|[{}]|[%]|[&?])", Punctuation), + (r'\n+', Text), + ], + 'numbers': [ + (r'\b([+-]?([2-9]|[12][0-9]|3[0-6])[rR][0-9a-zA-Z]+)\b', Number.Hex), + (r'[+-]?[0-9]*\.([0-9]*)([Ee][+-]?[0-9]*)?', Number.Float), + (r'\b([+-]?[0-9]+[KMGTPkmgtp]?)\b', Number.Integer), + ], + 'subprogram': [ + (r'\(', Punctuation, ('#pop', 'formal_part')), + (r';', Punctuation, '#pop'), + (r'"[^"]+"|\w+', Name.Function), + include('root'), + ], + 'type_def': [ + (r'\(', Punctuation, 'formal_part'), + ], + 'formal_part': [ + (r'\)', Punctuation, '#pop'), + (r'\w+', Name.Variable), + (r',', Punctuation), + (r'(:string|:integer|:real)\b', Keyword.Reserved), + include('root'), + ], + } + +class IconLexer(RegexLexer): + """ + Lexer for Icon + + .. versionadded:: 1.6 + """ + name = 'Icon' + aliases = ['icon'] + filenames = ['*.icon', '*.ICON'] + mimetypes = [] + flags = re.MULTILINE + + tokens = { + 'root': [ + (r'[^\S\n]+', Text), + (r'#.*?\n', Comment.Single), + (r'[^\S\n]+', Text), + (r'class|method|procedure', Keyword.Declaration, 'subprogram'), + (r'(record)(\s+)(\w+)', + bygroups(Keyword.Declaration, Text, Keyword.Type), 'type_def'), + (r'(#line|\$C|\$Cend|\$define|\$else|\$endif|\$error|\$ifdef|' + r'\$ifndef|\$include|\$line|\$undef)\b', Keyword.PreProc), + (r'(&null|&fail)\b', Keyword.Constant), + (r'&allocated|&ascii|&clock|&collections|&column|&col|&control|' + r'&cset|¤t|&dateline|&date|&digits|&dump|' + r'&errno|&errornumber|&errortext|&errorvalue|&error|&errout|' + r'&eventcode|&eventvalue|&eventsource|&e|' + r'&features|&file|&host|&input|&interval|&lcase|&letters|' + r'&level|&line|&ldrag|&lpress|&lrelease|' + r'&main|&mdrag|&meta|&mpress|&mrelease|&now|&output|' + r'&phi|&pick|&pi|&pos|&progname|' + r'&random|&rdrag|®ions|&resize|&row|&rpress|&rrelease|' + r'&shift|&source|&storage|&subject|' + r'&time|&trace|&ucase|&version|' + r'&window|&x|&y', Keyword.Reserved), + (r'(by|of|not|to)\b', Keyword.Reserved), + (r'(global|local|static)\b', Keyword.Reserved), + (r'link', Keyword.Declaration), + (words(( + 'break', 'case', 'create', 'default', 'end', 'all', + 'do', 'else', 'every', 'fail', 'if', 'initial', + 'invocable', 'next', + 'repeat', 'return', 'suspend', + 'then', 'until', 'while'), prefix=r'\b', suffix=r'\b'), + Keyword.Reserved), + (words(( + 'abs', 'acos', 'Active', 'Alert', 'any', + 'args', 'array', 'asin', 'atan', 'atanh', 'Attrib', + 'bal', 'Bg', + 'callout', 'center', 'char', 'chdir', 'chmod', 'chown', 'chroot', + 'Clip', 'Clone', 'close', 'cofail', 'collect', + 'Color', 'ColorValue', 'condvar', 'copy', + 'CopyArea', 'cos', 'Couple', 'crypt', 'cset', 'ctime', + 'delay', 'delete', 'detab', 'display', 'DrawArc', + 'DrawCircle', 'DrawCube', 'DrawCurve', 'DrawCylinder', + 'DrawDisk', 'DrawImage', 'DrawLine', 'DrawPoint', 'DrawPolygon', + 'DrawRectangle', 'DrawSegment', 'DrawSphere', 'DrawString', + 'DrawTorus', 'dtor', + 'entab', 'EraseArea', 'errorclear', 'Event', 'eventmask', + 'EvGet', 'EvSend', 'exec', 'exit', 'exp', 'Eye', + 'fcntl', 'fdup', 'fetch', 'Fg', 'fieldnames', + 'FillArc', 'FillCircle', 'FillPolygon', + 'FillRectangle', 'find', 'flock', 'flush', 'Font', + 'FreeColor', 'FreeSpace', 'function', + 'get', 'getch', 'getche', 'getenv', + 'GetSpace', 'gettimeofday', + 'getuid', 'globalnames', 'GotoRC', 'GotoXY', 'gtime', 'hardlink', + 'iand', 'icom', 'IdentityMatrix', 'image', 'InPort', 'insert', + 'Int86', 'integer', 'ioctl', 'ior', 'ishift', 'istate', 'ixor', + 'kbhit', 'key', 'keyword', 'kill', + 'left', 'Len', 'list', 'load', 'loadfunc', 'localnames', + 'lock', 'log', 'Lower', 'lstat', + 'many', 'map', 'match', 'MatrixMode', 'max', 'member', + 'membernames', 'methodnames', 'methods', 'min', 'mkdir', 'move', + 'MultMatrix', 'mutex', + 'name', 'NewColor', 'Normals', 'numeric', + 'open', 'opencl', 'oprec', 'ord', 'OutPort', + 'PaletteChars', 'PaletteColor', 'PaletteKey', 'paramnames', + 'parent', 'Pattern', 'Peek', 'Pending', 'pipe', 'Pixel', + 'Poke', 'pop', 'PopMatrix', 'Pos', 'pos', + 'proc', 'pull', 'push', 'PushMatrix', 'PushRotate', 'PushScale', + 'PushTranslate', 'put', + 'QueryPointer', + 'Raise', 'read', 'ReadImage', 'readlink', 'reads', 'ready', + 'real', 'receive', 'Refresh', 'Rem', 'remove', 'rename', + 'repl', 'reverse', 'right', 'rmdir', 'Rotate', 'Rpos', + 'rtod', 'runerr', + 'save', 'Scale', 'seek', 'select', 'send', 'seq', + 'serial', 'set', 'setenv', + 'setuid', 'signal', 'sin', 'sort', 'sortf', + 'spawn', 'sql', 'sqrt', 'stat', 'staticnames', 'stop', + 'string', 'structure', 'Swi', + 'symlink', 'sys_errstr', 'system', 'syswrite', + 'tab', 'table', 'tan', + 'Texcoord', 'Texture', 'TextWidth', 'Translate', + 'trap', 'trim', 'truncate', 'trylock', 'type', + 'umask', 'Uncouple', 'unlock', 'upto', 'utime', + 'variable', + 'wait', 'WAttrib', 'WDefault', 'WFlush', 'where', + 'WinAssociate', 'WinButton', 'WinColorDialog', 'WindowContents', + 'WinEditRegion', 'WinFontDialog', 'WinMenuBar', 'WinOpenDialog', + 'WinPlayMedia', 'WinSaveDialog', 'WinScrollBar', 'WinSelectDialog', + 'write', 'WriteImage', 'writes', 'WSection', + 'WSync'), prefix=r'\b', suffix=r'\b'), + Name.Function), + include('numbers'), + (r'\*\*|\+\+|\-\-|\.|\=|\~\=|<\=|>\=|\=\=|\~\=\=|<<|<<\=|>>|>>\=', Operator), + (r':\=|:\=:|<\-|<\->|\+:\=|\||\|\|', Operator), + (r'\=\=\=|\~\=\=\=', Operator), + (r'"(?:[^\\"]|\\.)*"', String), + (r"'(?:[^\\']|\\.)*'", String.Character), + (r'[*<>+=/&!?@~\\-]', Operator), + (r'(\w+)(\s*|[(,])', bygroups(Name, using(this))), + (r"([[\]])", Punctuation), + (r"(<>|=>|[()|:;,.'`]|[{}]|[%^]|[&?])", Punctuation), + (r'\n+', Text), + ], + 'numbers': [ + (r'\b([+-]?([2-9]|[12][0-9]|3[0-6])[rR][0-9a-zA-Z]+)\b', Number.Hex), + (r'[+-]?[0-9]*\.([0-9]*)([Ee][+-]?[0-9]*)?', Number.Float), + (r'\b([+-]?[0-9]+[KMGTPkmgtp]?)\b', Number.Integer), + ], + 'subprogram': [ + (r'\(', Punctuation, ('#pop', 'formal_part')), + (r';', Punctuation, '#pop'), + (r'"[^"]+"|\w+', Name.Function), + include('root'), + ], + 'type_def': [ + (r'\(', Punctuation, 'formal_part'), + ], + 'formal_part': [ + (r'\)', Punctuation, '#pop'), + (r'\w+', Name.Variable), + (r',', Punctuation), + (r'(:string|:integer|:real)\b', Keyword.Reserved), + include('root'), + ], + } + +class UcodeLexer(RegexLexer): + """ + Lexer for Icon ucode files + + .. versionadded:: 2.? + """ + name = 'ucode' + aliases = ['ucode'] + filenames = ['*.u', '*.u1', '*.u2'] + mimetypes = [] + flags = re.MULTILINE + + tokens = { + 'root': [ + (r'(#.*\n)', Comment), + (words(( + 'con', 'declend', 'end', + 'global', + 'impl', 'invocable', + 'lab', 'link', 'local', + 'record', + 'uid', 'unions', + 'version'), + prefix=r'\b', suffix=r'\b'), + Name.Function), + (words(( + 'colm', 'filen', 'line', 'synt'), + prefix=r'\b', suffix=r'\b'), + Comment), + (words(( + 'asgn', + 'bang', 'bscan', + 'cat', 'ccase', 'chfail', + 'coact', 'cofail', 'compl', + 'coret', 'create', 'cset', + 'diff', 'div', 'dup', + 'efail', 'einit', 'end', 'eqv', 'eret', + 'error', 'escan', 'esusp', + 'field', + 'goto', + 'init', 'int', 'inter', + 'invoke', + 'keywd', + 'lconcat', 'lexeq', 'lexge', + 'lexgt', 'lexle', 'lexlt', 'lexne', + 'limit', 'llist', 'lsusp', + 'mark', 'mark0', 'minus', 'mod', 'mult', + 'neg', 'neqv', 'nonnull', 'noop', 'null', + 'number', 'numeq', 'numge', 'numgt', + 'numle', 'numlt', 'numne', + 'pfail', 'plus', 'pnull', 'pop', 'power', + 'pret', 'proc', 'psusp', 'push1', 'pushn1', + 'random', 'rasgn', 'rcv', 'rcvbk', 'real', + 'refresh', 'rswap', + 'sdup', 'sect', 'size', 'snd', 'sndbk', + 'str', 'subsc', 'swap', + 'tabmat', 'tally', 'toby', 'trace', + 'unmark', + 'value', 'var'), prefix=r'\b', suffix=r'\b'), + Keyword.Declaration), + (words(( + 'any', + 'case', + 'endcase', 'endevery', 'endif', + 'endifelse', 'endrepeat', 'endsuspend', + 'enduntil', 'endwhile', 'every', + 'if', 'ifelse', + 'repeat', + 'suspend', + 'until', + 'while'), + prefix=r'\b', suffix=r'\b'), + Name.Constant), + (r'\d+(\s*|\.$|$)', Number.Integer), + (r'[+-]?\d*\.\d+(E[-+]?\d+)?', Number.Float), + (r'[+-]?\d+\.\d*(E[-+]?\d+)?', Number.Float), + (r"(<>|=>|[()|:;,.'`]|[{}]|[%^]|[&?])", Punctuation), + (r'\s+\b', Text), + (r'[\w-]+', Text), + ], +} -- cgit v1.2.1 From 2a2b6104c3d391ac5d391e754516c6046afe08e6 Mon Sep 17 00:00:00 2001 From: Brian Tiffin Date: Thu, 31 Aug 2017 04:27:22 +0000 Subject: example.icon created online with Bitbucket --- tests/examplefiles/example.icon | 381 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 381 insertions(+) create mode 100644 tests/examplefiles/example.icon diff --git a/tests/examplefiles/example.icon b/tests/examplefiles/example.icon new file mode 100644 index 00000000..29bc548b --- /dev/null +++ b/tests/examplefiles/example.icon @@ -0,0 +1,381 @@ +############################################################################ +# +# File: kaleid.icn +# +# Subject: Program to produce kaleidoscope +# +# Author: Stephen B. Wampler +# +# Date: May 2, 2001 +# +############################################################################ +# +# This file is in the public domain. +# +############################################################################ +# +# Lots of options, most easily set by with the interface after +# startup. The only one that isn't set that way is -wn where 'n' is +# the size of the kaleidoscope window (default is 600 square). +# +# Terminology (and options): +# +# Window_size (-wN): How big of a display window to use. +# At the current time, this can only be set via a +# command line argument. +# +# Density (-dN): How many circles per octant to keep on display +# at any one time. There is NO LIMIT to the density. +# +# Duration (-lN): How long to keep drawing circles (measured in +# in circles) once the density is reached. There is NO LIMIT +# to the duration. +# +# MaxRadius (-MN): Maximum radius of any circle. +# +# MinRadius (-mN): Preferred minimum radius. Circles with centers +# near the edge have their radii forced down to fit entirely +# on the display +# +# MaxOffset (-XN): Maximum offset from center of display (may wrap). +# +# MinOffset (-xN): Minimum offset +# +# Skew (-sN): Shift probability of placing a circle at a 'typical' +# offset. +# +# Fill (-F): Turns off filling the circles. +# +# Clear (-C): After the duration, reduces density back to 0 before +# quitting. +# +# Random Seed: (-rN): Sets the random number seed. +# +# Thanks to Jon Lipp for help on using vidgets, and to Mary Camaron +# for her Interface Builder. +# +############################################################################ +# +# Requires: Version 9 graphics +# +############################################################################ +# +# Links: vidgets, vslider, vtext, vbuttons, vradio, wopen, xcompat +# +############################################################################ + +link vidgets +link vslider +link vtext +link vbuttons +link vradio +link wopen +link xcompat + +global Clear, fill, duration, density, maxoff, minoff +global maxradius, minradius, r_seed, skew, win_size, mid_win +global root, check1, mainwin, use_dialog +global draw_circle + +global du_v, de_v, rs_v, sk_v + +procedure main (args) + + draw_circle := DrawCircle + + init_globs() + process_args(args) + + if \use_dialog then { # have vidgets, so use them for args. + mainwin := WOpen("label=Kaleidoscope", "width=404", "height=313", + "font=6x12") | + stop ("bad mainwin") + root := ui (mainwin) + GetEvents (root, quit) + } + else { # just rely on command line arguments + kaleidoscope(r_seed) + } + +end + +procedure init_globs() + + duration := 500 # set default characteristics + density := 30 + win_size := 600 + minoff := 1 + maxradius := 150 + minradius := 1 + skew := 1 + fill := "On" + draw_circle := FillCircle + Clear := "Off" + r_seed := map("HhMmYy", "Hh:Mm:Yy", &clock) + # See if the Vidget library is available or not + if \VSet then use_dialog := "yes" + else use_dialog := &null + +end + +procedure process_args(args) + local arg + + # really only needed if you don't use the dialog box + every arg := !args do case arg[1+:2] of { + "-w" : win_size := integer(arg[3:0]) # window size + "-d" : density := integer(arg[3:0]) # density of circles + "-l" : duration := integer(arg[3:0]) # duration + "-M" : maxradius := integer(arg[3:0]) # maximum radius + "-m" : minradius := integer(arg[3:0]) # minimum radius + "-X" : maxoff := integer(arg[3:0]) # maximum offset + "-x" : minoff := integer(arg[3:0]) # minimum offset + "-s" : skew := numeric(arg[3:0]) # set skewedness + "-F" : fill := &null # turn off fill + "-C" : Clear := "yes" # turn on clear mode + "-r" : r_seed := integer(arg[3:0]) # random seed + "-h" : stop("usage: kal [-wn] [-dn] [-ln] [-Mn] [-mn] [-Xn] [-xn] _ + [-sn] [-F] [-C] [-rn]") + } + # adjust parameters that depend on the window size... + mid_win := win_size/2 + maxoff := win_size-1 +end + +# Lorraine Callahan's kaleidoscope program, translated into icon. +# (some of the things she did were too sophisticated for me +# to spend time to figure out, so the output is square instead of +# round), and I use 'xor' to draw instead of writing to separate +# bit planes. + +global putcircle, clrcircle + +procedure kaleidoscope(r) + local colors + + # What colors to use? This can be changed to whatever! + colors := ["red","green","blue","cyan","magenta","yellow"] + + &window := WOpen("label=Kaleidoscope: 'q' quits", "width="||win_size, + "height="||win_size, "bg=black") + WAttrib("drawop=xor") + + # Create two *indentical* sequences of circles, one to use when + # when drawing, one for erasing. (Since 'xor' is used to + # place them, these both just draw the circles!) + + putcircle := create { # draws sequence of circles + &random :=: r + |{ + Fg(?colors) + outcircle() + &random <-> r + } + } + + clrcircle := create { # erases sequence of circles + &random :=: r + |{ + Fg(?colors) + outcircle() + &random <-> r + } + } + + every 1 to density do @putcircle # fill screen to density + + every 1 to duration do { # maintain steady state + @putcircle + @clrcircle + if *Pending(&window) > 0 then break + } + + every (Clear == "On") & 1 to density do @clrcircle + + close(&window) +end + + +procedure outcircle() # select a circle at random, +local radius, xoff, yoff # draw it in kaleidoscopic form + + # get a random center point and radius + xoff := (?(maxoff - minoff) + minoff) % mid_win + yoff := (?(maxoff - minoff) + minoff) % mid_win + radius := ?0 ^ skew + # force radius to 'fit' + radius := ((maxradius-minradius) * radius + minradius) % + (mid_win - ((xoff < yoff)|xoff)) + + # put into all 8 octants + draw_circle(mid_win+xoff, mid_win+yoff, radius) + draw_circle(mid_win+xoff, mid_win-yoff, radius) + draw_circle(mid_win-xoff, mid_win+yoff, radius) + draw_circle(mid_win-xoff, mid_win-yoff, radius) + + draw_circle(mid_win+yoff, mid_win+xoff, radius) + draw_circle(mid_win+yoff, mid_win-xoff, radius) + draw_circle(mid_win-yoff, mid_win+xoff, radius) + draw_circle(mid_win-yoff, mid_win-xoff, radius) + + return +end + + +############################################################################ +# +# Vidget-based user interface -- developed originally using Mary +# Camaron's XIB program. Don't expect this to be very readable - +# you should have to play with it! +# +############################################################################ +procedure ui (win) + local cv1, cv2, cv3, cv4 + local + radio_button2, + radio_button1, + text_input6, + text_input5, + slider4, + slider3, + text_input4, + text_input3, + slider2, + slider1 + + /win := WOpen("label=ui", "width=404", "height=313", "font=6x12") | + stop ("bad win") + root := Vroot_frame (win) + + VInsert (root, Vmessage(win, win_size/2), 168, 98) + VInsert (root, Vmessage(win, "1"), 108, 97) + + VInsert (root, sk_v := Vtext(win,"Skew:\\=1",get_skew,,6), 280, 39) + + VInsert (root, du_v := Vtext(win, "Duration:\\="||duration, get_duration,,9), + 237, 15) + + VInsert (root, Vmessage(win, "Clear at end?"), 232, 145) + VInsert (root, Vmessage(win, "Fill?"), 105, 142) + VInsert (root, Vmessage(win,"Quit?"), 267, 259) + VInsert (root, Vmessage(win,"Display it?"), 26, 260) + + VInsert (root, Vcheckbox(win, do_quit, "check2",20), 305, 255, 20, 20) + + VInsert (root, check1:=Vcheckbox(win, do_display, "check1",20), + 106, 258, 20, 20) + + radio_button2 := Vradio_buttons (win, ["On", "Off"], get_clear, , V_CIRCLE) + VSet(radio_button2,Clear) + VInsert (root, radio_button2, 253, 165) + + radio_button1 := Vradio_buttons (win, ["On", "Off"], get_fill, , V_CIRCLE) + VSet(radio_button1,fill) + VInsert (root, radio_button1, 99, 165) + + cv1 := Vcoupler() + VAddClient(cv1, get_max_offset) + text_input6 := Vtext (win, "Max Offset:\\="||(win_size-1), cv1, , 3) + VAddClient(cv1, text_input6) + slider4 := Vhoriz_slider (win, cv1, "slider4", 70, 12, 0, + win_size-1, win_size-1, ) + VAddClient(cv1, slider4) + VInsert (root, text_input6, 196, 103) + VInsert (root, slider4, 306, 106) + + cv2 := Vcoupler() + VAddClient(cv2, get_min_offset) + text_input5 := Vtext (win, "Min Offset\\=1", cv2, , 3) + VAddClient(cv2, text_input5) + slider3 := Vhoriz_slider (win, cv2, "slider3", 70, 12, 1, win_size-1, 1, ) + VAddClient(cv2, slider3) + VInsert (root, text_input5, 201, 80) + VInsert (root, slider3, 307, 82) + + cv3 := Vcoupler() + VAddClient(cv3, get_max_radius) + text_input4 := Vtext (win, "Max Radius\\="||(win_size/4), cv3, , 3) + VAddClient(cv3, text_input4) + slider2 := Vhoriz_slider (win, cv3, "slider2", 70, 12, 1, win_size/2, + win_size/4, ) + VAddClient(cv3, slider2) + VInsert (root, text_input4, 10, 104) + VInsert (root, slider2, 110, 108) + + cv4 := Vcoupler() + VAddClient(cv4, get_min_radius) + text_input3 := Vtext (win, "Min Radius\\=1", cv4, , 3) + VAddClient(cv4, text_input3) + slider1 := Vhoriz_slider (win, cv4, "slider1", 70, 12, 1, win_size/2, 1, ) + VAddClient(cv4, slider1) + VInsert (root, text_input3, 10, 81) + VInsert (root, slider1, 110, 84) + + VInsert (root, rs_v := Vtext(win,"Random Seed:\\="||r_seed, get_random,, 11), + 30, 41) + VInsert (root, de_v := Vtext(win,"Density:\\="||density, get_density,,8), + 71, 16) + + VResize (root) + return root +end + +procedure get_skew (wit, value) + skew := value +end + +procedure get_duration (wit, value) + duration := value +end + +procedure do_quit (wit, value) + stop() +end + +procedure do_display (wit, value) + r_seed := numeric(rs_v.data) + duration := integer(du_v.data) + density := integer(de_v.data) + skew := integer(sk_v.data) + kaleidoscope(r_seed) + wit.callback.value := &null + VDraw(check1) +end + +procedure get_clear (wit, value) + Clear := value +end + +procedure get_fill (wit, value) + fill := value + if fill == "Off" then draw_circle := DrawCircle + else draw_circle := FillCircle +end + +procedure get_max_offset (wit, value) + maxoff := value +end + +procedure get_min_offset (wit, value) + minoff := value +end + +procedure get_max_radius (wit, value) + maxradius := value +end + +procedure get_min_radius (wit, value) + minradius := value +end + +procedure get_random (wit, value) + r_seed := integer(value) +end + +procedure get_density (wit, value) + density := integer(value) +end + +procedure quit(e) + if e === "q" then stop ("Exiting Kaleidoscope") +end -- cgit v1.2.1 From 0fe8a0744d6c2cfa5c23ebd3164d6c3712e04391 Mon Sep 17 00:00:00 2001 From: Brian Tiffin Date: Thu, 31 Aug 2017 04:29:00 +0000 Subject: example.u created online with Bitbucket --- tests/examplefiles/example.u | 659 ++++++++----------------------------------- 1 file changed, 111 insertions(+), 548 deletions(-) diff --git a/tests/examplefiles/example.u b/tests/examplefiles/example.u index 42c85902..92c45365 100644 --- a/tests/examplefiles/example.u +++ b/tests/examplefiles/example.u @@ -1,548 +1,111 @@ - // This is a one line comment. - /* an inner comment */ - - /* nested /* comments */ */ - - /* - /* - Multi-line. - */ - */ - -// Binary blob escape. -//"some text \B(3)("\") ouhyeah" == "\"\\\""; -"some text \B(3)("\") ouhyeah" == "\"\\\""; -'some text \B(3)('\') ouhyeah' == '\'\\\''; - -//"\B(4)()"'()"; -"\B(4)()"'()"; -'\B(4)()'"()'; - -//blob size limits -"hey ! \B(0)() oh !" - -//blob format is wrong -"hey ! \B(2)(aaa) oh !" -"hey ! \B(100)(aaa) oh !" - -//multiple blob in a string -"hey ! \B(3)(aaa) hey ! \B(3)(aaa) oh !" - -// multiple digits blob size -"hey ! \B(10)(aaaaaaaaaa) !" -"hey ! \B(10)(aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa) !" -"hey ! \B(100)(a) !" - -// multiple digits blob size -"hey ! \B(007)(aaaaaaa) !" -"hey ! \B(007)(aa) !" -"hey ! \B(007)(aaaaaaaaaaaaaaaaaa) !" - -// deprecated and restricted keyworks -emit Event.new; -static int main(); - -loopn (2) {echo("a");}; - -foreach (var i : [1,2,3,4]) { - echo(i); -}; - -function() {}; - -var 'if'; -var this.'else'; - -var '%x'; -var '1 2 3'; -var this.'[]'; - -// angles -pi == 180deg; -pi == 200grad; - -// Dictionary -[ => ]; // The empty dictionary - -// duration -1d == 24h; -0.5d == 12h; -1h == 60min; -1min == 60s; -1s == 1000ms; - -1s == 1; -1s 2s 3s == 6; -1s 1ms == 1.001; -1ms 1s == 1.001; - - - 1 == 1; - 1 == 1.0; - 1.2 == 1.2000; - 1.234e6 == 1234000; - 1e+11 == 1E+11; - 1e10 == 10000000000; - 1e30 == 1e10 * 1e10 * 1e10; - - -0.000001; - -0.0000001; - -0.00000000001; - -1e+3; - -1E-5; - - -1.; -// [00004701:error] !!! syntax error: unexpected ; - - 0x2a == 42; - 0x2A == 42; - 0xabcdef == 11259375; - 0xABCDEF == 11259375; -0xFFFFFFFF == 4294967295; - - -//123foo; -//[00005658:error] !!! syntax error: invalid token: '123foo' -//12.3foo; -//[00018827:error] !!! syntax error: invalid token: '12.3foo' -0xabcdef; -//[00060432] 11259375 -//0xabcdefg; -//[00061848:error] !!! syntax error: invalid token: '0xabcdefg' - - -[]; // The empty list -[1, 2, 3]; - -// Special characters. -"\"" == "\""; -"\\" == "\\"; - -// ASCII characters. -"\a" == "\007"; "\a" == "\x07"; -"\b" == "\010"; "\b" == "\x08"; -"\f" == "\014"; "\f" == "\x0c"; -"\n" == "\012"; "\n" == "\x0a"; -"\r" == "\015"; "\r" == "\x0d"; -"\t" == "\011"; "\t" == "\x09"; -"\v" == "\013"; "\v" == "\x0b"; - -// Octal escapes. -"\0" == "\00"; "\0" == "\000"; -"\0000" == "\0""0"; -"\062\063" == "23"; - -// Hexadecimal escapes. -"\x00" == "\0"; -"\x32\x33" == "23"; - - - -"foo" "bar" "baz" == "foobarbaz"; - -// Tuples -(); -[00000000] () -(1,); -[00000000] (1,) -(1, 2); -[00000000] (1, 2) -(1, 2, 3, 4,); -[00000000] (1, 2, 3, 4) - -function Global.verboseId(var x) -{ - echo(x) | x -}|; -class verboseId(Global).math : verboseId(Math) -{ -}; - -{ - for (3) - { - sleep(1s); - echo("ping"); - }, - sleep(0.5s); - for (3) - { - sleep(1s); - echo("pong"); - }, -}; - - 1 + 1 == 2; - 1 - 2 == -1; - 2 * 3 == 6; - 10 / 2 == 5; - 2 ** 10 == 1024; - -(1 + 2) == -3; - 1 + 2 * 3 == 7; - (1 + 2) * 3 == 9; - -2 ** 2 == -4; - - - - - 1 == 1; - -a = b -a += b -a -= b -a *= b -a /= b -a %= b -a ^= b - - -var value = 0|; -var valueAlias = value|; -value += 10; -valueAlias; -var myList = []|; -var myList.specialFeature = 42|; -myList += [1, 2, 3]; -myList.specialFeature; -var myOtherList = myList + [4, 5]; -myOtherList.specialFeature; -var something = []|; -var somethingElse = something|; -something += [1, 2]; -somethingElse += [3, 4]; -something; - - -class Counter -{ - var count = 0; - function init (n) { var this.count = n }; - // Display the value, and the identity. - function asString() { "%s @ %s" % [count, uid ] }; - function '+'(var n) { new(count + n) }; - function '-'(var n) { new(count - n) }; -}|; - - -class ImmutableCounter : Counter -{ - function '+='(var n) { this + n }; - function '-='(var n) { this - n }; -}|; - -var ic1 = ImmutableCounter.new(0); -var ic2 = ic1; - -ic1 += 1; -ic1; -ic2; - - -a << b -a >> b -a ^ b - -4 << 2 == 16; -4 >> 2 == 1; - -!a -a && b -a || b - -true && true; -true || false; -!true == false; -true || (1 / 0); -(false && (1 / 0)) == false; - -a == b -a != b -a === b -a !== b -a ~= b -a =~= b -a < b -a <= b -a > b -a >= b - -assert{ - ! (0 < 0); - 0 <= 0; - 0 == 0; - 0 !== 0; -}; - -a in b -a not in b -a[args] -a[args] = v - -1 in [0, 1, 2]; -3 not in [0, 1, 2]; - -"one" in ["zero" => 0, "one" => 1, "two" => 2]; -"three" not in ["zero" => 0, "one" => 1, "two" => 2]; - -a.b -a.b(args) -a->b -a->b = v -a.&b - -var obj = Object.new|; -function obj.f() { 24 }|; - - -var f = function(a, b) { - echo(b + a); -}| -f(1, 0); - - -function g3() -{ - return; // Stop execution at this point and return void - echo(0); // This is not executed -}| - -Object.setProperty, to define/set a property. -Object.getProperty, to get a property. -Object.removeProperty, to delete a property. -Object.hasProperty, to test for the existence of a property. -Object.properties, to get all the properties of a slot. - -enum Suit -{ - hearts, - diamonds, - clubs, - spades, // Last comma is optional -}; - -for (var suit in Suit) - echo("%s the ace of %s." % [find_ace(suit), suit]); - -switch ( ("foo", [1, 2]) ) -{ - // The pattern does not match the values of the list. - case ("foo", [2, 1]): - echo("fail"); - - // The pattern does not match the tuple. - case ["foo", [1, 2]]: - echo("fail"); - - // The pattern matches and binds the variable "l" - // but the condition is not verified. - case ("foo", var l) if l.size == 0: - echo("fail"); - - // The pattern matches. - case ("foo", [var a, var b]): - echo("foo(%s, %s)" % [a, b]); -}; -//[00000000] *** foo(1, 2) - -{ - ["b" => var b, "a" => var a] = ["a" => 1, "b" => 2, "c" => 3]; - echo("a = %d, b = %d" % [a, b]); -}; -//[00000000] *** a = 1, b = 2 - - -switch (["speed" => 2, "time" => 6s]) -{ - case ["speed" => var s] if s > 3: - echo("Too fast"); - case ["speed" => var s, "time" => var t] if s * t > 10: - echo("Too far"); -}; -//[00000000] *** Too far - - -try -{ - throw ("message", 0) -} -catch (var e if e.isA(Exception)) -{ - echo(e.message) -} -catch ((var msg, var value) if value.isA(Float)) -{ - echo("%s: %d" % [msg, value]) -}; -//[00000000] *** message: 0 - - -{ - var e = Event.new; - at (e?(var msg, var value) if value % 2 == 0) - echo("%s: %d" % [msg, value]); - - // Does not trigger the "at" because the guard is not verified. - e!("message", 1); - - // Trigger the "at". - e!("message", 2); -}; -//[00000000] *** message: 2 - -for (var i = 0; i < 8; i++) -{ - if (i % 2 != 0) - continue; - echo(i); -}; - -do (1024) -{ - assert(this == 1024); - assert(sqrt == 32); - setSlot("y", 23); -}.y; - -{ - var n = 10|; - var res = []|; - loop;{ - n--; - res << n; - if (n == 0) - break - }; - res -} - - -{ - var n = 10|; - var res = []|; - loop|{ - n--; - res << n; - if (n == 0) - break - }; - res -} - - -var j = 3| -while (0 < j) -{ - echo(j); - j--; -}; - - -{ - var i = 4| - while| (true) - { - i -= 1; - echo ("in: " + i); - if (i == 1) - break - else if (i == 2) - continue; - echo ("out: " + i); - }; -}; - - - -function test(e) -{ - try - { throw e; } - catch (0) - { echo("zero") } - catch ([var x, var y]) - { echo(x + y) } -} | {}; - -try { echo("try") } -catch { echo("catch")} -else { echo("else")}; - - -try -{ - echo("inside"); -} -finally -{ - echo("finally"); -}; -//[00000001] *** inside -//[00000002] *** finally - -at (e?(var start) ~ 1s) - echo("in : %s" % (time - start).round) -onleave - echo("out: %s" % (time - start).round); - -// This emission is too short to trigger the at. -e!(time); - -// This one is long enough. -// The body triggers 1s after the emission started. -e!(time) ~ 2s; -//[00001000] *** in : 1 -//[00002000] *** out: 2 - - -timeout (2.1s) - every (1s) - echo("Are you still there?"); -//[00000000] *** Are you still there? -//[00001000] *** Are you still there? -//[00002000] *** Are you still there? - - every| (1s) - { - echo("aba"); - }; - -for, (var i = 3; 0 < i; i -= 1) -{ - echo (i); -}; - - -for& (var i: [0, 1, 2]) -{ - echo (i * i); -}; - -loop,{ -}; - - -waituntil (e?(1, var b)); - -whenever (e?("arg", var arg) if arg % 2) - echo("e (%s) on" % arg) -else - echo("e off"); - - - while, (i) - { - var j = i -= 1; - }| - - -var y = 0; -{ - sleep(0.5s); - y = 100 smooth:3s, -}, - - - - +version U12.1.00 +uid version.u1-1494453463-0 +impl local +global 1 + 0,000005,version,0 + + +proc version + local 0,000000,tab + local 1,000000,find + local 2,000000,many + con 0,010000,8,126,145,162,163,151,157,156,040 + con 1,002000,1,8 + con 2,020000,11,060,061,062,063,064,065,066,067,070,071,056 + con 3,002000,1,1 + declend + filen version.icn + line 23 + colm 11 + synt any + mark L1 + line 25 + colm 4 + synt any + keywd version + line 25 + colm 13 + synt any + bscan + mark L2 + mark L3 + var 0 + pnull + var 1 + str 0 + line 26 + colm 15 + synt any + invoke 1 + int 1 + line 26 + colm 28 + synt any + plus + line 26 + colm 10 + synt any + invoke 1 + line 26 + colm 33 + synt any + esusp + goto L4 +lab L3 + line 26 + colm 35 + synt any + pfail +lab L4 + unmark +lab L2 + var 0 + var 2 + cset 2 + line 27 + colm 15 + synt any + invoke 1 + line 27 + colm 10 + synt any + invoke 1 + line 27 + colm 32 + synt any + bscan + mark L5 + var 0 + pnull + int 3 + line 27 + colm 45 + synt any + neg + line 27 + colm 44 + synt any + invoke 1 + line 27 + colm 34 + synt any + pret +lab L5 + synt any + pfail + line 27 + colm 32 + synt any + escan + line 25 + colm 13 + synt any + escan + unmark +lab L1 + pnull + line 30 + colm 1 + synt any + pfail + end \ No newline at end of file -- cgit v1.2.1 From 84baa921c15946d7dc692f5bdfa20a5050e572cd Mon Sep 17 00:00:00 2001 From: Brian Tiffin Date: Thu, 31 Aug 2017 05:04:49 +0000 Subject: example.u edited online with Bitbucket --- tests/examplefiles/example.u | 658 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 547 insertions(+), 111 deletions(-) diff --git a/tests/examplefiles/example.u b/tests/examplefiles/example.u index 92c45365..8c6686eb 100644 --- a/tests/examplefiles/example.u +++ b/tests/examplefiles/example.u @@ -1,111 +1,547 @@ -version U12.1.00 -uid version.u1-1494453463-0 -impl local -global 1 - 0,000005,version,0 - - -proc version - local 0,000000,tab - local 1,000000,find - local 2,000000,many - con 0,010000,8,126,145,162,163,151,157,156,040 - con 1,002000,1,8 - con 2,020000,11,060,061,062,063,064,065,066,067,070,071,056 - con 3,002000,1,1 - declend - filen version.icn - line 23 - colm 11 - synt any - mark L1 - line 25 - colm 4 - synt any - keywd version - line 25 - colm 13 - synt any - bscan - mark L2 - mark L3 - var 0 - pnull - var 1 - str 0 - line 26 - colm 15 - synt any - invoke 1 - int 1 - line 26 - colm 28 - synt any - plus - line 26 - colm 10 - synt any - invoke 1 - line 26 - colm 33 - synt any - esusp - goto L4 -lab L3 - line 26 - colm 35 - synt any - pfail -lab L4 - unmark -lab L2 - var 0 - var 2 - cset 2 - line 27 - colm 15 - synt any - invoke 1 - line 27 - colm 10 - synt any - invoke 1 - line 27 - colm 32 - synt any - bscan - mark L5 - var 0 - pnull - int 3 - line 27 - colm 45 - synt any - neg - line 27 - colm 44 - synt any - invoke 1 - line 27 - colm 34 - synt any - pret -lab L5 - synt any - pfail - line 27 - colm 32 - synt any - escan - line 25 - colm 13 - synt any - escan - unmark -lab L1 - pnull - line 30 - colm 1 - synt any - pfail - end \ No newline at end of file + // This is a one line comment. + /* an inner comment */ + + /* nested /* comments */ */ + + /* + /* + Multi-line. + */ + */ + +// Binary blob escape. +//"some text \B(3)("\") ouhyeah" == "\"\\\""; +"some text \B(3)("\") ouhyeah" == "\"\\\""; +'some text \B(3)('\') ouhyeah' == '\'\\\''; + +//"\B(4)()"'()"; +"\B(4)()"'()"; +'\B(4)()'"()'; + +//blob size limits +"hey ! \B(0)() oh !" + +//blob format is wrong +"hey ! \B(2)(aaa) oh !" +"hey ! \B(100)(aaa) oh !" + +//multiple blob in a string +"hey ! \B(3)(aaa) hey ! \B(3)(aaa) oh !" + +// multiple digits blob size +"hey ! \B(10)(aaaaaaaaaa) !" +"hey ! \B(10)(aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa) !" +"hey ! \B(100)(a) !" + +// multiple digits blob size +"hey ! \B(007)(aaaaaaa) !" +"hey ! \B(007)(aa) !" +"hey ! \B(007)(aaaaaaaaaaaaaaaaaa) !" + +// deprecated and restricted keyworks +emit Event.new; +static int main(); + +loopn (2) {echo("a");}; + +foreach (var i : [1,2,3,4]) { + echo(i); +}; + +function() {}; + +var 'if'; +var this.'else'; + +var '%x'; +var '1 2 3'; +var this.'[]'; + +// angles +pi == 180deg; +pi == 200grad; + +// Dictionary +[ => ]; // The empty dictionary + +// duration +1d == 24h; +0.5d == 12h; +1h == 60min; +1min == 60s; +1s == 1000ms; + +1s == 1; +1s 2s 3s == 6; +1s 1ms == 1.001; +1ms 1s == 1.001; + + + 1 == 1; + 1 == 1.0; + 1.2 == 1.2000; + 1.234e6 == 1234000; + 1e+11 == 1E+11; + 1e10 == 10000000000; + 1e30 == 1e10 * 1e10 * 1e10; + + +0.000001; + +0.0000001; + +0.00000000001; + +1e+3; + +1E-5; + + +1.; +// [00004701:error] !!! syntax error: unexpected ; + + 0x2a == 42; + 0x2A == 42; + 0xabcdef == 11259375; + 0xABCDEF == 11259375; +0xFFFFFFFF == 4294967295; + + +//123foo; +//[00005658:error] !!! syntax error: invalid token: '123foo' +//12.3foo; +//[00018827:error] !!! syntax error: invalid token: '12.3foo' +0xabcdef; +//[00060432] 11259375 +//0xabcdefg; +//[00061848:error] !!! syntax error: invalid token: '0xabcdefg' + + +[]; // The empty list +[1, 2, 3]; + +// Special characters. +"\"" == "\""; +"\\" == "\\"; + +// ASCII characters. +"\a" == "\007"; "\a" == "\x07"; +"\b" == "\010"; "\b" == "\x08"; +"\f" == "\014"; "\f" == "\x0c"; +"\n" == "\012"; "\n" == "\x0a"; +"\r" == "\015"; "\r" == "\x0d"; +"\t" == "\011"; "\t" == "\x09"; +"\v" == "\013"; "\v" == "\x0b"; + +// Octal escapes. +"\0" == "\00"; "\0" == "\000"; +"\0000" == "\0""0"; +"\062\063" == "23"; + +// Hexadecimal escapes. +"\x00" == "\0"; +"\x32\x33" == "23"; + + + +"foo" "bar" "baz" == "foobarbaz"; + +// Tuples +(); +[00000000] () +(1,); +[00000000] (1,) +(1, 2); +[00000000] (1, 2) +(1, 2, 3, 4,); +[00000000] (1, 2, 3, 4) + +function Global.verboseId(var x) +{ + echo(x) | x +}|; +class verboseId(Global).math : verboseId(Math) +{ +}; + +{ + for (3) + { + sleep(1s); + echo("ping"); + }, + sleep(0.5s); + for (3) + { + sleep(1s); + echo("pong"); + }, +}; + + 1 + 1 == 2; + 1 - 2 == -1; + 2 * 3 == 6; + 10 / 2 == 5; + 2 ** 10 == 1024; + -(1 + 2) == -3; + 1 + 2 * 3 == 7; + (1 + 2) * 3 == 9; + -2 ** 2 == -4; + - - - - 1 == 1; + +a = b +a += b +a -= b +a *= b +a /= b +a %= b +a ^= b + + +var value = 0|; +var valueAlias = value|; +value += 10; +valueAlias; +var myList = []|; +var myList.specialFeature = 42|; +myList += [1, 2, 3]; +myList.specialFeature; +var myOtherList = myList + [4, 5]; +myOtherList.specialFeature; +var something = []|; +var somethingElse = something|; +something += [1, 2]; +somethingElse += [3, 4]; +something; + + +class Counter +{ + var count = 0; + function init (n) { var this.count = n }; + // Display the value, and the identity. + function asString() { "%s @ %s" % [count, uid ] }; + function '+'(var n) { new(count + n) }; + function '-'(var n) { new(count - n) }; +}|; + + +class ImmutableCounter : Counter +{ + function '+='(var n) { this + n }; + function '-='(var n) { this - n }; +}|; + +var ic1 = ImmutableCounter.new(0); +var ic2 = ic1; + +ic1 += 1; +ic1; +ic2; + + +a << b +a >> b +a ^ b + +4 << 2 == 16; +4 >> 2 == 1; + +!a +a && b +a || b + +true && true; +true || false; +!true == false; +true || (1 / 0); +(false && (1 / 0)) == false; + +a == b +a != b +a === b +a !== b +a ~= b +a =~= b +a < b +a <= b +a > b +a >= b + +assert{ + ! (0 < 0); + 0 <= 0; + 0 == 0; + 0 !== 0; +}; + +a in b +a not in b +a[args] +a[args] = v + +1 in [0, 1, 2]; +3 not in [0, 1, 2]; + +"one" in ["zero" => 0, "one" => 1, "two" => 2]; +"three" not in ["zero" => 0, "one" => 1, "two" => 2]; + +a.b +a.b(args) +a->b +a->b = v +a.&b + +var obj = Object.new|; +function obj.f() { 24 }|; + + +var f = function(a, b) { + echo(b + a); +}| +f(1, 0); + + +function g3() +{ + return; // Stop execution at this point and return void + echo(0); // This is not executed +}| + +Object.setProperty, to define/set a property. +Object.getProperty, to get a property. +Object.removeProperty, to delete a property. +Object.hasProperty, to test for the existence of a property. +Object.properties, to get all the properties of a slot. + +enum Suit +{ + hearts, + diamonds, + clubs, + spades, // Last comma is optional +}; + +for (var suit in Suit) + echo("%s the ace of %s." % [find_ace(suit), suit]); + +switch ( ("foo", [1, 2]) ) +{ + // The pattern does not match the values of the list. + case ("foo", [2, 1]): + echo("fail"); + + // The pattern does not match the tuple. + case ["foo", [1, 2]]: + echo("fail"); + + // The pattern matches and binds the variable "l" + // but the condition is not verified. + case ("foo", var l) if l.size == 0: + echo("fail"); + + // The pattern matches. + case ("foo", [var a, var b]): + echo("foo(%s, %s)" % [a, b]); +}; +//[00000000] *** foo(1, 2) + +{ + ["b" => var b, "a" => var a] = ["a" => 1, "b" => 2, "c" => 3]; + echo("a = %d, b = %d" % [a, b]); +}; +//[00000000] *** a = 1, b = 2 + + +switch (["speed" => 2, "time" => 6s]) +{ + case ["speed" => var s] if s > 3: + echo("Too fast"); + case ["speed" => var s, "time" => var t] if s * t > 10: + echo("Too far"); +}; +//[00000000] *** Too far + + +try +{ + throw ("message", 0) +} +catch (var e if e.isA(Exception)) +{ + echo(e.message) +} +catch ((var msg, var value) if value.isA(Float)) +{ + echo("%s: %d" % [msg, value]) +}; +//[00000000] *** message: 0 + + +{ + var e = Event.new; + at (e?(var msg, var value) if value % 2 == 0) + echo("%s: %d" % [msg, value]); + + // Does not trigger the "at" because the guard is not verified. + e!("message", 1); + + // Trigger the "at". + e!("message", 2); +}; +//[00000000] *** message: 2 + +for (var i = 0; i < 8; i++) +{ + if (i % 2 != 0) + continue; + echo(i); +}; + +do (1024) +{ + assert(this == 1024); + assert(sqrt == 32); + setSlot("y", 23); +}.y; + +{ + var n = 10|; + var res = []|; + loop;{ + n--; + res << n; + if (n == 0) + break + }; + res +} + + +{ + var n = 10|; + var res = []|; + loop|{ + n--; + res << n; + if (n == 0) + break + }; + res +} + + +var j = 3| +while (0 < j) +{ + echo(j); + j--; +}; + + +{ + var i = 4| + while| (true) + { + i -= 1; + echo ("in: " + i); + if (i == 1) + break + else if (i == 2) + continue; + echo ("out: " + i); + }; +}; + + + +function test(e) +{ + try + { throw e; } + catch (0) + { echo("zero") } + catch ([var x, var y]) + { echo(x + y) } +} | {}; + +try { echo("try") } +catch { echo("catch")} +else { echo("else")}; + + +try +{ + echo("inside"); +} +finally +{ + echo("finally"); +}; +//[00000001] *** inside +//[00000002] *** finally + +at (e?(var start) ~ 1s) + echo("in : %s" % (time - start).round) +onleave + echo("out: %s" % (time - start).round); + +// This emission is too short to trigger the at. +e!(time); + +// This one is long enough. +// The body triggers 1s after the emission started. +e!(time) ~ 2s; +//[00001000] *** in : 1 +//[00002000] *** out: 2 + + +timeout (2.1s) + every (1s) + echo("Are you still there?"); +//[00000000] *** Are you still there? +//[00001000] *** Are you still there? +//[00002000] *** Are you still there? + + every| (1s) + { + echo("aba"); + }; + +for, (var i = 3; 0 < i; i -= 1) +{ + echo (i); +}; + + +for& (var i: [0, 1, 2]) +{ + echo (i * i); +}; + +loop,{ +}; + + +waituntil (e?(1, var b)); + +whenever (e?("arg", var arg) if arg % 2) + echo("e (%s) on" % arg) +else + echo("e off"); + + + while, (i) + { + var j = i -= 1; + }| + + +var y = 0; +{ + sleep(0.5s); + y = 100 smooth:3s, +}, + + + -- cgit v1.2.1 From da8371503a37d730192221e9098728935b488aed Mon Sep 17 00:00:00 2001 From: Brian Tiffin Date: Thu, 31 Aug 2017 05:07:12 +0000 Subject: example.u1 created online with Bitbucket --- tests/examplefiles/example.u1 | 111 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 tests/examplefiles/example.u1 diff --git a/tests/examplefiles/example.u1 b/tests/examplefiles/example.u1 new file mode 100644 index 00000000..92c45365 --- /dev/null +++ b/tests/examplefiles/example.u1 @@ -0,0 +1,111 @@ +version U12.1.00 +uid version.u1-1494453463-0 +impl local +global 1 + 0,000005,version,0 + + +proc version + local 0,000000,tab + local 1,000000,find + local 2,000000,many + con 0,010000,8,126,145,162,163,151,157,156,040 + con 1,002000,1,8 + con 2,020000,11,060,061,062,063,064,065,066,067,070,071,056 + con 3,002000,1,1 + declend + filen version.icn + line 23 + colm 11 + synt any + mark L1 + line 25 + colm 4 + synt any + keywd version + line 25 + colm 13 + synt any + bscan + mark L2 + mark L3 + var 0 + pnull + var 1 + str 0 + line 26 + colm 15 + synt any + invoke 1 + int 1 + line 26 + colm 28 + synt any + plus + line 26 + colm 10 + synt any + invoke 1 + line 26 + colm 33 + synt any + esusp + goto L4 +lab L3 + line 26 + colm 35 + synt any + pfail +lab L4 + unmark +lab L2 + var 0 + var 2 + cset 2 + line 27 + colm 15 + synt any + invoke 1 + line 27 + colm 10 + synt any + invoke 1 + line 27 + colm 32 + synt any + bscan + mark L5 + var 0 + pnull + int 3 + line 27 + colm 45 + synt any + neg + line 27 + colm 44 + synt any + invoke 1 + line 27 + colm 34 + synt any + pret +lab L5 + synt any + pfail + line 27 + colm 32 + synt any + escan + line 25 + colm 13 + synt any + escan + unmark +lab L1 + pnull + line 30 + colm 1 + synt any + pfail + end \ No newline at end of file -- cgit v1.2.1 From 6736da739ef32a255d544508bb44905da086dd35 Mon Sep 17 00:00:00 2001 From: Brian Tiffin Date: Thu, 31 Aug 2017 05:10:31 +0000 Subject: example.icn created online with Bitbucket --- tests/examplefiles/example.icn | 283 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 283 insertions(+) create mode 100644 tests/examplefiles/example.icn diff --git a/tests/examplefiles/example.icn b/tests/examplefiles/example.icn new file mode 100644 index 00000000..c8fcf335 --- /dev/null +++ b/tests/examplefiles/example.icn @@ -0,0 +1,283 @@ +# +# $Id: button.icn,v 1.7 2006-07-09 23:43:07 rparlett Exp $ +# +# This file is in the public domain. +# +# Author: Robert Parlett (parlett@dial.pipex.com) +# + +package gui +link graphics + +$include "guih.icn" + + +# +# This is the parent class of the button classes, including +# checkboxes. +# +# A {Button} produces a BUTTON_PRESS_EVENT when the button is +# depressed, and code BUTTON_RELEASE_EVENT when it is released, +# as well as an ACTION_EVENT. +# +# By default, when a button holds the keyboard focus a dashed +# line appears just within the button. Then, when return is +# pressed an ACTION_EVENT is generated. The method +# {Dialog.set_initial_focus()} can be used to have the button +# have the focus when the dialog is first displayed. +# +# Buttons also repeatedly produce a BUTTON_HELD_EVENT whilst they +# are held down, rather like a repeating keyboard press. The +# delay between the initial repeat event and subsequent repeat +# events is set in the parent dialog (see above). +# +class Button : Toggle : Component( + is_down, # + is_held, # + is_checked_flag, # + label, + img_up, # + img_down, # + img_w, # + img_h, # + parent_check_box_group, # + parent_button_group, # + repeat_delay, + no_keyboard_flag, # + toggles_flag + ) + + method set_parent_button_group(x) + return self.parent_button_group := x + end + + # + # Invoking this method disables the keyboard control over the + # button described above. No dashed line will ever appear in + # the button display and return will have no effect on the + # button even if it has the focus. + # + method set_no_keyboard() + self.no_keyboard_flag := 1 + self.accepts_focus_flag := &null + end + + # + # Clear the no keyboard behaviour (the default) + # + method clear_no_keyboard() + self.no_keyboard_flag := &null + self.accepts_focus_flag := 1 + end + + method tick() + if dispatcher.curr_time_of_day() > self.repeat_delay then + fire(BUTTON_HELD_EVENT) + end + + method go_down() + self.is_down := 1 + set_ticker(self.parent_dialog.repeat_rate) + end + + method go_up() + self.is_down := &null + stop_ticker() + end + + method handle_press(e) + local b + if self.in_region() then { + go_down() + self.repeat_delay := dispatcher.curr_time_of_day() + self.parent_dialog.repeat_delay + self.is_held := 1 + every b := !(\self.parent_button_group).buttons do { + if b.is_unhidden() then { + b.is_held := 1 + b.repeat_delay := self.repeat_delay + } + } + self.invalidate() + fire(BUTTON_PRESS_EVENT, e) + } + end + + method handle_drag(e) + if \self.is_held then { + # + # Button held down; toggle on/off as it goes over the button + # + if self.in_region() then { + if /self.is_down then { + go_down() + invalidate() + } + } else { + if \self.is_down then { + go_up() + invalidate() + } + } + } + end + + method handle_release(e) + if \self.is_held then { + self.is_held := &null + if \self.is_down then { + go_up() + fire(BUTTON_RELEASE_EVENT, e) + on_action(e) + } + } + end + + method on_action(e) + if \self.toggles_flag then { + if \self.parent_check_box_group then + self.parent_check_box_group.set_which_one(self) + else + self.toggle_is_checked() + } + self.invalidate() + fire(ACTION_EVENT, e) + end + + method handle_accel(e) + self.Component.handle_accel(e) + on_action(e) + end + + method handle_default(e) + if \self.has_focus then { + if /self.no_keyboard_flag & e == ("\r" | "\l" | " ") then { + on_action(e) + } + } + end + + method handle_event(e) + if e === (&lpress | &rpress | &mpress) then { + handle_press(e) + } else if e === (&ldrag | &rdrag | &mdrag) then { + handle_drag(e) + } else if e === (&lrelease | &rrelease | &mrelease) then { + handle_release(e) + } else + handle_default(e) + end + + # + # Set the up/down images (if any) to the strings provided, + # which should be in Icon image format. + # The two images must have the same dimensions. + # @param x The up image + # @param y The down image + # + method set_imgs(x, y) + self.img_up := x + self.img_w := img_width(x) = img_width(y) | fatal("Image widths differ") + self.img_h := img_height(x) = img_height(y) | fatal("Image heights differ") + + self.img_down := y + + return + end + + # + # Set the image (if any) to the given string, which should be in Icon image + # format. + # @param x The image + # + method set_img(x) + self.img_up := self.img_down := x + self.img_w := img_width(x) + self.img_h := img_height(x) + return x + end + + # + # Toggle the checked status of the button. This method, and + # the following two methods, may be + # inappropriate for non-toggle styles of button. + # + method toggle_is_checked() + self.Toggle.toggle_is_checked() + self.invalidate() + end + + # + # Set the status to checked. + # + method set_is_checked() + self.Toggle.set_is_checked() + self.invalidate() + end + + # + # Set the status to unchecked. + # + method clear_is_checked() + self.Toggle.clear_is_checked() + self.invalidate() + end + + # + # Set the button so that when it is pressed, it toggles + # between two states, as indicated by the is_checked + # flag. + # + # Instances of Checkbox have this flag on by default, but + # TextButton and IconButton do not. When the flag is on, + # the latter classes indicate their checked status by + # showing the button as being "down". + # + method set_toggles() + self.toggles_flag := 1 + self.invalidate() + end + + # + # Clear the toggles flag. + # + method clear_toggles() + self.toggles_flag := &null + self.invalidate() + end + + # + # Set the label of the button, if any. + # @param x The label + # + method set_label(x) + self.label := x + self.invalidate() + return x + end + + method set_one(attr, val) + case attr of { + "label" : set_label(string_val(attr, val)) + "is_checked" : + if test_flag(attr, val) then + set_is_checked() + else + clear_is_checked() + "toggles" : + if test_flag(attr, val) then + set_toggles() + else + clear_toggles() + "no_keyboard" : + if test_flag(attr, val) then + set_no_keyboard() + else + clear_no_keyboard() + default: self.Component.set_one(attr, val) + } + end + + initially() + self.Component.initially() + self.accepts_focus_flag := 1 +end \ No newline at end of file -- cgit v1.2.1 From 11d027aa3ce1af6959a518024990926dba01ff93 Mon Sep 17 00:00:00 2001 From: Camil Staps Date: Wed, 20 Sep 2017 15:44:43 +0200 Subject: inline is only a keyword after code --- pygments/lexers/clean.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pygments/lexers/clean.py b/pygments/lexers/clean.py index b87f06ec..b1a49632 100644 --- a/pygments/lexers/clean.py +++ b/pygments/lexers/clean.py @@ -28,9 +28,9 @@ class CleanLexer(ExtendedRegexLexer): filenames = ['*.icl', '*.dcl'] keywords = ( - 'case', 'ccall', 'class', 'code', 'derive', 'export', 'foreign', - 'from', 'generic', 'if', 'import', 'in', 'infix', 'infixl', 'infixr', - 'inline', 'instance', 'let', 'of', 'otherwise', 'qualified', 'special', + 'case', 'ccall', 'class', 'code', 'code inline', 'derive', 'export', + 'foreign', 'from', 'generic', 'if', 'import', 'in', 'infix', 'infixl', + 'infixr', 'instance', 'let', 'of', 'otherwise', 'qualified', 'special', 'stdcall', 'where', 'with') modulewords = ('implementation', 'definition', 'system') -- cgit v1.2.1 From b60aca2cefb7cecfebf1834b4adf52e7a33f070b Mon Sep 17 00:00:00 2001 From: mppf5 Date: Mon, 25 Sep 2017 15:57:19 -0400 Subject: Updating Chapel highlighter * include new error handling keywords * fix problems with 1-character procs * include Type.method in method name * highlight iterators in addition to procs --- pygments/lexers/chapel.py | 14 +++++++++----- tests/examplefiles/99_bottles_of_beer.chpl | 20 ++++++++++++++++++++ 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/pygments/lexers/chapel.py b/pygments/lexers/chapel.py index 55bf0e1e..445f0a9d 100644 --- a/pygments/lexers/chapel.py +++ b/pygments/lexers/chapel.py @@ -42,16 +42,20 @@ class ChapelLexer(RegexLexer): (r'(bool|complex|imag|int|opaque|range|real|string|uint)\b', Keyword.Type), (words(( - 'align', 'as', 'atomic', 'begin', 'break', 'by', 'cobegin', + 'align', 'as', 'atomic', 'begin', 'break', 'by', 'catch', + 'cobegin', 'coforall', 'continue', 'delete', 'dmapped', 'do', 'domain', 'else', 'enum', 'except', 'export', 'extern', 'for', 'forall', - 'if', 'index', 'inline', 'iter', 'label', 'lambda', 'let', + 'if', 'index', 'inline', 'label', 'lambda', 'let', 'local', 'new', 'noinit', 'on', 'only', 'otherwise', 'pragma', - 'private', 'public', 'reduce', 'require', 'return', 'scan', + 'private', 'prototype', + 'public', 'reduce', 'require', 'return', 'scan', 'select', 'serial', 'single', 'sparse', 'subdomain', 'sync', - 'then', 'use', 'when', 'where', 'while', 'with', 'yield', + 'then', 'throw', 'throws', 'try', + 'use', 'when', 'where', 'while', 'with', 'yield', 'zip'), suffix=r'\b'), Keyword), + (r'(iter)((?:\s)+)', bygroups(Keyword, Text), 'procname'), (r'(proc)((?:\s)+)', bygroups(Keyword, Text), 'procname'), (r'(class|module|record|union)(\s+)', bygroups(Keyword, Text), 'classname'), @@ -96,7 +100,7 @@ class ChapelLexer(RegexLexer): (r'[a-zA-Z_][\w$]*', Name.Class, '#pop'), ], 'procname': [ - (r'([a-zA-Z_][\w$]+|\~[a-zA-Z_][\w$]+|[+*/!~%<>=&^|\-]{1,2})', + (r'([a-zA-Z_][.\w$]*|\~[a-zA-Z_][.\w$]*|[+*/!~%<>=&^|\-]{1,2})', Name.Function, '#pop'), ], } diff --git a/tests/examplefiles/99_bottles_of_beer.chpl b/tests/examplefiles/99_bottles_of_beer.chpl index cdc1e650..80d2a89e 100644 --- a/tests/examplefiles/99_bottles_of_beer.chpl +++ b/tests/examplefiles/99_bottles_of_beer.chpl @@ -177,3 +177,23 @@ private module M3 { private var x: int; } +prototype module X { + + proc f() throws { + throw new Error(); + } + + proc g() { + try { + f(); + try! f(); + } catch e { + writeln("Caught ", e); + } + } + + proc int.add() { } + + g(); +} + -- cgit v1.2.1 From 8c72421a642bcc5b3d1723a326cd827a92893ff9 Mon Sep 17 00:00:00 2001 From: alfonse Date: Sun, 29 Oct 2017 11:06:54 -0400 Subject: GLSL: Added keywords for GLSL versions > 1.20. Now current to GLSL 4.60. --- pygments/lexers/graphics.py | 108 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 90 insertions(+), 18 deletions(-) diff --git a/pygments/lexers/graphics.py b/pygments/lexers/graphics.py index c8af9f99..468f740c 100644 --- a/pygments/lexers/graphics.py +++ b/pygments/lexers/graphics.py @@ -46,28 +46,100 @@ class GLShaderLexer(RegexLexer): (r'0[0-7]*', Number.Oct), (r'[1-9][0-9]*', Number.Integer), (words(( - 'attribute', 'const', 'uniform', 'varying', 'centroid', 'break', - 'continue', 'do', 'for', 'while', 'if', 'else', 'in', 'out', - 'inout', 'float', 'int', 'void', 'bool', 'true', 'false', - 'invariant', 'discard', 'return', 'mat2', 'mat3' 'mat4', - 'mat2x2', 'mat3x2', 'mat4x2', 'mat2x3', 'mat3x3', 'mat4x3', - 'mat2x4', 'mat3x4', 'mat4x4', 'vec2', 'vec3', 'vec4', - 'ivec2', 'ivec3', 'ivec4', 'bvec2', 'bvec3', 'bvec4', - 'sampler1D', 'sampler2D', 'sampler3D' 'samplerCube', - 'sampler1DShadow', 'sampler2DShadow', 'struct'), + # Storage qualifiers + 'attribute', 'const', 'uniform', 'varying', + 'buffer', 'shared', 'in', 'out', + # Layout qualifiers + 'layout', + # Interpolation qualifiers + 'flat', 'smooth', 'noperspective', + # Auxiliary qualifiers + 'centroid', 'sample', 'patch', + # Parameter qualifiers. Some double as Storage qualifiers + 'inout', + # Precision qualifiers + 'lowp', 'mediump', 'highp', 'precision', + # Invariance qualifiers + 'invariant', + # Precise qualifiers + 'precise', + # Memory qualifiers + 'coherent', 'volatile', 'restrict', 'readonly', 'writeonly', + # Statements + 'break', 'continue', 'do', 'for', 'while', 'switch', + 'case', 'default', 'if', 'else', 'subroutine', + 'discard', 'return', 'struct'), prefix=r'\b', suffix=r'\b'), Keyword), (words(( - 'asm', 'class', 'union', 'enum', 'typedef', 'template', 'this', - 'packed', 'goto', 'switch', 'default', 'inline', 'noinline', - 'volatile', 'public', 'static', 'extern', 'external', 'interface', - 'long', 'short', 'double', 'half', 'fixed', 'unsigned', 'lowp', - 'mediump', 'highp', 'precision', 'input', 'output', - 'hvec2', 'hvec3', 'hvec4', 'dvec2', 'dvec3', 'dvec4', - 'fvec2', 'fvec3', 'fvec4', 'sampler2DRect', 'sampler3DRect', - 'sampler2DRectShadow', 'sizeof', 'cast', 'namespace', 'using'), + # Boolean values + 'true', 'false'), prefix=r'\b', suffix=r'\b'), - Keyword), # future use + Keyword.Constant), + (words(( + # Miscellaneous types + 'void', 'atomic_uint', + # Floating-point scalars and vectors + 'float', 'vec2', 'vec3', 'vec4', + 'double', 'dvec2', 'dvec3', 'dvec4', + # Integer scalars and vectors + 'int', 'ivec2', 'ivec3', 'ivec4', + 'uint', 'uvec2', 'uvec3', 'uvec4', + # Boolean scalars and vectors + 'bool', 'bvec2', 'bvec3', 'bvec4', + # Matrices + 'mat2', 'mat3', 'mat4', 'dmat2', 'dmat3', 'dmat4', + 'mat2x2', 'mat2x3', 'mat2x4', 'dmat2x2', 'dmat2x3', 'dmat2x4', + 'mat3x2', 'mat3x3', 'mat3x4', 'dmat3x2', 'dmat3x3', + 'dmat3x4', 'mat4x2', 'mat4x3', 'mat4x4', 'dmat4x2', 'dmat4x3', 'dmat4x4', + # Floating-point samplers + 'sampler1D', 'sampler2D', 'sampler3D', 'samplerCube', + 'sampler1DArray', 'sampler2DArray', 'samplerCubeArray', + 'sampler2DRect', 'samplerBuffer', + 'sampler2DMS', 'sampler2DMSArray', + # Shadow samplers + 'sampler1DShadow', 'sampler2DShadow', 'samplerCubeShadow', + 'sampler1DArrayShadow', 'sampler2DArrayShadow', + 'samplerCubeArrayShadow', 'sampler2DRectShadow', + # Signed integer samplers + 'isampler1D', 'isampler2D', 'isampler3D', 'isamplerCube', + 'isampler1DArray', 'isampler2DArray', 'isamplerCubeArray', + 'isampler2DRect', 'isamplerBuffer', + 'isampler2DMS', 'isampler2DMSArray', + # Unsigned integer samplers + 'usampler1D', 'usampler2D', 'usampler3D', 'usamplerCube', + 'usampler1DArray', 'usampler2DArray', 'usamplerCubeArray', + 'usampler2DRect', 'usamplerBuffer', + 'usampler2DMS', 'usampler2DMSArray', + # Floating-point image types + 'image1D', 'image2D', 'image3D', 'imageCube', + 'image1DArray', 'image2DArray', 'imageCubeArray', + 'image2DRect', 'imageBuffer', + 'image2DMS', 'image2DMSArray', + # Signed integer image types + 'iimage1D', 'iimage2D', 'iimage3D', 'iimageCube', + 'iimage1DArray', 'iimage2DArray', 'iimageCubeArray', + 'iimage2DRect', 'iimageBuffer', + 'iimage2DMS', 'iimage2DMSArray', + # Unsigned integer image types + 'uimage1D', 'uimage2D', 'uimage3D', 'uimageCube', + 'uimage1DArray', 'uimage2DArray', 'uimageCubeArray', + 'uimage2DRect', 'uimageBuffer', + 'uimage2DMS', 'uimage2DMSArray'), + prefix=r'\b', suffix=r'\b'), + Keyword.Type), + (words(( + # Reserved for future use. + 'common', 'partition', 'active', 'asm', 'class', + 'union', 'enum', 'typedef', 'template', 'this', + 'resource', 'goto', 'inline', 'noinline', 'public', + 'static', 'extern', 'external', 'interface', 'long', + 'short', 'half', 'fixed', 'unsigned', 'superp', 'input', + 'output', 'hvec2', 'hvec3', 'hvec4', 'fvec2', 'fvec3', + 'fvec4', 'sampler3DRect', 'filter', 'sizeof', 'cast', + 'namespace', 'using'), + prefix=r'\b', suffix=r'\b'), + Keyword.Reserved), (r'[a-zA-Z_]\w*', Name), (r'\.', Punctuation), (r'\s+', Text), -- cgit v1.2.1 From da6ec9620f8af6c9c8886e8b87245159ead6f99a Mon Sep 17 00:00:00 2001 From: alfonse Date: Sun, 29 Oct 2017 11:12:59 -0400 Subject: GLSL: Names that start with `gl_` are properly recognized as built-in variables. --- pygments/lexers/graphics.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pygments/lexers/graphics.py b/pygments/lexers/graphics.py index 468f740c..c7aa7df4 100644 --- a/pygments/lexers/graphics.py +++ b/pygments/lexers/graphics.py @@ -140,6 +140,8 @@ class GLShaderLexer(RegexLexer): 'namespace', 'using'), prefix=r'\b', suffix=r'\b'), Keyword.Reserved), + # All names beginning with "gl_" are reserved. + (r'gl_\w*', Name.Builtin), (r'[a-zA-Z_]\w*', Name), (r'\.', Punctuation), (r'\s+', Text), -- cgit v1.2.1 From 47fd1dc317a09cce2e16b8ccd8e52cd7dd5b37d9 Mon Sep 17 00:00:00 2001 From: Fredrik Larsen Date: Fri, 10 Nov 2017 17:22:12 +0100 Subject: Fix issue with markdown lexer code fences Fixes the bug reported in #1389, where the markdown lexer will not issue a token for the closing code fence. Issue: #1389 --- pygments/lexers/markup.py | 7 +++---- tests/test_markdown_lexer.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 4 deletions(-) create mode 100644 tests/test_markdown_lexer.py diff --git a/pygments/lexers/markup.py b/pygments/lexers/markup.py index 92dc9e7a..e6265f40 100644 --- a/pygments/lexers/markup.py +++ b/pygments/lexers/markup.py @@ -536,10 +536,9 @@ class MarkdownLexer(RegexLexer): # no lexer for this language. handle it like it was a code block if lexer is None: yield match.start(4), String, code - return - - for item in do_insertions([], lexer.get_tokens_unprocessed(code)): - yield item + else: + for item in do_insertions([], lexer.get_tokens_unprocessed(code)): + yield item yield match.start(5), String , match.group(5) diff --git a/tests/test_markdown_lexer.py b/tests/test_markdown_lexer.py new file mode 100644 index 00000000..16d1f28d --- /dev/null +++ b/tests/test_markdown_lexer.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +""" + Pygments regex lexer tests + ~~~~~~~~~~~~~~~~~~~~~~~~~~ + + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" +import unittest + +from pygments.lexers.markup import MarkdownLexer + + +class SameTextTests(unittest.TestCase): + + lexer = MarkdownLexer() + + def assert_same_text(self, text): + """Show that lexed markdown does not remove any content. """ + tokens = list(self.lexer.get_tokens_unprocessed(text)) + output = ''.join(t[2] for t in tokens) + self.assertEqual(text, output) + + def test_code_fence(self): + self.assert_same_text(r'```\nfoo\n```\n') + + def test_code_fence_gsm(self): + self.assert_same_text(r'```markdown\nfoo\n```\n') + + def test_code_fence_gsm_with_no_lexer(self): + self.assert_same_text(r'```invalid-lexer\nfoo\n```\n') -- cgit v1.2.1 From 14501bcab5687b5d94adb98e9c8548dcc3158242 Mon Sep 17 00:00:00 2001 From: Nobukazu Hanada Date: Fri, 8 Dec 2017 16:33:38 +0000 Subject: fixed Elm lexer about variable name enabled to analyze variable name including number. --- pygments/lexers/elm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pygments/lexers/elm.py b/pygments/lexers/elm.py index 0fa36367..22a10bd9 100644 --- a/pygments/lexers/elm.py +++ b/pygments/lexers/elm.py @@ -27,7 +27,7 @@ class ElmLexer(RegexLexer): filenames = ['*.elm'] mimetypes = ['text/x-elm'] - validName = r'[a-z_][a-zA-Z_\']*' + validName = r'[a-z_][a-zA-Z0-9_\']*' specialName = r'^main ' -- cgit v1.2.1 From 2934045c9ea5c20870f913e2d03f62a3a757d1be Mon Sep 17 00:00:00 2001 From: Takenobu Tani Date: Sat, 9 Dec 2017 22:42:05 +0900 Subject: Fix Haskell lexer for numeric literals. See also https://github.com/ghc-proposals/ghc-proposals/blob/master/proposals/0009-numeric-underscores.rst#new-syntax-this-proposal --- pygments/lexers/haskell.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/pygments/lexers/haskell.py b/pygments/lexers/haskell.py index 1a2f2217..b4624124 100644 --- a/pygments/lexers/haskell.py +++ b/pygments/lexers/haskell.py @@ -72,11 +72,14 @@ class HaskellLexer(RegexLexer): (r':[:!#$%&*+.\\/<=>?@^|~-]*', Keyword.Type), # Constructor operators (r'[:!#$%&*+.\\/<=>?@^|~-]+', Operator), # Other operators # Numbers - (r'\d+[eE][+-]?\d+', Number.Float), - (r'\d+\.\d+([eE][+-]?\d+)?', Number.Float), - (r'0[oO][0-7]+', Number.Oct), - (r'0[xX][\da-fA-F]+', Number.Hex), - (r'\d+', Number.Integer), + (r'0[xX]_*[\da-fA-F](_*[\da-fA-F])*_*[pP][+-]?\d(_*\d)*', Number.Float), + (r'0[xX]_*[\da-fA-F](_*[\da-fA-F])*\.[\da-fA-F](_*[\da-fA-F])*(_*[pP][+-]?\d(_*\d)*)?', Number.Float), + (r'\d(_*\d)*_*[eE][+-]?\d(_*\d)*', Number.Float), + (r'\d(_*\d)*\.\d(_*\d)*(_*[eE][+-]?\d(_*\d)*)?', Number.Float), + (r'0[bB]_*[01](_*[01])*', Number.Bin), + (r'0[oO]_*[0-7](_*[0-7])*', Number.Oct), + (r'0[xX]_*[\da-fA-F](_*[\da-fA-F])*', Number.Hex), + (r'\d(_*\d)*', Number.Integer), # Character/String Literals (r"'", String.Char, 'character'), (r'"', String, 'string'), -- cgit v1.2.1 From bd741767afbee76c168f7bdf3e7f5f010d2d150a Mon Sep 17 00:00:00 2001 From: Gabor Szarnyas Date: Thu, 14 Dec 2017 17:55:12 -0500 Subject: Add more clauses to Cypher lexer --- pygments/lexers/graph.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/pygments/lexers/graph.py b/pygments/lexers/graph.py index 1a338246..6e836bdd 100644 --- a/pygments/lexers/graph.py +++ b/pygments/lexers/graph.py @@ -22,9 +22,9 @@ __all__ = ['CypherLexer'] class CypherLexer(RegexLexer): """ For `Cypher Query Language - `_ + `_ - For the Cypher version in Neo4J 2.0 + For the Cypher version in Neo4j 3.3 .. versionadded:: 2.0 """ @@ -49,14 +49,19 @@ class CypherLexer(RegexLexer): ], 'keywords': [ (r'(create|order|match|limit|set|skip|start|return|with|where|' - r'delete|foreach|not|by)\b', Keyword), + r'delete|foreach|not|by|true|false)\b', Keyword), ], 'clauses': [ - # TODO: many missing ones, see http://docs.neo4j.org/refcard/2.0/ - (r'(all|any|as|asc|create|create\s+unique|delete|' - r'desc|distinct|foreach|in|is\s+null|limit|match|none|' - r'order\s+by|return|set|skip|single|start|union|where|with)\b', - Keyword), + # based on https://neo4j.com/docs/cypher-refcard/3.3/ + (r'(all|any|as|asc|ascending|assert|call|case|create|' + r'create\s+index|create\s+unique|delete|desc|descending|' + r'distinct|drop\s+constraint\s+on|drop\s+index\s+on|end|' + r'ends\s+with|fieldterminator|foreach|in|is\s+node\s+key|' + r'is\s+null|is\s+unique|limit|load\s+csv\s+from|match|merge|none|' + r'not|null|on\s+match|on\s+create|optional\s+match|order\s+by|' + r'remove|return|set|skip|single|start|starts\s+with|then|union|' + r'union\s+all|unwind|using\s+periodic\s+commit|yield|where|when|' + r'with)\b', Keyword), ], 'relations': [ (r'(-\[)(.*?)(\]->)', bygroups(Operator, using(this), Operator)), -- cgit v1.2.1 From 7f514de4ac133fe075af8956be9b9eec01011495 Mon Sep 17 00:00:00 2001 From: Nathan Whetsell Date: Sun, 24 Dec 2017 10:25:39 -0500 Subject: Update for Csound 6.10.0 --- pygments/lexers/_csound_builtins.py | 73 +++++++++++++++++++++---------------- 1 file changed, 42 insertions(+), 31 deletions(-) diff --git a/pygments/lexers/_csound_builtins.py b/pygments/lexers/_csound_builtins.py index 84c06956..74d6cffb 100644 --- a/pygments/lexers/_csound_builtins.py +++ b/pygments/lexers/_csound_builtins.py @@ -7,32 +7,47 @@ :license: BSD, see LICENSE for details. """ -# Opcodes in Csound 6.09.1 at commit 0815e64 from -# csound --list-opcodes3 +# Opcodes in Csound 6.10.0 at commit 71eedf01f9c8db6c9fbfdd59888eb3d5ef52f7a0 using +# python -c " +# import re, subprocess +# output = subprocess.Popen(['csound', '--list-opcodes0'], stderr=subprocess.PIPE).communicate()[1] +# opcodes = output[re.search(r'^\d+ opcodes$', output, re.M).end():re.search(r'^Usage:', output, re.M).start()].split() +# output = subprocess.Popen(['csound', '--list-opcodes2'], stderr=subprocess.PIPE).communicate()[1] +# all_opcodes = output[re.search(r'^\d+ opcodes$', output, re.M).end():re.search(r'^Usage:', output, re.M).start()].split() +# deprecated_opcodes = [opcode for opcode in all_opcodes if opcode not in opcodes] +# print '''OPCODES = set(\''' +# {} +# \'''.split()) +# +# DEPRECATED_OPCODES = set(\''' +# {} +# \'''.split()) +# '''.format('\n'.join(opcodes), '\n'.join(deprecated_opcodes)) +# " # except for -# cggoto https://csound.github.io/docs/manual/cggoto.html -# cigoto https://csound.github.io/docs/manual/cigoto.html +# cggoto csound.com/docs/manual/cggoto.html +# cigoto csound.com/docs/manual/cigoto.html # cingoto (undocumented) -# ckgoto https://csound.github.io/docs/manual/ckgoto.html -# cngoto https://csound.github.io/docs/manual/cngoto.html +# ckgoto csound.com/docs/manual/ckgoto.html +# cngoto csound.com/docs/manual/cngoto.html # cnkgoto (undocumented) -# endin https://csound.github.io/docs/manual/endin.html -# endop https://csound.github.io/docs/manual/endop.html -# goto https://csound.github.io/docs/manual/goto.html -# igoto https://csound.github.io/docs/manual/igoto.html -# instr https://csound.github.io/docs/manual/instr.html -# kgoto https://csound.github.io/docs/manual/kgoto.html -# loop_ge https://csound.github.io/docs/manual/loop_ge.html -# loop_gt https://csound.github.io/docs/manual/loop_gt.html -# loop_le https://csound.github.io/docs/manual/loop_le.html -# loop_lt https://csound.github.io/docs/manual/loop_lt.html -# opcode https://csound.github.io/docs/manual/opcode.html -# reinit https://csound.github.io/docs/manual/reinit.html -# return https://csound.github.io/docs/manual/return.html -# rireturn https://csound.github.io/docs/manual/rireturn.html -# rigoto https://csound.github.io/docs/manual/rigoto.html -# tigoto https://csound.github.io/docs/manual/tigoto.html -# timout https://csound.github.io/docs/manual/timout.html +# endin csound.com/docs/manual/endin.html +# endop csound.com/docs/manual/endop.html +# goto csound.com/docs/manual/goto.html +# igoto csound.com/docs/manual/igoto.html +# instr csound.com/docs/manual/instr.html +# kgoto csound.com/docs/manual/kgoto.html +# loop_ge csound.com/docs/manual/loop_ge.html +# loop_gt csound.com/docs/manual/loop_gt.html +# loop_le csound.com/docs/manual/loop_le.html +# loop_lt csound.com/docs/manual/loop_lt.html +# opcode csound.com/docs/manual/opcode.html +# reinit csound.com/docs/manual/reinit.html +# return csound.com/docs/manual/return.html +# rireturn csound.com/docs/manual/rireturn.html +# rigoto csound.com/docs/manual/rigoto.html +# tigoto csound.com/docs/manual/tigoto.html +# timout csound.com/docs/manual/timout.html # which are treated as keywords in csound.py. OPCODES = set(''' @@ -138,12 +153,9 @@ MixerSend MixerSetLevel MixerSetLevel_i OSCinit -OSCinitM OSClisten OSCraw OSCsend -OSCsendA -OSCsend_lo S STKBandedWG STKBeeThree @@ -242,9 +254,11 @@ chn_k chnclear chnexport chnget +chngetks chnmix chnparams chnset +chnsetks chuap clear clfilt @@ -821,6 +835,7 @@ midinoteonpch midion midion2 midiout +midiout_i midipgm midipitchbend midipolyaftertouch @@ -1302,6 +1317,7 @@ sprintf sprintfk spsend sqrt +squinewave statevar stix strcat @@ -1605,10 +1621,6 @@ ktableseg lentab maxtab mintab -pop -pop_f -push -push_f scalet sndload soundout @@ -1622,7 +1634,6 @@ specptrk specscal specsum spectrum -stack sumtab tabgen tabmap -- cgit v1.2.1 From 1c98199b06c059434b7b16c85b6568b6e6d6c9a7 Mon Sep 17 00:00:00 2001 From: Marshall Ward Date: Sat, 6 Jan 2018 11:39:40 +1100 Subject: Fortran E-notation float without decimal Added a regex to parse Fortran floating points which identify floating point values with exponents but without decimals (e.g. 1e4 vs 1.e4 or 1.0e4) --- pygments/lexers/fortran.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pygments/lexers/fortran.py b/pygments/lexers/fortran.py index 1a611c9d..61b8aaeb 100644 --- a/pygments/lexers/fortran.py +++ b/pygments/lexers/fortran.py @@ -158,6 +158,7 @@ class FortranLexer(RegexLexer): (r'\d+(?![.e])(_[a-z]\w+)?', Number.Integer), (r'[+-]?\d*\.\d+([ed][-+]?\d+)?(_[a-z]\w+)?', Number.Float), (r'[+-]?\d+\.\d*([ed][-+]?\d+)?(_[a-z]\w+)?', Number.Float), + (r'[+-]?\d+(\.\d*)?([ed][-+]?\d+)?(_[a-z]\w+)?', Number.Float), ], } -- cgit v1.2.1 From c5629b93b8128c9a4925c1d118b6cb9b1d468f7a Mon Sep 17 00:00:00 2001 From: Marshall Ward Date: Sat, 6 Jan 2018 11:45:30 +1100 Subject: Safer Fortran E-notation without decimal Fortran e-notation values without decimal now require the presence of e (or d) and at least one exponent value. --- pygments/lexers/fortran.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pygments/lexers/fortran.py b/pygments/lexers/fortran.py index 61b8aaeb..5165bac0 100644 --- a/pygments/lexers/fortran.py +++ b/pygments/lexers/fortran.py @@ -158,7 +158,7 @@ class FortranLexer(RegexLexer): (r'\d+(?![.e])(_[a-z]\w+)?', Number.Integer), (r'[+-]?\d*\.\d+([ed][-+]?\d+)?(_[a-z]\w+)?', Number.Float), (r'[+-]?\d+\.\d*([ed][-+]?\d+)?(_[a-z]\w+)?', Number.Float), - (r'[+-]?\d+(\.\d*)?([ed][-+]?\d+)?(_[a-z]\w+)?', Number.Float), + (r'[+-]?\d+(\.\d*)?[ed][-+]?\d+(_[a-z]\w+)?', Number.Float), ], } -- cgit v1.2.1 From 74830b41f20a36ed7736c3bc79f28fd8acd363f2 Mon Sep 17 00:00:00 2001 From: 0486 Date: Sat, 3 Feb 2018 01:33:47 +0200 Subject: Added FloScript lexer --- pygments/lexers/_mapping.py | 1 + pygments/lexers/floscript.py | 85 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+) create mode 100644 pygments/lexers/floscript.py diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py index b48ee1d1..fbfc5230 100644 --- a/pygments/lexers/_mapping.py +++ b/pygments/lexers/_mapping.py @@ -155,6 +155,7 @@ LEXERS = { 'FelixLexer': ('pygments.lexers.felix', 'Felix', ('felix', 'flx'), ('*.flx', '*.flxh'), ('text/x-felix',)), 'FishShellLexer': ('pygments.lexers.shell', 'Fish', ('fish', 'fishshell'), ('*.fish', '*.load'), ('application/x-fish',)), 'FlatlineLexer': ('pygments.lexers.dsls', 'Flatline', ('flatline',), (), ('text/x-flatline',)), + 'FloScriptLexer': ('pygments.lexers.floscript', 'FloScript', ('floscript|flo',), ('*.flo',), ()), 'ForthLexer': ('pygments.lexers.forth', 'Forth', ('forth',), ('*.frt', '*.fs'), ('application/x-forth',)), 'FortranFixedLexer': ('pygments.lexers.fortran', 'FortranFixed', ('fortranfixed',), ('*.f', '*.F'), ()), 'FortranLexer': ('pygments.lexers.fortran', 'Fortran', ('fortran',), ('*.f03', '*.f90', '*.F03', '*.F90'), ('text/x-fortran',)), diff --git a/pygments/lexers/floscript.py b/pygments/lexers/floscript.py new file mode 100644 index 00000000..b123d74c --- /dev/null +++ b/pygments/lexers/floscript.py @@ -0,0 +1,85 @@ +# -*- coding: utf-8 -*- +""" + pygments.lexers.python + ~~~~~~~~~~~~~~~~~~~~~~ + + Lexer for FloScript + + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import re + +from pygments.lexer import Lexer, RegexLexer, include, bygroups, using, \ + default, words, combined, do_insertions +from pygments.util import get_bool_opt, shebang_matches +from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ + Number, Punctuation, Generic, Other, Error +from pygments import unistring as uni + +__all__ = ['FloScriptLexer',] + +class FloScriptLexer(RegexLexer): + """ + For `FloScript `_ configuration language source code. + """ + + name = 'FloScript' + aliases = ['floscript|flo'] + filenames = ['*.flo'] + + def innerstring_rules(ttype): + return [ + # the old style '%s' % (...) string formatting + (r'%(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?' + '[hlL]?[E-GXc-giorsux%]', String.Interpol), + # backslashes, quotes and formatting signs must be parsed one at a time + (r'[^\\\'"%\n]+', ttype), + (r'[\'"\\]', ttype), + # unhandled string formatting sign + (r'%', ttype), + # newlines are an error (use "nl" state) + ] + + + tokens = { + 'root': [ + (r'\n', Text), + (r'[^\S\n]+', Text), + + (r'[]{}:(),;[]', Punctuation), + (r'\\\n', Text), + (r'\\', Text), + (r'(to|by|with|from|per|for|cum|qua|via|as|at|in|of|on|re|is|if|be|into|and|not)\b', Operator.Word), + (r'!=|==|<<|>>|[-~+/*%=<>&^|.]', Operator), + (r'(load|init|server|logger|log|loggee|first|over|under|next|done|timeout|repeat|native|benter|enter|recur|exit|precur|renter|rexit|print|put|inc|copy|set|aux|rear|raze|go|let|do|bid|ready|start|stop|run|abort|use|flo|give|take)\b', Name.Builtin), + (r'(frame|framer|house)\b', Keyword), + ('"', String, 'string'), + + include('name'), + include('numbers'), + (r'#.+$', Comment.Singleline), + ], + 'string': [ + ('[^"]+', String), + ('"', String, '#pop'), + ], + 'numbers': [ + (r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?j?', Number.Float), + (r'\d+[eE][+-]?[0-9]+j?', Number.Float), + (r'0[0-7]+j?', Number.Oct), + (r'0[bB][01]+', Number.Bin), + (r'0[xX][a-fA-F0-9]+', Number.Hex), + (r'\d+L', Number.Integer.Long), + (r'\d+j?', Number.Integer) + ], + + 'name': [ + (r'@[\w.]+', Name.Decorator), + (r'[a-zA-Z_]\w*', Name), + ], + + + + } -- cgit v1.2.1 From 4bd2f6efad25f2f50d0d39f6677576766322664f Mon Sep 17 00:00:00 2001 From: Rostyslav Golda Date: Sat, 3 Feb 2018 01:35:20 +0200 Subject: Added myself to Authors. --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index f9ba2675..ccff46b3 100644 --- a/AUTHORS +++ b/AUTHORS @@ -216,5 +216,6 @@ Other contributors, listed alphabetically, are: * Alex Zimin -- Nemerle lexer * Rob Zimmerman -- Kal lexer * Vincent Zurczak -- Roboconf lexer +* Rostyslav Golda -- FloScript lexer Many thanks for all contributions! -- cgit v1.2.1 From b2176869817dfb0b43b760c243b1ecaf8202cf3c Mon Sep 17 00:00:00 2001 From: Rostyslav Golda Date: Sat, 3 Feb 2018 01:40:38 +0200 Subject: Corrected one word... --- pygments/lexers/floscript.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pygments/lexers/floscript.py b/pygments/lexers/floscript.py index b123d74c..9efee613 100644 --- a/pygments/lexers/floscript.py +++ b/pygments/lexers/floscript.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ - pygments.lexers.python + pygments.lexers.floscript ~~~~~~~~~~~~~~~~~~~~~~ Lexer for FloScript -- cgit v1.2.1 From 6037043e01b2a096d39883f9f7dd27cdcfe8d0ab Mon Sep 17 00:00:00 2001 From: Colin Watson Date: Thu, 22 Feb 2018 14:15:07 +0000 Subject: Markdown: add support for reference-style links --- pygments/lexers/markup.py | 5 +++++ tests/examplefiles/example.md | 3 +++ 2 files changed, 8 insertions(+) diff --git a/pygments/lexers/markup.py b/pygments/lexers/markup.py index 92dc9e7a..6b104bbe 100644 --- a/pygments/lexers/markup.py +++ b/pygments/lexers/markup.py @@ -583,6 +583,11 @@ class MarkdownLexer(RegexLexer): (r'[@#][\w/:]+', Name.Entity), # (image?) links eg: ![Image of Yaktocat](https://octodex.github.com/images/yaktocat.png) (r'(!?\[)([^]]+)(\])(\()([^)]+)(\))', bygroups(Text, Name.Tag, Text, Text, Name.Attribute, Text)), + # reference-style links, e.g.: + # [an example][id] + # [id]: http://example.com/ + (r'(\[)([^]]+)(\])(\[)([^]]*)(\])', bygroups(Text, Name.Tag, Text, Text, Name.Label, Text)), + (r'^(\s*\[)([^]]*)(\]:\s*)(.+)', bygroups(Text, Name.Label, Text, Name.Attribute)), # general text, must come last! (r'[^\\\s]+', Text), diff --git a/tests/examplefiles/example.md b/tests/examplefiles/example.md index 2befb107..e2bbacf1 100644 --- a/tests/examplefiles/example.md +++ b/tests/examplefiles/example.md @@ -46,6 +46,9 @@ this sentence @tweets a person about a #topic. [google](https://google.com/some/path.html) ![Image of Yaktocat](https://octodex.github.com/images/yaktocat.png) +[reference link][id] +[id]: http://example.com/ + ``` * this is just unformated __text__ -- cgit v1.2.1 From f55a60e07815480f4b1c2bb306df41d90d6280bc Mon Sep 17 00:00:00 2001 From: Simon Gomizelj Date: Thu, 5 Apr 2018 00:39:10 -0400 Subject: Hy lexer should accept colons inside identifiers Colons inside identifiers are completely legal in this Lisp dialect. Also prevents the lexer from catching on Python literals when lexing REPL output. --- pygments/lexers/lisp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pygments/lexers/lisp.py b/pygments/lexers/lisp.py index e258c347..64ac3657 100644 --- a/pygments/lexers/lisp.py +++ b/pygments/lexers/lisp.py @@ -382,7 +382,7 @@ class HyLexer(RegexLexer): # valid names for identifiers # well, names can only not consist fully of numbers # but this should be good enough for now - valid_name = r'(?!#)[\w!$%*+<=>?/.#-]+' + valid_name = r'(?!#)[\w!$%*+<=>?/.#-:]+' def _multi_escape(entries): return words(entries, suffix=' ') -- cgit v1.2.1 From f7e7251a479043014a512597f544169be033539b Mon Sep 17 00:00:00 2001 From: Nathan Whetsell Date: Fri, 11 May 2018 09:41:06 -0400 Subject: Update for Csound 6.11.0 --- pygments/lexers/_csound_builtins.py | 112 +++++++++++++++++++----------------- 1 file changed, 60 insertions(+), 52 deletions(-) diff --git a/pygments/lexers/_csound_builtins.py b/pygments/lexers/_csound_builtins.py index 74d6cffb..c5c338c6 100644 --- a/pygments/lexers/_csound_builtins.py +++ b/pygments/lexers/_csound_builtins.py @@ -3,27 +3,27 @@ pygments.lexers._csound_builtins ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2018 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ -# Opcodes in Csound 6.10.0 at commit 71eedf01f9c8db6c9fbfdd59888eb3d5ef52f7a0 using -# python -c " -# import re, subprocess -# output = subprocess.Popen(['csound', '--list-opcodes0'], stderr=subprocess.PIPE).communicate()[1] -# opcodes = output[re.search(r'^\d+ opcodes$', output, re.M).end():re.search(r'^Usage:', output, re.M).start()].split() -# output = subprocess.Popen(['csound', '--list-opcodes2'], stderr=subprocess.PIPE).communicate()[1] -# all_opcodes = output[re.search(r'^\d+ opcodes$', output, re.M).end():re.search(r'^Usage:', output, re.M).start()].split() -# deprecated_opcodes = [opcode for opcode in all_opcodes if opcode not in opcodes] -# print '''OPCODES = set(\''' -# {} -# \'''.split()) -# -# DEPRECATED_OPCODES = set(\''' -# {} -# \'''.split()) -# '''.format('\n'.join(opcodes), '\n'.join(deprecated_opcodes)) -# " +# Opcodes in Csound 6.11.0 at commit 25b2e8e53bc924526eaad34e0768a5e866638e94 using +python -c " +import re, subprocess +output = subprocess.Popen(['csound', '--list-opcodes0'], stderr=subprocess.PIPE).communicate()[1] +opcodes = output[re.search(r'^\d+ opcodes$', output, re.M).end():re.search(r'^(?:WARNING|Usage):', output, re.M).start()].split() +output = subprocess.Popen(['csound', '--list-opcodes2'], stderr=subprocess.PIPE).communicate()[1] +all_opcodes = output[re.search(r'^\d+ opcodes$', output, re.M).end():re.search(r'^(?:WARNING|Usage):', output, re.M).start()].split() +deprecated_opcodes = [opcode for opcode in all_opcodes if opcode not in opcodes] +print '''OPCODES = set(\''' +{} +\'''.split()) + +DEPRECATED_OPCODES = set(\''' +{} +\'''.split()) +'''.format('\n'.join(opcodes), '\n'.join(deprecated_opcodes)) +" # except for # cggoto csound.com/docs/manual/cggoto.html # cigoto csound.com/docs/manual/cigoto.html @@ -153,9 +153,11 @@ MixerSend MixerSetLevel MixerSetLevel_i OSCinit +OSCinitM OSClisten OSCraw OSCsend +OSCsend_lo S STKBandedWG STKBeeThree @@ -205,6 +207,7 @@ atonek atonex babo balance +balance2 bamboo barmodel bbcutm @@ -765,6 +768,8 @@ loopxseg lorenz loscil loscil3 +loscil3phs +loscilphs loscilx lowpass2 lowres @@ -1064,7 +1069,6 @@ pvsftr pvsftw pvsfwrite pvsgain -pvsgendy pvshift pvsifd pvsin @@ -1302,7 +1306,6 @@ sndwarpst sockrecv sockrecvs socksend -socksend_k socksends sorta sortd @@ -1403,38 +1406,6 @@ tan tanh taninv taninv2 -tb0 -tb0_init -tb1 -tb10 -tb10_init -tb11 -tb11_init -tb12 -tb12_init -tb13 -tb13_init -tb14 -tb14_init -tb15 -tb15_init -tb1_init -tb2 -tb2_init -tb3 -tb3_init -tb4 -tb4_init -tb5 -tb5_init -tb6 -tb6_init -tb7 -tb7_init -tb8 -tb8_init -tb9 -tb9_init tbvcf tempest tempo @@ -1621,6 +1592,10 @@ ktableseg lentab maxtab mintab +pop +pop_f +push +push_f scalet sndload soundout @@ -1634,11 +1609,44 @@ specptrk specscal specsum spectrum +stack sumtab tabgen tabmap tabmap_i tabslice +tb0 +tb0_init +tb1 +tb10 +tb10_init +tb11 +tb11_init +tb12 +tb12_init +tb13 +tb13_init +tb14 +tb14_init +tb15 +tb15_init +tb1_init +tb2 +tb2_init +tb3 +tb3_init +tb4 +tb4_init +tb5 +tb5_init +tb6 +tb6_init +tb7 +tb7_init +tb8 +tb8_init +tb9 +tb9_init vbap16 vbap4 vbap4move -- cgit v1.2.1 From 337e754de703d5e52989dd11981b89e0083103f6 Mon Sep 17 00:00:00 2001 From: Nathan Whetsell Date: Fri, 11 May 2018 09:45:45 -0400 Subject: Comment out Python script --- pygments/lexers/_csound_builtins.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/pygments/lexers/_csound_builtins.py b/pygments/lexers/_csound_builtins.py index c5c338c6..856daa29 100644 --- a/pygments/lexers/_csound_builtins.py +++ b/pygments/lexers/_csound_builtins.py @@ -8,22 +8,22 @@ """ # Opcodes in Csound 6.11.0 at commit 25b2e8e53bc924526eaad34e0768a5e866638e94 using -python -c " -import re, subprocess -output = subprocess.Popen(['csound', '--list-opcodes0'], stderr=subprocess.PIPE).communicate()[1] -opcodes = output[re.search(r'^\d+ opcodes$', output, re.M).end():re.search(r'^(?:WARNING|Usage):', output, re.M).start()].split() -output = subprocess.Popen(['csound', '--list-opcodes2'], stderr=subprocess.PIPE).communicate()[1] -all_opcodes = output[re.search(r'^\d+ opcodes$', output, re.M).end():re.search(r'^(?:WARNING|Usage):', output, re.M).start()].split() -deprecated_opcodes = [opcode for opcode in all_opcodes if opcode not in opcodes] -print '''OPCODES = set(\''' -{} -\'''.split()) - -DEPRECATED_OPCODES = set(\''' -{} -\'''.split()) -'''.format('\n'.join(opcodes), '\n'.join(deprecated_opcodes)) -" +# python -c " +# import re, subprocess +# output = subprocess.Popen(['csound', '--list-opcodes0'], stderr=subprocess.PIPE).communicate()[1] +# opcodes = output[re.search(r'^\d+ opcodes$', output, re.M).end():re.search(r'^(?:WARNING|Usage):', output, re.M).start()].split() +# output = subprocess.Popen(['csound', '--list-opcodes2'], stderr=subprocess.PIPE).communicate()[1] +# all_opcodes = output[re.search(r'^\d+ opcodes$', output, re.M).end():re.search(r'^(?:WARNING|Usage):', output, re.M).start()].split() +# deprecated_opcodes = [opcode for opcode in all_opcodes if opcode not in opcodes] +# print '''OPCODES = set(\''' +# {} +# \'''.split()) +# +# DEPRECATED_OPCODES = set(\''' +# {} +# \'''.split()) +# '''.format('\n'.join(opcodes), '\n'.join(deprecated_opcodes)) +# " # except for # cggoto csound.com/docs/manual/cggoto.html # cigoto csound.com/docs/manual/cigoto.html -- cgit v1.2.1 From fff637a82567bb30fa56e4d03f47ca0cc1ec7f35 Mon Sep 17 00:00:00 2001 From: Nathan Whetsell Date: Fri, 11 May 2018 09:52:52 -0400 Subject: Use spaces instead of tabs --- pygments/lexers/_csound_builtins.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/pygments/lexers/_csound_builtins.py b/pygments/lexers/_csound_builtins.py index 856daa29..bdd8df37 100644 --- a/pygments/lexers/_csound_builtins.py +++ b/pygments/lexers/_csound_builtins.py @@ -8,22 +8,22 @@ """ # Opcodes in Csound 6.11.0 at commit 25b2e8e53bc924526eaad34e0768a5e866638e94 using -# python -c " -# import re, subprocess -# output = subprocess.Popen(['csound', '--list-opcodes0'], stderr=subprocess.PIPE).communicate()[1] -# opcodes = output[re.search(r'^\d+ opcodes$', output, re.M).end():re.search(r'^(?:WARNING|Usage):', output, re.M).start()].split() -# output = subprocess.Popen(['csound', '--list-opcodes2'], stderr=subprocess.PIPE).communicate()[1] -# all_opcodes = output[re.search(r'^\d+ opcodes$', output, re.M).end():re.search(r'^(?:WARNING|Usage):', output, re.M).start()].split() -# deprecated_opcodes = [opcode for opcode in all_opcodes if opcode not in opcodes] -# print '''OPCODES = set(\''' -# {} -# \'''.split()) -# -# DEPRECATED_OPCODES = set(\''' -# {} -# \'''.split()) -# '''.format('\n'.join(opcodes), '\n'.join(deprecated_opcodes)) -# " +# python -c " +# import re, subprocess +# output = subprocess.Popen(['csound', '--list-opcodes0'], stderr=subprocess.PIPE).communicate()[1] +# opcodes = output[re.search(r'^\d+ opcodes$', output, re.M).end():re.search(r'^(?:WARNING|Usage):', output, re.M).start()].split() +# output = subprocess.Popen(['csound', '--list-opcodes2'], stderr=subprocess.PIPE).communicate()[1] +# all_opcodes = output[re.search(r'^\d+ opcodes$', output, re.M).end():re.search(r'^(?:WARNING|Usage):', output, re.M).start()].split() +# deprecated_opcodes = [opcode for opcode in all_opcodes if opcode not in opcodes] +# print '''OPCODES = set(\''' +# {} +# \'''.split()) +# +# DEPRECATED_OPCODES = set(\''' +# {} +# \'''.split()) +# '''.format('\n'.join(opcodes), '\n'.join(deprecated_opcodes)) +# " # except for # cggoto csound.com/docs/manual/cggoto.html # cigoto csound.com/docs/manual/cigoto.html -- cgit v1.2.1 From ead0ddfe75a8ae8e051d3c4cbc780a472e107654 Mon Sep 17 00:00:00 2001 From: Brandon Cook Date: Fri, 11 May 2018 11:28:11 -0700 Subject: added Slurm script parser --- pygments/lexers/_mapping.py | 1 + pygments/lexers/shell.py | 22 +++++++++++++++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py index b48ee1d1..4368b27e 100644 --- a/pygments/lexers/_mapping.py +++ b/pygments/lexers/_mapping.py @@ -380,6 +380,7 @@ LEXERS = { 'ShenLexer': ('pygments.lexers.lisp', 'Shen', ('shen',), ('*.shen',), ('text/x-shen', 'application/x-shen')), 'SilverLexer': ('pygments.lexers.verification', 'Silver', ('silver',), ('*.sil', '*.vpr'), ()), 'SlimLexer': ('pygments.lexers.webmisc', 'Slim', ('slim',), ('*.slim',), ('text/x-slim',)), + 'SlurmBashLexer': ('pygments.lexers.shell', 'Slurm', ('slurm', 'sbatch'), ('*.sl',), ('application/x-sh', 'application/x-shellscript')), 'SmaliLexer': ('pygments.lexers.dalvik', 'Smali', ('smali',), ('*.smali',), ('text/smali',)), 'SmalltalkLexer': ('pygments.lexers.smalltalk', 'Smalltalk', ('smalltalk', 'squeak', 'st'), ('*.st',), ('text/x-smalltalk',)), 'SmartyLexer': ('pygments.lexers.templates', 'Smarty', ('smarty',), ('*.tpl',), ('application/x-smarty',)), diff --git a/pygments/lexers/shell.py b/pygments/lexers/shell.py index ceb6f14d..8b7989a3 100644 --- a/pygments/lexers/shell.py +++ b/pygments/lexers/shell.py @@ -19,7 +19,7 @@ from pygments.util import shebang_matches __all__ = ['BashLexer', 'BashSessionLexer', 'TcshLexer', 'BatchLexer', - 'MSDOSSessionLexer', 'PowerShellLexer', + 'SlurmBashLexer', 'MSDOSSessionLexer', 'PowerShellLexer', 'PowerShellSessionLexer', 'TcshSessionLexer', 'FishShellLexer'] line_re = re.compile('.*?\n') @@ -126,6 +126,26 @@ class BashLexer(RegexLexer): return 0.2 +class SlurmBashLexer(BashLexer): + """ + Lexer for (ba|k|z|)sh Slurm scripts. + """ + + name = 'Slurm' + aliases = ['slurm', 'sbatch'] + filenames = ['*.sl'] + EXTRA_KEYWORDS = {'srun'} + + def get_tokens_unprocessed(self, text): + for index, token, value in BashLexer.get_tokens_unprocessed(self, text): + print(index, token, value) + if token is Text and value in self.EXTRA_KEYWORDS: + yield index, Name.Builtin, value + elif token is Comment.Single and 'SBATCH' in value: + yield index, Keyword.Pseudo, value + else: + yield index, token, value + class ShellSessionBaseLexer(Lexer): """ Base lexer for simplistic shell sessions. -- cgit v1.2.1 From 51cc817c44c907cd5db1357195d91c626cd63bdc Mon Sep 17 00:00:00 2001 From: Thomas Braun Date: Mon, 14 May 2018 11:12:21 +0200 Subject: Igor Pro lexer: Raise copyright year --- pygments/lexers/igor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pygments/lexers/igor.py b/pygments/lexers/igor.py index 1a21fe87..fe02b4ae 100644 --- a/pygments/lexers/igor.py +++ b/pygments/lexers/igor.py @@ -5,7 +5,7 @@ Lexers for Igor Pro. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2018 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ -- cgit v1.2.1 From 7316e2bb033596c513eaece1274b69d581caa133 Mon Sep 17 00:00:00 2001 From: Thomas Braun Date: Mon, 14 May 2018 11:18:11 +0200 Subject: Igor Pro lexer: Update for Igor Pro 8 --- pygments/lexers/igor.py | 359 +++++++++++++++++++++++++++++++++--------------- 1 file changed, 248 insertions(+), 111 deletions(-) diff --git a/pygments/lexers/igor.py b/pygments/lexers/igor.py index fe02b4ae..e2e2cdfa 100644 --- a/pygments/lexers/igor.py +++ b/pygments/lexers/igor.py @@ -49,51 +49,71 @@ class IgorLexer(RegexLexer): ) operations = ( 'Abort', 'AddFIFOData', 'AddFIFOVectData', 'AddMovieAudio', 'AddMovieFrame', - 'AdoptFiles', 'APMath', 'Append', 'AppendImage', 'AppendLayoutObject', - 'AppendMatrixContour', 'AppendText', 'AppendToGizmo', 'AppendToGraph', - 'AppendToLayout', 'AppendToTable', 'AppendXYZContour', 'AutoPositionWindow', - 'BackgroundInfo', 'Beep', 'BoundingBall', 'BoxSmooth', 'BrowseURL', 'BuildMenu', - 'Button', 'cd', 'Chart', 'CheckBox', 'CheckDisplayed', 'ChooseColor', 'Close', - 'CloseHelp', 'CloseMovie', 'CloseProc', 'ColorScale', 'ColorTab2Wave', - 'Concatenate', 'ControlBar', 'ControlInfo', 'ControlUpdate', - 'ConvertGlobalStringTextEncoding', 'ConvexHull', 'Convolve', 'CopyFile', - 'CopyFolder', 'CopyScales', 'Correlate', 'CreateAliasShortcut', 'CreateBrowser', - 'Cross', 'CtrlBackground', 'CtrlFIFO', 'CtrlNamedBackground', 'Cursor', - 'CurveFit', 'CustomControl', 'CWT', 'Debugger', 'DebuggerOptions', 'DefaultFont', - 'DefaultGuiControls', 'DefaultGuiFont', 'DefaultTextEncoding', 'DefineGuide', - 'DelayUpdate', 'DeleteAnnotations', 'DeleteFile', 'DeleteFolder', 'DeletePoints', - 'Differentiate', 'dir', 'Display', 'DisplayHelpTopic', 'DisplayProcedure', - 'DoAlert', 'DoIgorMenu', 'DoUpdate', 'DoWindow', 'DoXOPIdle', 'DPSS', - 'DrawAction', 'DrawArc', 'DrawBezier', 'DrawLine', 'DrawOval', 'DrawPICT', - 'DrawPoly', 'DrawRect', 'DrawRRect', 'DrawText', 'DrawUserShape', 'DSPDetrend', - 'DSPPeriodogram', 'Duplicate', 'DuplicateDataFolder', 'DWT', 'EdgeStats', 'Edit', - 'ErrorBars', 'EstimatePeakSizes', 'Execute', 'ExecuteScriptText', + 'AddWavesToBoxPlot', 'AddWavesToViolinPlot', 'AdoptFiles', 'APMath', 'Append', + 'AppendBoxPlot', 'AppendImage', 'AppendLayoutObject', 'AppendMatrixContour', + 'AppendText', 'AppendToGizmo', 'AppendToGraph', 'AppendToLayout', 'AppendToTable', + 'AppendViolinPlot', 'AppendXYZContour', 'AutoPositionWindow', + 'AxonTelegraphFindServers', 'BackgroundInfo', 'Beep', 'BoundingBall', 'BoxSmooth', + 'BrowseURL', 'BuildMenu', 'Button', 'cd', 'Chart', 'CheckBox', 'CheckDisplayed', + 'ChooseColor', 'Close', 'CloseHelp', 'CloseMovie', 'CloseProc', 'ColorScale', + 'ColorTab2Wave', 'Concatenate', 'ControlBar', 'ControlInfo', 'ControlUpdate', + 'ConvertGlobalStringTextEncoding', 'ConvexHull', 'Convolve', 'CopyDimLabels', + 'CopyFile', 'CopyFolder', 'CopyScales', 'Correlate', 'CreateAliasShortcut', + 'CreateBrowser', 'Cross', 'CtrlBackground', 'CtrlFIFO', 'CtrlNamedBackground', + 'Cursor', 'CurveFit', 'CustomControl', 'CWT', 'DAQmx_AI_SetupReader', + 'DAQmx_AO_SetOutputs', 'DAQmx_CTR_CountEdges', 'DAQmx_CTR_OutputPulse', + 'DAQmx_CTR_Period', 'DAQmx_CTR_PulseWidth', 'DAQmx_DIO_Config', + 'DAQmx_DIO_WriteNewData', 'DAQmx_Scan', 'DAQmx_WaveformGen', 'Debugger', + 'DebuggerOptions', 'DefaultFont', 'DefaultGuiControls', 'DefaultGuiFont', + 'DefaultTextEncoding', 'DefineGuide', 'DelayUpdate', 'DeleteAnnotations', + 'DeleteFile', 'DeleteFolder', 'DeletePoints', 'Differentiate', 'dir', 'Display', + 'DisplayHelpTopic', 'DisplayProcedure', 'DoAlert', 'DoIgorMenu', 'DoUpdate', + 'DoWindow', 'DoXOPIdle', 'DPSS', 'DrawAction', 'DrawArc', 'DrawBezier', + 'DrawLine', 'DrawOval', 'DrawPICT', 'DrawPoly', 'DrawRect', 'DrawRRect', + 'DrawText', 'DrawUserShape', 'DSPDetrend', 'DSPPeriodogram', 'Duplicate', + 'DuplicateDataFolder', 'DWT', 'EdgeStats', 'Edit', 'ErrorBars', + 'EstimatePeakSizes', 'Execute', 'ExecuteScriptText', 'ExperimentInfo', 'ExperimentModified', 'ExportGizmo', 'Extract', 'FastGaussTransform', 'FastOp', - 'FBinRead', 'FBinWrite', 'FFT', 'FIFOStatus', 'FIFO2Wave', 'FilterFIR', + 'FBinRead', 'FBinWrite', 'FFT', 'FGetPos', 'FIFOStatus', 'FIFO2Wave', 'FilterFIR', 'FilterIIR', 'FindAPeak', 'FindContour', 'FindDuplicates', 'FindLevel', 'FindLevels', 'FindPeak', 'FindPointsInPoly', 'FindRoots', 'FindSequence', - 'FindValue', 'FPClustering', 'fprintf', 'FReadLine', 'FSetPos', 'FStatus', - 'FTPCreateDirectory', 'FTPDelete', 'FTPDownload', 'FTPUpload', 'FuncFit', - 'FuncFitMD', 'GBLoadWave', 'GetAxis', 'GetCamera', 'GetFileFolderInfo', + 'FindValue', 'FMaxFlat', 'FPClustering', 'fprintf', 'FReadLine', 'FSetPos', + 'FStatus', 'FTPCreateDirectory', 'FTPDelete', 'FTPDownload', 'FTPUpload', + 'FuncFit', 'FuncFitMD', 'GBLoadWave', 'GetAxis', 'GetCamera', 'GetFileFolderInfo', 'GetGizmo', 'GetLastUserMenuInfo', 'GetMarquee', 'GetMouse', 'GetSelection', - 'GetWindow', 'GPIBReadBinaryWave2', 'GPIBReadBinary2', 'GPIBReadWave2', - 'GPIBRead2', 'GPIBWriteBinaryWave2', 'GPIBWriteBinary2', 'GPIBWriteWave2', - 'GPIBWrite2', 'GPIB2', 'GraphNormal', 'GraphWaveDraw', 'GraphWaveEdit', 'Grep', - 'GroupBox', 'Hanning', 'HDF5CloseFile', 'HDF5CloseGroup', 'HDF5ConvertColors', - 'HDF5CreateFile', 'HDF5CreateGroup', 'HDF5CreateLink', 'HDF5Dump', - 'HDF5DumpErrors', 'HDF5DumpState', 'HDF5ListAttributes', 'HDF5ListGroup', - 'HDF5LoadData', 'HDF5LoadGroup', 'HDF5LoadImage', 'HDF5OpenFile', 'HDF5OpenGroup', - 'HDF5SaveData', 'HDF5SaveGroup', 'HDF5SaveImage', 'HDF5TestOperation', - 'HDF5UnlinkObject', 'HideIgorMenus', 'HideInfo', 'HideProcedures', 'HideTools', - 'HilbertTransform', 'Histogram', 'ICA', 'IFFT', 'ImageAnalyzeParticles', - 'ImageBlend', 'ImageBoundaryToMask', 'ImageEdgeDetection', 'ImageFileInfo', - 'ImageFilter', 'ImageFocus', 'ImageFromXYZ', 'ImageGenerateROIMask', 'ImageGLCM', + 'GetWindow', 'GISCreateVectorLayer', 'GISGetRasterInfo', + 'GISGetRegisteredFileInfo', 'GISGetVectorLayerInfo', 'GISLoadRasterData', + 'GISLoadVectorData', 'GISRasterizeVectorData', 'GISRegisterFile', + 'GISTransformCoords', 'GISUnRegisterFile', 'GISWriteFieldData', + 'GISWriteGeometryData', 'GISWriteRaster', 'GPIBReadBinaryWave2', + 'GPIBReadBinary2', 'GPIBReadWave2', 'GPIBRead2', 'GPIBWriteBinaryWave2', + 'GPIBWriteBinary2', 'GPIBWriteWave2', 'GPIBWrite2', 'GPIB2', 'GraphNormal', + 'GraphWaveDraw', 'GraphWaveEdit', 'Grep', 'GroupBox', 'Hanning', 'HDFInfo', + 'HDFReadImage', 'HDFReadSDS', 'HDFReadVset', 'HDF5CloseFile', 'HDF5CloseGroup', + 'HDF5ConvertColors', 'HDF5CreateFile', 'HDF5CreateGroup', 'HDF5CreateLink', + 'HDF5Dump', 'HDF5DumpErrors', 'HDF5DumpState', 'HDF5FlushFile', + 'HDF5ListAttributes', 'HDF5ListGroup', 'HDF5LoadData', 'HDF5LoadGroup', + 'HDF5LoadImage', 'HDF5OpenFile', 'HDF5OpenGroup', 'HDF5SaveData', 'HDF5SaveGroup', + 'HDF5SaveImage', 'HDF5TestOperation', 'HDF5UnlinkObject', 'HideIgorMenus', + 'HideInfo', 'HideProcedures', 'HideTools', 'HilbertTransform', 'Histogram', 'ICA', + 'IFFT', 'ImageAnalyzeParticles', 'ImageBlend', 'ImageBoundaryToMask', + 'ImageComposite', 'ImageEdgeDetection', 'ImageFileInfo', 'ImageFilter', + 'ImageFocus', 'ImageFromXYZ', 'ImageGenerateROIMask', 'ImageGLCM', 'ImageHistModification', 'ImageHistogram', 'ImageInterpolate', 'ImageLineProfile', 'ImageLoad', 'ImageMorphology', 'ImageRegistration', 'ImageRemoveBackground', 'ImageRestore', 'ImageRotate', 'ImageSave', 'ImageSeedFill', 'ImageSkeleton3d', 'ImageSnake', 'ImageStats', 'ImageThreshold', 'ImageTransform', 'ImageUnwrapPhase', 'ImageWindow', 'IndexSort', 'InsertPoints', 'Integrate', 'IntegrateODE', 'Integrate2D', 'Interpolate2', 'Interpolate3D', 'Interp3DPath', + 'ITCCloseAll2', 'ITCCloseDevice2', 'ITCConfigAllChannels2', + 'ITCConfigChannelReset2', 'ITCConfigChannelUpload2', 'ITCConfigChannel2', + 'ITCFIFOAvailableAll2', 'ITCFIFOAvailable2', 'ITCGetAllChannelsConfig2', + 'ITCGetChannelConfig2', 'ITCGetCurrentDevice2', 'ITCGetDeviceInfo2', + 'ITCGetDevices2', 'ITCGetErrorString2', 'ITCGetSerialNumber2', 'ITCGetState2', + 'ITCGetVersions2', 'ITCInitialize2', 'ITCOpenDevice2', 'ITCReadADC2', + 'ITCReadDigital2', 'ITCReadTimer2', 'ITCSelectDevice2', 'ITCSetDAC2', + 'ITCSetGlobals2', 'ITCSetModes2', 'ITCSetState2', 'ITCStartAcq2', 'ITCStopAcq2', + 'ITCUpdateFIFOPositionAll2', 'ITCUpdateFIFOPosition2', 'ITCWriteDigital2', 'JCAMPLoadWave', 'JointHistogram', 'KillBackground', 'KillControl', 'KillDataFolder', 'KillFIFO', 'KillFreeAxis', 'KillPath', 'KillPICTs', 'KillStrings', 'KillVariables', 'KillWaves', 'KillWindow', 'KMeans', 'Label', @@ -104,39 +124,48 @@ class IgorLexer(RegexLexer): 'MatrixFilter', 'MatrixGaussJ', 'MatrixGLM', 'MatrixInverse', 'MatrixLinearSolve', 'MatrixLinearSolveTD', 'MatrixLLS', 'MatrixLUBkSub', 'MatrixLUD', 'MatrixLUDTD', 'MatrixMultiply', 'MatrixOP', 'MatrixSchur', 'MatrixSolve', 'MatrixSVBkSub', - 'MatrixSVD', 'MatrixTranspose', 'MeasureStyledText', 'MLLoadWave', 'Modify', - 'ModifyBrowser', 'ModifyCamera', 'ModifyContour', 'ModifyControl', - 'ModifyControlList', 'ModifyFreeAxis', 'ModifyGizmo', 'ModifyGraph', - 'ModifyImage', 'ModifyLayout', 'ModifyPanel', 'ModifyTable', 'ModifyWaterfall', - 'MoveDataFolder', 'MoveFile', 'MoveFolder', 'MoveString', 'MoveSubwindow', - 'MoveVariable', 'MoveWave', 'MoveWindow', 'MultiTaperPSD', - 'MultiThreadingControl', 'NeuralNetworkRun', 'NeuralNetworkTrain', 'NewCamera', - 'NewDataFolder', 'NewFIFO', 'NewFIFOChan', 'NewFreeAxis', 'NewGizmo', 'NewImage', - 'NewLayout', 'NewMovie', 'NewNotebook', 'NewPanel', 'NewPath', 'NewWaterfall', - 'NI4882', 'Note', 'Notebook', 'NotebookAction', 'Open', 'OpenHelp', - 'OpenNotebook', 'Optimize', 'ParseOperationTemplate', 'PathInfo', 'PauseForUser', - 'PauseUpdate', 'PCA', 'PlayMovie', 'PlayMovieAction', 'PlaySound', - 'PopupContextualMenu', 'PopupMenu', 'Preferences', 'PrimeFactors', 'Print', - 'printf', 'PrintGraphs', 'PrintLayout', 'PrintNotebook', 'PrintSettings', - 'PrintTable', 'Project', 'PulseStats', 'PutScrapText', 'pwd', 'Quit', - 'RatioFromNumber', 'Redimension', 'Remove', 'RemoveContour', 'RemoveFromGizmo', - 'RemoveFromGraph', 'RemoveFromLayout', 'RemoveFromTable', 'RemoveImage', - 'RemoveLayoutObjects', 'RemovePath', 'Rename', 'RenameDataFolder', 'RenamePath', - 'RenamePICT', 'RenameWindow', 'ReorderImages', 'ReorderTraces', 'ReplaceText', - 'ReplaceWave', 'Resample', 'ResumeUpdate', 'Reverse', 'Rotate', 'Save', - 'SaveData', 'SaveExperiment', 'SaveGraphCopy', 'SaveNotebook', + 'MatrixSVD', 'MatrixTranspose', 'MCC_FindServers', 'MeasureStyledText', + 'MFR_CheckForNewBricklets', + 'MFR_CloseResultFile', 'MFR_CreateOverviewTable', 'MFR_GetBrickletCount', + 'MFR_GetBrickletData', 'MFR_GetBrickletDeployData', 'MFR_GetBrickletMetaData', + 'MFR_GetBrickletRawData', 'MFR_GetReportTemplate', 'MFR_GetResultFileMetaData', + 'MFR_GetResultFileName', 'MFR_GetVernissageVersion', 'MFR_GetVersion', + 'MFR_GetXOPErrorMessage', 'MFR_OpenResultFile', + 'MLLoadWave', 'Modify', 'ModifyBoxPlot', 'ModifyBrowser', 'ModifyCamera', + 'ModifyContour', 'ModifyControl', 'ModifyControlList', 'ModifyFreeAxis', + 'ModifyGizmo', 'ModifyGraph', 'ModifyImage', 'ModifyLayout', 'ModifyPanel', + 'ModifyTable', 'ModifyViolinPlot', 'ModifyWaterfall', 'MoveDataFolder', + 'MoveFile', 'MoveFolder', 'MoveString', 'MoveSubwindow', 'MoveVariable', + 'MoveWave', 'MoveWindow', 'MultiTaperPSD', 'MultiThreadingControl', + 'NC_CloseFile', 'NC_DumpErrors', 'NC_Inquire', 'NC_ListAttributes', + 'NC_ListObjects', 'NC_LoadData', 'NC_OpenFile', 'NeuralNetworkRun', + 'NeuralNetworkTrain', 'NewCamera', 'NewDataFolder', 'NewFIFO', 'NewFIFOChan', + 'NewFreeAxis', 'NewGizmo', 'NewImage', 'NewLayout', 'NewMovie', 'NewNotebook', + 'NewPanel', 'NewPath', 'NewWaterfall', 'NILoadWave', 'NI4882', 'Note', 'Notebook', + 'NotebookAction', 'Open', 'OpenHelp', 'OpenNotebook', 'Optimize', + 'ParseOperationTemplate', 'PathInfo', 'PauseForUser', 'PauseUpdate', 'PCA', + 'PlayMovie', 'PlayMovieAction', 'PlaySound', 'PopupContextualMenu', 'PopupMenu', + 'Preferences', 'PrimeFactors', 'Print', 'printf', 'PrintGraphs', 'PrintLayout', + 'PrintNotebook', 'PrintSettings', 'PrintTable', 'Project', 'PulseStats', + 'PutScrapText', 'pwd', 'Quit', 'RatioFromNumber', 'Redimension', 'Remez', + 'Remove', 'RemoveContour', 'RemoveFromGizmo', 'RemoveFromGraph', + 'RemoveFromLayout', 'RemoveFromTable', 'RemoveImage', 'RemoveLayoutObjects', + 'RemovePath', 'Rename', 'RenameDataFolder', 'RenamePath', 'RenamePICT', + 'RenameWindow', 'ReorderImages', 'ReorderTraces', 'ReplaceText', 'ReplaceWave', + 'Resample', 'ResumeUpdate', 'Reverse', 'Rotate', 'Save', 'SaveData', + 'SaveExperiment', 'SaveGizmoCopy', 'SaveGraphCopy', 'SaveNotebook', 'SavePackagePreferences', 'SavePICT', 'SaveTableCopy', 'SetActiveSubwindow', 'SetAxis', 'SetBackground', 'SetDashPattern', 'SetDataFolder', 'SetDimLabel', - 'SetDrawEnv', 'SetDrawLayer', 'SetFileFolderInfo', 'SetFormula', 'SetIgorHook', - 'SetIgorMenuMode', 'SetIgorOption', 'SetMarquee', 'SetProcessSleep', - 'SetRandomSeed', 'SetScale', 'SetVariable', 'SetWaveLock', 'SetWaveTextEncoding', - 'SetWindow', 'ShowIgorMenus', 'ShowInfo', 'ShowTools', 'Silent', 'Sleep', - 'Slider', 'Smooth', 'SmoothCustom', 'Sort', 'SortColumns', 'SoundInRecord', - 'SoundInSet', 'SoundInStartChart', 'SoundInStatus', 'SoundInStopChart', - 'SoundLoadWave', 'SoundSaveWave', 'SphericalInterpolate', 'SphericalTriangulate', - 'SplitString', 'SplitWave', 'sprintf', 'sscanf', 'Stack', 'StackWindows', - 'StatsAngularDistanceTest', 'StatsANOVA1Test', 'StatsANOVA2NRTest', - 'StatsANOVA2RMTest', 'StatsANOVA2Test', 'StatsChiTest', + 'SetDrawEnv', 'SetDrawLayer', 'SetFileFolderInfo', 'SetFormula', 'SetIdlePeriod', + 'SetIgorHook', 'SetIgorMenuMode', 'SetIgorOption', 'SetMarquee', + 'SetProcessSleep', 'SetRandomSeed', 'SetScale', 'SetVariable', 'SetWaveLock', + 'SetWaveTextEncoding', 'SetWindow', 'ShowIgorMenus', 'ShowInfo', 'ShowTools', + 'Silent', 'Sleep', 'Slider', 'Smooth', 'SmoothCustom', 'Sort', 'SortColumns', + 'SoundInRecord', 'SoundInSet', 'SoundInStartChart', 'SoundInStatus', + 'SoundInStopChart', 'SoundLoadWave', 'SoundSaveWave', 'SphericalInterpolate', + 'SphericalTriangulate', 'SplitString', 'SplitWave', 'sprintf', 'SQLHighLevelOp', + 'sscanf', 'Stack', 'StackWindows', 'StatsAngularDistanceTest', 'StatsANOVA1Test', + 'StatsANOVA2NRTest', 'StatsANOVA2RMTest', 'StatsANOVA2Test', 'StatsChiTest', 'StatsCircularCorrelationTest', 'StatsCircularMeans', 'StatsCircularMoments', 'StatsCircularTwoSampleTest', 'StatsCochranTest', 'StatsContingencyTable', 'StatsDIPTest', 'StatsDunnettTest', 'StatsFriedmanTest', 'StatsFTest', @@ -148,23 +177,30 @@ class IgorLexer(RegexLexer): 'StatsSignTest', 'StatsSRTest', 'StatsTTest', 'StatsTukeyTest', 'StatsVariancesTest', 'StatsWatsonUSquaredTest', 'StatsWatsonWilliamsTest', 'StatsWheelerWatsonTest', 'StatsWilcoxonRankTest', 'StatsWRCorrelationTest', - 'String', 'StructGet', 'StructPut', 'SumDimension', 'SumSeries', 'TabControl', - 'Tag', 'TextBox', 'ThreadGroupPutDF', 'ThreadStart', 'Tile', 'TileWindows', + 'STFT', 'String', 'StructFill', 'StructGet', 'StructPut', 'SumDimension', + 'SumSeries', 'TabControl', 'Tag', 'TDMLoadData', 'TDMSaveData', 'TextBox', + 'ThreadGroupPutDF', 'ThreadStart', 'TickWavesFromAxis', 'Tile', 'TileWindows', 'TitleBox', 'ToCommandLine', 'ToolsGrid', 'Triangulate3d', 'Unwrap', 'URLRequest', 'ValDisplay', 'Variable', 'VDTClosePort2', 'VDTGetPortList2', 'VDTGetStatus2', 'VDTOpenPort2', 'VDTOperationsPort2', 'VDTReadBinaryWave2', 'VDTReadBinary2', 'VDTReadHexWave2', 'VDTReadHex2', 'VDTReadWave2', 'VDTRead2', 'VDTTerminalPort2', 'VDTWriteBinaryWave2', 'VDTWriteBinary2', 'VDTWriteHexWave2', 'VDTWriteHex2', - 'VDTWriteWave2', 'VDTWrite2', 'VDT2', 'WaveMeanStdv', 'WaveStats', + 'VDTWriteWave2', 'VDTWrite2', 'VDT2', 'VISAControl', 'VISARead', 'VISAReadBinary', + 'VISAReadBinaryWave', 'VISAReadWave', 'VISAWrite', 'VISAWriteBinary', + 'VISAWriteBinaryWave', 'VISAWriteWave', 'WaveMeanStdv', 'WaveStats', 'WaveTransform', 'wfprintf', 'WignerTransform', 'WindowFunction', 'XLLoadWave' ) functions = ( 'abs', 'acos', 'acosh', 'AddListItem', 'AiryA', 'AiryAD', 'AiryB', 'AiryBD', 'alog', 'AnnotationInfo', 'AnnotationList', 'area', 'areaXY', 'asin', 'asinh', - 'atan', 'atanh', 'atan2', 'AxisInfo', 'AxisList', 'AxisValFromPixel', 'Besseli', - 'Besselj', 'Besselk', 'Bessely', 'beta', 'betai', 'BinarySearch', - 'BinarySearchInterp', 'binomial', 'binomialln', 'binomialNoise', 'cabs', - 'CaptureHistory', 'CaptureHistoryStart', 'ceil', 'cequal', 'char2num', + 'atan', 'atanh', 'atan2', 'AxisInfo', 'AxisList', 'AxisValFromPixel', + 'AxonTelegraphAGetDataNum', 'AxonTelegraphAGetDataString', + 'AxonTelegraphAGetDataStruct', 'AxonTelegraphGetDataNum', + 'AxonTelegraphGetDataString', 'AxonTelegraphGetDataStruct', + 'AxonTelegraphGetTimeoutMs', 'AxonTelegraphSetTimeoutMs', 'Base64Decode', + 'Base64Encode', 'Besseli', 'Besselj', 'Besselk', 'Bessely', 'beta', 'betai', + 'BinarySearch', 'BinarySearchInterp', 'binomial', 'binomialln', 'binomialNoise', + 'cabs', 'CaptureHistory', 'CaptureHistoryStart', 'ceil', 'cequal', 'char2num', 'chebyshev', 'chebyshevU', 'CheckName', 'ChildWindowList', 'CleanupName', 'cmplx', 'cmpstr', 'conj', 'ContourInfo', 'ContourNameList', 'ContourNameToWaveRef', 'ContourZ', 'ControlNameList', 'ConvertTextEncoding', 'cos', 'cosh', @@ -172,37 +208,70 @@ class IgorLexer(RegexLexer): 'CreationDate', 'csc', 'csch', 'CsrInfo', 'CsrWave', 'CsrWaveRef', 'CsrXWave', 'CsrXWaveRef', 'CTabList', 'DataFolderDir', 'DataFolderExists', 'DataFolderRefsEqual', 'DataFolderRefStatus', 'date', 'datetime', 'DateToJulian', - 'date2secs', 'Dawson', 'DDERequestString', 'defined', 'deltax', 'digamma', - 'dilogarithm', 'DimDelta', 'DimOffset', 'DimSize', 'ei', 'enoise', 'equalWaves', - 'erf', 'erfc', 'erfcw', 'exists', 'exp', 'ExpConvExp', 'ExpConvExpFit', - 'ExpConvExpFitBL', 'ExpConvExpFit1Shape', 'ExpConvExpFit1ShapeBL', 'ExpGauss', - 'ExpGaussFit', 'ExpGaussFitBL', 'ExpGaussFit1Shape', 'ExpGaussFit1ShapeBL', - 'expInt', 'expIntegralE1', 'expNoise', 'factorial', 'fakedata', 'faverage', - 'faverageXY', 'FetchURL', 'FindDimLabel', 'FindListItem', 'floor', 'FontList', - 'FontSizeHeight', 'FontSizeStringWidth', 'FresnelCos', 'FresnelSin', + 'date2secs', 'Dawson', 'defined', 'deltax', 'digamma', 'dilogarithm', 'DimDelta', + 'DimOffset', 'DimSize', 'ei', 'enoise', 'equalWaves', 'erf', 'erfc', 'erfcw', + 'exists', 'exp', 'expInt', 'expIntegralE1', 'expNoise', 'factorial', 'Faddeeva', + 'fakedata', 'faverage', 'faverageXY', 'fDAQmx_AI_GetReader', + 'fDAQmx_AO_UpdateOutputs', 'fDAQmx_ConnectTerminals', 'fDAQmx_CTR_Finished', + 'fDAQmx_CTR_IsFinished', 'fDAQmx_CTR_IsPulseFinished', 'fDAQmx_CTR_ReadCounter', + 'fDAQmx_CTR_ReadWithOptions', 'fDAQmx_CTR_SetPulseFrequency', 'fDAQmx_CTR_Start', + 'fDAQmx_DeviceNames', 'fDAQmx_DIO_Finished', 'fDAQmx_DIO_PortWidth', + 'fDAQmx_DIO_Read', 'fDAQmx_DIO_Write', 'fDAQmx_DisconnectTerminals', + 'fDAQmx_ErrorString', 'fDAQmx_ExternalCalDate', 'fDAQmx_NumAnalogInputs', + 'fDAQmx_NumAnalogOutputs', 'fDAQmx_NumCounters', 'fDAQmx_NumDIOPorts', + 'fDAQmx_ReadChan', 'fDAQmx_ReadNamedChan', 'fDAQmx_ResetDevice', + 'fDAQmx_ScanGetAvailable', 'fDAQmx_ScanGetNextIndex', 'fDAQmx_ScanStart', + 'fDAQmx_ScanStop', 'fDAQmx_ScanWait', 'fDAQmx_ScanWaitWithTimeout', + 'fDAQmx_SelfCalDate', 'fDAQmx_SelfCalibration', 'fDAQmx_WaveformStart', + 'fDAQmx_WaveformStop', 'fDAQmx_WF_IsFinished', 'fDAQmx_WF_WaitUntilFinished', + 'fDAQmx_WriteChan', 'FetchURL', 'FindDimLabel', 'FindListItem', 'floor', + 'FontList', 'FontSizeHeight', 'FontSizeStringWidth', 'FresnelCos', 'FresnelSin', 'FuncRefInfo', 'FunctionInfo', 'FunctionList', 'FunctionPath', 'gamma', 'gammaEuler', 'gammaInc', 'gammaNoise', 'gammln', 'gammp', 'gammq', 'Gauss', - 'GaussFit', 'GaussFitBL', 'GaussFit1Width', 'GaussFit1WidthBL', 'Gauss1D', - 'Gauss2D', 'gcd', 'GetBrowserLine', 'GetBrowserSelection', 'GetDataFolder', - 'GetDataFolderDFR', 'GetDefaultFont', 'GetDefaultFontSize', 'GetDefaultFontStyle', - 'GetDimLabel', 'GetEnvironmentVariable', 'GetErrMessage', 'GetFormula', - 'GetIndependentModuleName', 'GetIndexedObjName', 'GetIndexedObjNameDFR', - 'GetKeyState', 'GetRTErrMessage', 'GetRTError', 'GetRTLocation', 'GetRTLocInfo', - 'GetRTStackInfo', 'GetScrapText', 'GetUserData', 'GetWavesDataFolder', - 'GetWavesDataFolderDFR', 'GizmoInfo', 'GizmoScale', 'gnoise', 'GrepList', - 'GrepString', 'GuideInfo', 'GuideNameList', 'Hash', 'hcsr', 'HDF5AttributeInfo', + 'Gauss1D', 'Gauss2D', 'gcd', 'GetBrowserLine', 'GetBrowserSelection', + 'GetDataFolder', 'GetDataFolderDFR', 'GetDefaultFont', 'GetDefaultFontSize', + 'GetDefaultFontStyle', 'GetDimLabel', 'GetEnvironmentVariable', 'GetErrMessage', + 'GetFormula', 'GetIndependentModuleName', 'GetIndexedObjName', + 'GetIndexedObjNameDFR', 'GetKeyState', 'GetRTErrMessage', 'GetRTError', + 'GetRTLocation', 'GetRTLocInfo', 'GetRTStackInfo', 'GetScrapText', 'GetUserData', + 'GetWavesDataFolder', 'GetWavesDataFolderDFR', 'GISGetAllFileFormats', + 'GISSRefsAreEqual', 'GizmoInfo', 'GizmoScale', 'gnoise', 'GrepList', 'GrepString', + 'GuideInfo', 'GuideNameList', 'Hash', 'hcsr', 'HDF5AttributeInfo', 'HDF5DatasetInfo', 'HDF5LibraryInfo', 'HDF5TypeInfo', 'hermite', 'hermiteGauss', 'HyperGNoise', 'HyperGPFQ', 'HyperG0F1', 'HyperG1F1', 'HyperG2F1', 'IgorInfo', 'IgorVersion', 'imag', 'ImageInfo', 'ImageNameList', 'ImageNameToWaveRef', - 'IndependentModuleList', 'IndexedDir', 'IndexedFile', 'Inf', 'Integrate1D', - 'interp', 'Interp2D', 'Interp3D', 'inverseERF', 'inverseERFC', 'ItemsInList', - 'JacobiCn', 'JacobiSn', 'JulianToDate', 'Laguerre', 'LaguerreA', 'LaguerreGauss', - 'LambertW', 'LayoutInfo', 'leftx', 'LegendreA', 'limit', 'ListMatch', - 'ListToTextWave', 'ListToWaveRefWave', 'ln', 'log', 'logNormalNoise', - 'LorentzianFit', 'LorentzianFitBL', 'LorentzianFit1Width', - 'LorentzianFit1WidthBL', 'lorentzianNoise', 'LowerStr', 'MacroList', 'magsqr', - 'MandelbrotPoint', 'MarcumQ', 'MatrixCondition', 'MatrixDet', 'MatrixDot', - 'MatrixRank', 'MatrixTrace', 'max', 'mean', 'median', 'min', 'mod', 'ModDate', + 'IndependentModuleList', 'IndexedDir', 'IndexedFile', 'IndexToScale', 'Inf', + 'Integrate1D', 'interp', 'Interp2D', 'Interp3D', 'inverseERF', 'inverseERFC', + 'ItemsInList', 'JacobiCn', 'JacobiSn', 'JulianToDate', 'Laguerre', 'LaguerreA', + 'LaguerreGauss', 'LambertW', 'LayoutInfo', 'leftx', 'LegendreA', 'limit', + 'ListMatch', 'ListToTextWave', 'ListToWaveRefWave', 'ln', 'log', 'logNormalNoise', + 'lorentzianNoise', 'LowerStr', 'MacroList', 'magsqr', 'MandelbrotPoint', + 'MarcumQ', 'MatrixCondition', 'MatrixDet', 'MatrixDot', 'MatrixRank', + 'MatrixTrace', 'max', 'MCC_AutoBridgeBal', 'MCC_AutoFastComp', + 'MCC_AutoPipetteOffset', 'MCC_AutoSlowComp', 'MCC_AutoWholeCellComp', + 'MCC_GetBridgeBalEnable', 'MCC_GetBridgeBalResist', 'MCC_GetFastCompCap', + 'MCC_GetFastCompTau', 'MCC_GetHolding', 'MCC_GetHoldingEnable', 'MCC_GetMode', + 'MCC_GetNeutralizationCap', 'MCC_GetNeutralizationEnable', + 'MCC_GetOscKillerEnable', 'MCC_GetPipetteOffset', 'MCC_GetPrimarySignalGain', + 'MCC_GetPrimarySignalHPF', 'MCC_GetPrimarySignalLPF', 'MCC_GetRsCompBandwidth', + 'MCC_GetRsCompCorrection', 'MCC_GetRsCompEnable', 'MCC_GetRsCompPrediction', + 'MCC_GetSecondarySignalGain', 'MCC_GetSecondarySignalLPF', 'MCC_GetSlowCompCap', + 'MCC_GetSlowCompTau', 'MCC_GetSlowCompTauX20Enable', + 'MCC_GetSlowCurrentInjEnable', 'MCC_GetSlowCurrentInjLevel', + 'MCC_GetSlowCurrentInjSetlTime', 'MCC_GetWholeCellCompCap', + 'MCC_GetWholeCellCompEnable', 'MCC_GetWholeCellCompResist', + 'MCC_SelectMultiClamp700B', 'MCC_SetBridgeBalEnable', 'MCC_SetBridgeBalResist', + 'MCC_SetFastCompCap', 'MCC_SetFastCompTau', 'MCC_SetHolding', + 'MCC_SetHoldingEnable', 'MCC_SetMode', 'MCC_SetNeutralizationCap', + 'MCC_SetNeutralizationEnable', 'MCC_SetOscKillerEnable', 'MCC_SetPipetteOffset', + 'MCC_SetPrimarySignalGain', 'MCC_SetPrimarySignalHPF', 'MCC_SetPrimarySignalLPF', + 'MCC_SetRsCompBandwidth', 'MCC_SetRsCompCorrection', 'MCC_SetRsCompEnable', + 'MCC_SetRsCompPrediction', 'MCC_SetSecondarySignalGain', + 'MCC_SetSecondarySignalLPF', 'MCC_SetSlowCompCap', 'MCC_SetSlowCompTau', + 'MCC_SetSlowCompTauX20Enable', 'MCC_SetSlowCurrentInjEnable', + 'MCC_SetSlowCurrentInjLevel', 'MCC_SetSlowCurrentInjSetlTime', 'MCC_SetTimeoutMs', + 'MCC_SetWholeCellCompCap', 'MCC_SetWholeCellCompEnable', + 'MCC_SetWholeCellCompResist', 'mean', 'median', 'min', 'mod', 'ModDate', 'MPFXEMGPeak', 'MPFXExpConvExpPeak', 'MPFXGaussPeak', 'MPFXLorenzianPeak', 'MPFXVoigtPeak', 'NameOfWave', 'NaN', 'NewFreeDataFolder', 'NewFreeWave', 'norm', 'NormalizeUnicode', 'note', 'NumberByKey', 'numpnts', 'numtype', @@ -217,9 +286,30 @@ class IgorLexer(RegexLexer): 'SelectNumber', 'SelectString', 'SetEnvironmentVariable', 'sign', 'sin', 'sinc', 'sinh', 'sinIntegral', 'SortList', 'SpecialCharacterInfo', 'SpecialCharacterList', 'SpecialDirPath', 'SphericalBessJ', 'SphericalBessJD', 'SphericalBessY', - 'SphericalBessYD', 'SphericalHarmonics', 'sqrt', 'StartMSTimer', 'StatsBetaCDF', - 'StatsBetaPDF', 'StatsBinomialCDF', 'StatsBinomialPDF', 'StatsCauchyCDF', - 'StatsCauchyPDF', 'StatsChiCDF', 'StatsChiPDF', 'StatsCMSSDCDF', + 'SphericalBessYD', 'SphericalHarmonics', 'SQLAllocHandle', 'SQLAllocStmt', + 'SQLBinaryWavesToTextWave', 'SQLBindCol', 'SQLBindParameter', 'SQLBrowseConnect', + 'SQLBulkOperations', 'SQLCancel', 'SQLCloseCursor', 'SQLColAttributeNum', + 'SQLColAttributeStr', 'SQLColumnPrivileges', 'SQLColumns', 'SQLConnect', + 'SQLDataSources', 'SQLDescribeCol', 'SQLDescribeParam', 'SQLDisconnect', + 'SQLDriverConnect', 'SQLDrivers', 'SQLEndTran', 'SQLError', 'SQLExecDirect', + 'SQLExecute', 'SQLFetch', 'SQLFetchScroll', 'SQLForeignKeys', 'SQLFreeConnect', + 'SQLFreeEnv', 'SQLFreeHandle', 'SQLFreeStmt', 'SQLGetConnectAttrNum', + 'SQLGetConnectAttrStr', 'SQLGetCursorName', 'SQLGetDataNum', 'SQLGetDataStr', + 'SQLGetDescFieldNum', 'SQLGetDescFieldStr', 'SQLGetDescRec', 'SQLGetDiagFieldNum', + 'SQLGetDiagFieldStr', 'SQLGetDiagRec', 'SQLGetEnvAttrNum', 'SQLGetEnvAttrStr', + 'SQLGetFunctions', 'SQLGetInfoNum', 'SQLGetInfoStr', 'SQLGetStmtAttrNum', + 'SQLGetStmtAttrStr', 'SQLGetTypeInfo', 'SQLMoreResults', 'SQLNativeSql', + 'SQLNumParams', 'SQLNumResultCols', 'SQLNumResultRowsIfKnown', + 'SQLNumRowsFetched', 'SQLParamData', 'SQLPrepare', 'SQLPrimaryKeys', + 'SQLProcedureColumns', 'SQLProcedures', 'SQLPutData', 'SQLReinitialize', + 'SQLRowCount', 'SQLSetConnectAttrNum', 'SQLSetConnectAttrStr', 'SQLSetCursorName', + 'SQLSetDescFieldNum', 'SQLSetDescFieldStr', 'SQLSetDescRec', 'SQLSetEnvAttrNum', + 'SQLSetEnvAttrStr', 'SQLSetPos', 'SQLSetStmtAttrNum', 'SQLSetStmtAttrStr', + 'SQLSpecialColumns', 'SQLStatistics', 'SQLTablePrivileges', 'SQLTables', + 'SQLTextWaveToBinaryWaves', 'SQLTextWaveTo2DBinaryWave', 'SQLUpdateBoundValues', + 'SQLXOPCheckState', 'SQL2DBinaryWaveToTextWave', 'sqrt', 'StartMSTimer', + 'StatsBetaCDF', 'StatsBetaPDF', 'StatsBinomialCDF', 'StatsBinomialPDF', + 'StatsCauchyCDF', 'StatsCauchyPDF', 'StatsChiCDF', 'StatsChiPDF', 'StatsCMSSDCDF', 'StatsCorrelation', 'StatsDExpCDF', 'StatsDExpPDF', 'StatsErlangCDF', 'StatsErlangPDF', 'StatsErrorPDF', 'StatsEValueCDF', 'StatsEValuePDF', 'StatsExpCDF', 'StatsExpPDF', 'StatsFCDF', 'StatsFPDF', 'StatsFriedmanCDF', @@ -250,19 +340,66 @@ class IgorLexer(RegexLexer): 'StopMSTimer', 'StringByKey', 'stringCRC', 'StringFromList', 'StringList', 'stringmatch', 'strlen', 'strsearch', 'StrVarOrDefault', 'str2num', 'StudentA', 'StudentT', 'sum', 'SVAR_Exists', 'TableInfo', 'TagVal', 'TagWaveRef', 'tan', - 'tanh', 'TextEncodingCode', 'TextEncodingName', 'TextFile', 'ThreadGroupCreate', + 'tango_close_device', 'tango_command_inout', 'tango_compute_image_proj', + 'tango_get_dev_attr_list', 'tango_get_dev_black_box', 'tango_get_dev_cmd_list', + 'tango_get_dev_status', 'tango_get_dev_timeout', 'tango_get_error_stack', + 'tango_open_device', 'tango_ping_device', 'tango_read_attribute', + 'tango_read_attributes', 'tango_reload_dev_interface', + 'tango_resume_attr_monitor', 'tango_set_attr_monitor_period', + 'tango_set_dev_timeout', 'tango_start_attr_monitor', 'tango_stop_attr_monitor', + 'tango_suspend_attr_monitor', 'tango_write_attribute', 'tango_write_attributes', + 'tanh', 'TDMAddChannel', 'TDMAddGroup', 'TDMAppendDataValues', + 'TDMAppendDataValuesTime', 'TDMChannelPropertyExists', 'TDMCloseChannel', + 'TDMCloseFile', 'TDMCloseGroup', 'TDMCreateChannelProperty', 'TDMCreateFile', + 'TDMCreateFileProperty', 'TDMCreateGroupProperty', 'TDMFilePropertyExists', + 'TDMGetChannelPropertyNames', 'TDMGetChannelPropertyNum', + 'TDMGetChannelPropertyStr', 'TDMGetChannelPropertyTime', + 'TDMGetChannelPropertyType', 'TDMGetChannels', 'TDMGetChannelStringPropertyLen', + 'TDMGetDataType', 'TDMGetDataValues', 'TDMGetDataValuesTime', + 'TDMGetFilePropertyNames', 'TDMGetFilePropertyNum', 'TDMGetFilePropertyStr', + 'TDMGetFilePropertyTime', 'TDMGetFilePropertyType', 'TDMGetFileStringPropertyLen', + 'TDMGetGroupPropertyNames', 'TDMGetGroupPropertyNum', 'TDMGetGroupPropertyStr', + 'TDMGetGroupPropertyTime', 'TDMGetGroupPropertyType', 'TDMGetGroups', + 'TDMGetGroupStringPropertyLen', 'TDMGetLibraryErrorDescription', + 'TDMGetNumChannelProperties', 'TDMGetNumChannels', 'TDMGetNumDataValues', + 'TDMGetNumFileProperties', 'TDMGetNumGroupProperties', 'TDMGetNumGroups', + 'TDMGroupPropertyExists', 'TDMOpenFile', 'TDMOpenFileEx', 'TDMRemoveChannel', + 'TDMRemoveGroup', 'TDMReplaceDataValues', 'TDMReplaceDataValuesTime', + 'TDMSaveFile', 'TDMSetChannelPropertyNum', 'TDMSetChannelPropertyStr', + 'TDMSetChannelPropertyTime', 'TDMSetDataValues', 'TDMSetDataValuesTime', + 'TDMSetFilePropertyNum', 'TDMSetFilePropertyStr', 'TDMSetFilePropertyTime', + 'TDMSetGroupPropertyNum', 'TDMSetGroupPropertyStr', 'TDMSetGroupPropertyTime', + 'TextEncodingCode', 'TextEncodingName', 'TextFile', 'ThreadGroupCreate', 'ThreadGroupGetDF', 'ThreadGroupGetDFR', 'ThreadGroupRelease', 'ThreadGroupWait', 'ThreadProcessorCount', 'ThreadReturnValue', 'ticks', 'time', 'TraceFromPixel', - 'TraceInfo', 'TraceNameList', 'TraceNameToWaveRef', 'trunc', 'UniqueName', - 'UnPadString', 'UnsetEnvironmentVariable', 'UpperStr', 'URLDecode', 'URLEncode', - 'VariableList', 'Variance', 'vcsr', 'Voigt', 'VoigtFit', 'VoigtFitBL', - 'VoigtFit1Shape', 'VoigtFit1ShapeBL', 'VoigtFit1Shape1Width', - 'VoigtFit1Shape1WidthBL', 'VoigtFunc', 'WaveCRC', 'WaveDims', 'WaveExists', - 'WaveInfo', 'WaveList', 'WaveMax', 'WaveMin', 'WaveName', 'WaveRefIndexed', + 'TraceInfo', 'TraceNameList', 'TraceNameToWaveRef', 'TrimString', 'trunc', + 'UniqueName', 'UnPadString', 'UnsetEnvironmentVariable', 'UpperStr', 'URLDecode', + 'URLEncode', 'VariableList', 'Variance', 'vcsr', 'viAssertIntrSignal', + 'viAssertTrigger', 'viAssertUtilSignal', 'viClear', 'viClose', 'viDisableEvent', + 'viDiscardEvents', 'viEnableEvent', 'viFindNext', 'viFindRsrc', 'viGetAttribute', + 'viGetAttributeString', 'viGpibCommand', 'viGpibControlATN', 'viGpibControlREN', + 'viGpibPassControl', 'viGpibSendIFC', 'viIn8', 'viIn16', 'viIn32', 'viLock', + 'viMapAddress', 'viMapTrigger', 'viMemAlloc', 'viMemFree', 'viMoveIn8', + 'viMoveIn16', 'viMoveIn32', 'viMoveOut8', 'viMoveOut16', 'viMoveOut32', 'viOpen', + 'viOpenDefaultRM', 'viOut8', 'viOut16', 'viOut32', 'viPeek8', 'viPeek16', + 'viPeek32', 'viPoke8', 'viPoke16', 'viPoke32', 'viRead', 'viReadSTB', + 'viSetAttribute', 'viSetAttributeString', 'viStatusDesc', 'viTerminate', + 'viUnlock', 'viUnmapAddress', 'viUnmapTrigger', 'viUsbControlIn', + 'viUsbControlOut', 'viVxiCommandQuery', 'viWaitOnEvent', 'viWrite', 'VoigtFunc', + 'VoigtPeak', 'WaveCRC', 'WaveDims', 'WaveExists', 'WaveHash', 'WaveInfo', + 'WaveList', 'WaveMax', 'WaveMin', 'WaveName', 'WaveRefIndexed', 'WaveRefIndexedDFR', 'WaveRefsEqual', 'WaveRefWaveToList', 'WaveTextEncoding', 'WaveType', 'WaveUnits', 'WhichListItem', 'WinList', 'WinName', 'WinRecreation', - 'WinType', 'WMFindWholeWord', 'WNoise', 'xcsr', 'XWaveName', 'XWaveRefFromTrace', - 'x2pnt', 'zcsr', 'ZernikeR', 'zeta' + 'WinType', 'wnoise', 'xcsr', 'XWaveName', 'XWaveRefFromTrace', 'x2pnt', 'zcsr', + 'ZernikeR', 'zeromq_client_connect', 'zeromq_client_connect', + 'zeromq_client_recv', 'zeromq_client_recv', 'zeromq_client_send', + 'zeromq_client_send', 'zeromq_handler_start', 'zeromq_handler_start', + 'zeromq_handler_stop', 'zeromq_handler_stop', 'zeromq_server_bind', + 'zeromq_server_bind', 'zeromq_server_recv', 'zeromq_server_recv', + 'zeromq_server_send', 'zeromq_server_send', 'zeromq_set', 'zeromq_set', + 'zeromq_stop', 'zeromq_stop', 'zeromq_test_callfunction', + 'zeromq_test_callfunction', 'zeromq_test_serializeWave', + 'zeromq_test_serializeWave', 'zeta' ) tokens = { -- cgit v1.2.1 From ee6237e744729b3d2d58112dba6fff4bad4b9bfa Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Sat, 19 May 2018 16:12:37 +0200 Subject: Improving lexing of YAML mappings. --- pygments/lexers/data.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pygments/lexers/data.py b/pygments/lexers/data.py index 296366c2..abe87160 100644 --- a/pygments/lexers/data.py +++ b/pygments/lexers/data.py @@ -232,6 +232,9 @@ class YamlLexer(ExtendedRegexLexer): (r'[ ]*(?=#|$)', something(Text), '#pop'), # whitespaces separating tokens (r'[ ]+', Text), + # key with colon + (r'([^,:?\[\]{}\n]+)(:)(?=[ ]|$)', + bygroups(Name.Tag, set_indent(Punctuation, implicit=True))), # tags, anchors and aliases, include('descriptors'), # block collections and scalars @@ -308,6 +311,9 @@ class YamlLexer(ExtendedRegexLexer): # a flow mapping indicated by '{' and '}' 'flow-mapping': [ + # key with colon + (r'([^,:?\[\]{}\n]+)(:)(?=[ ]|$)', + bygroups(Name.Tag, Punctuation)), # include flow collection rules include('flow-collection'), # the closing indicator -- cgit v1.2.1 From b9ae9547d99e4219df253ddeb61598c26b0f3032 Mon Sep 17 00:00:00 2001 From: Andreas Kloeckner Date: Tue, 19 Jun 2018 14:05:00 -0500 Subject: Fix remaining 'DeprecationWarning: invalid escape sequence' occurrences in lexer files --- pygments/lexers/actionscript.py | 4 ++-- pygments/lexers/html.py | 32 ++++++++++++++++---------------- pygments/lexers/iolang.py | 2 +- pygments/lexers/javascript.py | 6 +++--- pygments/lexers/jvm.py | 6 +++--- pygments/lexers/lisp.py | 8 ++++---- pygments/lexers/perl.py | 16 ++++++++-------- pygments/lexers/php.py | 4 ++-- pygments/lexers/python.py | 20 ++++++++++---------- pygments/lexers/ruby.py | 12 ++++++------ pygments/lexers/scripting.py | 6 +++--- pygments/lexers/webmisc.py | 10 +++++----- 12 files changed, 63 insertions(+), 63 deletions(-) diff --git a/pygments/lexers/actionscript.py b/pygments/lexers/actionscript.py index 84607e68..fc3b90cd 100644 --- a/pygments/lexers/actionscript.py +++ b/pygments/lexers/actionscript.py @@ -125,7 +125,7 @@ class ActionScript3Lexer(RegexLexer): 'text/actionscript3'] identifier = r'[$a-zA-Z_]\w*' - typeidentifier = identifier + '(?:\.<\w+>)?' + typeidentifier = identifier + r'(?:\.<\w+>)?' flags = re.DOTALL | re.MULTILINE tokens = { @@ -232,7 +232,7 @@ class MxmlLexer(RegexLexer): (r'/?\s*>', Name.Tag, '#pop'), ], 'attr': [ - ('\s+', Text), + (r'\s+', Text), ('".*?"', String, '#pop'), ("'.*?'", String, '#pop'), (r'[^\s>]+', String, '#pop'), diff --git a/pygments/lexers/html.py b/pygments/lexers/html.py index 73f020fa..091379ce 100644 --- a/pygments/lexers/html.py +++ b/pygments/lexers/html.py @@ -220,7 +220,7 @@ class XmlLexer(RegexLexer): (r'/?\s*>', Name.Tag, '#pop'), ], 'attr': [ - ('\s+', Text), + (r'\s+', Text), ('".*?"', String, '#pop'), ("'.*?'", String, '#pop'), (r'[^\s>]+', String, '#pop'), @@ -313,7 +313,7 @@ class HamlLexer(ExtendedRegexLexer): include('css'), (r'%[\w:-]+', Name.Tag, 'tag'), (r'!!!' + _dot + r'*\n', Name.Namespace, '#pop'), - (r'(/)(\[' + _dot + '*?\])(' + _dot + r'*\n)', + (r'(/)(\[' + _dot + r'*?\])(' + _dot + r'*\n)', bygroups(Comment, Comment.Special, Comment), '#pop'), (r'/' + _dot + r'*\n', _starts_block(Comment, 'html-comment-block'), @@ -330,8 +330,8 @@ class HamlLexer(ExtendedRegexLexer): 'tag': [ include('css'), - (r'\{(,\n|' + _dot + ')*?\}', using(RubyLexer)), - (r'\[' + _dot + '*?\]', using(RubyLexer)), + (r'\{(,\n|' + _dot + r')*?\}', using(RubyLexer)), + (r'\[' + _dot + r'*?\]', using(RubyLexer)), (r'\(', Text, 'html-attributes'), (r'/[ \t]*\n', Punctuation, '#pop:2'), (r'[<>]{1,2}(?=[ \t=])', Punctuation), @@ -340,7 +340,7 @@ class HamlLexer(ExtendedRegexLexer): 'plain': [ (r'([^#\n]|#[^{\n]|(\\\\)*\\#\{)+', Text), - (r'(#\{)(' + _dot + '*?)(\})', + (r'(#\{)(' + _dot + r'*?)(\})', bygroups(String.Interpol, using(RubyLexer), String.Interpol)), (r'\n', Text, 'root'), ], @@ -373,7 +373,7 @@ class HamlLexer(ExtendedRegexLexer): 'filter-block': [ (r'([^#\n]|#[^{\n]|(\\\\)*\\#\{)+', Name.Decorator), - (r'(#\{)(' + _dot + '*?)(\})', + (r'(#\{)(' + _dot + r'*?)(\})', bygroups(String.Interpol, using(RubyLexer), String.Interpol)), (r'\n', Text, 'root'), ], @@ -422,7 +422,7 @@ class ScamlLexer(ExtendedRegexLexer): include('css'), (r'%[\w:-]+', Name.Tag, 'tag'), (r'!!!' + _dot + r'*\n', Name.Namespace, '#pop'), - (r'(/)(\[' + _dot + '*?\])(' + _dot + r'*\n)', + (r'(/)(\[' + _dot + r'*?\])(' + _dot + r'*\n)', bygroups(Comment, Comment.Special, Comment), '#pop'), (r'/' + _dot + r'*\n', _starts_block(Comment, 'html-comment-block'), @@ -442,8 +442,8 @@ class ScamlLexer(ExtendedRegexLexer): 'tag': [ include('css'), - (r'\{(,\n|' + _dot + ')*?\}', using(ScalaLexer)), - (r'\[' + _dot + '*?\]', using(ScalaLexer)), + (r'\{(,\n|' + _dot + r')*?\}', using(ScalaLexer)), + (r'\[' + _dot + r'*?\]', using(ScalaLexer)), (r'\(', Text, 'html-attributes'), (r'/[ \t]*\n', Punctuation, '#pop:2'), (r'[<>]{1,2}(?=[ \t=])', Punctuation), @@ -452,7 +452,7 @@ class ScamlLexer(ExtendedRegexLexer): 'plain': [ (r'([^#\n]|#[^{\n]|(\\\\)*\\#\{)+', Text), - (r'(#\{)(' + _dot + '*?)(\})', + (r'(#\{)(' + _dot + r'*?)(\})', bygroups(String.Interpol, using(ScalaLexer), String.Interpol)), (r'\n', Text, 'root'), ], @@ -485,7 +485,7 @@ class ScamlLexer(ExtendedRegexLexer): 'filter-block': [ (r'([^#\n]|#[^{\n]|(\\\\)*\\#\{)+', Name.Decorator), - (r'(#\{)(' + _dot + '*?)(\})', + (r'(#\{)(' + _dot + r'*?)(\})', bygroups(String.Interpol, using(ScalaLexer), String.Interpol)), (r'\n', Text, 'root'), ], @@ -530,7 +530,7 @@ class PugLexer(ExtendedRegexLexer): 'content': [ include('css'), (r'!!!' + _dot + r'*\n', Name.Namespace, '#pop'), - (r'(/)(\[' + _dot + '*?\])(' + _dot + r'*\n)', + (r'(/)(\[' + _dot + r'*?\])(' + _dot + r'*\n)', bygroups(Comment, Comment.Special, Comment), '#pop'), (r'/' + _dot + r'*\n', _starts_block(Comment, 'html-comment-block'), @@ -551,8 +551,8 @@ class PugLexer(ExtendedRegexLexer): 'tag': [ include('css'), - (r'\{(,\n|' + _dot + ')*?\}', using(ScalaLexer)), - (r'\[' + _dot + '*?\]', using(ScalaLexer)), + (r'\{(,\n|' + _dot + r')*?\}', using(ScalaLexer)), + (r'\[' + _dot + r'*?\]', using(ScalaLexer)), (r'\(', Text, 'html-attributes'), (r'/[ \t]*\n', Punctuation, '#pop:2'), (r'[<>]{1,2}(?=[ \t=])', Punctuation), @@ -561,7 +561,7 @@ class PugLexer(ExtendedRegexLexer): 'plain': [ (r'([^#\n]|#[^{\n]|(\\\\)*\\#\{)+', Text), - (r'(#\{)(' + _dot + '*?)(\})', + (r'(#\{)(' + _dot + r'*?)(\})', bygroups(String.Interpol, using(ScalaLexer), String.Interpol)), (r'\n', Text, 'root'), ], @@ -594,7 +594,7 @@ class PugLexer(ExtendedRegexLexer): 'filter-block': [ (r'([^#\n]|#[^{\n]|(\\\\)*\\#\{)+', Name.Decorator), - (r'(#\{)(' + _dot + '*?)(\})', + (r'(#\{)(' + _dot + r'*?)(\})', bygroups(String.Interpol, using(ScalaLexer), String.Interpol)), (r'\n', Text, 'root'), ], diff --git a/pygments/lexers/iolang.py b/pygments/lexers/iolang.py index bbc17faf..26f44e27 100644 --- a/pygments/lexers/iolang.py +++ b/pygments/lexers/iolang.py @@ -49,7 +49,7 @@ class IoLexer(RegexLexer): # names (r'(Object|list|List|Map|args|Sequence|Coroutine|File)\b', Name.Builtin), - ('[a-zA-Z_]\w*', Name), + (r'[a-zA-Z_]\w*', Name), # numbers (r'(\d+\.?\d*|\d*\.\d+)([eE][+-]?[0-9]+)?', Number.Float), (r'\d+', Number.Integer) diff --git a/pygments/lexers/javascript.py b/pygments/lexers/javascript.py index 862535c9..6b69a86a 100644 --- a/pygments/lexers/javascript.py +++ b/pygments/lexers/javascript.py @@ -535,8 +535,8 @@ class TypeScriptLexer(RegexLexer): } def analyse_text(text): - if re.search('^(import.+(from\s+)?["\']|' - '(export\s*)?(interface|class|function)\s+)', + if re.search(r'^(import.+(from\s+)?["\']|' + r'(export\s*)?(interface|class|function)\s+)', text, re.MULTILINE): return 1.0 @@ -1015,7 +1015,7 @@ class ObjectiveJLexer(RegexLexer): } def analyse_text(text): - if re.search('^\s*@import\s+[<"]', text, re.MULTILINE): + if re.search(r'^\s*@import\s+[<"]', text, re.MULTILINE): # special directive found in most Objective-J files return True return False diff --git a/pygments/lexers/jvm.py b/pygments/lexers/jvm.py index f4392839..3843fdac 100644 --- a/pygments/lexers/jvm.py +++ b/pygments/lexers/jvm.py @@ -257,7 +257,7 @@ class ScalaLexer(RegexLexer): u'\ua77d-\ua77e\ua780\ua782\ua784\ua786\ua78b\uff21-\uff3a]') idrest = u'%s(?:%s|[0-9])*(?:(?<=_)%s)?' % (letter, letter, op) - letter_letter_digit = u'%s(?:%s|\d)*' % (letter, letter) + letter_letter_digit = u'%s(?:%s|\\d)*' % (letter, letter) tokens = { 'root': [ @@ -689,7 +689,7 @@ class IokeLexer(RegexLexer): # functions (u'(generateMatchMethod|aliasMethod|\u03bb|\u028E|fnx|fn|method|' u'dmacro|dlecro|syntax|macro|dlecrox|lecrox|lecro|syntax)' - u'(?![\w!:?])', Name.Function), + u'(?![\\w!:?])', Name.Function), # Numbers (r'-?0[xX][0-9a-fA-F]+', Number.Hex), @@ -1258,7 +1258,7 @@ class GoloLexer(RegexLexer): (r'-?\d[\d_]*L', Number.Integer.Long), (r'-?\d[\d_]*', Number.Integer), - ('`?[a-zA-Z_][\w$]*', Name), + (r'`?[a-zA-Z_][\w$]*', Name), (r'@[a-zA-Z_][\w$.]*', Name.Decorator), (r'"""', String, combined('stringescape', 'triplestring')), diff --git a/pygments/lexers/lisp.py b/pygments/lexers/lisp.py index e258c347..b4c26659 100644 --- a/pygments/lexers/lisp.py +++ b/pygments/lexers/lisp.py @@ -139,7 +139,7 @@ class SchemeLexer(RegexLexer): (r"(?<=#\()" + valid_name, Name.Variable), # highlight the builtins - ("(?<=\()(%s)" % '|'.join(re.escape(entry) + ' ' for entry in builtins), + (r"(?<=\()(%s)" % '|'.join(re.escape(entry) + ' ' for entry in builtins), Name.Builtin), # the remaining functions @@ -321,7 +321,7 @@ class CommonLispLexer(RegexLexer): (r'#\d+#', Operator), # read-time comment - (r'#+nil' + terminated + '\s*\(', Comment.Preproc, 'commented-form'), + (r'#+nil' + terminated + r'\s*\(', Comment.Preproc, 'commented-form'), # read-time conditional (r'#[+-]', Operator), @@ -333,7 +333,7 @@ class CommonLispLexer(RegexLexer): (r'(t|nil)' + terminated, Name.Constant), # functions and variables - (r'\*' + symbol + '\*', Name.Variable.Global), + (r'\*' + symbol + r'\*', Name.Variable.Global), (symbol, Name.Variable), # parentheses @@ -2154,7 +2154,7 @@ class EmacsLispLexer(RegexLexer): (r'(t|nil)' + terminated, Name.Constant), # functions and variables - (r'\*' + symbol + '\*', Name.Variable.Global), + (r'\*' + symbol + r'\*', Name.Variable.Global), (symbol, Name.Variable), # parentheses diff --git a/pygments/lexers/perl.py b/pygments/lexers/perl.py index db5a9361..27e3cc79 100644 --- a/pygments/lexers/perl.py +++ b/pygments/lexers/perl.py @@ -208,7 +208,7 @@ class PerlLexer(RegexLexer): def analyse_text(text): if shebang_matches(text, r'perl'): return True - if re.search('(?:my|our)\s+[$@%(]', text): + if re.search(r'(?:my|our)\s+[$@%(]', text): return 0.9 @@ -226,7 +226,7 @@ class Perl6Lexer(ExtendedRegexLexer): mimetypes = ['text/x-perl6', 'application/x-perl6'] flags = re.MULTILINE | re.DOTALL | re.UNICODE - PERL6_IDENTIFIER_RANGE = "['\w:-]" + PERL6_IDENTIFIER_RANGE = r"['\w:-]" PERL6_KEYWORDS = ( 'BEGIN', 'CATCH', 'CHECK', 'CONTROL', 'END', 'ENTER', 'FIRST', 'INIT', @@ -495,7 +495,7 @@ class Perl6Lexer(ExtendedRegexLexer): (r'^=.*?\n\s*?\n', Comment.Multiline), (r'(regex|token|rule)(\s*' + PERL6_IDENTIFIER_RANGE + '+:sym)', bygroups(Keyword, Name), 'token-sym-brackets'), - (r'(regex|token|rule)(?!' + PERL6_IDENTIFIER_RANGE + ')(\s*' + PERL6_IDENTIFIER_RANGE + '+)?', + (r'(regex|token|rule)(?!' + PERL6_IDENTIFIER_RANGE + r')(\s*' + PERL6_IDENTIFIER_RANGE + '+)?', bygroups(Keyword, Name), 'pre-token'), # deal with a special case in the Perl 6 grammar (role q { ... }) (r'(role)(\s+)(q)(\s*)', bygroups(Keyword, Text, Name, Text)), @@ -591,21 +591,21 @@ class Perl6Lexer(ExtendedRegexLexer): rating = False # check for my/our/has declarations - if re.search("(?:my|our|has)\s+(?:" + Perl6Lexer.PERL6_IDENTIFIER_RANGE + - "+\s+)?[$@%&(]", text): + if re.search(r"(?:my|our|has)\s+(?:" + Perl6Lexer.PERL6_IDENTIFIER_RANGE + + r"+\s+)?[$@%&(]", text): rating = 0.8 saw_perl_decl = True for line in lines: line = re.sub('#.*', '', line) - if re.match('^\s*$', line): + if re.match(r'^\s*$', line): continue # match v6; use v6; use v6.0; use v6.0.0; - if re.match('^\s*(?:use\s+)?v6(?:\.\d(?:\.\d)?)?;', line): + if re.match(r'^\s*(?:use\s+)?v6(?:\.\d(?:\.\d)?)?;', line): return True # match class, module, role, enum, grammar declarations - class_decl = re.match('^\s*(?:(?Pmy|our)\s+)?(?:module|class|role|enum|grammar)', line) + class_decl = re.match(r'^\s*(?:(?Pmy|our)\s+)?(?:module|class|role|enum|grammar)', line) if class_decl: if saw_perl_decl or class_decl.group('scope') is not None: return True diff --git a/pygments/lexers/php.py b/pygments/lexers/php.py index f618b5fd..f959fb1f 100644 --- a/pygments/lexers/php.py +++ b/pygments/lexers/php.py @@ -173,7 +173,7 @@ class PhpLexer(RegexLexer): r'finally)\b', Keyword), (r'(true|false|null)\b', Keyword.Constant), include('magicconstants'), - (r'\$\{\$+' + _ident_inner + '\}', Name.Variable), + (r'\$\{\$+' + _ident_inner + r'\}', Name.Variable), (r'\$+' + _ident_inner, Name.Variable), (_ident_inner, Name.Other), (r'(\d+\.\d*|\d*\.\d+)(e[+-]?[0-9]+)?', Number.Float), @@ -214,7 +214,7 @@ class PhpLexer(RegexLexer): (r'"', String.Double, '#pop'), (r'[^{$"\\]+', String.Double), (r'\\([nrt"$\\]|[0-7]{1,3}|x[0-9a-f]{1,2})', String.Escape), - (r'\$' + _ident_inner + '(\[\S+?\]|->' + _ident_inner + ')?', + (r'\$' + _ident_inner + r'(\[\S+?\]|->' + _ident_inner + ')?', String.Interpol), (r'(\{\$\{)(.*?)(\}\})', bygroups(String.Interpol, using(this, _startinline=True), diff --git a/pygments/lexers/python.py b/pygments/lexers/python.py index 390eafe8..f4a7404e 100644 --- a/pygments/lexers/python.py +++ b/pygments/lexers/python.py @@ -180,15 +180,15 @@ class PythonLexer(RegexLexer): ], 'name': [ (r'@[\w.]+', Name.Decorator), - ('[a-zA-Z_]\w*', Name), + (r'[a-zA-Z_]\w*', Name), ], 'funcname': [ include('magicfuncs'), - ('[a-zA-Z_]\w*', Name.Function, '#pop'), + (r'[a-zA-Z_]\w*', Name.Function, '#pop'), default('#pop'), ], 'classname': [ - ('[a-zA-Z_]\w*', Name.Class, '#pop') + (r'[a-zA-Z_]\w*', Name.Class, '#pop') ], 'import': [ (r'(?:[ \t]|\\\n)+', Text), @@ -265,10 +265,10 @@ class Python3Lexer(RegexLexer): '[hlL]?[E-GXc-giorsux%]', String.Interpol), # the new style '{}'.format(...) string formatting (r'\{' - '((\w+)((\.\w+)|(\[[^\]]+\]))*)?' # field name - '(\![sra])?' # conversion - '(\:(.?[<>=\^])?[-+ ]?#?0?(\d+)?,?(\.\d+)?[E-GXb-gnosx%]?)?' - '\}', String.Interpol), + r'((\w+)((\.\w+)|(\[[^\]]+\]))*)?' # field name + r'(\![sra])?' # conversion + r'(\:(.?[<>=\^])?[-+ ]?#?0?(\d+)?,?(\.\d+)?[E-GXb-gnosx%]?)?' + r'\}', String.Interpol), # backslashes, quotes and formatting signs must be parsed one at a time (r'[^\\\'"%{\n]+', ttype), @@ -671,10 +671,10 @@ class CythonLexer(RegexLexer): ], 'name': [ (r'@\w+', Name.Decorator), - ('[a-zA-Z_]\w*', Name), + (r'[a-zA-Z_]\w*', Name), ], 'funcname': [ - ('[a-zA-Z_]\w*', Name.Function, '#pop') + (r'[a-zA-Z_]\w*', Name.Function, '#pop') ], 'cdef': [ (r'(public|readonly|extern|api|inline)\b', Keyword.Reserved), @@ -691,7 +691,7 @@ class CythonLexer(RegexLexer): (r'.', Text), ], 'classname': [ - ('[a-zA-Z_]\w*', Name.Class, '#pop') + (r'[a-zA-Z_]\w*', Name.Class, '#pop') ], 'import': [ (r'(\s+)(as)(\s+)', bygroups(Text, Keyword, Text)), diff --git a/pygments/lexers/ruby.py b/pygments/lexers/ruby.py index fe750f1a..ce2fc7a7 100644 --- a/pygments/lexers/ruby.py +++ b/pygments/lexers/ruby.py @@ -403,8 +403,8 @@ class RubyConsoleLexer(Lexer): aliases = ['rbcon', 'irb'] mimetypes = ['text/x-ruby-shellsession'] - _prompt_re = re.compile('irb\([a-zA-Z_]\w*\):\d{3}:\d+[>*"\'] ' - '|>> |\?> ') + _prompt_re = re.compile(r'irb\([a-zA-Z_]\w*\):\d{3}:\d+[>*"\'] ' + r'|>> |\?> ') def get_tokens_unprocessed(self, text): rblexer = RubyLexer(**self.options) @@ -498,11 +498,11 @@ class FancyLexer(RegexLexer): (r'[a-zA-Z](\w|[-+?!=*/^><%])*:', Name.Function), # operators, must be below functions (r'[-+*/~,<>=&!?%^\[\].$]+', Operator), - ('[A-Z]\w*', Name.Constant), - ('@[a-zA-Z_]\w*', Name.Variable.Instance), - ('@@[a-zA-Z_]\w*', Name.Variable.Class), + (r'[A-Z]\w*', Name.Constant), + (r'@[a-zA-Z_]\w*', Name.Variable.Instance), + (r'@@[a-zA-Z_]\w*', Name.Variable.Class), ('@@?', Operator), - ('[a-zA-Z_]\w*', Name), + (r'[a-zA-Z_]\w*', Name), # numbers - / checks are necessary to avoid mismarking regexes, # see comment in RubyLexer (r'(0[oO]?[0-7]+(?:_[0-7]+)*)(\s*)([/?])?', diff --git a/pygments/lexers/scripting.py b/pygments/lexers/scripting.py index b3af606e..28c37db8 100644 --- a/pygments/lexers/scripting.py +++ b/pygments/lexers/scripting.py @@ -104,7 +104,7 @@ class LuaLexer(RegexLexer): (r'%s(?=%s*[.:])' % (_name, _s), Name.Class), (_name, Name.Function, '#pop'), # inline function - ('\(', Punctuation, '#pop'), + (r'\(', Punctuation, '#pop'), ], 'goto': [ @@ -696,8 +696,8 @@ class AppleScriptLexer(RegexLexer): (r'[-+]?\d+', Number.Integer), ], 'comment': [ - ('\(\*', Comment.Multiline, '#push'), - ('\*\)', Comment.Multiline, '#pop'), + (r'\(\*', Comment.Multiline, '#push'), + (r'\*\)', Comment.Multiline, '#pop'), ('[^*(]+', Comment.Multiline), ('[*(]', Comment.Multiline), ], diff --git a/pygments/lexers/webmisc.py b/pygments/lexers/webmisc.py index 712c8246..304c8e78 100644 --- a/pygments/lexers/webmisc.py +++ b/pygments/lexers/webmisc.py @@ -438,7 +438,7 @@ class XQueryLexer(ExtendedRegexLexer): ], 'varname': [ (r'\(:', Comment, 'comment'), - (r'(' + qname + ')(\()?', bygroups(Name, Punctuation), 'operator'), + (r'(' + qname + r')(\()?', bygroups(Name, Punctuation), 'operator'), ], 'singletype': [ include('whitespace'), @@ -643,9 +643,9 @@ class XQueryLexer(ExtendedRegexLexer): bygroups(Keyword.Declaration, Text, Keyword.Declaration, Text, Keyword.Declaration), 'operator'), (r'(declare)(\s+)(context)(\s+)(item)', bygroups(Keyword.Declaration, Text, Keyword.Declaration, Text, Keyword.Declaration), 'operator'), - (ncname + ':\*', Name, 'operator'), - ('\*:'+ncname, Name.Tag, 'operator'), - ('\*', Name.Tag, 'operator'), + (ncname + r':\*', Name, 'operator'), + (r'\*:'+ncname, Name.Tag, 'operator'), + (r'\*', Name.Tag, 'operator'), (stringdouble, String.Double, 'operator'), (stringsingle, String.Single, 'operator'), @@ -861,7 +861,7 @@ class QmlLexer(RegexLexer): class CirruLexer(RegexLexer): - """ + r""" Syntax rules of Cirru can be found at: http://cirru.org/ -- cgit v1.2.1 From 5927c118d3556f19199fe502c0ffcc8584652518 Mon Sep 17 00:00:00 2001 From: Miro Hron?ok Date: Wed, 4 Jul 2018 18:08:48 +0200 Subject: PEP 479: Raising StopIteration from a generator is now an error So we return instead. Fix needed for Python 3.7. Fixes https://bitbucket.org/birkenfeld/pygments-main/issues/1457 --- pygments/lexers/lisp.py | 4 ++-- pygments/lexers/sql.py | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/pygments/lexers/lisp.py b/pygments/lexers/lisp.py index e258c347..258916df 100644 --- a/pygments/lexers/lisp.py +++ b/pygments/lexers/lisp.py @@ -2327,13 +2327,13 @@ class ShenLexer(RegexLexer): token = Name.Function if token == Literal else token yield index, token, value - raise StopIteration + return def _process_signature(self, tokens): for index, token, value in tokens: if token == Literal and value == '}': yield index, Punctuation, value - raise StopIteration + return elif token in (Literal, Name.Function): token = Name.Variable if value.istitle() else Keyword.Type yield index, token, value diff --git a/pygments/lexers/sql.py b/pygments/lexers/sql.py index 7507c0fc..05af51ba 100644 --- a/pygments/lexers/sql.py +++ b/pygments/lexers/sql.py @@ -347,7 +347,10 @@ class PostgresConsoleLexer(Lexer): # Emit the output lines out_token = Generic.Output while 1: - line = next(lines) + try: + line = next(lines) + except StopIteration: + return mprompt = re_prompt.match(line) if mprompt is not None: # push the line back to have it processed by the prompt -- cgit v1.2.1 From b909c279a4cf4a93c5244e2c3046d769ba737dc0 Mon Sep 17 00:00:00 2001 From: Miro Hron?ok Date: Wed, 4 Jul 2018 18:31:40 +0200 Subject: Be more Pythonic, use a for instead of while 1: try: next --- pygments/lexers/sql.py | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/pygments/lexers/sql.py b/pygments/lexers/sql.py index 05af51ba..3f7dfdb8 100644 --- a/pygments/lexers/sql.py +++ b/pygments/lexers/sql.py @@ -308,14 +308,7 @@ class PostgresConsoleLexer(Lexer): # and continue until the end of command is detected curcode = '' insertions = [] - while 1: - try: - line = next(lines) - except StopIteration: - # allow the emission of partially collected items - # the repl loop will be broken below - break - + for line in lines: # Identify a shell prompt in case of psql commandline example if line.startswith('$') and not curcode: lexer = get_lexer_by_name('console', **self.options) @@ -346,11 +339,7 @@ class PostgresConsoleLexer(Lexer): # Emit the output lines out_token = Generic.Output - while 1: - try: - line = next(lines) - except StopIteration: - return + for line in lines: mprompt = re_prompt.match(line) if mprompt is not None: # push the line back to have it processed by the prompt @@ -366,6 +355,8 @@ class PostgresConsoleLexer(Lexer): yield (mmsg.start(2), out_token, mmsg.group(2)) else: yield (0, out_token, line) + else: + return class SqlLexer(RegexLexer): -- cgit v1.2.1 From bcd5d3e83bb2edc0cf61781ebc042008d48bab45 Mon Sep 17 00:00:00 2001 From: Jeffrey Arnold Date: Fri, 6 Jul 2018 11:52:43 -0700 Subject: Update Stan lexer Update Stan lexer to language version 2.17.0; add builtin-functions, new keywords, fix bug in highlighting numbers. --- pygments/lexers/_stan_builtins.py | 276 +++++++++++++++++++++----------------- pygments/lexers/modeling.py | 32 +++-- 2 files changed, 170 insertions(+), 138 deletions(-) diff --git a/pygments/lexers/_stan_builtins.py b/pygments/lexers/_stan_builtins.py index a189647a..7f1e0ce3 100644 --- a/pygments/lexers/_stan_builtins.py +++ b/pygments/lexers/_stan_builtins.py @@ -4,24 +4,23 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ This file contains the names of functions for Stan used by - ``pygments.lexers.math.StanLexer. This is for Stan language version 2.8.0. + ``pygments.lexers.math.StanLexer. This is for Stan language version 2.17.0. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2018 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ KEYWORDS = ( + 'break', + 'continue', 'else', 'for', 'if', 'in', - 'increment_log_prob', - 'integrate_ode', - 'lp__', 'print', 'reject', 'return', - 'while' + 'while', ) TYPES = ( @@ -35,18 +34,18 @@ TYPES = ( 'positive_ordered', 'real', 'row_vector', - 'row_vectormatrix', 'simplex', 'unit_vector', 'vector', - 'void') + 'void', +) FUNCTIONS = ( - 'Phi', - 'Phi_approx', 'abs', 'acos', 'acosh', + 'algebra_solver', + 'append_array', 'append_col', 'append_row', 'asin', @@ -54,55 +53,59 @@ FUNCTIONS = ( 'atan', 'atan2', 'atanh', - 'bernoulli_ccdf_log', 'bernoulli_cdf', - 'bernoulli_cdf_log', - 'bernoulli_log', - 'bernoulli_logit_log', + 'bernoulli_lccdf', + 'bernoulli_lcdf', + 'bernoulli_logit_lpmf', + 'bernoulli_logit_rng', + 'bernoulli_lpmf', 'bernoulli_rng', 'bessel_first_kind', 'bessel_second_kind', - 'beta_binomial_ccdf_log', 'beta_binomial_cdf', - 'beta_binomial_cdf_log', - 'beta_binomial_log', + 'beta_binomial_lccdf', + 'beta_binomial_lcdf', + 'beta_binomial_lpmf', 'beta_binomial_rng', - 'beta_ccdf_log', 'beta_cdf', - 'beta_cdf_log', - 'beta_log', + 'beta_lccdf', + 'beta_lcdf', + 'beta_lpdf', 'beta_rng', 'binary_log_loss', - 'binomial_ccdf_log', 'binomial_cdf', - 'binomial_cdf_log', 'binomial_coefficient_log', - 'binomial_log', - 'binomial_logit_log', + 'binomial_lccdf', + 'binomial_lcdf', + 'binomial_logit_lpmf', + 'binomial_lpmf', 'binomial_rng', 'block', - 'categorical_log', - 'categorical_logit_log', + 'categorical_logit_lpmf', + 'categorical_logit_rng', + 'categorical_lpmf', 'categorical_rng', - 'cauchy_ccdf_log', 'cauchy_cdf', - 'cauchy_cdf_log', - 'cauchy_log', + 'cauchy_lccdf', + 'cauchy_lcdf', + 'cauchy_lpdf', 'cauchy_rng', 'cbrt', 'ceil', - 'chi_square_ccdf_log', 'chi_square_cdf', - 'chi_square_cdf_log', - 'chi_square_log', + 'chi_square_lccdf', + 'chi_square_lcdf', + 'chi_square_lpdf', 'chi_square_rng', 'cholesky_decompose', + 'choose', 'col', 'cols', 'columns_dot_product', 'columns_dot_self', 'cos', 'cosh', + 'cov_exp_quad', 'crossprod', 'csr_extract_u', 'csr_extract_v', @@ -117,15 +120,15 @@ FUNCTIONS = ( 'diagonal', 'digamma', 'dims', - 'dirichlet_log', + 'dirichlet_lpdf', 'dirichlet_rng', 'distance', 'dot_product', 'dot_self', - 'double_exponential_ccdf_log', 'double_exponential_cdf', - 'double_exponential_cdf_log', - 'double_exponential_log', + 'double_exponential_lccdf', + 'double_exponential_lcdf', + 'double_exponential_lpdf', 'double_exponential_rng', 'e', 'eigenvalues_sym', @@ -134,16 +137,16 @@ FUNCTIONS = ( 'erfc', 'exp', 'exp2', - 'exp_mod_normal_ccdf_log', 'exp_mod_normal_cdf', - 'exp_mod_normal_cdf_log', - 'exp_mod_normal_log', + 'exp_mod_normal_lccdf', + 'exp_mod_normal_lcdf', + 'exp_mod_normal_lpdf', 'exp_mod_normal_rng', 'expm1', - 'exponential_ccdf_log', 'exponential_cdf', - 'exponential_cdf_log', - 'exponential_log', + 'exponential_lccdf', + 'exponential_lcdf', + 'exponential_lpdf', 'exponential_rng', 'fabs', 'falling_factorial', @@ -153,60 +156,65 @@ FUNCTIONS = ( 'fmax', 'fmin', 'fmod', - 'frechet_ccdf_log', 'frechet_cdf', - 'frechet_cdf_log', - 'frechet_log', + 'frechet_lccdf', + 'frechet_lcdf', + 'frechet_lpdf', 'frechet_rng', - 'gamma_ccdf_log', 'gamma_cdf', - 'gamma_cdf_log', - 'gamma_log', + 'gamma_lccdf', + 'gamma_lcdf', + 'gamma_lpdf', 'gamma_p', 'gamma_q', 'gamma_rng', - 'gaussian_dlm_obs_log', + 'gaussian_dlm_obs_lpdf', 'get_lp', - 'gumbel_ccdf_log', 'gumbel_cdf', - 'gumbel_cdf_log', - 'gumbel_log', + 'gumbel_lccdf', + 'gumbel_lcdf', + 'gumbel_lpdf', 'gumbel_rng', 'head', - 'hypergeometric_log', + 'hypergeometric_lpmf', 'hypergeometric_rng', 'hypot', - 'if_else', + 'inc_beta', 'int_step', + 'integrate_ode', + 'integrate_ode_bdf', + 'integrate_ode_rk45', 'inv', - 'inv_chi_square_ccdf_log', 'inv_chi_square_cdf', - 'inv_chi_square_cdf_log', - 'inv_chi_square_log', + 'inv_chi_square_lccdf', + 'inv_chi_square_lcdf', + 'inv_chi_square_lpdf', 'inv_chi_square_rng', 'inv_cloglog', - 'inv_gamma_ccdf_log', 'inv_gamma_cdf', - 'inv_gamma_cdf_log', - 'inv_gamma_log', + 'inv_gamma_lccdf', + 'inv_gamma_lcdf', + 'inv_gamma_lpdf', 'inv_gamma_rng', 'inv_logit', - 'inv_phi', + 'inv_Phi', 'inv_sqrt', 'inv_square', - 'inv_wishart_log', + 'inv_wishart_lpdf', 'inv_wishart_rng', 'inverse', 'inverse_spd', 'is_inf', 'is_nan', 'lbeta', + 'lchoose', 'lgamma', - 'lkj_corr_cholesky_log', + 'lkj_corr_cholesky_lpdf', 'lkj_corr_cholesky_rng', - 'lkj_corr_log', + 'lkj_corr_lpdf', 'lkj_corr_rng', 'lmgamma', + 'lmultiply', 'log', 'log10', 'log1m', @@ -223,81 +231,87 @@ FUNCTIONS = ( 'log_rising_factorial', 'log_softmax', 'log_sum_exp', - 'logistic_ccdf_log', 'logistic_cdf', - 'logistic_cdf_log', - 'logistic_log', + 'logistic_lccdf', + 'logistic_lcdf', + 'logistic_lpdf', 'logistic_rng', 'logit', - 'lognormal_ccdf_log', 'lognormal_cdf', - 'lognormal_cdf_log', - 'lognormal_log', + 'lognormal_lccdf', + 'lognormal_lcdf', + 'lognormal_lpdf', 'lognormal_rng', 'machine_precision', + 'matrix_exp', 'max', + 'mdivide_left_spd', 'mdivide_left_tri_low', + 'mdivide_right_spd', 'mdivide_right_tri_low', 'mean', 'min', 'modified_bessel_first_kind', 'modified_bessel_second_kind', - 'multi_gp_cholesky_log', - 'multi_gp_log', - 'multi_normal_cholesky_log', + 'multi_gp_cholesky_lpdf', + 'multi_gp_lpdf', + 'multi_normal_cholesky_lpdf', 'multi_normal_cholesky_rng', - 'multi_normal_log', - 'multi_normal_prec_log', + 'multi_normal_lpdf', + 'multi_normal_prec_lpdf', 'multi_normal_rng', - 'multi_student_t_log', + 'multi_student_t_lpdf', 'multi_student_t_rng', - 'multinomial_log', + 'multinomial_lpmf', 'multinomial_rng', 'multiply_log', 'multiply_lower_tri_self_transpose', - 'neg_binomial_2_ccdf_log', 'neg_binomial_2_cdf', - 'neg_binomial_2_cdf_log', - 'neg_binomial_2_log', - 'neg_binomial_2_log_log', + 'neg_binomial_2_lccdf', + 'neg_binomial_2_lcdf', + 'neg_binomial_2_log_lpmf', 'neg_binomial_2_log_rng', + 'neg_binomial_2_lpmf', 'neg_binomial_2_rng', - 'neg_binomial_ccdf_log', 'neg_binomial_cdf', - 'neg_binomial_cdf_log', - 'neg_binomial_log', + 'neg_binomial_lccdf', + 'neg_binomial_lcdf', + 'neg_binomial_lpmf', 'neg_binomial_rng', 'negative_infinity', - 'normal_ccdf_log', 'normal_cdf', - 'normal_cdf_log', - 'normal_log', + 'normal_lccdf', + 'normal_lcdf', + 'normal_lpdf', 'normal_rng', 'not_a_number', 'num_elements', - 'ordered_logistic_log', + 'ordered_logistic_lpmf', 'ordered_logistic_rng', 'owens_t', - 'pareto_ccdf_log', 'pareto_cdf', - 'pareto_cdf_log', - 'pareto_log', + 'pareto_lccdf', + 'pareto_lcdf', + 'pareto_lpdf', 'pareto_rng', - 'pareto_type_2_ccdf_log', 'pareto_type_2_cdf', - 'pareto_type_2_cdf_log', - 'pareto_type_2_log', + 'pareto_type_2_lccdf', + 'pareto_type_2_lcdf', + 'pareto_type_2_lpdf', 'pareto_type_2_rng', + 'Phi', + 'Phi_approx', 'pi', - 'poisson_ccdf_log', 'poisson_cdf', - 'poisson_cdf_log', - 'poisson_log', - 'poisson_log_log', + 'poisson_lccdf', + 'poisson_lcdf', + 'poisson_log_lpmf', 'poisson_log_rng', + 'poisson_lpmf', 'poisson_rng', 'positive_infinity', 'pow', + 'print', 'prod', 'qr_Q', 'qr_R', @@ -305,11 +319,12 @@ FUNCTIONS = ( 'quad_form_diag', 'quad_form_sym', 'rank', - 'rayleigh_ccdf_log', 'rayleigh_cdf', - 'rayleigh_cdf_log', - 'rayleigh_log', + 'rayleigh_lccdf', + 'rayleigh_lcdf', + 'rayleigh_lpdf', 'rayleigh_rng', + 'reject', 'rep_array', 'rep_matrix', 'rep_row_vector', @@ -320,10 +335,10 @@ FUNCTIONS = ( 'rows', 'rows_dot_product', 'rows_dot_self', - 'scaled_inv_chi_square_ccdf_log', 'scaled_inv_chi_square_cdf', - 'scaled_inv_chi_square_cdf_log', - 'scaled_inv_chi_square_log', + 'scaled_inv_chi_square_lccdf', + 'scaled_inv_chi_square_lcdf', + 'scaled_inv_chi_square_lpdf', 'scaled_inv_chi_square_rng', 'sd', 'segment', @@ -331,10 +346,10 @@ FUNCTIONS = ( 'singular_values', 'sinh', 'size', - 'skew_normal_ccdf_log', 'skew_normal_cdf', - 'skew_normal_cdf_log', - 'skew_normal_log', + 'skew_normal_lccdf', + 'skew_normal_lcdf', + 'skew_normal_lpdf', 'skew_normal_rng', 'softmax', 'sort_asc', @@ -346,10 +361,10 @@ FUNCTIONS = ( 'square', 'squared_distance', 'step', - 'student_t_ccdf_log', 'student_t_cdf', - 'student_t_cdf_log', - 'student_t_log', + 'student_t_lccdf', + 'student_t_lcdf', + 'student_t_lpdf', 'student_t_rng', 'sub_col', 'sub_row', @@ -357,6 +372,7 @@ FUNCTIONS = ( 'tail', 'tan', 'tanh', + 'target', 'tcrossprod', 'tgamma', 'to_array_1d', @@ -369,22 +385,22 @@ FUNCTIONS = ( 'trace_quad_form', 'trigamma', 'trunc', - 'uniform_ccdf_log', 'uniform_cdf', - 'uniform_cdf_log', - 'uniform_log', + 'uniform_lccdf', + 'uniform_lcdf', + 'uniform_lpdf', 'uniform_rng', 'variance', - 'von_mises_log', + 'von_mises_lpdf', 'von_mises_rng', - 'weibull_ccdf_log', 'weibull_cdf', - 'weibull_cdf_log', - 'weibull_log', + 'weibull_lccdf', + 'weibull_lcdf', + 'weibull_lpdf', 'weibull_rng', - 'wiener_log', - 'wishart_log', - 'wishart_rng' + 'wiener_lpdf', + 'wishart_lpdf', + 'wishart_rng', ) DISTRIBUTIONS = ( @@ -438,7 +454,7 @@ DISTRIBUTIONS = ( 'von_mises', 'weibull', 'wiener', - 'wishart' + 'wishart', ) RESERVED = ( @@ -469,19 +485,23 @@ RESERVED = ( 'do', 'double', 'dynamic_cast', + 'else', 'enum', 'explicit', 'export', 'extern', 'false', - 'false', 'float', + 'for', 'friend', 'fvar', 'goto', + 'if', + 'in', 'inline', 'int', 'long', + 'lp__', 'mutable', 'namespace', 'new', @@ -498,9 +518,16 @@ RESERVED = ( 'register', 'reinterpret_cast', 'repeat', + 'return', 'short', 'signed', 'sizeof', + 'STAN_MAJOR', + 'STAN_MATH_MAJOR', + 'STAN_MATH_MINOR', + 'STAN_MATH_PATCH', + 'STAN_MINOR', + 'STAN_PATCH', 'static', 'static_assert', 'static_cast', @@ -512,7 +539,6 @@ RESERVED = ( 'thread_local', 'throw', 'true', - 'true', 'try', 'typedef', 'typeid', @@ -526,7 +552,7 @@ RESERVED = ( 'void', 'volatile', 'wchar_t', + 'while', 'xor', - 'xor_eq' + 'xor_eq', ) - diff --git a/pygments/lexers/modeling.py b/pygments/lexers/modeling.py index b354f1cf..49d98d1b 100644 --- a/pygments/lexers/modeling.py +++ b/pygments/lexers/modeling.py @@ -284,8 +284,8 @@ class StanLexer(RegexLexer): """Pygments Lexer for Stan models. The Stan modeling language is specified in the *Stan Modeling Language - User's Guide and Reference Manual, v2.8.0*, - `pdf `__. + User's Guide and Reference Manual, v2.17.0*, + `pdf `__. .. versionadded:: 1.6 """ @@ -316,19 +316,24 @@ class StanLexer(RegexLexer): 'parameters', r'transformed\s+parameters', 'model', r'generated\s+quantities')), bygroups(Keyword.Namespace, Text, Punctuation)), + # target keyword + (r'target\s*\+=', Keyword), # Reserved Words (r'(%s)\b' % r'|'.join(_stan_builtins.KEYWORDS), Keyword), # Truncation (r'T(?=\s*\[)', Keyword), # Data types (r'(%s)\b' % r'|'.join(_stan_builtins.TYPES), Keyword.Type), + # < should be punctuation, but elsewhere I can't tell if it is in + # a range constraint + (r'(<)\s*(upper|lower)\s*(=)', bygroups(Operator, Keyword, Punctuation)), + (r'(,)\s*(upper)\s*(=)', bygroups(Punctuation, Keyword, Punctuation)), # Punctuation - (r"[;:,\[\]()]", Punctuation), + (r"[;,\[\]()]", Punctuation), # Builtin - (r'(%s)(?=\s*\()' - % r'|'.join(_stan_builtins.FUNCTIONS - + _stan_builtins.DISTRIBUTIONS), - Name.Builtin), + (r'(%s)(?=\s*\()' % '|'.join(_stan_builtins.FUNCTIONS), Name.Builtin), + (r'(~)\s*(%s)(?=\s*\()' % '|'.join(_stan_builtins.DISTRIBUTIONS), + bygroups(Operator, Name.Builtin)), # Special names ending in __, like lp__ (r'[A-Za-z]\w*__\b', Name.Builtin.Pseudo), (r'(%s)\b' % r'|'.join(_stan_builtins.RESERVED), Keyword.Reserved), @@ -337,17 +342,18 @@ class StanLexer(RegexLexer): # Regular variable names (r'[A-Za-z]\w*\b', Name), # Real Literals - (r'-?[0-9]+(\.[0-9]+)?[eE]-?[0-9]+', Number.Float), - (r'-?[0-9]*\.[0-9]*', Number.Float), + (r'[0-9]+(\.[0-9]*)?([eE][+-]?[0-9]+)?', Number.Float), + (r'\.[0-9]+([eE][+-]?[0-9]+)?', Number.Float), # Integer Literals - (r'-?[0-9]+', Number.Integer), + (r'[0-9]+', Number.Integer), # Assignment operators - # SLexer makes these tokens Operators. - (r'<-|~', Operator), + (r'<-|(?:\+|-|\.?/|\.?\*|=)?=|~', Operator), # Infix, prefix and postfix operators (and = ) - (r"\+|-|\.?\*|\.?/|\\|'|\^|==?|!=?|<=?|>=?|\|\||&&", Operator), + (r"\+|-|\.?\*|\.?/|\\|'|\^|!=?|<=?|>=?|\|\||&&|%|\?|:", Operator), # Block delimiters (r'[{}]', Punctuation), + # Distribution | + (r'\|', Punctuation) ] } -- cgit v1.2.1 From cc4e9a47ff2ff69f5b3c55b582c7793c4f83f951 Mon Sep 17 00:00:00 2001 From: James Martindale Date: Mon, 16 Jul 2018 14:36:48 -0500 Subject: Fix external links in RdLexer and NewLispLexer --- pygments/lexers/lisp.py | 2 +- pygments/lexers/r.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pygments/lexers/lisp.py b/pygments/lexers/lisp.py index e258c347..50c56373 100644 --- a/pygments/lexers/lisp.py +++ b/pygments/lexers/lisp.py @@ -1400,7 +1400,7 @@ class RacketLexer(RegexLexer): class NewLispLexer(RegexLexer): """ - For `newLISP. `_ source code (version 10.3.0). + For `newLISP. `_ source code (version 10.3.0). .. versionadded:: 1.5 """ diff --git a/pygments/lexers/r.py b/pygments/lexers/r.py index dce61969..f4e76f6f 100644 --- a/pygments/lexers/r.py +++ b/pygments/lexers/r.py @@ -421,7 +421,7 @@ class RdLexer(RegexLexer): This is a very minimal implementation, highlighting little more than the macros. A description of Rd syntax is found in `Writing R Extensions `_ - and `Parsing Rd files `_. + and `Parsing Rd files `_. .. versionadded:: 1.6 """ -- cgit v1.2.1 From 9ddc62fd449592eaee42b4026884b976b2fb3b64 Mon Sep 17 00:00:00 2001 From: Morten Enemark Lund Date: Sat, 4 Aug 2018 19:43:17 +0200 Subject: Change ansi color names to more saying names The ansi color names are changed to names which are easier to understand and align with color names of other projects and terminals. (e.g. ``#ansifuchsia`` to ``ansibrightmagenta``) This also drops the # prefix to the color names. Hashtag # is usually used for hex colors and the names are already prefixed with `ansi`. Furthermore, it allows the new and old set of names to be exclusive. --- doc/docs/styles.rst | 49 ++++++++++++++++++++++++----- pygments/formatters/terminal256.py | 8 ++++- pygments/style.py | 63 ++++++++++++++++++++++++++------------ tests/test_terminal_formatter.py | 10 +++--- 4 files changed, 97 insertions(+), 33 deletions(-) diff --git a/doc/docs/styles.rst b/doc/docs/styles.rst index 1094a270..cd0144ab 100644 --- a/doc/docs/styles.rst +++ b/doc/docs/styles.rst @@ -153,17 +153,17 @@ Terminal Styles .. versionadded:: 2.2 Custom styles used with the 256-color terminal formatter can also map colors to -use the 8 default ANSI colors. To do so, use ``#ansigreen``, ``#ansired`` or +use the 8 default ANSI colors. To do so, use ``ansigreen``, ``ansibrightred`` or any other colors defined in :attr:`pygments.style.ansicolors`. Foreground ANSI colors will be mapped to the corresponding `escape codes 30 to 37 `_ thus respecting any custom color mapping and themes provided by many terminal emulators. Light variants are treated as foreground color with and an added bold flag. -``bg:#ansi`` will also be respected, except the light variant will be the +``bg:ansi`` will also be respected, except the light variant will be the same shade as their dark variant. See the following example where the color of the string ``"hello world"`` is -governed by the escape sequence ``\x1b[34;01m`` (Ansi Blue, Bold, 41 being red +governed by the escape sequence ``\x1b[34;01m`` (Ansi bright blue, Bold, 41 being red background) instead of an extended foreground & background color. .. sourcecode:: pycon @@ -176,7 +176,7 @@ background) instead of an extended foreground & background color. >>> class MyStyle(Style): styles = { - Token.String: '#ansiblue bg:#ansired', + Token.String: 'ansibrightblue bg:ansibrightred', } >>> code = 'print("Hello World")' @@ -184,18 +184,51 @@ background) instead of an extended foreground & background color. >>> print(result.encode()) b'\x1b[34;41;01m"\x1b[39;49;00m\x1b[34;41;01mHello World\x1b[39;49;00m\x1b[34;41;01m"\x1b[39;49;00m' -Colors specified using ``#ansi*`` are converted to a default set of RGB colors +Colors specified using ``ansi*`` are converted to a default set of RGB colors when used with formatters other than the terminal-256 formatter. By definition of ANSI, the following colors are considered "light" colors, and will be rendered by most terminals as bold: -- "darkgray", "red", "green", "yellow", "blue", "fuchsia", "turquoise", "white" +- "brightblack" (darkgrey), "brightred", "brightgreen", "brightyellow", "brightblue", + "brightmagenta", "brightcyan", "white" The following are considered "dark" colors and will be rendered as non-bold: -- "black", "darkred", "darkgreen", "brown", "darkblue", "purple", "teal", - "lightgray" +- "black", "red", "green", "yellow", "blue", "magenta", "cyan", + "gray" Exact behavior might depends on the terminal emulator you are using, and its settings. + +.. _NewAnsiColorNames: + +.. versionchanged:: 2.3 + +The definition of the ansi color names has changed. +New names are easier to understand and align to the colors used in other projects. + + ++-------------------------+--------------------------+ +| New names | Pygments 2.2 | ++=======================+============================+ +| ``ansiblack`` | ``#ansiblack`` | +| ``ansired`` | ``#ansidarkred`` | +| ``ansigreen`` | ``#ansidarkgreen`` | +| ``ansiyellow`` | ``#ansibrown`` | +| ``ansiblue`` | ``#ansidarkblue`` | +| ``ansimagenta`` | ``#ansipurple`` | +| ``ansicyan`` | ``#ansiteal`` | +| ``ansigray`` | ``#ansilightgray`` | +| ``ansibrightblack`` | ``#ansidarkgray`` | +| ``ansibrightred`` | ``#ansired`` | +| ``ansibrightgreen`` | ``#ansigreen`` | +| ``ansibrightyellow`` | ``#ansiyellow`` | +| ``ansibrightblue`` | ``#ansiblue`` | +| ``ansibrightmagenta`` | ``#ansifuchsia`` | +| ``ansibrightcyan`` | ``#ansiturquoise`` | +| ``ansiwhite`` | ``#ansiwhite`` | ++=========================+==========================+ + +Old ansi color names are deprecated but will still work. + diff --git a/pygments/formatters/terminal256.py b/pygments/formatters/terminal256.py index b80dc7dd..88bc23ce 100644 --- a/pygments/formatters/terminal256.py +++ b/pygments/formatters/terminal256.py @@ -110,6 +110,12 @@ class Terminal256Formatter(Formatter): `Terminal256Formatter` will map these to non extended foreground color. See :ref:`AnsiTerminalStyle` for more information. + .. versionchanged:: 2.3 + The ansi color names have been updated with names that are easier to + understand and align with colornames of other projects and terminals. + See :ref:`NewAnsiColorNames` for more information. + + Options accepted: `style` @@ -189,7 +195,7 @@ class Terminal256Formatter(Formatter): def _color_index(self, color): index = self.best_match.get(color, None) if color in ansicolors: - # strip the `#ansi` part and look up code + # strip the `ansi/#ansi` part and look up code index = color self.best_match[color] = index if index is None: diff --git a/pygments/style.py b/pygments/style.py index 879c4e05..26812f84 100644 --- a/pygments/style.py +++ b/pygments/style.py @@ -12,26 +12,47 @@ from pygments.token import Token, STANDARD_TYPES from pygments.util import add_metaclass -# Default mapping of #ansixxx to RGB colors. +# Default mapping of ansixxx to RGB colors. _ansimap = { # dark - '#ansiblack': '000000', - '#ansidarkred': '7f0000', - '#ansidarkgreen': '007f00', - '#ansibrown': '7f7fe0', - '#ansidarkblue': '00007f', - '#ansipurple': '7f007f', - '#ansiteal': '007f7f', - '#ansilightgray': 'e5e5e5', + 'ansiblack': '000000', + 'ansired': '7f0000', + 'ansigreen': '007f00', + 'ansiyellow': '7f7fe0', + 'ansiblue': '00007f', + 'ansimagenta': '7f007f', + 'ansicyan': '007f7f', + 'ansigray': 'e5e5e5', # normal - '#ansidarkgray': '555555', - '#ansired': 'ff0000', - '#ansigreen': '00ff00', - '#ansiyellow': 'ffff00', - '#ansiblue': '0000ff', - '#ansifuchsia': 'ff00ff', - '#ansiturquoise': '00ffff', - '#ansiwhite': 'ffffff', + '#ansibrightblack': '555555', + 'ansibrightred': 'ff0000', + 'ansibrightgreen': '00ff00', + 'ansibrightyellow': 'ffff00', + 'ansibrightblue': '0000ff', + 'ansibrightmagenta': 'ff00ff', + 'ansibrightcyan': '00ffff', + 'ansiwhite': 'ffffff', +} +# mapping of deprecated #ansixxx colors to new color names +_deprecated_ansicolors = { + # dark + '#ansiblack': 'ansiblack', + '#ansidarkred': 'ansired', + '#ansidarkgreen': 'ansigreen', + '#ansibrown': 'ansiyellow', + '#ansidarkblue': 'ansiblue', + '#ansipurple': 'ansimagenta', + '#ansiteal': 'ansicyan', + '#ansilightgray': 'ansigray', + # normal + '#ansidarkgray': 'ansibrightblack', + '#ansired': 'ansibrightred', + '#ansigreen': 'ansibrightgreen', + '#ansiyellow': 'ansibrightyellow', + '#ansiblue': 'ansibrightblue', + '#ansifuchsia': 'ansibrightmagenta', + '#ansiturquoise': 'ansibrightcyan', + '#ansiwhite': 'ansiwhite', } ansicolors = set(_ansimap) @@ -106,11 +127,15 @@ class StyleMeta(type): t = cls._styles[token] ansicolor = bgansicolor = None color = t[0] - if color.startswith('#ansi'): + if color in _deprecated_ansicolors: + color = _deprecated_ansicolors[color] + if color in ansicolors: ansicolor = color color = _ansimap[color] bgcolor = t[4] - if bgcolor.startswith('#ansi'): + if bgcolor in _deprecated_ansicolors: + bgcolor = _deprecated_ansicolors[color] + if bgcolor in ansicolors: bgansicolor = bgcolor bgcolor = _ansimap[bgcolor] diff --git a/tests/test_terminal_formatter.py b/tests/test_terminal_formatter.py index ee0ac380..1f44807d 100644 --- a/tests/test_terminal_formatter.py +++ b/tests/test_terminal_formatter.py @@ -61,10 +61,10 @@ class TerminalFormatterTest(unittest.TestCase): class MyStyle(Style): styles = { - Token.Comment: '#ansidarkgray', - Token.String: '#ansiblue bg:#ansidarkred', - Token.Number: '#ansigreen bg:#ansidarkgreen', - Token.Number.Hex: '#ansidarkgreen bg:#ansired', + Token.Comment: 'ansibrightblack', + Token.String: 'ansibrightblue bg:ansired', + Token.Number: 'ansibrightgreen bg:ansigreen', + Token.Number.Hex: 'ansigreen bg:ansibrightred', } @@ -90,7 +90,7 @@ async def function(a,b,c, *d, **kwarg:Bool)->Bool: def test_256esc_seq(self): """ - test that a few escape sequences are actualy used when using #ansi<> color codes + test that a few escape sequences are actualy used when using ansi<> color codes """ def termtest(x): return highlight(x, Python3Lexer(), -- cgit v1.2.1 From e22d30b61416fc024cde4f262b36383ff9b19e3b Mon Sep 17 00:00:00 2001 From: Morten Enemark Lund Date: Tue, 21 Aug 2018 08:09:07 +0200 Subject: Ensure terminal formatter works with new ansi colors names --- pygments/console.py | 11 ++++------- pygments/formatters/terminal256.py | 4 ++-- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/pygments/console.py b/pygments/console.py index 31b6839d..27d46e7a 100644 --- a/pygments/console.py +++ b/pygments/console.py @@ -22,10 +22,10 @@ codes["underline"] = esc + "04m" codes["blink"] = esc + "05m" codes["overline"] = esc + "06m" -dark_colors = ["black", "darkred", "darkgreen", "brown", "darkblue", - "purple", "teal", "lightgray"] -light_colors = ["darkgray", "red", "green", "yellow", "blue", - "fuchsia", "turquoise", "white"] +dark_colors = ["black", "red", "green", "yellow", "blue", + "magenta", "cyan", "gray"] +light_colors = ["brightblack", "brightgreen", "brightgreen", "brightyellow", "brightblue", + "brightmagenta", "brightcyan", "white"] x = 30 for d, l in zip(dark_colors, light_colors): @@ -35,9 +35,6 @@ for d, l in zip(dark_colors, light_colors): del d, l, x -codes["darkteal"] = codes["turquoise"] -codes["darkyellow"] = codes["brown"] -codes["fuscia"] = codes["fuchsia"] codes["white"] = codes["bold"] diff --git a/pygments/formatters/terminal256.py b/pygments/formatters/terminal256.py index 88bc23ce..7e9214bc 100644 --- a/pygments/formatters/terminal256.py +++ b/pygments/formatters/terminal256.py @@ -50,7 +50,7 @@ class EscapeSequence: attrs = [] if self.fg is not None: if self.fg in ansicolors: - esc = codes[self.fg[5:]] + esc = codes[self.fg.lstrip('ansi')] if ';01m' in esc: self.bold = True # extract fg color code. @@ -59,7 +59,7 @@ class EscapeSequence: attrs.extend(("38", "5", "%i" % self.fg)) if self.bg is not None: if self.bg in ansicolors: - esc = codes[self.bg[5:]] + esc = codes[self.bg.lstrip('ansi')] # extract fg color code, add 10 for bg. attrs.append(str(int(esc[2:4])+10)) else: -- cgit v1.2.1 From d85884690446646a59834a510157b4fffbb3ff9e Mon Sep 17 00:00:00 2001 From: Morten Enemark Lund Date: Tue, 21 Aug 2018 08:34:47 +0200 Subject: Use replace instead of strip --- pygments/formatters/terminal256.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pygments/formatters/terminal256.py b/pygments/formatters/terminal256.py index 7e9214bc..aec19804 100644 --- a/pygments/formatters/terminal256.py +++ b/pygments/formatters/terminal256.py @@ -50,7 +50,7 @@ class EscapeSequence: attrs = [] if self.fg is not None: if self.fg in ansicolors: - esc = codes[self.fg.lstrip('ansi')] + esc = codes[self.fg.replace('ansi','')] if ';01m' in esc: self.bold = True # extract fg color code. @@ -59,7 +59,7 @@ class EscapeSequence: attrs.extend(("38", "5", "%i" % self.fg)) if self.bg is not None: if self.bg in ansicolors: - esc = codes[self.bg.lstrip('ansi')] + esc = codes[self.bg.replace('ansi','')] # extract fg color code, add 10 for bg. attrs.append(str(int(esc[2:4])+10)) else: -- cgit v1.2.1 From 303a40b0d3f585bfd992208755c99f4c94055527 Mon Sep 17 00:00:00 2001 From: Morten Enemark Lund Date: Tue, 21 Aug 2018 13:05:32 +0200 Subject: Fix bug with leading # in color name --- pygments/style.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pygments/style.py b/pygments/style.py index 26812f84..89766d8c 100644 --- a/pygments/style.py +++ b/pygments/style.py @@ -24,7 +24,7 @@ _ansimap = { 'ansicyan': '007f7f', 'ansigray': 'e5e5e5', # normal - '#ansibrightblack': '555555', + 'ansibrightblack': '555555', 'ansibrightred': 'ff0000', 'ansibrightgreen': '00ff00', 'ansibrightyellow': 'ffff00', -- cgit v1.2.1 From d564ab9227416838ffb0363a716fa86f20025a4d Mon Sep 17 00:00:00 2001 From: Morten Enemark Lund Date: Tue, 21 Aug 2018 13:24:50 +0200 Subject: fix missing brightred color in console colors --- pygments/console.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pygments/console.py b/pygments/console.py index 27d46e7a..e61744ce 100644 --- a/pygments/console.py +++ b/pygments/console.py @@ -24,7 +24,7 @@ codes["overline"] = esc + "06m" dark_colors = ["black", "red", "green", "yellow", "blue", "magenta", "cyan", "gray"] -light_colors = ["brightblack", "brightgreen", "brightgreen", "brightyellow", "brightblue", +light_colors = ["brightblack", "brightred", "brightgreen", "brightyellow", "brightblue", "brightmagenta", "brightcyan", "white"] x = 30 -- cgit v1.2.1 From 17a7965e047228c139f4bb488d8aab087650d8f7 Mon Sep 17 00:00:00 2001 From: mppf5 Date: Mon, 10 Sep 2018 14:45:56 -0400 Subject: Keywords grouped by starting letter --- pygments/lexers/chapel.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/pygments/lexers/chapel.py b/pygments/lexers/chapel.py index 445f0a9d..a7f8ea1a 100644 --- a/pygments/lexers/chapel.py +++ b/pygments/lexers/chapel.py @@ -42,17 +42,23 @@ class ChapelLexer(RegexLexer): (r'(bool|complex|imag|int|opaque|range|real|string|uint)\b', Keyword.Type), (words(( - 'align', 'as', 'atomic', 'begin', 'break', 'by', 'catch', - 'cobegin', - 'coforall', 'continue', 'delete', 'dmapped', 'do', 'domain', - 'else', 'enum', 'except', 'export', 'extern', 'for', 'forall', - 'if', 'index', 'inline', 'label', 'lambda', 'let', - 'local', 'new', 'noinit', 'on', 'only', 'otherwise', 'pragma', - 'private', 'prototype', - 'public', 'reduce', 'require', 'return', 'scan', - 'select', 'serial', 'single', 'sparse', 'subdomain', 'sync', + 'align', 'as', 'atomic', + 'begin', 'break', 'by', + 'catch', 'cobegin', 'coforall', 'continue', + 'delete', 'dmapped', 'do', 'domain', + 'else', 'enum', 'except', 'export', 'extern', + 'for', 'forall', + 'if', 'index', 'inline', + 'label', 'lambda', 'let', 'local', + 'new', 'noinit', + 'on', 'only', 'otherwise', + 'pragma', 'private', 'prototype', 'public', + 'reduce', 'require', 'return', + 'scan', 'select', 'serial', 'single', 'sparse', 'subdomain', 'sync', 'then', 'throw', 'throws', 'try', - 'use', 'when', 'where', 'while', 'with', 'yield', + 'use', + 'when', 'where', 'while', 'with', + 'yield', 'zip'), suffix=r'\b'), Keyword), (r'(iter)((?:\s)+)', bygroups(Keyword, Text), 'procname'), -- cgit v1.2.1 From 0e7ce1e9b77da20d2d4bdee7f93b1ee3b0852869 Mon Sep 17 00:00:00 2001 From: mppf5 Date: Mon, 10 Sep 2018 14:48:13 -0400 Subject: Add new keywords --- pygments/lexers/chapel.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pygments/lexers/chapel.py b/pygments/lexers/chapel.py index a7f8ea1a..16ce720b 100644 --- a/pygments/lexers/chapel.py +++ b/pygments/lexers/chapel.py @@ -43,7 +43,7 @@ class ChapelLexer(RegexLexer): Keyword.Type), (words(( 'align', 'as', 'atomic', - 'begin', 'break', 'by', + 'begin', 'borrowed', 'break', 'by', 'catch', 'cobegin', 'coforall', 'continue', 'delete', 'dmapped', 'do', 'domain', 'else', 'enum', 'except', 'export', 'extern', @@ -51,12 +51,12 @@ class ChapelLexer(RegexLexer): 'if', 'index', 'inline', 'label', 'lambda', 'let', 'local', 'new', 'noinit', - 'on', 'only', 'otherwise', + 'on', 'only', 'otherwise', 'override', 'owned', 'pragma', 'private', 'prototype', 'public', 'reduce', 'require', 'return', - 'scan', 'select', 'serial', 'single', 'sparse', 'subdomain', 'sync', + 'scan', 'select', 'serial', 'shared', 'single', 'sparse', 'subdomain', 'sync', 'then', 'throw', 'throws', 'try', - 'use', + 'unmanaged', 'use', 'when', 'where', 'while', 'with', 'yield', 'zip'), suffix=r'\b'), -- cgit v1.2.1 From cda7b0a58d2b80293e365de9ccd38dadeb957720 Mon Sep 17 00:00:00 2001 From: mppf5 Date: Mon, 10 Sep 2018 14:48:28 -0400 Subject: example shows new keywords --- tests/examplefiles/99_bottles_of_beer.chpl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/examplefiles/99_bottles_of_beer.chpl b/tests/examplefiles/99_bottles_of_beer.chpl index 80d2a89e..ff50b294 100644 --- a/tests/examplefiles/99_bottles_of_beer.chpl +++ b/tests/examplefiles/99_bottles_of_beer.chpl @@ -195,5 +195,12 @@ prototype module X { proc int.add() { } g(); + + override proc test() throws { + var a = new borrowed IntPair(); + var b = new owned IntPair(); + var c = new shared IntPair(); + throw new unmanaged Error(); + } } -- cgit v1.2.1 From 473d784cddb1c6907d6d01f437b2934d48e63fe2 Mon Sep 17 00:00:00 2001 From: "Stephane\"" Date: Sun, 21 Oct 2018 17:25:33 +0200 Subject: Add SGF lexer --- AUTHORS | 1 + pygments/lexers/sgf.py | 52 ++++++++++++++++++++++++++++++++++++++++++ pygments/lexers/text.py | 1 + tests/examplefiles/example.sgf | 35 ++++++++++++++++++++++++++++ 4 files changed, 89 insertions(+) create mode 100644 pygments/lexers/sgf.py create mode 100644 tests/examplefiles/example.sgf diff --git a/AUTHORS b/AUTHORS index f9ba2675..a6a31b81 100644 --- a/AUTHORS +++ b/AUTHORS @@ -31,6 +31,7 @@ Other contributors, listed alphabetically, are: * Sébastien Bigaret -- QVT Operational lexer * Jarrett Billingsley -- MiniD lexer * Adam Blinkinsop -- Haskell, Redcode lexers +* Stéphane Blondon -- SGF lexer * Frits van Bommel -- assembler lexers * Pierre Bourdon -- bugfixes * Matthias Bussonnier -- ANSI style handling for terminal-256 formatter diff --git a/pygments/lexers/sgf.py b/pygments/lexers/sgf.py new file mode 100644 index 00000000..8dd9beaa --- /dev/null +++ b/pygments/lexers/sgf.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- +""" + pygments.lexers.sgf + ~~~~~~~~~~~~~~~~~~~~ + + Lexer for Smart Game Format (sgf) file format. + + The format is used to store game records of board games for two players + (mainly Go game). + For more information about the definition of the format, see: + https://www.red-bean.com/sgf/ + + :copyright: Copyright 2006-2018 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.lexer import RegexLexer, bygroups +from pygments.token import * + +__all__ = ["SmartGameFormatLexer"] + + +class SmartGameFormatLexer(RegexLexer): + name = 'SmartGameFormat' + aliases = ['sgf'] + filenames = ['*.sgf'] + + tokens = { + 'root': [ + (r'([\(\):;\s])', Punctuation), + # tokens: + (r'(A[BW]|AE|AN|AP|AR|AS|[BW]L|BM|[BW]R|[BW]S|[BW]T|CA|CH|CP|CR|DD|DM|DO|DT|EL|EV|EX|FF|FG|G[BW]|GC|GM|GN|HA|HO|ID|IP|IT|IY|KM|KO|L|LB|LN|LT|M|MA|MN|N|OB|OM|ON|OP|OT|OV|P[BW]|PC|PL|PM|RE|RG|RO|RU|SO|SC|SE|SI|SL|SO|SQ|ST|SU|SZ|T[BW]|TC|TE|TM|TR|UC|US|V|VW|[BW]|C)', + Name.Builtin), + # number: + (r'(\[)([0-9.]+)(\])', + bygroups(Punctuation, Literal.Number, Punctuation)), + # date: + (r'(\[)([0-9]{4}-[0-9]{2}-[0-9]{2})(\])', + bygroups(Punctuation, Literal.Date, Punctuation)), + # point: + (r'(\[)([a-z]{2})(\])', + bygroups(Punctuation, String, Punctuation)), + # double points: + (r'(\[)([a-z]{2})(:)([a-z]{2})(\])', + bygroups(Punctuation, String, Punctuation, String, Punctuation)), + + (r'(\[)([\w\s\(\)\.\-\:,\#\+\?]+)(\])', + bygroups(Punctuation, String, Punctuation)), + (r'(\[)(\s.*)(\])', + bygroups(Punctuation, Text, Punctuation)), + ], + } diff --git a/pygments/lexers/text.py b/pygments/lexers/text.py index 9b3b5fea..bb1dccf2 100644 --- a/pygments/lexers/text.py +++ b/pygments/lexers/text.py @@ -18,6 +18,7 @@ from pygments.lexers.markup import BBCodeLexer, MoinWikiLexer, RstLexer, \ from pygments.lexers.installers import DebianControlLexer, SourcesListLexer from pygments.lexers.make import MakefileLexer, BaseMakefileLexer, CMakeLexer from pygments.lexers.haxe import HxmlLexer +from pygments.lexers.sgf import SmartGameFormatLexer from pygments.lexers.diff import DiffLexer, DarcsPatchLexer from pygments.lexers.data import YamlLexer from pygments.lexers.textfmts import IrcLogsLexer, GettextLexer, HttpLexer diff --git a/tests/examplefiles/example.sgf b/tests/examplefiles/example.sgf new file mode 100644 index 00000000..024a461e --- /dev/null +++ b/tests/examplefiles/example.sgf @@ -0,0 +1,35 @@ +(;FF[4]GM[1]SZ[19]FG[257:Figure 1]PM[1] +PB[Takemiya Masaki]BR[9 dan]PW[Cho Chikun] +WR[9 dan]RE[W+Resign]KM[5.5]TM[28800]DT[1996-10-18,19] +EV[21st Meijin]RO[2 (final)]SO[Go World #78]US[Arno Hollosi] +;B[pd];W[dp];B[pp];W[dd];B[pj];W[nc];B[oe];W[qc];B[pc];W[qd] +(;B[qf];W[rf];B[rg];W[re];B[qg];W[pb];B[ob];W[qb] +(;B[mp];W[fq];B[ci];W[cg];B[dl];W[cn];B[qo];W[ec];B[jp];W[jd] +;B[ei];W[eg];B[kk]LB[qq:a][dj:b][ck:c][qp:d]N[Figure 1] + +;W[me]FG[257:Figure 2];B[kf];W[ke];B[lf];W[jf];B[jg] +(;W[mf];B[if];W[je];B[ig];W[mg];B[mj];W[mq];B[lq];W[nq] +(;B[lr];W[qq];B[pq];W[pr];B[rq];W[rr];B[rp];W[oq];B[mr];W[oo];B[mn] +(;W[nr];B[qp]LB[kd:a][kh:b]N[Figure 2] + +;W[pk]FG[257:Figure 3];B[pm];W[oj];B[ok];W[qr];B[os];W[ol];B[nk];W[qj] +;B[pi];W[pl];B[qm];W[ns];B[sr];W[om];B[op];W[qi];B[oi] +(;W[rl];B[qh];W[rm];B[rn];W[ri];B[ql];W[qk];B[sm];W[sk];B[sh];W[og] +;B[oh];W[np];B[no];W[mm];B[nn];W[lp];B[kp];W[lo];B[ln];W[ko];B[mo] +;W[jo];B[km]N[Figure 3]) + +(;W[ql]VW[ja:ss]FG[257:Dia. 6]MN[1];B[rm];W[ph];B[oh];W[pg];B[og];W[pf] +;B[qh];W[qe];B[sh];W[of];B[sj]TR[oe][pd][pc][ob]LB[pe:a][sg:b][si:c] +N[Diagram 6])) + +(;W[no]VW[jj:ss]FG[257:Dia. 5]MN[1];B[pn]N[Diagram 5])) + +(;B[pr]FG[257:Dia. 4]MN[1];W[kq];B[lp];W[lr];B[jq];W[jr];B[kp];W[kr];B[ir] +;W[hr]LB[is:a][js:b][or:c]N[Diagram 4])) + +(;W[if]FG[257:Dia. 3]MN[1];B[mf];W[ig];B[jh]LB[ki:a]N[Diagram 3])) + +(;W[oc]VW[aa:sk]FG[257:Dia. 2]MN[1];B[md];W[mc];B[ld]N[Diagram 2])) + +(;B[qe]VW[aa:sj]FG[257:Dia. 1]MN[1];W[re];B[qf];W[rf];B[qg];W[pb];B[ob] +;W[qb]LB[rg:a]N[Diagram 1])) -- cgit v1.2.1 From 42f986a2427d9c572a33b8b4acef60245ab88536 Mon Sep 17 00:00:00 2001 From: Nathan Whetsell Date: Thu, 25 Oct 2018 07:08:01 -0400 Subject: Update for Csound 6.12.0 --- pygments/lexers/_csound_builtins.py | 44 ++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/pygments/lexers/_csound_builtins.py b/pygments/lexers/_csound_builtins.py index bdd8df37..56b5a452 100644 --- a/pygments/lexers/_csound_builtins.py +++ b/pygments/lexers/_csound_builtins.py @@ -7,13 +7,13 @@ :license: BSD, see LICENSE for details. """ -# Opcodes in Csound 6.11.0 at commit 25b2e8e53bc924526eaad34e0768a5e866638e94 using +# Opcodes in Csound 6.12.0 at commit 6ca322bd31f1ca907c008616b40a5f237ff449db using # python -c " # import re, subprocess # output = subprocess.Popen(['csound', '--list-opcodes0'], stderr=subprocess.PIPE).communicate()[1] -# opcodes = output[re.search(r'^\d+ opcodes$', output, re.M).end():re.search(r'^(?:WARNING|Usage):', output, re.M).start()].split() +# opcodes = output[re.search(r'^$', output, re.M).end():re.search(r'^\d+ opcodes$', output, re.M).start()].split() # output = subprocess.Popen(['csound', '--list-opcodes2'], stderr=subprocess.PIPE).communicate()[1] -# all_opcodes = output[re.search(r'^\d+ opcodes$', output, re.M).end():re.search(r'^(?:WARNING|Usage):', output, re.M).start()].split() +# all_opcodes = output[re.search(r'^$', output, re.M).end():re.search(r'^\d+ opcodes$', output, re.M).start()].split() # deprecated_opcodes = [opcode for opcode in all_opcodes if opcode not in opcodes] # print '''OPCODES = set(\''' # {} @@ -152,6 +152,8 @@ MixerReceive MixerSend MixerSetLevel MixerSetLevel_i +OSCbundle +OSCcount OSCinit OSCinitM OSClisten @@ -212,6 +214,8 @@ bamboo barmodel bbcutm bbcuts +beadsynt +beosc betarand bexprnd bformdec1 @@ -221,8 +225,8 @@ biquad biquada birnd bpf +bpfcos bqrez -buchla butbp butbr buthp @@ -326,6 +330,7 @@ dcblock2 dconv dct dctinv +deinterleave delay delay1 delayk @@ -389,7 +394,9 @@ fareyleni faustaudio faustcompile faustctl +faustdsp faustgen +faustplay fft fftinv ficlose @@ -411,16 +418,6 @@ flashtxt flooper flooper2 floor -fluidAllOut -fluidCCi -fluidCCk -fluidControl -fluidEngine -fluidLoad -fluidNote -fluidOut -fluidProgramSelect -fluidSetInterpMethod fmanal fmax fmb3 @@ -451,6 +448,7 @@ frac fractalnoise framebuffer freeverb +ftaudio ftchnls ftconv ftcps @@ -464,11 +462,13 @@ ftloadk ftlptim ftmorf ftom +ftprint ftresize ftresizei ftsamplebank ftsave ftsavek +ftslice ftsr gain gainslider @@ -485,6 +485,7 @@ getcfg getcol getftargs getrow +getrowlin getseed gogobel grain @@ -540,6 +541,7 @@ insglobal insremot int integ +interleave interp invalue inx @@ -734,6 +736,7 @@ lenarray lfo limit limit1 +lincos line linen linenr @@ -868,11 +871,6 @@ mp3in mp3len mp3nchnls mp3scal -mp3scal_check -mp3scal_load -mp3scal_load2 -mp3scal_play -mp3scal_play2 mp3sr mpulse mrtmsg @@ -1019,6 +1017,7 @@ prealloc prepiano print print_type +printarray printf printf_i printk @@ -1197,6 +1196,7 @@ release remoteport remove repluck +reshapearray reson resonk resonr @@ -1275,7 +1275,6 @@ sfpreset shaker shiftin shiftout -signalflowgraph signum sin sinh @@ -1283,6 +1282,7 @@ sininv sinsyn sleighbells slicearray +slicearray_i slider16 slider16f slider16table @@ -1364,6 +1364,7 @@ syncphasor system system_i tab +tab2array tab2pvs tab_i tabifd @@ -1398,6 +1399,7 @@ tabmorphak tabmorphi tabplay tabrec +tabrowlin tabsum tabw tabw_i @@ -1431,6 +1433,8 @@ trfilter trhighest trigger trigseq +trim +trim_i trirand trlowest trmix -- cgit v1.2.1 From 0342044ec7dcd26d6ab911c87977aa5055bb3336 Mon Sep 17 00:00:00 2001 From: Edward Betts Date: Sun, 4 Nov 2018 07:51:02 +0000 Subject: Use print as a function so the example works in Python 3. --- doc/docs/quickstart.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/docs/quickstart.rst b/doc/docs/quickstart.rst index dba7698a..d994f74e 100644 --- a/doc/docs/quickstart.rst +++ b/doc/docs/quickstart.rst @@ -39,7 +39,7 @@ Here is a small example for highlighting Python code: from pygments.formatters import HtmlFormatter code = 'print "Hello World"' - print highlight(code, PythonLexer(), HtmlFormatter()) + print(highlight(code, PythonLexer(), HtmlFormatter())) which prints something like this: -- cgit v1.2.1 From c1255470b48207d060454abcf2a0b3709db5904b Mon Sep 17 00:00:00 2001 From: Phil Hagelberg Date: Fri, 16 Nov 2018 18:11:03 -0800 Subject: Add support for the Fennel programming language This is a pretty straightforward language in the lisp family with a small number of special forms. Since Fennel runs on the Lua runtime, the list of builtins is the same as that of Lua, so it might be possible to re-use the definition from the Lua lexer, but since I don't know Python I couldn't figure out how that would work; maybe someone else could add that. --- pygments/lexers/_mapping.py | 1 + pygments/lexers/lisp.py | 71 ++++++++++++++++- tests/examplefiles/fennelview.fnl | 156 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 227 insertions(+), 1 deletion(-) create mode 100644 tests/examplefiles/fennelview.fnl diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py index b48ee1d1..6ed0852d 100644 --- a/pygments/lexers/_mapping.py +++ b/pygments/lexers/_mapping.py @@ -153,6 +153,7 @@ LEXERS = { 'FancyLexer': ('pygments.lexers.ruby', 'Fancy', ('fancy', 'fy'), ('*.fy', '*.fancypack'), ('text/x-fancysrc',)), 'FantomLexer': ('pygments.lexers.fantom', 'Fantom', ('fan',), ('*.fan',), ('application/x-fantom',)), 'FelixLexer': ('pygments.lexers.felix', 'Felix', ('felix', 'flx'), ('*.flx', '*.flxh'), ('text/x-felix',)), + 'FennelLexer': ('pygments.lexers.lisp', 'Fennel', (), ('*.fnl',), ()), 'FishShellLexer': ('pygments.lexers.shell', 'Fish', ('fish', 'fishshell'), ('*.fish', '*.load'), ('application/x-fish',)), 'FlatlineLexer': ('pygments.lexers.dsls', 'Flatline', ('flatline',), (), ('text/x-flatline',)), 'ForthLexer': ('pygments.lexers.forth', 'Forth', ('forth',), ('*.frt', '*.fs'), ('application/x-forth',)), diff --git a/pygments/lexers/lisp.py b/pygments/lexers/lisp.py index e258c347..345dabfb 100644 --- a/pygments/lexers/lisp.py +++ b/pygments/lexers/lisp.py @@ -19,7 +19,7 @@ from pygments.lexers.python import PythonLexer __all__ = ['SchemeLexer', 'CommonLispLexer', 'HyLexer', 'RacketLexer', 'NewLispLexer', 'EmacsLispLexer', 'ShenLexer', 'CPSALexer', - 'XtlangLexer'] + 'XtlangLexer', 'FennelLexer'] class SchemeLexer(RegexLexer): @@ -2619,3 +2619,72 @@ class XtlangLexer(RegexLexer): include('scheme') ], } + +class FennelLexer(RegexLexer): + """A lexer for the Fennel programming language + + Fennel compiles to Lua, so all the Lua builtins are recognized as well + as the special forms that are particular to the Fennel compiler. + """ + name = 'Fennel' + aliases = ['fennel', 'fnl'] + filenames = ['*.fnl'] + + # these two lists are taken from fennel-mode.el: + # https://gitlab.com/technomancy/fennel-mode + # this list is current as of Fennel version 0.1.0. + special_forms = ( + u'require-macros', u'eval-compiler', + u'do', u'values', u'if', u'when', u'each', u'for', u'fn', u'lambda', + u'λ', u'set', u'global', u'var', u'local', u'let', u'tset', u'doto', + u'set-forcibly!', u'defn', u'partial', u'while', u'or', u'and', u'true', + u'false', u'nil', u'.', u'+', u'..', u'^', u'-', u'*', u'%', u'/', u'>', + u'<', u'>=', u'<=', u'=', u'~=', u'#', u'...', u':', u'->', u'->>', + ) + + # Might be nicer to use the list from _lua_builtins.py but it's unclear how? + builtins = ( + u'_G', u'_VERSION', u'arg', u'assert', u'bit32', u'collectgarbage', + u'coroutine', u'debug', u'dofile', u'error', u'getfenv', + u'getmetatable', u'io', u'ipairs', u'load', u'loadfile', u'loadstring', + u'math', u'next', u'os', u'package', u'pairs', u'pcall', u'print', + u'rawequal', u'rawget', u'rawlen', u'rawset', u'require', u'select', + u'setfenv', u'setmetatable', u'string', u'table', u'tonumber', + u'tostring', u'type', u'unpack', u'xpcall' + ) + + # based on the scheme definition, but disallowing leading digits and commas + valid_name = r'[a-zA-Z_!$%&*+/:<=>?@^~|-][\w!$%&*+/:<=>?@^~|\.-]*' + + tokens = { + 'root': [ + # the only comment form is a semicolon; goes to the end of the line + (r';.*$', Comment.Single), + + (r'[,\s]+', Text), + (r'-?\d+\.\d+', Number.Float), + (r'-?\d+', Number.Integer), + + (r'"(\\\\|\\"|[^"])*"', String), + (r"'(\\\\|\\'|[^'])*'", String), + + # these are technically strings, but it's worth visually + # distinguishing them because their intent is different + # from regular strings. + (r':' + valid_name, String.Symbol), + + # special forms are keywords + (words(special_forms, suffix=' '), Keyword), + # lua standard library are builtins + (words(builtins, suffix=' '), Name.Builtin), + # special-case the vararg symbol + (r'\.\.\.', Name.Variable), + # regular identifiers + (valid_name, Name.Variable), + + # all your normal paired delimiters for your programming enjoyment + (r'(\(|\))', Punctuation), + (r'(\[|\])', Punctuation), + (r'(\{|\})', Punctuation), + ] + } diff --git a/tests/examplefiles/fennelview.fnl b/tests/examplefiles/fennelview.fnl new file mode 100644 index 00000000..fd0fc648 --- /dev/null +++ b/tests/examplefiles/fennelview.fnl @@ -0,0 +1,156 @@ +;; A pretty-printer that outputs tables in Fennel syntax. +;; Loosely based on inspect.lua: http://github.com/kikito/inspect.lua + +(local quote (fn [str] (.. '"' (: str :gsub '"' '\\"') '"'))) + +(local short-control-char-escapes + {"\a" "\\a" "\b" "\\b" "\f" "\\f" "\n" "\\n" + "\r" "\\r" "\t" "\\t" "\v" "\\v"}) + +(local long-control-char-esapes + (let [long {}] + (for [i 0 31] + (let [ch (string.char i)] + (when (not (. short-control-char-escapes ch)) + (tset short-control-char-escapes ch (.. "\\" i)) + (tset long ch (: "\\%03d" :format i))))) + long)) + +(fn escape [str] + (let [str (: str :gsub "\\" "\\\\") + str (: str :gsub "(%c)%f[0-9]" long-control-char-esapes)] + (: str :gsub "%c" short-control-char-escapes))) + +(fn sequence-key? [k len] + (and (= (type k) "number") + (<= 1 k) + (<= k len) + (= (math.floor k) k))) + +(local type-order {:number 1 :boolean 2 :string 3 :table 4 + :function 5 :userdata 6 :thread 7}) + +(fn sort-keys [a b] + (let [ta (type a) tb (type b)] + (if (and (= ta tb) (~= ta "boolean") + (or (= ta "string") (= ta "number"))) + (< a b) + (let [dta (. type-order a) + dtb (. type-order b)] + (if (and dta dtb) + (< dta dtb) + dta true + dtb false + :else (< ta tb)))))) + +(fn get-sequence-length [t] + (var len 1) + (each [i (ipairs t)] (set len i)) + len) + +(fn get-nonsequential-keys [t] + (let [keys {} + sequence-length (get-sequence-length t)] + (each [k (pairs t)] + (when (not (sequence-key? k sequence-length)) + (table.insert keys k))) + (table.sort keys sort-keys) + (values keys sequence-length))) + +(fn count-table-appearances [t appearances] + (if (= (type t) "table") + (when (not (. appearances t)) + (tset appearances t 1) + (each [k v (pairs t)] + (count-table-appearances k appearances) + (count-table-appearances v appearances))) + (when (and t (= t t)) ; no nans please + (tset appearances t (+ (or (. appearances t) 0) 1)))) + appearances) + + + +(var put-value nil) ; mutual recursion going on; defined below + +(fn puts [self ...] + (each [_ v (ipairs [...])] + (table.insert self.buffer v))) + +(fn tabify [self] (puts self "\n" (: self.indent :rep self.level))) + +(fn already-visited? [self v] (~= (. self.ids v) nil)) + +(fn get-id [self v] + (var id (. self.ids v)) + (when (not id) + (let [tv (type v)] + (set id (+ (or (. self.max-ids tv) 0) 1)) + (tset self.max-ids tv id) + (tset self.ids v id))) + (tostring id)) + +(fn put-sequential-table [self t length] + (puts self "[") + (set self.level (+ self.level 1)) + (for [i 1 length] + (puts self " ") + (put-value self (. t i))) + (set self.level (- self.level 1)) + (puts self " ]")) + +(fn put-key [self k] + (if (and (= (type k) "string") + (: k :find "^[-%w?\\^_`!#$%&*+./@~:|<=>]+$")) + (puts self ":" k) + (put-value self k))) + +(fn put-kv-table [self t] + (puts self "{") + (set self.level (+ self.level 1)) + (each [k v (pairs t)] + (tabify self) + (put-key self k) + (puts self " ") + (put-value self v)) + (set self.level (- self.level 1)) + (tabify self) + (puts self "}")) + +(fn put-table [self t] + (if (already-visited? self t) + (puts self "#") + (>= self.level self.depth) + (puts self "{...}") + :else + (let [(non-seq-keys length) (get-nonsequential-keys t) + id (get-id self t)] + (if (> (. self.appearances t) 1) + (puts self "#<" id ">") + (and (= (# non-seq-keys) 0) (= (# t) 0)) + (puts self "{}") + (= (# non-seq-keys) 0) + (put-sequential-table self t length) + :else + (put-kv-table self t))))) + +(set put-value (fn [self v] + (let [tv (type v)] + (if (= tv "string") + (puts self (quote (escape v))) + (or (= tv "number") (= tv "boolean") (= tv "nil")) + (puts self (tostring v)) + (= tv "table") + (put-table self v) + :else + (puts self "#<" (tostring v) ">"))))) + + + +(fn fennelview [root options] + (let [options (or options {}) + inspector {:appearances (count-table-appearances root {}) + :depth (or options.depth 128) + :level 0 :buffer {} :ids {} :max-ids {} + :indent (or options.indent " ")}] + (put-value inspector root) + (table.concat inspector.buffer))) -- cgit v1.2.1 From bd2cb3c85d549bee2bcc6e4fa882470b4acbda5d Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sat, 24 Nov 2018 15:26:06 +0100 Subject: add release checklist --- scripts/release-checklist | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 scripts/release-checklist diff --git a/scripts/release-checklist b/scripts/release-checklist new file mode 100644 index 00000000..f18e6376 --- /dev/null +++ b/scripts/release-checklist @@ -0,0 +1,24 @@ +Release checklist +================= + +* Check hg status +* Make check +* Make pylint +* Make test from clean checkout with all supported Python versions +* Update ez_setup.py +* Update version info in setup.py/__init__.py +* Check setup.py metadata: long description, trove classifiers +* Update release date/code name in CHANGES +* hg commit +* make clean +* For every supported version: + pythonX.Y setup.py release bdist_egg sdist upload +* Check PyPI release page for obvious errors +* hg tag +* Make a maintenance branch if applicable +* Update homepage (release info), regenerate docs (+printable!) +* Add new version/milestone to tracker categories +* Write announcement and send to mailing list/python-announce +* Update version info, add new CHANGES entry for next version +* hg commit +* hg push -- cgit v1.2.1 From 60afd6c7897f2e0b5326645efd48c9b70db744d8 Mon Sep 17 00:00:00 2001 From: "Matth?us G. Chajdas" Date: Sat, 24 Nov 2018 17:49:57 +0100 Subject: Fix the tests, and enable them for Python 2.7, 3.5, 3.6, 3.7. Updated the documentation as well to make clear which Python versions are officially supported. --- doc/faq.rst | 4 ++-- pygments/lexers/xorg.py | 4 ++-- tox.ini | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/faq.rst b/doc/faq.rst index f375828b..172929e0 100644 --- a/doc/faq.rst +++ b/doc/faq.rst @@ -35,8 +35,8 @@ and in this case, source code! What are the system requirements? --------------------------------- -Pygments only needs a standard Python install, version 2.6 or higher or version -3.3 or higher for Python 3. No additional libraries are needed. +Pygments only needs a standard Python install, version 2.7 or higher or version +3.5 or higher for Python 3. No additional libraries are needed. How can I use Pygments? ----------------------- diff --git a/pygments/lexers/xorg.py b/pygments/lexers/xorg.py index 89475a80..e6383b30 100644 --- a/pygments/lexers/xorg.py +++ b/pygments/lexers/xorg.py @@ -26,8 +26,8 @@ class XorgLexer(RegexLexer): (r'\s+', Text), (r'#.*$', Comment), - (r'((|Sub)Section)(\s+)("\w+")', - bygroups(String.Escape, String.Escape, Text, String.Escape)), + (r'((?:Sub)?Section)(\s+)("\w+")', + bygroups(String.Escape, Text, String.Escape)), (r'(End(|Sub)Section)', String.Escape), (r'(\w+)(\s+)([^\n#]+)', diff --git a/tox.ini b/tox.ini index 8a33f99c..2c63c292 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py26, py27, py33, py34 +envlist = py27, py35, py36, py37 [testenv] deps = nose -- cgit v1.2.1 From 72b98953d2dd8db2a909595eea718744ed7ee318 Mon Sep 17 00:00:00 2001 From: "Matth?us G. Chajdas" Date: Sat, 24 Nov 2018 17:50:46 +0100 Subject: Fix another instance of print in the quickstart documentation. --- doc/docs/quickstart.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/docs/quickstart.rst b/doc/docs/quickstart.rst index d994f74e..3a823e7f 100644 --- a/doc/docs/quickstart.rst +++ b/doc/docs/quickstart.rst @@ -56,7 +56,7 @@ can be produced by: .. sourcecode:: python - print HtmlFormatter().get_style_defs('.highlight') + print(HtmlFormatter().get_style_defs('.highlight')) The argument to :func:`get_style_defs` is used as an additional CSS selector: the output may look like this: -- cgit v1.2.1 From 6bc8c0de737b181566fa66bc29f5fef4bc6ab11c Mon Sep 17 00:00:00 2001 From: "Matth?us G. Chajdas" Date: Sat, 24 Nov 2018 17:52:06 +0100 Subject: Fix documentation not building with latest Sphinx. --- pygments/lexers/xorg.py | 1 + pygments/sphinxext.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pygments/lexers/xorg.py b/pygments/lexers/xorg.py index e6383b30..3bba930f 100644 --- a/pygments/lexers/xorg.py +++ b/pygments/lexers/xorg.py @@ -16,6 +16,7 @@ __all__ = ['XorgLexer'] class XorgLexer(RegexLexer): + """Lexer for xorg.conf file.""" name = 'Xorg' aliases = ['xorg.conf'] filenames = ['xorg.conf'] diff --git a/pygments/sphinxext.py b/pygments/sphinxext.py index f962f8c6..5c8ac8b9 100644 --- a/pygments/sphinxext.py +++ b/pygments/sphinxext.py @@ -16,7 +16,7 @@ import sys from docutils import nodes from docutils.statemachine import ViewList -from sphinx.util.compat import Directive +from docutils.parsers.rst import Directive from sphinx.util.nodes import nested_parse_with_titles -- cgit v1.2.1 From aa765094469cb90eff740b46a7c5c5903204c098 Mon Sep 17 00:00:00 2001 From: "Matth?us G. Chajdas" Date: Sat, 24 Nov 2018 17:55:37 +0100 Subject: Add Fennel and HLSL to the list of supported languages. --- AUTHORS | 3 ++- doc/languages.rst | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index f39bbaa1..18e642f1 100644 --- a/AUTHORS +++ b/AUTHORS @@ -72,7 +72,7 @@ Other contributors, listed alphabetically, are: * Alex Gosse -- TrafficScript lexer * Patrick Gotthardt -- PHP namespaces support * Olivier Guibe -- Asymptote lexer -* Jordi Gutiérrez Hermoso -- Octave lexer +* Phil Hagelberg -- Fennel lexer * Florian Hahn -- Boogie lexer * Martin Harriman -- SNOBOL lexer * Matthew Harrison -- SVG formatter @@ -81,6 +81,7 @@ Other contributors, listed alphabetically, are: * Aslak Hellesøy -- Gherkin lexer * Greg Hendershott -- Racket lexer * Justin Hendrick -- ParaSail lexer +* Jordi Gutiérrez Hermoso -- Octave lexer * David Hess, Fish Software, Inc. -- Objective-J lexer * Varun Hiremath -- Debian control lexer * Rob Hoelz -- Perl 6 lexer diff --git a/doc/languages.rst b/doc/languages.rst index 7fa8eb2f..e5399403 100644 --- a/doc/languages.rst +++ b/doc/languages.rst @@ -37,6 +37,7 @@ Programming languages * `Ezhil `_ Ezhil - A Tamil programming language * Factor * Fancy +* `Fennel `_ * Fortran * F# * GAP @@ -44,6 +45,7 @@ Programming languages * GL shaders * Groovy * `Haskell `_ (incl. Literate Haskell) +* HLSL * IDL * Io * Java -- cgit v1.2.1 From a37f5614f11f3d5af660a024f1749dc7eee7e8ad Mon Sep 17 00:00:00 2001 From: "Matth?us G. Chajdas" Date: Sat, 24 Nov 2018 17:56:44 +0100 Subject: Add Bitbucket CI configuration. --- bitbucket-pipelines.yml | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 bitbucket-pipelines.yml diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml new file mode 100644 index 00000000..e3c74d6e --- /dev/null +++ b/bitbucket-pipelines.yml @@ -0,0 +1,34 @@ +pipelines: + default: + - step: + name: Test on Python 2.7 + image: python:2.7 + caches: + - pip + script: + - pip install -r requirements.txt + - tox -e py27 + - step: + name: Test on Python 3.4 + image: python:3.4 + caches: + - pip + script: + - pip install -r requirements.txt + - tox -e py34 + - step: + name: Test on Python 3.6 + image: python:3.6 + caches: + - pip + script: + - pip install -r requirements.txt + - tox -e py36 + - step: + name: Test on Python 3.7 + image: python:3.7 + caches: + - pip + script: + - pip install -r requirements.txt + - tox -e py37 \ No newline at end of file -- cgit v1.2.1 From 4792cccf7886199bd6d49dbe487f213c0f8094ce Mon Sep 17 00:00:00 2001 From: "Matth?us G. Chajdas" Date: Sat, 24 Nov 2018 18:04:30 +0100 Subject: Update changelog and version number. --- CHANGES | 13 +++++++++++++ pygments/__init__.py | 2 +- setup.py | 2 +- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index 0bab9118..af4141f2 100644 --- a/CHANGES +++ b/CHANGES @@ -6,6 +6,19 @@ Issue numbers refer to the tracker at pull request numbers to the requests at . +Version 2.3.0 +------------- +(not yet released) + +- Added lexers: + + * Fennel (PR#783) + * HLSL (PR#675) + +- Added Python 3.7 support (PR#772) +- Fix incorrect token type in SCSS for single-quote strings (#1322) +- Use `terminal256` formatter if `TERM` contains `256` (PR#666) + Version 2.2.0 ------------- (release Jan 22, 2017) diff --git a/pygments/__init__.py b/pygments/__init__.py index 394a85f2..19aafdeb 100644 --- a/pygments/__init__.py +++ b/pygments/__init__.py @@ -29,7 +29,7 @@ import sys from pygments.util import StringIO, BytesIO -__version__ = '2.2.0' +__version__ = '2.3.0' __docformat__ = 'restructuredtext' __all__ = ['lex', 'format', 'highlight'] diff --git a/setup.py b/setup.py index 1705923c..7e6eca8c 100755 --- a/setup.py +++ b/setup.py @@ -48,7 +48,7 @@ else: setup( name = 'Pygments', - version = '2.2.0', + version = '2.3.0', url = 'http://pygments.org/', license = 'BSD License', author = 'Georg Brandl', -- cgit v1.2.1 From 67ca36ddb9b8b00c9158db00fcb3e10bd6e78c58 Mon Sep 17 00:00:00 2001 From: "Matth?us G. Chajdas" Date: Sat, 24 Nov 2018 18:32:53 +0100 Subject: Update changelog. --- CHANGES | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES b/CHANGES index af4141f2..dc078635 100644 --- a/CHANGES +++ b/CHANGES @@ -18,6 +18,8 @@ Version 2.3.0 - Added Python 3.7 support (PR#772) - Fix incorrect token type in SCSS for single-quote strings (#1322) - Use `terminal256` formatter if `TERM` contains `256` (PR#666) +- Fix incorrect handling of GitHub style fences in Markdown (PR#741, #1389) +- Fix `%a` not being highlighted in Python3 strings (PR#727) Version 2.2.0 ------------- -- cgit v1.2.1 From 6866f8b98f4051603b560e6f6313b74eee7a12a9 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 25 Nov 2018 08:37:32 +0100 Subject: Minimal fixup changes for the release, add release date. --- .hgignore | 1 + CHANGES | 12 +++++++++--- Makefile | 6 ++++++ bitbucket-pipelines.yml | 2 +- pygments/lexers/_mapping.py | 2 +- pygments/lexers/graphics.py | 2 +- pygments/lexers/javascript.py | 6 ++++-- pygments/lexers/lisp.py | 5 ++++- pygments/lexers/webmisc.py | 3 ++- scripts/release-checklist | 32 ++++++++++++++++---------------- 10 files changed, 45 insertions(+), 26 deletions(-) diff --git a/.hgignore b/.hgignore index 850baf13..b564df83 100644 --- a/.hgignore +++ b/.hgignore @@ -8,6 +8,7 @@ syntax: glob .project .tags .tox +.cache/ Pygments.egg-info/* TAGS build/* diff --git a/CHANGES b/CHANGES index dc078635..c82ca1cf 100644 --- a/CHANGES +++ b/CHANGES @@ -8,22 +8,28 @@ pull request numbers to the requests at Version 2.3.0 ------------- -(not yet released) +(released Nov 25, 2018) - Added lexers: * Fennel (PR#783) * HLSL (PR#675) -- Added Python 3.7 support (PR#772) +- Updated lexers: + + * Dockerfile (PR#714) + +- Minimum Python versions changed to 2.7 and 3.5 +- Added support for Python 3.7 generator changes (PR#772) - Fix incorrect token type in SCSS for single-quote strings (#1322) - Use `terminal256` formatter if `TERM` contains `256` (PR#666) - Fix incorrect handling of GitHub style fences in Markdown (PR#741, #1389) - Fix `%a` not being highlighted in Python3 strings (PR#727) + Version 2.2.0 ------------- -(release Jan 22, 2017) +(released Jan 22, 2017) - Added lexers: diff --git a/Makefile b/Makefile index 82c4a124..878b94b7 100644 --- a/Makefile +++ b/Makefile @@ -63,3 +63,9 @@ tox-test: tox-test-coverage: @tox -- --with-coverage --cover-package=pygments --cover-erase $(TEST) + +RLMODULES = pygments.lexers + +regexlint: + @if [ -z "$(REGEXLINT)" ]; then echo "Please set REGEXLINT=checkout path"; exit 1; fi + PYTHONPATH=`pwd`:$(REGEXLINT) $(REGEXLINT)/regexlint/cmdline.py $(RLMODULES) diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index e3c74d6e..fc745f0f 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -31,4 +31,4 @@ pipelines: - pip script: - pip install -r requirements.txt - - tox -e py37 \ No newline at end of file + - tox -e py37 diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py index f9513e28..9ccc7615 100644 --- a/pygments/lexers/_mapping.py +++ b/pygments/lexers/_mapping.py @@ -153,7 +153,7 @@ LEXERS = { 'FancyLexer': ('pygments.lexers.ruby', 'Fancy', ('fancy', 'fy'), ('*.fy', '*.fancypack'), ('text/x-fancysrc',)), 'FantomLexer': ('pygments.lexers.fantom', 'Fantom', ('fan',), ('*.fan',), ('application/x-fantom',)), 'FelixLexer': ('pygments.lexers.felix', 'Felix', ('felix', 'flx'), ('*.flx', '*.flxh'), ('text/x-felix',)), - 'FennelLexer': ('pygments.lexers.lisp', 'Fennel', (), ('*.fnl',), ()), + 'FennelLexer': ('pygments.lexers.lisp', 'Fennel', ('fennel', 'fnl'), ('*.fnl',), ()), 'FishShellLexer': ('pygments.lexers.shell', 'Fish', ('fish', 'fishshell'), ('*.fish', '*.load'), ('application/x-fish',)), 'FlatlineLexer': ('pygments.lexers.dsls', 'Flatline', ('flatline',), (), ('text/x-flatline',)), 'ForthLexer': ('pygments.lexers.forth', 'Forth', ('forth',), ('*.frt', '*.fs'), ('application/x-forth',)), diff --git a/pygments/lexers/graphics.py b/pygments/lexers/graphics.py index 5c3ed7ee..fa5aa372 100644 --- a/pygments/lexers/graphics.py +++ b/pygments/lexers/graphics.py @@ -79,7 +79,7 @@ class HLSLShaderLexer(RegexLexer): """ HLSL (Microsoft Direct3D Shader) lexer. - .. versionadded:: 2.2 + .. versionadded:: 2.3 """ name = 'HLSL' aliases = ['hlsl'] diff --git a/pygments/lexers/javascript.py b/pygments/lexers/javascript.py index 862535c9..663557ab 100644 --- a/pygments/lexers/javascript.py +++ b/pygments/lexers/javascript.py @@ -1500,8 +1500,10 @@ class JuttleLexer(RegexLexer): (r'^(?=\s|/)', Text, 'slashstartsregex'), include('commentsandwhitespace'), (r':\d{2}:\d{2}:\d{2}(\.\d*)?:', String.Moment), - (r':(now|beginning|end|forever|yesterday|today|tomorrow|(\d+(\.\d*)?|\.\d+)(ms|[smhdwMy])?):', String.Moment), - (r':\d{4}-\d{2}-\d{2}(T\d{2}:\d{2}:\d{2}(\.\d*)?)?(Z|[+-]\d{2}:\d{2}|[+-]\d{4})?:', String.Moment), + (r':(now|beginning|end|forever|yesterday|today|tomorrow|' + r'(\d+(\.\d*)?|\.\d+)(ms|[smhdwMy])?):', String.Moment), + (r':\d{4}-\d{2}-\d{2}(T\d{2}:\d{2}:\d{2}(\.\d*)?)?' + r'(Z|[+-]\d{2}:\d{2}|[+-]\d{4})?:', String.Moment), (r':((\d+(\.\d*)?|\.\d+)[ ]+)?(millisecond|second|minute|hour|day|week|month|year)[s]?' r'(([ ]+and[ ]+(\d+[ ]+)?(millisecond|second|minute|hour|day|week|month|year)[s]?)' r'|[ ]+(ago|from[ ]+now))*:', String.Moment), diff --git a/pygments/lexers/lisp.py b/pygments/lexers/lisp.py index f1494b13..4a1ce137 100644 --- a/pygments/lexers/lisp.py +++ b/pygments/lexers/lisp.py @@ -2620,11 +2620,14 @@ class XtlangLexer(RegexLexer): ], } + class FennelLexer(RegexLexer): - """A lexer for the Fennel programming language + """A lexer for the `Fennel programming language `_. Fennel compiles to Lua, so all the Lua builtins are recognized as well as the special forms that are particular to the Fennel compiler. + + .. versionadded:: 2.3 """ name = 'Fennel' aliases = ['fennel', 'fnl'] diff --git a/pygments/lexers/webmisc.py b/pygments/lexers/webmisc.py index 712c8246..30dd3717 100644 --- a/pygments/lexers/webmisc.py +++ b/pygments/lexers/webmisc.py @@ -661,7 +661,8 @@ class XQueryLexer(ExtendedRegexLexer): # NAMESPACE KEYWORD (r'(declare)(\s+)(default)(\s+)(element|function)', - bygroups(Keyword.Declaration, Text, Keyword.Declaration, Text, Keyword.Declaration), 'namespacekeyword'), + bygroups(Keyword.Declaration, Text, Keyword.Declaration, Text, Keyword.Declaration), + 'namespacekeyword'), (r'(import)(\s+)(schema|module)', bygroups(Keyword.Pseudo, Text, Keyword.Pseudo), 'namespacekeyword'), (r'(declare)(\s+)(copy-namespaces)', diff --git a/scripts/release-checklist b/scripts/release-checklist index f18e6376..efc1e1e8 100644 --- a/scripts/release-checklist +++ b/scripts/release-checklist @@ -1,24 +1,24 @@ Release checklist ================= -* Check hg status -* Make check -* Make pylint -* Make test from clean checkout with all supported Python versions -* Update ez_setup.py -* Update version info in setup.py/__init__.py +* Check ``hg status`` +* ``make check`` +* LATER when configured properly: ``make pylint`` +* ``tox`` +* Update version info in ``setup.py/__init__.py`` * Check setup.py metadata: long description, trove classifiers -* Update release date/code name in CHANGES -* hg commit -* make clean -* For every supported version: - pythonX.Y setup.py release bdist_egg sdist upload +* Update release date/code name in ``CHANGES`` +* ``hg commit`` +* ``make clean`` +* ``python2 setup.py release bdist_wheel`` +* ``python3 setup.py release bdist_wheel sdist`` +* ``twine upload dist/Pygments-$NEWVER*`` * Check PyPI release page for obvious errors -* hg tag -* Make a maintenance branch if applicable +* ``hg tag`` +* Merge default into stable if this was a ``x.y.0`` * Update homepage (release info), regenerate docs (+printable!) * Add new version/milestone to tracker categories * Write announcement and send to mailing list/python-announce -* Update version info, add new CHANGES entry for next version -* hg commit -* hg push +* Update version info, add new ``CHANGES`` entry for next version +* ``hg commit`` +* ``hg push`` -- cgit v1.2.1 From 7fa421fd738fb31a4fb7148f089251fb55a21583 Mon Sep 17 00:00:00 2001 From: Camil Staps Date: Sun, 25 Nov 2018 10:41:28 +0100 Subject: Clean lexer: better support for qualified imports; add tests --- pygments/lexers/clean.py | 84 ++++++++++++++++++++++++++++++++++----- tests/examplefiles/StdGeneric.icl | 44 +++++++++++++++++++- 2 files changed, 116 insertions(+), 12 deletions(-) diff --git a/pygments/lexers/clean.py b/pygments/lexers/clean.py index b1a49632..dc973bea 100644 --- a/pygments/lexers/clean.py +++ b/pygments/lexers/clean.py @@ -29,23 +29,26 @@ class CleanLexer(ExtendedRegexLexer): keywords = ( 'case', 'ccall', 'class', 'code', 'code inline', 'derive', 'export', - 'foreign', 'from', 'generic', 'if', 'import', 'in', 'infix', 'infixl', - 'infixr', 'instance', 'let', 'of', 'otherwise', 'qualified', 'special', - 'stdcall', 'where', 'with') + 'foreign', 'generic', 'if', 'in', 'infix', 'infixl', 'infixr', + 'instance', 'let', 'of', 'otherwise', 'special', 'stdcall', 'where', + 'with') modulewords = ('implementation', 'definition', 'system') - lowerId = r'[a-z][\w\d`]*' - upperId = r'[A-Z][\w\d`]*' + lowerId = r'[a-z`][\w\d`]*' + upperId = r'[A-Z`][\w\d`]*' funnyId = r'[~@#\$%\^?!+\-*<>\\/|&=:]+' scoreUpperId = r'_' + upperId scoreLowerId = r'_' + lowerId + moduleId = r'[a-zA-Z_][a-zA-Z0-9_.`]+' + classId = '|'.join([lowerId, upperId, funnyId]) tokens = { 'root': [ include('comments'), include('keywords'), include('module'), + include('import'), include('whitespace'), include('literals'), include('operators'), @@ -75,12 +78,71 @@ class CleanLexer(ExtendedRegexLexer): (r'\bmodule\b', Keyword.Namespace, 'module.name'), ], 'module.name': [ + include('whitespace'), + (moduleId, Name.Class, '#pop'), + ], + 'import': [ + (r'\b(import)\b(\s*)', bygroups(Keyword, Whitespace), 'import.module'), + (r'\b(from)\b(\s*)\b(' + moduleId + r')\b(\s*)\b(import)\b', + bygroups(Keyword, Whitespace, Name.Class, Whitespace, Keyword), + 'import.what'), + ], + 'import.module': [ + (r'\b(qualified)\b(\s*)', bygroups(Keyword, Whitespace)), + (r'(\s*)\b(as)\b', bygroups(Whitespace, Keyword), ('#pop', 'import.module.as')), + (moduleId, Name.Class), + (r'(\s*)(,)(\s*)', bygroups(Whitespace, Punctuation, Whitespace)), + (r'\s*', Whitespace, '#pop'), + ], + 'import.module.as': [ include('whitespace'), (lowerId, Name.Class, '#pop'), (upperId, Name.Class, '#pop'), - (funnyId, Name.Class, '#pop'), - (scoreLowerId, Name.Class, '#pop'), - (scoreUpperId, Name.Class, '#pop'), + ], + 'import.what': [ + (r'\b(class)\b(\s+)(' + classId + r')', + bygroups(Keyword, Whitespace, Name.Class), 'import.what.class'), + (r'\b(instance)(\s+)(' + classId + r')(\s+)', + bygroups(Keyword, Whitespace, Name.Class, Whitespace), 'import.what.instance'), + (r'(::)(\s*)\b(' + upperId + r')\b', + bygroups(Punctuation, Whitespace, Name.Class), 'import.what.type'), + (r'\b(generic)\b(\s+)\b(' + lowerId + '|' + upperId + r')\b', + bygroups(Keyword, Whitespace, Name)), + include('names'), + (r'(,)(\s+)', bygroups(Punctuation, Whitespace)), + (r'$', Whitespace, '#pop'), + include('whitespace'), + ], + 'import.what.class': [ + (r',', Punctuation, '#pop'), + (r'\(', Punctuation, 'import.what.class.members'), + (r'$', Whitespace, '#pop:2'), + include('whitespace'), + ], + 'import.what.class.members': [ + (r',', Punctuation), + (r'\.\.', Punctuation), + (r'\)', Punctuation, '#pop'), + include('names'), + ], + 'import.what.instance': [ + (r'[,)]', Punctuation, '#pop'), + (r'\(', Punctuation, 'import.what.instance'), + (r'$', Whitespace, '#pop:2'), + include('whitespace'), + include('names'), + ], + 'import.what.type': [ + (r',', Punctuation, '#pop'), + (r'[({]', Punctuation, 'import.what.type.consesandfields'), + (r'$', Whitespace, '#pop:2'), + include('whitespace'), + ], + 'import.what.type.consesandfields': [ + (r',', Punctuation), + (r'\.\.', Punctuation), + (r'[)}]', Punctuation, '#pop'), + include('names'), ], 'literals': [ (r'\'([^\'\\]|\\(x[\da-fA-F]+|\d+|.))\'', Literal.Char), @@ -107,10 +169,10 @@ class CleanLexer(ExtendedRegexLexer): bygroups(Punctuation, Name.Class, Punctuation)), ], 'names': [ - (r'\b' + lowerId, Name), + (lowerId, Name), (scoreLowerId, Name), - (r'\b' + funnyId, Name.Function), - (r'\b' + upperId, Name.Class), + (funnyId, Name.Function), + (upperId, Name.Class), (scoreUpperId, Name.Class), ] } diff --git a/tests/examplefiles/StdGeneric.icl b/tests/examplefiles/StdGeneric.icl index 2e6c3931..891b510a 100644 --- a/tests/examplefiles/StdGeneric.icl +++ b/tests/examplefiles/StdGeneric.icl @@ -1,5 +1,13 @@ implementation module StdGeneric +/** + * NOTE: this is a collection of different tricky parts of Clean modules (even + * though the file is simply called StdGeneric.icl). The code is taken from: + * + * - StdGeneric (StdEnv) + * - Graphics.Scalable.Image (Platform) + */ + import StdInt, StdMisc, StdClass, StdFunc generic bimap a b :: Bimap .a .b @@ -89,4 +97,38 @@ where = [ ConsLeft : doit i (n/2) ] | otherwise = [ ConsRight : doit (i - (n/2)) (n - (n/2)) ] - \ No newline at end of file + +:: NoAttr m = NoAttr +:: DashAttr m = { dash :: ![Int] } +:: FillAttr m = { fill :: !SVGColor } +:: LineEndMarker m = { endmarker :: !Image m } +:: LineMidMarker m = { midmarker :: !Image m } +:: LineStartMarker m = { startmarker :: !Image m } +:: MaskAttr m = { mask :: !Image m } +:: OpacityAttr m = { opacity :: !Real } +:: StrokeAttr m = { stroke :: !SVGColor } +:: StrokeWidthAttr m = { strokewidth :: !Span } +:: XRadiusAttr m = { xradius :: !Span } +:: YRadiusAttr m = { yradius :: !Span } + + +instance tuneImage NoAttr where tuneImage image _ = image +instance tuneImage DashAttr where tuneImage image attr = Attr` (BasicImageAttr` (BasicImgDashAttr attr.DashAttr.dash)) image +instance tuneImage FillAttr where tuneImage image attr = Attr` (BasicImageAttr` (BasicImgFillAttr attr.FillAttr.fill)) image +instance tuneImage LineEndMarker where tuneImage image attr = Attr` (LineMarkerAttr` {LineMarkerAttr | markerImg = attr.LineEndMarker.endmarker, markerPos = LineMarkerEnd}) image +instance tuneImage LineMidMarker where tuneImage image attr = Attr` (LineMarkerAttr` {LineMarkerAttr | markerImg = attr.LineMidMarker.midmarker, markerPos = LineMarkerMid}) image +instance tuneImage LineStartMarker where tuneImage image attr = Attr` (LineMarkerAttr` {LineMarkerAttr | markerImg = attr.LineStartMarker.startmarker, markerPos = LineMarkerStart}) image +instance tuneImage MaskAttr where tuneImage image attr = Attr` (MaskAttr` attr.MaskAttr.mask) image +instance tuneImage OpacityAttr where tuneImage image attr = Attr` (BasicImageAttr` (BasicImgFillOpacityAttr attr.OpacityAttr.opacity)) image +instance tuneImage StrokeAttr where tuneImage image attr = Attr` (BasicImageAttr` (BasicImgStrokeAttr attr.StrokeAttr.stroke)) image +instance tuneImage StrokeWidthAttr where tuneImage image attr = Attr` (BasicImageAttr` (BasicImgStrokeWidthAttr attr.StrokeWidthAttr.strokewidth)) image +instance tuneImage XRadiusAttr where tuneImage image attr = Attr` (BasicImageAttr` (BasicImgXRadiusAttr attr.XRadiusAttr.xradius)) image +instance tuneImage YRadiusAttr where tuneImage image attr = Attr` (BasicImageAttr` (BasicImgYRadiusAttr attr.YRadiusAttr.yradius)) image + +instance tuneImage DraggableAttr where tuneImage image attr = Attr` (HandlerAttr` (ImgEventhandlerDraggableAttr attr)) image +instance tuneImage OnClickAttr where tuneImage image attr = Attr` (HandlerAttr` (ImgEventhandlerOnClickAttr attr)) image +instance tuneImage OnMouseDownAttr where tuneImage image attr = Attr` (HandlerAttr` (ImgEventhandlerOnMouseDownAttr attr)) image +instance tuneImage OnMouseMoveAttr where tuneImage image attr = Attr` (HandlerAttr` (ImgEventhandlerOnMouseMoveAttr attr)) image +instance tuneImage OnMouseOutAttr where tuneImage image attr = Attr` (HandlerAttr` (ImgEventhandlerOnMouseOutAttr attr)) image +instance tuneImage OnMouseOverAttr where tuneImage image attr = Attr` (HandlerAttr` (ImgEventhandlerOnMouseOverAttr attr)) image +instance tuneImage OnMouseUpAttr where tuneImage image attr = Attr` (HandlerAttr` (ImgEventhandlerOnMouseUpAttr attr)) image -- cgit v1.2.1 -- cgit v1.2.1 From a1ae0fb71c07a83c3e07356ef22d249a7f965dc8 Mon Sep 17 00:00:00 2001 From: michal sojka Date: Mon, 26 Nov 2018 16:03:12 +0100 Subject: Fix handling of end-of-line comments in asm lexer When there is a comment after an instruction, e.g. mov $1,%eax ; move 1 to eax ret the next instruction (ret above) is not properly highlighted. This patch fixes that. This was also reported in #1477. --- pygments/lexers/asm.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pygments/lexers/asm.py b/pygments/lexers/asm.py index 5b8cab80..411b7cc4 100644 --- a/pygments/lexers/asm.py +++ b/pygments/lexers/asm.py @@ -53,6 +53,7 @@ class GasLexer(RegexLexer): ('@' + identifier, Name.Attribute), (number, Number.Integer), (r'[\r\n]+', Text, '#pop'), + (r'[;#].*?\n', Comment, '#pop'), include('punctuation'), include('whitespace') @@ -76,6 +77,7 @@ class GasLexer(RegexLexer): ('$'+number, Number.Integer), (r"$'(.|\\')'", String.Char), (r'[\r\n]+', Text, '#pop'), + (r'[;#].*?\n', Comment, '#pop'), include('punctuation'), include('whitespace') -- cgit v1.2.1 From 8acfcefb45af3a7fce54c6f573106b738b004c92 Mon Sep 17 00:00:00 2001 From: "Matth?us G. Chajdas" Date: Tue, 27 Nov 2018 21:06:54 +0100 Subject: Remove duplicated number definition. --- pygments/lexers/python.py | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/pygments/lexers/python.py b/pygments/lexers/python.py index 212f38af..0756d009 100644 --- a/pygments/lexers/python.py +++ b/pygments/lexers/python.py @@ -361,12 +361,12 @@ class Python3Lexer(RegexLexer): Name.Variable.Magic), ] tokens['numbers'] = [ - (r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?', Number.Float), - (r'\d+[eE][+-]?[0-9]+j?', Number.Float), - (r'0[oO][0-7]+', Number.Oct), - (r'0[bB][01]+', Number.Bin), - (r'0[xX][a-fA-F0-9]+', Number.Hex), - (r'\d+', Number.Integer) + (r'(\d(?:_?\d)*\.(?:\d(?:_?\d)*)?|(?:\d(?:_?\d)*)?\.\d(?:_?\d)*)([eE][+-]?\d(?:_?\d)*)?', Number.Float), + (r'\d(?:_?\d)*[eE][+-]?\d(?:_?\d)*j?', Number.Float), + (r'0[oO](?:_?[0-7])+', Number.Oct), + (r'0[bB](?:_?[01])+', Number.Bin), + (r'0[xX](?:_?[a-fA-F0-9])+', Number.Hex), + (r'\d(?:_?\d)*', Number.Integer) ] tokens['backtick'] = [] tokens['name'] = [ @@ -395,14 +395,6 @@ class Python3Lexer(RegexLexer): ] tokens['strings-single'] = innerstring_rules(String.Single) tokens['strings-double'] = innerstring_rules(String.Double) - tokens['numbers'] = [ - (r'(\d(?:_?\d)*\.(?:\d(?:_?\d)*)?|(?:\d(?:_?\d)*)?\.\d(?:_?\d)*)([eE][+-]?\d(?:_?\d)*)?', Number.Float), - (r'\d(?:_?\d)*[eE][+-]?\d(?:_?\d)*j?', Number.Float), - (r'0[oO](?:_?[0-7])+', Number.Oct), - (r'0[bB](?:_?[01])+', Number.Bin), - (r'0[xX](?:_?[a-fA-F0-9])+', Number.Hex), - (r'\d(?:_?\d)*', Number.Integer) - ] def analyse_text(text): -- cgit v1.2.1 From 46c8092c1804b2d19d275024614a22e22f963aef Mon Sep 17 00:00:00 2001 From: "Matth?us G. Chajdas" Date: Tue, 27 Nov 2018 22:03:28 +0100 Subject: Fix Csound name callback cutting off part of the name. Previously, a name like P:C would get parsed as P, breaking the roundtrip. This would result in random test failures during roundtrip testing. --- pygments/lexers/csound.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pygments/lexers/csound.py b/pygments/lexers/csound.py index b86de091..c30b87cc 100644 --- a/pygments/lexers/csound.py +++ b/pygments/lexers/csound.py @@ -227,6 +227,9 @@ class CsoundOrchestraLexer(CsoundLexer): yield nameMatch.start(2), Name, nameMatch.group(2) else: yield match.start(), Name, name + if match.group(2): + yield match.start(2), Punctuation, match.group(2) + yield match.start(3), Name, match.group(3) tokens = { 'root': [ -- cgit v1.2.1 From 075938a7748906f9d20981b90d07cab30b3a3121 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Wed, 28 Nov 2018 05:54:28 +0100 Subject: Handle regex syntax FutureWarnings. Make sure they are converted to errors in test runs, to catch new introductions immediately. --- pygments/lexers/capnproto.py | 12 ++++++------ pygments/lexers/dsls.py | 2 +- pygments/lexers/int_fiction.py | 2 +- pygments/lexers/julia.py | 2 +- tests/run.py | 4 ++++ 5 files changed, 13 insertions(+), 9 deletions(-) diff --git a/pygments/lexers/capnproto.py b/pygments/lexers/capnproto.py index 203523a1..2615dcaf 100644 --- a/pygments/lexers/capnproto.py +++ b/pygments/lexers/capnproto.py @@ -44,34 +44,34 @@ class CapnProtoLexer(RegexLexer): ], 'type': [ (r'[^][=;,(){}$]+', Name.Class), - (r'[[(]', Name.Class, 'parentype'), + (r'[\[(]', Name.Class, 'parentype'), default('#pop'), ], 'parentype': [ (r'[^][;()]+', Name.Class), - (r'[[(]', Name.Class, '#push'), + (r'[\[(]', Name.Class, '#push'), (r'[])]', Name.Class, '#pop'), default('#pop'), ], 'expression': [ (r'[^][;,(){}$]+', Literal), - (r'[[(]', Literal, 'parenexp'), + (r'[\[(]', Literal, 'parenexp'), default('#pop'), ], 'parenexp': [ (r'[^][;()]+', Literal), - (r'[[(]', Literal, '#push'), + (r'[\[(]', Literal, '#push'), (r'[])]', Literal, '#pop'), default('#pop'), ], 'annotation': [ (r'[^][;,(){}=:]+', Name.Attribute), - (r'[[(]', Name.Attribute, 'annexp'), + (r'[\[(]', Name.Attribute, 'annexp'), default('#pop'), ], 'annexp': [ (r'[^][;()]+', Name.Attribute), - (r'[[(]', Name.Attribute, '#push'), + (r'[\[(]', Name.Attribute, '#push'), (r'[])]', Name.Attribute, '#pop'), default('#pop'), ], diff --git a/pygments/lexers/dsls.py b/pygments/lexers/dsls.py index a1426bd6..b2afc875 100644 --- a/pygments/lexers/dsls.py +++ b/pygments/lexers/dsls.py @@ -688,7 +688,7 @@ class CrmshLexer(RegexLexer): (r'([\w#$-]+)(?:(:)(%s))?(?![\w#$-])' % rsc_role_action, bygroups(Name, Punctuation, Operator.Word)), # punctuation - (r'(\\(?=\n)|[[\](){}/:@])', Punctuation), + (r'(\\(?=\n)|[\[\](){}/:@])', Punctuation), (r'\s+|\n', Whitespace), ], } diff --git a/pygments/lexers/int_fiction.py b/pygments/lexers/int_fiction.py index f280a56d..57ace259 100644 --- a/pygments/lexers/int_fiction.py +++ b/pygments/lexers/int_fiction.py @@ -911,7 +911,7 @@ class Tads3Lexer(RegexLexer): 'block?/root': [ (r'\{', Punctuation, ('#pop', 'block')), include('whitespace'), - (r'(?=[[\'"<(:])', Text, # It might be a VerbRule macro. + (r'(?=[\[\'"<(:])', Text, # It might be a VerbRule macro. ('#pop', 'object-body/no-braces', 'grammar', 'grammar-rules')), # It might be a macro like DefineAction. default(('#pop', 'object-body/no-braces')) diff --git a/pygments/lexers/julia.py b/pygments/lexers/julia.py index 67453aba..56578270 100644 --- a/pygments/lexers/julia.py +++ b/pygments/lexers/julia.py @@ -181,7 +181,7 @@ class JuliaLexer(RegexLexer): # prec-dot u'.', # unary op - u'+', u'-', u'!', u'~', u'√', u'∛', u'∜' + u'+', u'-', u'!', u'√', u'∛', u'∜' ]), Operator), # chars diff --git a/tests/run.py b/tests/run.py index 07665b2a..bdaceb7e 100644 --- a/tests/run.py +++ b/tests/run.py @@ -16,6 +16,7 @@ from __future__ import print_function import os import sys +import warnings # only find tests in this directory if os.path.dirname(__file__): @@ -31,6 +32,9 @@ except ImportError: # make sure the current source is first on sys.path sys.path.insert(0, '..') +# make FutureWarnings (coming from Regex syntax most likely) an error +warnings.filterwarnings("error", category=FutureWarning) + if '--with-coverage' not in sys.argv: # if running with coverage, pygments should not be imported before coverage # is started, otherwise it will count already executed lines as uncovered -- cgit v1.2.1 From cdb2a89a25131f46a09a5497e56196c2fe69e184 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Wed, 28 Nov 2018 15:49:16 +0100 Subject: Fix invalid escapes due to missing raw string prefix. --- pygments/lexers/clean.py | 2 +- pygments/lexers/configs.py | 2 +- pygments/lexers/csound.py | 2 +- pygments/lexers/dsls.py | 4 ++-- pygments/lexers/graphics.py | 10 +++++----- pygments/lexers/julia.py | 2 +- pygments/lexers/lisp.py | 12 ++++++------ pygments/lexers/rdf.py | 8 ++++---- pygments/lexers/sql.py | 4 ++-- pygments/lexers/templates.py | 44 +++++++++++++++++++++---------------------- pygments/lexers/typoscript.py | 4 ++-- tests/run.py | 6 ++++++ tests/test_html_formatter.py | 4 ++-- tests/test_rtf_formatter.py | 2 +- 14 files changed, 56 insertions(+), 50 deletions(-) diff --git a/pygments/lexers/clean.py b/pygments/lexers/clean.py index ba2569f6..4330dcc0 100644 --- a/pygments/lexers/clean.py +++ b/pygments/lexers/clean.py @@ -203,7 +203,7 @@ class CleanLexer(ExtendedRegexLexer): 'import': [ include('common'), (words(('from', 'import', 'as', 'qualified'), - prefix='(?<=\s)', suffix='(?=\s)'), Keyword.Namespace), + prefix=r'(?<=\s)', suffix=r'(?=\s)'), Keyword.Namespace), (r'[\w`.]+', Name.Class), (r'\n', Whitespace, '#pop'), (r',', Punctuation), diff --git a/pygments/lexers/configs.py b/pygments/lexers/configs.py index c39b1a52..c3035f30 100644 --- a/pygments/lexers/configs.py +++ b/pygments/lexers/configs.py @@ -586,7 +586,7 @@ class TerraformLexer(RegexLexer): prefix=r'\b', suffix=r'\b'), Keyword.Reserved, 'function'), (words(('ingress', 'egress', 'listener', 'default', 'connection', 'alias'), prefix=r'\b', suffix=r'\b'), Keyword.Declaration), - ('\$\{', String.Interpol, 'var_builtin'), + (r'\$\{', String.Interpol, 'var_builtin'), ], 'function': [ (r'(\s+)(".*")(\s+)', bygroups(Text, String, Text)), diff --git a/pygments/lexers/csound.py b/pygments/lexers/csound.py index 858aa348..1d97f54a 100644 --- a/pygments/lexers/csound.py +++ b/pygments/lexers/csound.py @@ -153,7 +153,7 @@ class CsoundOrchestraLexer(CsoundLexer): def name_callback(lexer, match): name = match.group(0) - if re.match('p\d+$', name) or name in OPCODES: + if re.match(r'p\d+$', name) or name in OPCODES: yield match.start(), Name.Builtin, name elif name in lexer.user_defined_opcodes: yield match.start(), Name.Function, name diff --git a/pygments/lexers/dsls.py b/pygments/lexers/dsls.py index a1426bd6..045a37cf 100644 --- a/pygments/lexers/dsls.py +++ b/pygments/lexers/dsls.py @@ -66,7 +66,7 @@ class ProtoBufLexer(RegexLexer): (r'[+-=]', Operator), (r'([a-zA-Z_][\w.]*)([ \t]*)(=)', bygroups(Name.Attribute, Text, Operator)), - ('[a-zA-Z_][\w.]*', Name), + (r'[a-zA-Z_][\w.]*', Name), ], 'package': [ (r'[a-zA-Z_]\w*', Name.Namespace, '#pop'), @@ -300,7 +300,7 @@ class PuppetLexer(RegexLexer): ], 'names': [ - ('[a-zA-Z_]\w*', Name.Attribute), + (r'[a-zA-Z_]\w*', Name.Attribute), (r'(\$\S+)(\[)(\S+)(\])', bygroups(Name.Variable, Punctuation, String, Punctuation)), (r'\$\S+', Name.Variable), diff --git a/pygments/lexers/graphics.py b/pygments/lexers/graphics.py index c8af9f99..ecf8717e 100644 --- a/pygments/lexers/graphics.py +++ b/pygments/lexers/graphics.py @@ -233,8 +233,8 @@ class AsymptoteLexer(RegexLexer): r'bounds|coord|frame|guide|horner|int|linefit|marginT|pair|pen|' r'picture|position|real|revolution|slice|splitface|ticksgridT|' r'tickvalues|tree|triple|vertex|void)\b', Keyword.Type), - ('[a-zA-Z_]\w*:(?!:)', Name.Label), - ('[a-zA-Z_]\w*', Name), + (r'[a-zA-Z_]\w*:(?!:)', Name.Label), + (r'[a-zA-Z_]\w*', Name), ], 'root': [ include('whitespace'), @@ -334,9 +334,9 @@ class GnuplotLexer(RegexLexer): (_shortened_many('pwd$', 're$read', 'res$et', 'scr$eendump', 'she$ll', 'test$'), Keyword, 'noargs'), - ('([a-zA-Z_]\w*)(\s*)(=)', + (r'([a-zA-Z_]\w*)(\s*)(=)', bygroups(Name.Variable, Text, Operator), 'genericargs'), - ('([a-zA-Z_]\w*)(\s*\(.*?\)\s*)(=)', + (r'([a-zA-Z_]\w*)(\s*\(.*?\)\s*)(=)', bygroups(Name.Function, Text, Operator), 'genericargs'), (r'@[a-zA-Z_]\w*', Name.Constant), # macros (r';', Keyword), @@ -382,7 +382,7 @@ class GnuplotLexer(RegexLexer): (r'(\d+\.\d*|\.\d+)', Number.Float), (r'-?\d+', Number.Integer), ('[,.~!%^&*+=|?:<>/-]', Operator), - ('[{}()\[\]]', Punctuation), + (r'[{}()\[\]]', Punctuation), (r'(eq|ne)\b', Operator.Word), (r'([a-zA-Z_]\w*)(\s*)(\()', bygroups(Name.Function, Text, Punctuation)), diff --git a/pygments/lexers/julia.py b/pygments/lexers/julia.py index 67453aba..333c4d59 100644 --- a/pygments/lexers/julia.py +++ b/pygments/lexers/julia.py @@ -146,7 +146,7 @@ class JuliaLexer(RegexLexer): (words([ # prec-assignment u'=', u':=', u'+=', u'-=', u'*=', u'/=', u'//=', u'.//=', u'.*=', u'./=', - u'\=', u'.\=', u'^=', u'.^=', u'÷=', u'.÷=', u'%=', u'.%=', u'|=', u'&=', + u'\\=', u'.\\=', u'^=', u'.^=', u'÷=', u'.÷=', u'%=', u'.%=', u'|=', u'&=', u'$=', u'=>', u'<<=', u'>>=', u'>>>=', u'~', u'.+=', u'.-=', # prec-conditional u'?', diff --git a/pygments/lexers/lisp.py b/pygments/lexers/lisp.py index b4c26659..12c3f9b9 100644 --- a/pygments/lexers/lisp.py +++ b/pygments/lexers/lisp.py @@ -1249,7 +1249,7 @@ class RacketLexer(RegexLexer): _opening_parenthesis = r'[([{]' _closing_parenthesis = r'[)\]}]' _delimiters = r'()[\]{}",\'`;\s' - _symbol = r'(?u)(?:\|[^|]*\||\\[\w\W]|[^|\\%s]+)+' % _delimiters + _symbol = r'(?:\|[^|]*\||\\[\w\W]|[^|\\%s]+)+' % _delimiters _exact_decimal_prefix = r'(?:#e)?(?:#d)?(?:#e)?' _exponent = r'(?:[defls][-+]?\d+)' _inexact_simple_no_hashes = r'(?:\d+(?:/\d+|\.\d*)?|\.\d+)' @@ -1301,16 +1301,16 @@ class RacketLexer(RegexLexer): (_inexact_simple, _delimiters), Number.Float, '#pop'), # #b - (r'(?i)(#[ei])?#b%s' % _symbol, Number.Bin, '#pop'), + (r'(?iu)(#[ei])?#b%s' % _symbol, Number.Bin, '#pop'), # #o - (r'(?i)(#[ei])?#o%s' % _symbol, Number.Oct, '#pop'), + (r'(?iu)(#[ei])?#o%s' % _symbol, Number.Oct, '#pop'), # #x - (r'(?i)(#[ei])?#x%s' % _symbol, Number.Hex, '#pop'), + (r'(?iu)(#[ei])?#x%s' % _symbol, Number.Hex, '#pop'), # #i is always inexact, i.e. float - (r'(?i)(#d)?#i%s' % _symbol, Number.Float, '#pop'), + (r'(?iu)(#d)?#i%s' % _symbol, Number.Float, '#pop'), # Strings and characters (r'#?"', String.Double, ('#pop', 'string')), @@ -1323,7 +1323,7 @@ class RacketLexer(RegexLexer): (r'#(true|false|[tTfF])', Name.Constant, '#pop'), # Keyword argument names (e.g. #:keyword) - (r'#:%s' % _symbol, Keyword.Declaration, '#pop'), + (r'(?u)#:%s' % _symbol, Keyword.Declaration, '#pop'), # Reader extensions (r'(#lang |#!)(\S+)', diff --git a/pygments/lexers/rdf.py b/pygments/lexers/rdf.py index d0f8778a..27bbe154 100644 --- a/pygments/lexers/rdf.py +++ b/pygments/lexers/rdf.py @@ -97,7 +97,7 @@ class SparqlLexer(RegexLexer): 'root': [ (r'\s+', Text), # keywords :: - (r'((?i)select|construct|describe|ask|where|filter|group\s+by|minus|' + (r'(?i)(select|construct|describe|ask|where|filter|group\s+by|minus|' r'distinct|reduced|from\s+named|from|order\s+by|desc|asc|limit|' r'offset|bindings|load|clear|drop|create|add|move|copy|' r'insert\s+data|delete\s+data|delete\s+where|delete|insert|' @@ -111,10 +111,10 @@ class SparqlLexer(RegexLexer): # # variables :: ('[?$]' + VARNAME, Name.Variable), # prefixed names :: - (r'(' + PN_PREFIX + ')?(\:)(' + PN_LOCAL + ')?', + (r'(' + PN_PREFIX + r')?(\:)(' + PN_LOCAL + r')?', bygroups(Name.Namespace, Punctuation, Name.Tag)), # function names :: - (r'((?i)str|lang|langmatches|datatype|bound|iri|uri|bnode|rand|abs|' + (r'(?i)(str|lang|langmatches|datatype|bound|iri|uri|bnode|rand|abs|' r'ceil|floor|round|concat|strlen|ucase|lcase|encode_for_uri|' r'contains|strstarts|strends|strbefore|strafter|year|month|day|' r'hours|minutes|seconds|timezone|tz|now|md5|sha1|sha256|sha384|' @@ -125,7 +125,7 @@ class SparqlLexer(RegexLexer): # boolean literals :: (r'(true|false)', Keyword.Constant), # double literals :: - (r'[+\-]?(\d+\.\d*' + EXPONENT + '|\.?\d+' + EXPONENT + ')', Number.Float), + (r'[+\-]?(\d+\.\d*' + EXPONENT + r'|\.?\d+' + EXPONENT + ')', Number.Float), # decimal literals :: (r'[+\-]?(\d+\.\d*|\.\d+)', Number.Float), # integer literals :: diff --git a/pygments/lexers/sql.py b/pygments/lexers/sql.py index 7507c0fc..bb216778 100644 --- a/pygments/lexers/sql.py +++ b/pygments/lexers/sql.py @@ -155,7 +155,7 @@ class PostgresLexer(PostgresBase, RegexLexer): (r'\s+', Text), (r'--.*\n?', Comment.Single), (r'/\*', Comment.Multiline, 'multiline-comments'), - (r'(' + '|'.join(s.replace(" ", "\s+") + (r'(' + '|'.join(s.replace(" ", r"\s+") for s in DATATYPES + PSEUDO_TYPES) + r')\b', Name.Builtin), (words(KEYWORDS, suffix=r'\b'), Keyword), @@ -499,7 +499,7 @@ class TransactSqlLexer(RegexLexer): tokens = { 'root': [ (r'\s+', Whitespace), - (r'--(?m).*?$\n?', Comment.Single), + (r'(?m)--.*?$\n?', Comment.Single), (r'/\*', Comment.Multiline, 'multiline-comments'), (words(_tsql_builtins.OPERATORS), Operator), (words(_tsql_builtins.OPERATOR_WORDS, suffix=r'\b'), Operator.Word), diff --git a/pygments/lexers/templates.py b/pygments/lexers/templates.py index 83c57db8..c184b2dd 100644 --- a/pygments/lexers/templates.py +++ b/pygments/lexers/templates.py @@ -187,13 +187,13 @@ class SmartyLexer(RegexLexer): def analyse_text(text): rv = 0.0 - if re.search('\{if\s+.*?\}.*?\{/if\}', text): + if re.search(r'\{if\s+.*?\}.*?\{/if\}', text): rv += 0.15 - if re.search('\{include\s+file=.*?\}', text): + if re.search(r'\{include\s+file=.*?\}', text): rv += 0.15 - if re.search('\{foreach\s+.*?\}.*?\{/foreach\}', text): + if re.search(r'\{foreach\s+.*?\}.*?\{/foreach\}', text): rv += 0.15 - if re.search('\{\$.*?\}', text): + if re.search(r'\{\$.*?\}', text): rv += 0.01 return rv @@ -421,18 +421,18 @@ class MyghtyLexer(RegexLexer): tokens = { 'root': [ (r'\s+', Text), - (r'(<%(?:def|method))(\s*)(.*?)(>)(.*?)()(?s)', + (r'(?s)(<%(?:def|method))(\s*)(.*?)(>)(.*?)()', bygroups(Name.Tag, Text, Name.Function, Name.Tag, using(this), Name.Tag)), - (r'(<%\w+)(.*?)(>)(.*?)()(?s)', + (r'(?s)(<%\w+)(.*?)(>)(.*?)()', bygroups(Name.Tag, Name.Function, Name.Tag, using(PythonLexer), Name.Tag)), (r'(<&[^|])(.*?)(,.*?)?(&>)', bygroups(Name.Tag, Name.Function, using(PythonLexer), Name.Tag)), - (r'(<&\|)(.*?)(,.*?)?(&>)(?s)', + (r'(?s)(<&\|)(.*?)(,.*?)?(&>)', bygroups(Name.Tag, Name.Function, using(PythonLexer), Name.Tag)), (r'', Name.Tag), - (r'(<%!?)(.*?)(%>)(?s)', + (r'(?s)(<%!?)(.*?)(%>)', bygroups(Name.Tag, using(PythonLexer), Name.Tag)), (r'(?<=^)#[^\n]*(\n|\Z)', Comment), (r'(?<=^)(%)([^\n]*)(\n|\Z)', @@ -538,20 +538,20 @@ class MasonLexer(RegexLexer): tokens = { 'root': [ (r'\s+', Text), - (r'(<%doc>)(.*?)()(?s)', + (r'(?s)(<%doc>)(.*?)()', bygroups(Name.Tag, Comment.Multiline, Name.Tag)), - (r'(<%(?:def|method))(\s*)(.*?)(>)(.*?)()(?s)', + (r'(?s)(<%(?:def|method))(\s*)(.*?)(>)(.*?)()', bygroups(Name.Tag, Text, Name.Function, Name.Tag, using(this), Name.Tag)), - (r'(<%\w+)(.*?)(>)(.*?)()(?s)', + (r'(?s)(<%\w+)(.*?)(>)(.*?)()', bygroups(Name.Tag, Name.Function, Name.Tag, using(PerlLexer), Name.Tag)), - (r'(<&[^|])(.*?)(,.*?)?(&>)(?s)', + (r'(?s)(<&[^|])(.*?)(,.*?)?(&>)', bygroups(Name.Tag, Name.Function, using(PerlLexer), Name.Tag)), - (r'(<&\|)(.*?)(,.*?)?(&>)(?s)', + (r'(?s)(<&\|)(.*?)(,.*?)?(&>)', bygroups(Name.Tag, Name.Function, using(PerlLexer), Name.Tag)), (r'', Name.Tag), - (r'(<%!?)(.*?)(%>)(?s)', + (r'(?s)(<%!?)(.*?)(%>)', bygroups(Name.Tag, using(PerlLexer), Name.Tag)), (r'(?<=^)#[^\n]*(\n|\Z)', Comment), (r'(?<=^)(%)([^\n]*)(\n|\Z)', @@ -607,7 +607,7 @@ class MakoLexer(RegexLexer): (r'()', bygroups(Comment.Preproc, Name.Builtin, Comment.Preproc)), (r'<%(?=([\w.:]+))', Comment.Preproc, 'ondeftags'), - (r'(<%(?:!?))(.*?)(%>)(?s)', + (r'(?s)(<%(?:!?))(.*?)(%>)', bygroups(Comment.Preproc, using(PythonLexer), Comment.Preproc)), (r'(\$\{)(.*?)(\})', bygroups(Comment.Preproc, using(PythonLexer), Comment.Preproc)), @@ -759,7 +759,7 @@ class CheetahLexer(RegexLexer): # TODO support other Python syntax like $foo['bar'] (r'(\$)([a-zA-Z_][\w.]*\w)', bygroups(Comment.Preproc, using(CheetahPythonLexer))), - (r'(\$\{!?)(.*?)(\})(?s)', + (r'(?s)(\$\{!?)(.*?)(\})', bygroups(Comment.Preproc, using(CheetahPythonLexer), Comment.Preproc)), (r'''(?sx) @@ -942,9 +942,9 @@ class HtmlGenshiLexer(DelegatingLexer): def analyse_text(text): rv = 0.0 - if re.search('\$\{.*?\}', text) is not None: + if re.search(r'\$\{.*?\}', text) is not None: rv += 0.2 - if re.search('py:(.*?)=["\']', text) is not None: + if re.search(r'py:(.*?)=["\']', text) is not None: rv += 0.2 return rv + HtmlLexer.analyse_text(text) - 0.01 @@ -967,9 +967,9 @@ class GenshiLexer(DelegatingLexer): def analyse_text(text): rv = 0.0 - if re.search('\$\{.*?\}', text) is not None: + if re.search(r'\$\{.*?\}', text) is not None: rv += 0.2 - if re.search('py:(.*?)=["\']', text) is not None: + if re.search(r'py:(.*?)=["\']', text) is not None: rv += 0.2 return rv + XmlLexer.analyse_text(text) - 0.01 @@ -1627,7 +1627,7 @@ class SspLexer(DelegatingLexer): def analyse_text(text): rv = 0.0 - if re.search('val \w+\s*:', text): + if re.search(r'val \w+\s*:', text): rv += 0.6 if looks_like_xml(text): rv += 0.2 @@ -1955,7 +1955,7 @@ class LiquidLexer(RegexLexer): 'output': [ include('whitespace'), - ('\}\}', Punctuation, '#pop'), # end of output + (r'\}\}', Punctuation, '#pop'), # end of output (r'\|', Punctuation, 'filters') ], diff --git a/pygments/lexers/typoscript.py b/pygments/lexers/typoscript.py index e358af07..7da87c75 100644 --- a/pygments/lexers/typoscript.py +++ b/pygments/lexers/typoscript.py @@ -132,7 +132,7 @@ class TypoScriptLexer(RegexLexer): ], 'keywords': [ # Conditions - (r'(\[)(?i)(browser|compatVersion|dayofmonth|dayofweek|dayofyear|' + (r'(?i)(\[)(browser|compatVersion|dayofmonth|dayofweek|dayofyear|' r'device|ELSE|END|GLOBAL|globalString|globalVar|hostname|hour|IP|' r'language|loginUser|loginuser|minute|month|page|PIDinRootline|' r'PIDupinRootline|system|treeLevel|useragent|userFunc|usergroup|' @@ -172,7 +172,7 @@ class TypoScriptLexer(RegexLexer): 'html': [ (r'<\S[^\n>]*>', using(TypoScriptHtmlDataLexer)), (r'&[^;\n]*;', String), - (r'(_CSS_DEFAULT_STYLE)(\s*)(\()(?s)(.*(?=\n\)))', + (r'(?s)(_CSS_DEFAULT_STYLE)(\s*)(\()(.*(?=\n\)))', bygroups(Name.Class, Text, String.Symbol, using(TypoScriptCssDataLexer))), ], 'literal': [ diff --git a/tests/run.py b/tests/run.py index 07665b2a..41279bef 100644 --- a/tests/run.py +++ b/tests/run.py @@ -16,6 +16,7 @@ from __future__ import print_function import os import sys +import warnings # only find tests in this directory if os.path.dirname(__file__): @@ -31,6 +32,11 @@ except ImportError: # make sure the current source is first on sys.path sys.path.insert(0, '..') +# make FutureWarnings (coming from Regex syntax most likely) and +# DeprecationWarnings (coming from invalid escapes due to non-raw strings) +# an error +warnings.filterwarnings("error", category=DeprecationWarning) + if '--with-coverage' not in sys.argv: # if running with coverage, pygments should not be imported before coverage # is started, otherwise it will count already executed lines as uncovered diff --git a/tests/test_html_formatter.py b/tests/test_html_formatter.py index 79990edd..10450c56 100644 --- a/tests/test_html_formatter.py +++ b/tests/test_html_formatter.py @@ -100,7 +100,7 @@ class HtmlFormatterTest(unittest.TestCase): fmt = HtmlFormatter(**optdict) fmt.format(tokensource, outfile) html = outfile.getvalue() - self.assertTrue(re.search("
\s+1\s+2\s+3", html))
+        self.assertTrue(re.search(r"
\s+1\s+2\s+3", html))
 
     def test_linenos_with_startnum(self):
         optdict = dict(linenos=True, linenostart=5)
@@ -108,7 +108,7 @@ class HtmlFormatterTest(unittest.TestCase):
         fmt = HtmlFormatter(**optdict)
         fmt.format(tokensource, outfile)
         html = outfile.getvalue()
-        self.assertTrue(re.search("
\s+5\s+6\s+7", html))
+        self.assertTrue(re.search(r"
\s+5\s+6\s+7", html))
 
     def test_lineanchors(self):
         optdict = dict(lineanchors="foo")
diff --git a/tests/test_rtf_formatter.py b/tests/test_rtf_formatter.py
index 756c03a9..44da5768 100644
--- a/tests/test_rtf_formatter.py
+++ b/tests/test_rtf_formatter.py
@@ -80,7 +80,7 @@ class RtfFormatterTest(StringTests, unittest.TestCase):
         self.assertEndsWith(result, expected+self.foot, msg)
 
     def test_escape_characters(self):
-        t = u'\ {{'
+        t = u'\\ {{'
         result = self.format_rtf(t)
         expected = (r'\\ \{\{')
         if not result.endswith(self.foot):
-- 
cgit v1.2.1


From 42bcf76904ad6ccf1005296a8d1688ebe1bc562d Mon Sep 17 00:00:00 2001
From: Georg Brandl 
Date: Wed, 28 Nov 2018 16:26:35 +0100
Subject: Update the unistring data.

---
 pygments/unistring.py | 76 +++++++++++++++++++++++++++------------------------
 1 file changed, 40 insertions(+), 36 deletions(-)

diff --git a/pygments/unistring.py b/pygments/unistring.py
index 6096d110..ba0e41de 100644
--- a/pygments/unistring.py
+++ b/pygments/unistring.py
@@ -16,9 +16,9 @@ import sys
 
 Cc = u'\x00-\x1f\x7f-\x9f'
 
-Cf = u'\xad\u0600-\u0604\u061c\u06dd\u070f\u180e\u200b-\u200f\u202a-\u202e\u2060-\u2064\u2066-\u206f\ufeff\ufff9-\ufffb'
+Cf = u'\xad\u0600-\u0605\u061c\u06dd\u070f\u08e2\u180e\u200b-\u200f\u202a-\u202e\u2060-\u2064\u2066-\u206f\ufeff\ufff9-\ufffb'
 
-Cn = u'\u0378-\u0379\u037f-\u0383\u038b\u038d\u03a2\u0528-\u0530\u0557-\u0558\u0560\u0588\u058b-\u058e\u0590\u05c8-\u05cf\u05eb-\u05ef\u05f5-\u05ff\u0605\u061d\u070e\u074b-\u074c\u07b2-\u07bf\u07fb-\u07ff\u082e-\u082f\u083f\u085c-\u085d\u085f-\u089f\u08a1\u08ad-\u08e3\u08ff\u0978\u0980\u0984\u098d-\u098e\u0991-\u0992\u09a9\u09b1\u09b3-\u09b5\u09ba-\u09bb\u09c5-\u09c6\u09c9-\u09ca\u09cf-\u09d6\u09d8-\u09db\u09de\u09e4-\u09e5\u09fc-\u0a00\u0a04\u0a0b-\u0a0e\u0a11-\u0a12\u0a29\u0a31\u0a34\u0a37\u0a3a-\u0a3b\u0a3d\u0a43-\u0a46\u0a49-\u0a4a\u0a4e-\u0a50\u0a52-\u0a58\u0a5d\u0a5f-\u0a65\u0a76-\u0a80\u0a84\u0a8e\u0a92\u0aa9\u0ab1\u0ab4\u0aba-\u0abb\u0ac6\u0aca\u0ace-\u0acf\u0ad1-\u0adf\u0ae4-\u0ae5\u0af2-\u0b00\u0b04\u0b0d-\u0b0e\u0b11-\u0b12\u0b29\u0b31\u0b34\u0b3a-\u0b3b\u0b45-\u0b46\u0b49-\u0b4a\u0b4e-\u0b55\u0b58-\u0b5b\u0b5e\u0b64-\u0b65\u0b78-\u0b81\u0b84\u0b8b-\u0b8d\u0b91\u0b96-\u0b98\u0b9b\u0b9d\u0ba0-\u0ba2\u0ba5-\u0ba7\u0bab-\u0bad\u0bba-\u0bbd\u0bc3-\u0bc5\u0bc9\u0bce-\u0bcf\u0bd1-\u0bd6\u0bd8-\u0be5\u0bfb-\u0c00\u0c04\u0c0d\u0c11\u0c29\u0c34\u0c3a-\u0c3c\u0c45\u0c49\u0c4e-\u0c54\u0c57\u0c5a-\u0c5f\u0c64-\u0c65\u0c70-\u0c77\u0c80-\u0c81\u0c84\u0c8d\u0c91\u0ca9\u0cb4\u0cba-\u0cbb\u0cc5\u0cc9\u0cce-\u0cd4\u0cd7-\u0cdd\u0cdf\u0ce4-\u0ce5\u0cf0\u0cf3-\u0d01\u0d04\u0d0d\u0d11\u0d3b-\u0d3c\u0d45\u0d49\u0d4f-\u0d56\u0d58-\u0d5f\u0d64-\u0d65\u0d76-\u0d78\u0d80-\u0d81\u0d84\u0d97-\u0d99\u0db2\u0dbc\u0dbe-\u0dbf\u0dc7-\u0dc9\u0dcb-\u0dce\u0dd5\u0dd7\u0de0-\u0df1\u0df5-\u0e00\u0e3b-\u0e3e\u0e5c-\u0e80\u0e83\u0e85-\u0e86\u0e89\u0e8b-\u0e8c\u0e8e-\u0e93\u0e98\u0ea0\u0ea4\u0ea6\u0ea8-\u0ea9\u0eac\u0eba\u0ebe-\u0ebf\u0ec5\u0ec7\u0ece-\u0ecf\u0eda-\u0edb\u0ee0-\u0eff\u0f48\u0f6d-\u0f70\u0f98\u0fbd\u0fcd\u0fdb-\u0fff\u10c6\u10c8-\u10cc\u10ce-\u10cf\u1249\u124e-\u124f\u1257\u1259\u125e-\u125f\u1289\u128e-\u128f\u12b1\u12b6-\u12b7\u12bf\u12c1\u12c6-\u12c7\u12d7\u1311\u1316-\u1317\u135b-\u135c\u137d-\u137f\u139a-\u139f\u13f5-\u13ff\u169d-\u169f\u16f1-\u16ff\u170d\u1715-\u171f\u1737-\u173f\u1754-\u175f\u176d\u1771\u1774-\u177f\u17de-\u17df\u17ea-\u17ef\u17fa-\u17ff\u180f\u181a-\u181f\u1878-\u187f\u18ab-\u18af\u18f6-\u18ff\u191d-\u191f\u192c-\u192f\u193c-\u193f\u1941-\u1943\u196e-\u196f\u1975-\u197f\u19ac-\u19af\u19ca-\u19cf\u19db-\u19dd\u1a1c-\u1a1d\u1a5f\u1a7d-\u1a7e\u1a8a-\u1a8f\u1a9a-\u1a9f\u1aae-\u1aff\u1b4c-\u1b4f\u1b7d-\u1b7f\u1bf4-\u1bfb\u1c38-\u1c3a\u1c4a-\u1c4c\u1c80-\u1cbf\u1cc8-\u1ccf\u1cf7-\u1cff\u1de7-\u1dfb\u1f16-\u1f17\u1f1e-\u1f1f\u1f46-\u1f47\u1f4e-\u1f4f\u1f58\u1f5a\u1f5c\u1f5e\u1f7e-\u1f7f\u1fb5\u1fc5\u1fd4-\u1fd5\u1fdc\u1ff0-\u1ff1\u1ff5\u1fff\u2065\u2072-\u2073\u208f\u209d-\u209f\u20bb-\u20cf\u20f1-\u20ff\u218a-\u218f\u23f4-\u23ff\u2427-\u243f\u244b-\u245f\u2700\u2b4d-\u2b4f\u2b5a-\u2bff\u2c2f\u2c5f\u2cf4-\u2cf8\u2d26\u2d28-\u2d2c\u2d2e-\u2d2f\u2d68-\u2d6e\u2d71-\u2d7e\u2d97-\u2d9f\u2da7\u2daf\u2db7\u2dbf\u2dc7\u2dcf\u2dd7\u2ddf\u2e3c-\u2e7f\u2e9a\u2ef4-\u2eff\u2fd6-\u2fef\u2ffc-\u2fff\u3040\u3097-\u3098\u3100-\u3104\u312e-\u3130\u318f\u31bb-\u31bf\u31e4-\u31ef\u321f\u32ff\u4db6-\u4dbf\u9fcd-\u9fff\ua48d-\ua48f\ua4c7-\ua4cf\ua62c-\ua63f\ua698-\ua69e\ua6f8-\ua6ff\ua78f\ua794-\ua79f\ua7ab-\ua7f7\ua82c-\ua82f\ua83a-\ua83f\ua878-\ua87f\ua8c5-\ua8cd\ua8da-\ua8df\ua8fc-\ua8ff\ua954-\ua95e\ua97d-\ua97f\ua9ce\ua9da-\ua9dd\ua9e0-\ua9ff\uaa37-\uaa3f\uaa4e-\uaa4f\uaa5a-\uaa5b\uaa7c-\uaa7f\uaac3-\uaada\uaaf7-\uab00\uab07-\uab08\uab0f-\uab10\uab17-\uab1f\uab27\uab2f-\uabbf\uabee-\uabef\uabfa-\uabff\ud7a4-\ud7af\ud7c7-\ud7ca\ud7fc-\ud7ff\ufa6e-\ufa6f\ufada-\ufaff\ufb07-\ufb12\ufb18-\ufb1c\ufb37\ufb3d\ufb3f\ufb42\ufb45\ufbc2-\ufbd2\ufd40-\ufd4f\ufd90-\ufd91\ufdc8-\ufdef\ufdfe-\ufdff\ufe1a-\ufe1f\ufe27-\ufe2f\ufe53\ufe67\ufe6c-\ufe6f\ufe75\ufefd-\ufefe\uff00\uffbf-\uffc1\uffc8-\uffc9\uffd0-\uffd1\uffd8-\uffd9\uffdd-\uffdf\uffe7\uffef-\ufff8\ufffe-\uffff'
+Cn = u'\u0378-\u0379\u0380-\u0383\u038b\u038d\u03a2\u0530\u0557-\u0558\u058b-\u058c\u0590\u05c8-\u05cf\u05eb-\u05ee\u05f5-\u05ff\u061d\u070e\u074b-\u074c\u07b2-\u07bf\u07fb-\u07fc\u082e-\u082f\u083f\u085c-\u085d\u085f\u086b-\u089f\u08b5\u08be-\u08d2\u0984\u098d-\u098e\u0991-\u0992\u09a9\u09b1\u09b3-\u09b5\u09ba-\u09bb\u09c5-\u09c6\u09c9-\u09ca\u09cf-\u09d6\u09d8-\u09db\u09de\u09e4-\u09e5\u09ff-\u0a00\u0a04\u0a0b-\u0a0e\u0a11-\u0a12\u0a29\u0a31\u0a34\u0a37\u0a3a-\u0a3b\u0a3d\u0a43-\u0a46\u0a49-\u0a4a\u0a4e-\u0a50\u0a52-\u0a58\u0a5d\u0a5f-\u0a65\u0a77-\u0a80\u0a84\u0a8e\u0a92\u0aa9\u0ab1\u0ab4\u0aba-\u0abb\u0ac6\u0aca\u0ace-\u0acf\u0ad1-\u0adf\u0ae4-\u0ae5\u0af2-\u0af8\u0b00\u0b04\u0b0d-\u0b0e\u0b11-\u0b12\u0b29\u0b31\u0b34\u0b3a-\u0b3b\u0b45-\u0b46\u0b49-\u0b4a\u0b4e-\u0b55\u0b58-\u0b5b\u0b5e\u0b64-\u0b65\u0b78-\u0b81\u0b84\u0b8b-\u0b8d\u0b91\u0b96-\u0b98\u0b9b\u0b9d\u0ba0-\u0ba2\u0ba5-\u0ba7\u0bab-\u0bad\u0bba-\u0bbd\u0bc3-\u0bc5\u0bc9\u0bce-\u0bcf\u0bd1-\u0bd6\u0bd8-\u0be5\u0bfb-\u0bff\u0c0d\u0c11\u0c29\u0c3a-\u0c3c\u0c45\u0c49\u0c4e-\u0c54\u0c57\u0c5b-\u0c5f\u0c64-\u0c65\u0c70-\u0c77\u0c8d\u0c91\u0ca9\u0cb4\u0cba-\u0cbb\u0cc5\u0cc9\u0cce-\u0cd4\u0cd7-\u0cdd\u0cdf\u0ce4-\u0ce5\u0cf0\u0cf3-\u0cff\u0d04\u0d0d\u0d11\u0d45\u0d49\u0d50-\u0d53\u0d64-\u0d65\u0d80-\u0d81\u0d84\u0d97-\u0d99\u0db2\u0dbc\u0dbe-\u0dbf\u0dc7-\u0dc9\u0dcb-\u0dce\u0dd5\u0dd7\u0de0-\u0de5\u0df0-\u0df1\u0df5-\u0e00\u0e3b-\u0e3e\u0e5c-\u0e80\u0e83\u0e85-\u0e86\u0e89\u0e8b-\u0e8c\u0e8e-\u0e93\u0e98\u0ea0\u0ea4\u0ea6\u0ea8-\u0ea9\u0eac\u0eba\u0ebe-\u0ebf\u0ec5\u0ec7\u0ece-\u0ecf\u0eda-\u0edb\u0ee0-\u0eff\u0f48\u0f6d-\u0f70\u0f98\u0fbd\u0fcd\u0fdb-\u0fff\u10c6\u10c8-\u10cc\u10ce-\u10cf\u1249\u124e-\u124f\u1257\u1259\u125e-\u125f\u1289\u128e-\u128f\u12b1\u12b6-\u12b7\u12bf\u12c1\u12c6-\u12c7\u12d7\u1311\u1316-\u1317\u135b-\u135c\u137d-\u137f\u139a-\u139f\u13f6-\u13f7\u13fe-\u13ff\u169d-\u169f\u16f9-\u16ff\u170d\u1715-\u171f\u1737-\u173f\u1754-\u175f\u176d\u1771\u1774-\u177f\u17de-\u17df\u17ea-\u17ef\u17fa-\u17ff\u180f\u181a-\u181f\u1879-\u187f\u18ab-\u18af\u18f6-\u18ff\u191f\u192c-\u192f\u193c-\u193f\u1941-\u1943\u196e-\u196f\u1975-\u197f\u19ac-\u19af\u19ca-\u19cf\u19db-\u19dd\u1a1c-\u1a1d\u1a5f\u1a7d-\u1a7e\u1a8a-\u1a8f\u1a9a-\u1a9f\u1aae-\u1aaf\u1abf-\u1aff\u1b4c-\u1b4f\u1b7d-\u1b7f\u1bf4-\u1bfb\u1c38-\u1c3a\u1c4a-\u1c4c\u1c89-\u1c8f\u1cbb-\u1cbc\u1cc8-\u1ccf\u1cfa-\u1cff\u1dfa\u1f16-\u1f17\u1f1e-\u1f1f\u1f46-\u1f47\u1f4e-\u1f4f\u1f58\u1f5a\u1f5c\u1f5e\u1f7e-\u1f7f\u1fb5\u1fc5\u1fd4-\u1fd5\u1fdc\u1ff0-\u1ff1\u1ff5\u1fff\u2065\u2072-\u2073\u208f\u209d-\u209f\u20c0-\u20cf\u20f1-\u20ff\u218c-\u218f\u2427-\u243f\u244b-\u245f\u2b74-\u2b75\u2b96-\u2b97\u2bc9\u2bff\u2c2f\u2c5f\u2cf4-\u2cf8\u2d26\u2d28-\u2d2c\u2d2e-\u2d2f\u2d68-\u2d6e\u2d71-\u2d7e\u2d97-\u2d9f\u2da7\u2daf\u2db7\u2dbf\u2dc7\u2dcf\u2dd7\u2ddf\u2e4f-\u2e7f\u2e9a\u2ef4-\u2eff\u2fd6-\u2fef\u2ffc-\u2fff\u3040\u3097-\u3098\u3100-\u3104\u3130\u318f\u31bb-\u31bf\u31e4-\u31ef\u321f\u32ff\u4db6-\u4dbf\u9ff0-\u9fff\ua48d-\ua48f\ua4c7-\ua4cf\ua62c-\ua63f\ua6f8-\ua6ff\ua7ba-\ua7f6\ua82c-\ua82f\ua83a-\ua83f\ua878-\ua87f\ua8c6-\ua8cd\ua8da-\ua8df\ua954-\ua95e\ua97d-\ua97f\ua9ce\ua9da-\ua9dd\ua9ff\uaa37-\uaa3f\uaa4e-\uaa4f\uaa5a-\uaa5b\uaac3-\uaada\uaaf7-\uab00\uab07-\uab08\uab0f-\uab10\uab17-\uab1f\uab27\uab2f\uab66-\uab6f\uabee-\uabef\uabfa-\uabff\ud7a4-\ud7af\ud7c7-\ud7ca\ud7fc-\ud7ff\ufa6e-\ufa6f\ufada-\ufaff\ufb07-\ufb12\ufb18-\ufb1c\ufb37\ufb3d\ufb3f\ufb42\ufb45\ufbc2-\ufbd2\ufd40-\ufd4f\ufd90-\ufd91\ufdc8-\ufdef\ufdfe-\ufdff\ufe1a-\ufe1f\ufe53\ufe67\ufe6c-\ufe6f\ufe75\ufefd-\ufefe\uff00\uffbf-\uffc1\uffc8-\uffc9\uffd0-\uffd1\uffd8-\uffd9\uffdd-\uffdf\uffe7\uffef-\ufff8\ufffe-\uffff'
 
 Co = u'\ue000-\uf8ff'
 
@@ -27,49 +27,49 @@ try:
 except UnicodeDecodeError:
     Cs = ''  # Jython can't handle isolated surrogates
 
-Ll = u'a-z\xb5\xdf-\xf6\xf8-\xff\u0101\u0103\u0105\u0107\u0109\u010b\u010d\u010f\u0111\u0113\u0115\u0117\u0119\u011b\u011d\u011f\u0121\u0123\u0125\u0127\u0129\u012b\u012d\u012f\u0131\u0133\u0135\u0137-\u0138\u013a\u013c\u013e\u0140\u0142\u0144\u0146\u0148-\u0149\u014b\u014d\u014f\u0151\u0153\u0155\u0157\u0159\u015b\u015d\u015f\u0161\u0163\u0165\u0167\u0169\u016b\u016d\u016f\u0171\u0173\u0175\u0177\u017a\u017c\u017e-\u0180\u0183\u0185\u0188\u018c-\u018d\u0192\u0195\u0199-\u019b\u019e\u01a1\u01a3\u01a5\u01a8\u01aa-\u01ab\u01ad\u01b0\u01b4\u01b6\u01b9-\u01ba\u01bd-\u01bf\u01c6\u01c9\u01cc\u01ce\u01d0\u01d2\u01d4\u01d6\u01d8\u01da\u01dc-\u01dd\u01df\u01e1\u01e3\u01e5\u01e7\u01e9\u01eb\u01ed\u01ef-\u01f0\u01f3\u01f5\u01f9\u01fb\u01fd\u01ff\u0201\u0203\u0205\u0207\u0209\u020b\u020d\u020f\u0211\u0213\u0215\u0217\u0219\u021b\u021d\u021f\u0221\u0223\u0225\u0227\u0229\u022b\u022d\u022f\u0231\u0233-\u0239\u023c\u023f-\u0240\u0242\u0247\u0249\u024b\u024d\u024f-\u0293\u0295-\u02af\u0371\u0373\u0377\u037b-\u037d\u0390\u03ac-\u03ce\u03d0-\u03d1\u03d5-\u03d7\u03d9\u03db\u03dd\u03df\u03e1\u03e3\u03e5\u03e7\u03e9\u03eb\u03ed\u03ef-\u03f3\u03f5\u03f8\u03fb-\u03fc\u0430-\u045f\u0461\u0463\u0465\u0467\u0469\u046b\u046d\u046f\u0471\u0473\u0475\u0477\u0479\u047b\u047d\u047f\u0481\u048b\u048d\u048f\u0491\u0493\u0495\u0497\u0499\u049b\u049d\u049f\u04a1\u04a3\u04a5\u04a7\u04a9\u04ab\u04ad\u04af\u04b1\u04b3\u04b5\u04b7\u04b9\u04bb\u04bd\u04bf\u04c2\u04c4\u04c6\u04c8\u04ca\u04cc\u04ce-\u04cf\u04d1\u04d3\u04d5\u04d7\u04d9\u04db\u04dd\u04df\u04e1\u04e3\u04e5\u04e7\u04e9\u04eb\u04ed\u04ef\u04f1\u04f3\u04f5\u04f7\u04f9\u04fb\u04fd\u04ff\u0501\u0503\u0505\u0507\u0509\u050b\u050d\u050f\u0511\u0513\u0515\u0517\u0519\u051b\u051d\u051f\u0521\u0523\u0525\u0527\u0561-\u0587\u1d00-\u1d2b\u1d6b-\u1d77\u1d79-\u1d9a\u1e01\u1e03\u1e05\u1e07\u1e09\u1e0b\u1e0d\u1e0f\u1e11\u1e13\u1e15\u1e17\u1e19\u1e1b\u1e1d\u1e1f\u1e21\u1e23\u1e25\u1e27\u1e29\u1e2b\u1e2d\u1e2f\u1e31\u1e33\u1e35\u1e37\u1e39\u1e3b\u1e3d\u1e3f\u1e41\u1e43\u1e45\u1e47\u1e49\u1e4b\u1e4d\u1e4f\u1e51\u1e53\u1e55\u1e57\u1e59\u1e5b\u1e5d\u1e5f\u1e61\u1e63\u1e65\u1e67\u1e69\u1e6b\u1e6d\u1e6f\u1e71\u1e73\u1e75\u1e77\u1e79\u1e7b\u1e7d\u1e7f\u1e81\u1e83\u1e85\u1e87\u1e89\u1e8b\u1e8d\u1e8f\u1e91\u1e93\u1e95-\u1e9d\u1e9f\u1ea1\u1ea3\u1ea5\u1ea7\u1ea9\u1eab\u1ead\u1eaf\u1eb1\u1eb3\u1eb5\u1eb7\u1eb9\u1ebb\u1ebd\u1ebf\u1ec1\u1ec3\u1ec5\u1ec7\u1ec9\u1ecb\u1ecd\u1ecf\u1ed1\u1ed3\u1ed5\u1ed7\u1ed9\u1edb\u1edd\u1edf\u1ee1\u1ee3\u1ee5\u1ee7\u1ee9\u1eeb\u1eed\u1eef\u1ef1\u1ef3\u1ef5\u1ef7\u1ef9\u1efb\u1efd\u1eff-\u1f07\u1f10-\u1f15\u1f20-\u1f27\u1f30-\u1f37\u1f40-\u1f45\u1f50-\u1f57\u1f60-\u1f67\u1f70-\u1f7d\u1f80-\u1f87\u1f90-\u1f97\u1fa0-\u1fa7\u1fb0-\u1fb4\u1fb6-\u1fb7\u1fbe\u1fc2-\u1fc4\u1fc6-\u1fc7\u1fd0-\u1fd3\u1fd6-\u1fd7\u1fe0-\u1fe7\u1ff2-\u1ff4\u1ff6-\u1ff7\u210a\u210e-\u210f\u2113\u212f\u2134\u2139\u213c-\u213d\u2146-\u2149\u214e\u2184\u2c30-\u2c5e\u2c61\u2c65-\u2c66\u2c68\u2c6a\u2c6c\u2c71\u2c73-\u2c74\u2c76-\u2c7b\u2c81\u2c83\u2c85\u2c87\u2c89\u2c8b\u2c8d\u2c8f\u2c91\u2c93\u2c95\u2c97\u2c99\u2c9b\u2c9d\u2c9f\u2ca1\u2ca3\u2ca5\u2ca7\u2ca9\u2cab\u2cad\u2caf\u2cb1\u2cb3\u2cb5\u2cb7\u2cb9\u2cbb\u2cbd\u2cbf\u2cc1\u2cc3\u2cc5\u2cc7\u2cc9\u2ccb\u2ccd\u2ccf\u2cd1\u2cd3\u2cd5\u2cd7\u2cd9\u2cdb\u2cdd\u2cdf\u2ce1\u2ce3-\u2ce4\u2cec\u2cee\u2cf3\u2d00-\u2d25\u2d27\u2d2d\ua641\ua643\ua645\ua647\ua649\ua64b\ua64d\ua64f\ua651\ua653\ua655\ua657\ua659\ua65b\ua65d\ua65f\ua661\ua663\ua665\ua667\ua669\ua66b\ua66d\ua681\ua683\ua685\ua687\ua689\ua68b\ua68d\ua68f\ua691\ua693\ua695\ua697\ua723\ua725\ua727\ua729\ua72b\ua72d\ua72f-\ua731\ua733\ua735\ua737\ua739\ua73b\ua73d\ua73f\ua741\ua743\ua745\ua747\ua749\ua74b\ua74d\ua74f\ua751\ua753\ua755\ua757\ua759\ua75b\ua75d\ua75f\ua761\ua763\ua765\ua767\ua769\ua76b\ua76d\ua76f\ua771-\ua778\ua77a\ua77c\ua77f\ua781\ua783\ua785\ua787\ua78c\ua78e\ua791\ua793\ua7a1\ua7a3\ua7a5\ua7a7\ua7a9\ua7fa\ufb00-\ufb06\ufb13-\ufb17\uff41-\uff5a'
+Ll = u'a-z\xb5\xdf-\xf6\xf8-\xff\u0101\u0103\u0105\u0107\u0109\u010b\u010d\u010f\u0111\u0113\u0115\u0117\u0119\u011b\u011d\u011f\u0121\u0123\u0125\u0127\u0129\u012b\u012d\u012f\u0131\u0133\u0135\u0137-\u0138\u013a\u013c\u013e\u0140\u0142\u0144\u0146\u0148-\u0149\u014b\u014d\u014f\u0151\u0153\u0155\u0157\u0159\u015b\u015d\u015f\u0161\u0163\u0165\u0167\u0169\u016b\u016d\u016f\u0171\u0173\u0175\u0177\u017a\u017c\u017e-\u0180\u0183\u0185\u0188\u018c-\u018d\u0192\u0195\u0199-\u019b\u019e\u01a1\u01a3\u01a5\u01a8\u01aa-\u01ab\u01ad\u01b0\u01b4\u01b6\u01b9-\u01ba\u01bd-\u01bf\u01c6\u01c9\u01cc\u01ce\u01d0\u01d2\u01d4\u01d6\u01d8\u01da\u01dc-\u01dd\u01df\u01e1\u01e3\u01e5\u01e7\u01e9\u01eb\u01ed\u01ef-\u01f0\u01f3\u01f5\u01f9\u01fb\u01fd\u01ff\u0201\u0203\u0205\u0207\u0209\u020b\u020d\u020f\u0211\u0213\u0215\u0217\u0219\u021b\u021d\u021f\u0221\u0223\u0225\u0227\u0229\u022b\u022d\u022f\u0231\u0233-\u0239\u023c\u023f-\u0240\u0242\u0247\u0249\u024b\u024d\u024f-\u0293\u0295-\u02af\u0371\u0373\u0377\u037b-\u037d\u0390\u03ac-\u03ce\u03d0-\u03d1\u03d5-\u03d7\u03d9\u03db\u03dd\u03df\u03e1\u03e3\u03e5\u03e7\u03e9\u03eb\u03ed\u03ef-\u03f3\u03f5\u03f8\u03fb-\u03fc\u0430-\u045f\u0461\u0463\u0465\u0467\u0469\u046b\u046d\u046f\u0471\u0473\u0475\u0477\u0479\u047b\u047d\u047f\u0481\u048b\u048d\u048f\u0491\u0493\u0495\u0497\u0499\u049b\u049d\u049f\u04a1\u04a3\u04a5\u04a7\u04a9\u04ab\u04ad\u04af\u04b1\u04b3\u04b5\u04b7\u04b9\u04bb\u04bd\u04bf\u04c2\u04c4\u04c6\u04c8\u04ca\u04cc\u04ce-\u04cf\u04d1\u04d3\u04d5\u04d7\u04d9\u04db\u04dd\u04df\u04e1\u04e3\u04e5\u04e7\u04e9\u04eb\u04ed\u04ef\u04f1\u04f3\u04f5\u04f7\u04f9\u04fb\u04fd\u04ff\u0501\u0503\u0505\u0507\u0509\u050b\u050d\u050f\u0511\u0513\u0515\u0517\u0519\u051b\u051d\u051f\u0521\u0523\u0525\u0527\u0529\u052b\u052d\u052f\u0560-\u0588\u10d0-\u10fa\u10fd-\u10ff\u13f8-\u13fd\u1c80-\u1c88\u1d00-\u1d2b\u1d6b-\u1d77\u1d79-\u1d9a\u1e01\u1e03\u1e05\u1e07\u1e09\u1e0b\u1e0d\u1e0f\u1e11\u1e13\u1e15\u1e17\u1e19\u1e1b\u1e1d\u1e1f\u1e21\u1e23\u1e25\u1e27\u1e29\u1e2b\u1e2d\u1e2f\u1e31\u1e33\u1e35\u1e37\u1e39\u1e3b\u1e3d\u1e3f\u1e41\u1e43\u1e45\u1e47\u1e49\u1e4b\u1e4d\u1e4f\u1e51\u1e53\u1e55\u1e57\u1e59\u1e5b\u1e5d\u1e5f\u1e61\u1e63\u1e65\u1e67\u1e69\u1e6b\u1e6d\u1e6f\u1e71\u1e73\u1e75\u1e77\u1e79\u1e7b\u1e7d\u1e7f\u1e81\u1e83\u1e85\u1e87\u1e89\u1e8b\u1e8d\u1e8f\u1e91\u1e93\u1e95-\u1e9d\u1e9f\u1ea1\u1ea3\u1ea5\u1ea7\u1ea9\u1eab\u1ead\u1eaf\u1eb1\u1eb3\u1eb5\u1eb7\u1eb9\u1ebb\u1ebd\u1ebf\u1ec1\u1ec3\u1ec5\u1ec7\u1ec9\u1ecb\u1ecd\u1ecf\u1ed1\u1ed3\u1ed5\u1ed7\u1ed9\u1edb\u1edd\u1edf\u1ee1\u1ee3\u1ee5\u1ee7\u1ee9\u1eeb\u1eed\u1eef\u1ef1\u1ef3\u1ef5\u1ef7\u1ef9\u1efb\u1efd\u1eff-\u1f07\u1f10-\u1f15\u1f20-\u1f27\u1f30-\u1f37\u1f40-\u1f45\u1f50-\u1f57\u1f60-\u1f67\u1f70-\u1f7d\u1f80-\u1f87\u1f90-\u1f97\u1fa0-\u1fa7\u1fb0-\u1fb4\u1fb6-\u1fb7\u1fbe\u1fc2-\u1fc4\u1fc6-\u1fc7\u1fd0-\u1fd3\u1fd6-\u1fd7\u1fe0-\u1fe7\u1ff2-\u1ff4\u1ff6-\u1ff7\u210a\u210e-\u210f\u2113\u212f\u2134\u2139\u213c-\u213d\u2146-\u2149\u214e\u2184\u2c30-\u2c5e\u2c61\u2c65-\u2c66\u2c68\u2c6a\u2c6c\u2c71\u2c73-\u2c74\u2c76-\u2c7b\u2c81\u2c83\u2c85\u2c87\u2c89\u2c8b\u2c8d\u2c8f\u2c91\u2c93\u2c95\u2c97\u2c99\u2c9b\u2c9d\u2c9f\u2ca1\u2ca3\u2ca5\u2ca7\u2ca9\u2cab\u2cad\u2caf\u2cb1\u2cb3\u2cb5\u2cb7\u2cb9\u2cbb\u2cbd\u2cbf\u2cc1\u2cc3\u2cc5\u2cc7\u2cc9\u2ccb\u2ccd\u2ccf\u2cd1\u2cd3\u2cd5\u2cd7\u2cd9\u2cdb\u2cdd\u2cdf\u2ce1\u2ce3-\u2ce4\u2cec\u2cee\u2cf3\u2d00-\u2d25\u2d27\u2d2d\ua641\ua643\ua645\ua647\ua649\ua64b\ua64d\ua64f\ua651\ua653\ua655\ua657\ua659\ua65b\ua65d\ua65f\ua661\ua663\ua665\ua667\ua669\ua66b\ua66d\ua681\ua683\ua685\ua687\ua689\ua68b\ua68d\ua68f\ua691\ua693\ua695\ua697\ua699\ua69b\ua723\ua725\ua727\ua729\ua72b\ua72d\ua72f-\ua731\ua733\ua735\ua737\ua739\ua73b\ua73d\ua73f\ua741\ua743\ua745\ua747\ua749\ua74b\ua74d\ua74f\ua751\ua753\ua755\ua757\ua759\ua75b\ua75d\ua75f\ua761\ua763\ua765\ua767\ua769\ua76b\ua76d\ua76f\ua771-\ua778\ua77a\ua77c\ua77f\ua781\ua783\ua785\ua787\ua78c\ua78e\ua791\ua793-\ua795\ua797\ua799\ua79b\ua79d\ua79f\ua7a1\ua7a3\ua7a5\ua7a7\ua7a9\ua7af\ua7b5\ua7b7\ua7b9\ua7fa\uab30-\uab5a\uab60-\uab65\uab70-\uabbf\ufb00-\ufb06\ufb13-\ufb17\uff41-\uff5a'
 
-Lm = u'\u02b0-\u02c1\u02c6-\u02d1\u02e0-\u02e4\u02ec\u02ee\u0374\u037a\u0559\u0640\u06e5-\u06e6\u07f4-\u07f5\u07fa\u081a\u0824\u0828\u0971\u0e46\u0ec6\u10fc\u17d7\u1843\u1aa7\u1c78-\u1c7d\u1d2c-\u1d6a\u1d78\u1d9b-\u1dbf\u2071\u207f\u2090-\u209c\u2c7c-\u2c7d\u2d6f\u2e2f\u3005\u3031-\u3035\u303b\u309d-\u309e\u30fc-\u30fe\ua015\ua4f8-\ua4fd\ua60c\ua67f\ua717-\ua71f\ua770\ua788\ua7f8-\ua7f9\ua9cf\uaa70\uaadd\uaaf3-\uaaf4\uff70\uff9e-\uff9f'
+Lm = u'\u02b0-\u02c1\u02c6-\u02d1\u02e0-\u02e4\u02ec\u02ee\u0374\u037a\u0559\u0640\u06e5-\u06e6\u07f4-\u07f5\u07fa\u081a\u0824\u0828\u0971\u0e46\u0ec6\u10fc\u17d7\u1843\u1aa7\u1c78-\u1c7d\u1d2c-\u1d6a\u1d78\u1d9b-\u1dbf\u2071\u207f\u2090-\u209c\u2c7c-\u2c7d\u2d6f\u2e2f\u3005\u3031-\u3035\u303b\u309d-\u309e\u30fc-\u30fe\ua015\ua4f8-\ua4fd\ua60c\ua67f\ua69c-\ua69d\ua717-\ua71f\ua770\ua788\ua7f8-\ua7f9\ua9cf\ua9e6\uaa70\uaadd\uaaf3-\uaaf4\uab5c-\uab5f\uff70\uff9e-\uff9f'
 
-Lo = u'\xaa\xba\u01bb\u01c0-\u01c3\u0294\u05d0-\u05ea\u05f0-\u05f2\u0620-\u063f\u0641-\u064a\u066e-\u066f\u0671-\u06d3\u06d5\u06ee-\u06ef\u06fa-\u06fc\u06ff\u0710\u0712-\u072f\u074d-\u07a5\u07b1\u07ca-\u07ea\u0800-\u0815\u0840-\u0858\u08a0\u08a2-\u08ac\u0904-\u0939\u093d\u0950\u0958-\u0961\u0972-\u0977\u0979-\u097f\u0985-\u098c\u098f-\u0990\u0993-\u09a8\u09aa-\u09b0\u09b2\u09b6-\u09b9\u09bd\u09ce\u09dc-\u09dd\u09df-\u09e1\u09f0-\u09f1\u0a05-\u0a0a\u0a0f-\u0a10\u0a13-\u0a28\u0a2a-\u0a30\u0a32-\u0a33\u0a35-\u0a36\u0a38-\u0a39\u0a59-\u0a5c\u0a5e\u0a72-\u0a74\u0a85-\u0a8d\u0a8f-\u0a91\u0a93-\u0aa8\u0aaa-\u0ab0\u0ab2-\u0ab3\u0ab5-\u0ab9\u0abd\u0ad0\u0ae0-\u0ae1\u0b05-\u0b0c\u0b0f-\u0b10\u0b13-\u0b28\u0b2a-\u0b30\u0b32-\u0b33\u0b35-\u0b39\u0b3d\u0b5c-\u0b5d\u0b5f-\u0b61\u0b71\u0b83\u0b85-\u0b8a\u0b8e-\u0b90\u0b92-\u0b95\u0b99-\u0b9a\u0b9c\u0b9e-\u0b9f\u0ba3-\u0ba4\u0ba8-\u0baa\u0bae-\u0bb9\u0bd0\u0c05-\u0c0c\u0c0e-\u0c10\u0c12-\u0c28\u0c2a-\u0c33\u0c35-\u0c39\u0c3d\u0c58-\u0c59\u0c60-\u0c61\u0c85-\u0c8c\u0c8e-\u0c90\u0c92-\u0ca8\u0caa-\u0cb3\u0cb5-\u0cb9\u0cbd\u0cde\u0ce0-\u0ce1\u0cf1-\u0cf2\u0d05-\u0d0c\u0d0e-\u0d10\u0d12-\u0d3a\u0d3d\u0d4e\u0d60-\u0d61\u0d7a-\u0d7f\u0d85-\u0d96\u0d9a-\u0db1\u0db3-\u0dbb\u0dbd\u0dc0-\u0dc6\u0e01-\u0e30\u0e32-\u0e33\u0e40-\u0e45\u0e81-\u0e82\u0e84\u0e87-\u0e88\u0e8a\u0e8d\u0e94-\u0e97\u0e99-\u0e9f\u0ea1-\u0ea3\u0ea5\u0ea7\u0eaa-\u0eab\u0ead-\u0eb0\u0eb2-\u0eb3\u0ebd\u0ec0-\u0ec4\u0edc-\u0edf\u0f00\u0f40-\u0f47\u0f49-\u0f6c\u0f88-\u0f8c\u1000-\u102a\u103f\u1050-\u1055\u105a-\u105d\u1061\u1065-\u1066\u106e-\u1070\u1075-\u1081\u108e\u10d0-\u10fa\u10fd-\u1248\u124a-\u124d\u1250-\u1256\u1258\u125a-\u125d\u1260-\u1288\u128a-\u128d\u1290-\u12b0\u12b2-\u12b5\u12b8-\u12be\u12c0\u12c2-\u12c5\u12c8-\u12d6\u12d8-\u1310\u1312-\u1315\u1318-\u135a\u1380-\u138f\u13a0-\u13f4\u1401-\u166c\u166f-\u167f\u1681-\u169a\u16a0-\u16ea\u1700-\u170c\u170e-\u1711\u1720-\u1731\u1740-\u1751\u1760-\u176c\u176e-\u1770\u1780-\u17b3\u17dc\u1820-\u1842\u1844-\u1877\u1880-\u18a8\u18aa\u18b0-\u18f5\u1900-\u191c\u1950-\u196d\u1970-\u1974\u1980-\u19ab\u19c1-\u19c7\u1a00-\u1a16\u1a20-\u1a54\u1b05-\u1b33\u1b45-\u1b4b\u1b83-\u1ba0\u1bae-\u1baf\u1bba-\u1be5\u1c00-\u1c23\u1c4d-\u1c4f\u1c5a-\u1c77\u1ce9-\u1cec\u1cee-\u1cf1\u1cf5-\u1cf6\u2135-\u2138\u2d30-\u2d67\u2d80-\u2d96\u2da0-\u2da6\u2da8-\u2dae\u2db0-\u2db6\u2db8-\u2dbe\u2dc0-\u2dc6\u2dc8-\u2dce\u2dd0-\u2dd6\u2dd8-\u2dde\u3006\u303c\u3041-\u3096\u309f\u30a1-\u30fa\u30ff\u3105-\u312d\u3131-\u318e\u31a0-\u31ba\u31f0-\u31ff\u3400-\u4db5\u4e00-\u9fcc\ua000-\ua014\ua016-\ua48c\ua4d0-\ua4f7\ua500-\ua60b\ua610-\ua61f\ua62a-\ua62b\ua66e\ua6a0-\ua6e5\ua7fb-\ua801\ua803-\ua805\ua807-\ua80a\ua80c-\ua822\ua840-\ua873\ua882-\ua8b3\ua8f2-\ua8f7\ua8fb\ua90a-\ua925\ua930-\ua946\ua960-\ua97c\ua984-\ua9b2\uaa00-\uaa28\uaa40-\uaa42\uaa44-\uaa4b\uaa60-\uaa6f\uaa71-\uaa76\uaa7a\uaa80-\uaaaf\uaab1\uaab5-\uaab6\uaab9-\uaabd\uaac0\uaac2\uaadb-\uaadc\uaae0-\uaaea\uaaf2\uab01-\uab06\uab09-\uab0e\uab11-\uab16\uab20-\uab26\uab28-\uab2e\uabc0-\uabe2\uac00-\ud7a3\ud7b0-\ud7c6\ud7cb-\ud7fb\uf900-\ufa6d\ufa70-\ufad9\ufb1d\ufb1f-\ufb28\ufb2a-\ufb36\ufb38-\ufb3c\ufb3e\ufb40-\ufb41\ufb43-\ufb44\ufb46-\ufbb1\ufbd3-\ufd3d\ufd50-\ufd8f\ufd92-\ufdc7\ufdf0-\ufdfb\ufe70-\ufe74\ufe76-\ufefc\uff66-\uff6f\uff71-\uff9d\uffa0-\uffbe\uffc2-\uffc7\uffca-\uffcf\uffd2-\uffd7\uffda-\uffdc'
+Lo = u'\xaa\xba\u01bb\u01c0-\u01c3\u0294\u05d0-\u05ea\u05ef-\u05f2\u0620-\u063f\u0641-\u064a\u066e-\u066f\u0671-\u06d3\u06d5\u06ee-\u06ef\u06fa-\u06fc\u06ff\u0710\u0712-\u072f\u074d-\u07a5\u07b1\u07ca-\u07ea\u0800-\u0815\u0840-\u0858\u0860-\u086a\u08a0-\u08b4\u08b6-\u08bd\u0904-\u0939\u093d\u0950\u0958-\u0961\u0972-\u0980\u0985-\u098c\u098f-\u0990\u0993-\u09a8\u09aa-\u09b0\u09b2\u09b6-\u09b9\u09bd\u09ce\u09dc-\u09dd\u09df-\u09e1\u09f0-\u09f1\u09fc\u0a05-\u0a0a\u0a0f-\u0a10\u0a13-\u0a28\u0a2a-\u0a30\u0a32-\u0a33\u0a35-\u0a36\u0a38-\u0a39\u0a59-\u0a5c\u0a5e\u0a72-\u0a74\u0a85-\u0a8d\u0a8f-\u0a91\u0a93-\u0aa8\u0aaa-\u0ab0\u0ab2-\u0ab3\u0ab5-\u0ab9\u0abd\u0ad0\u0ae0-\u0ae1\u0af9\u0b05-\u0b0c\u0b0f-\u0b10\u0b13-\u0b28\u0b2a-\u0b30\u0b32-\u0b33\u0b35-\u0b39\u0b3d\u0b5c-\u0b5d\u0b5f-\u0b61\u0b71\u0b83\u0b85-\u0b8a\u0b8e-\u0b90\u0b92-\u0b95\u0b99-\u0b9a\u0b9c\u0b9e-\u0b9f\u0ba3-\u0ba4\u0ba8-\u0baa\u0bae-\u0bb9\u0bd0\u0c05-\u0c0c\u0c0e-\u0c10\u0c12-\u0c28\u0c2a-\u0c39\u0c3d\u0c58-\u0c5a\u0c60-\u0c61\u0c80\u0c85-\u0c8c\u0c8e-\u0c90\u0c92-\u0ca8\u0caa-\u0cb3\u0cb5-\u0cb9\u0cbd\u0cde\u0ce0-\u0ce1\u0cf1-\u0cf2\u0d05-\u0d0c\u0d0e-\u0d10\u0d12-\u0d3a\u0d3d\u0d4e\u0d54-\u0d56\u0d5f-\u0d61\u0d7a-\u0d7f\u0d85-\u0d96\u0d9a-\u0db1\u0db3-\u0dbb\u0dbd\u0dc0-\u0dc6\u0e01-\u0e30\u0e32-\u0e33\u0e40-\u0e45\u0e81-\u0e82\u0e84\u0e87-\u0e88\u0e8a\u0e8d\u0e94-\u0e97\u0e99-\u0e9f\u0ea1-\u0ea3\u0ea5\u0ea7\u0eaa-\u0eab\u0ead-\u0eb0\u0eb2-\u0eb3\u0ebd\u0ec0-\u0ec4\u0edc-\u0edf\u0f00\u0f40-\u0f47\u0f49-\u0f6c\u0f88-\u0f8c\u1000-\u102a\u103f\u1050-\u1055\u105a-\u105d\u1061\u1065-\u1066\u106e-\u1070\u1075-\u1081\u108e\u1100-\u1248\u124a-\u124d\u1250-\u1256\u1258\u125a-\u125d\u1260-\u1288\u128a-\u128d\u1290-\u12b0\u12b2-\u12b5\u12b8-\u12be\u12c0\u12c2-\u12c5\u12c8-\u12d6\u12d8-\u1310\u1312-\u1315\u1318-\u135a\u1380-\u138f\u1401-\u166c\u166f-\u167f\u1681-\u169a\u16a0-\u16ea\u16f1-\u16f8\u1700-\u170c\u170e-\u1711\u1720-\u1731\u1740-\u1751\u1760-\u176c\u176e-\u1770\u1780-\u17b3\u17dc\u1820-\u1842\u1844-\u1878\u1880-\u1884\u1887-\u18a8\u18aa\u18b0-\u18f5\u1900-\u191e\u1950-\u196d\u1970-\u1974\u1980-\u19ab\u19b0-\u19c9\u1a00-\u1a16\u1a20-\u1a54\u1b05-\u1b33\u1b45-\u1b4b\u1b83-\u1ba0\u1bae-\u1baf\u1bba-\u1be5\u1c00-\u1c23\u1c4d-\u1c4f\u1c5a-\u1c77\u1ce9-\u1cec\u1cee-\u1cf1\u1cf5-\u1cf6\u2135-\u2138\u2d30-\u2d67\u2d80-\u2d96\u2da0-\u2da6\u2da8-\u2dae\u2db0-\u2db6\u2db8-\u2dbe\u2dc0-\u2dc6\u2dc8-\u2dce\u2dd0-\u2dd6\u2dd8-\u2dde\u3006\u303c\u3041-\u3096\u309f\u30a1-\u30fa\u30ff\u3105-\u312f\u3131-\u318e\u31a0-\u31ba\u31f0-\u31ff\u3400-\u4db5\u4e00-\u9fef\ua000-\ua014\ua016-\ua48c\ua4d0-\ua4f7\ua500-\ua60b\ua610-\ua61f\ua62a-\ua62b\ua66e\ua6a0-\ua6e5\ua78f\ua7f7\ua7fb-\ua801\ua803-\ua805\ua807-\ua80a\ua80c-\ua822\ua840-\ua873\ua882-\ua8b3\ua8f2-\ua8f7\ua8fb\ua8fd-\ua8fe\ua90a-\ua925\ua930-\ua946\ua960-\ua97c\ua984-\ua9b2\ua9e0-\ua9e4\ua9e7-\ua9ef\ua9fa-\ua9fe\uaa00-\uaa28\uaa40-\uaa42\uaa44-\uaa4b\uaa60-\uaa6f\uaa71-\uaa76\uaa7a\uaa7e-\uaaaf\uaab1\uaab5-\uaab6\uaab9-\uaabd\uaac0\uaac2\uaadb-\uaadc\uaae0-\uaaea\uaaf2\uab01-\uab06\uab09-\uab0e\uab11-\uab16\uab20-\uab26\uab28-\uab2e\uabc0-\uabe2\uac00-\ud7a3\ud7b0-\ud7c6\ud7cb-\ud7fb\uf900-\ufa6d\ufa70-\ufad9\ufb1d\ufb1f-\ufb28\ufb2a-\ufb36\ufb38-\ufb3c\ufb3e\ufb40-\ufb41\ufb43-\ufb44\ufb46-\ufbb1\ufbd3-\ufd3d\ufd50-\ufd8f\ufd92-\ufdc7\ufdf0-\ufdfb\ufe70-\ufe74\ufe76-\ufefc\uff66-\uff6f\uff71-\uff9d\uffa0-\uffbe\uffc2-\uffc7\uffca-\uffcf\uffd2-\uffd7\uffda-\uffdc'
 
 Lt = u'\u01c5\u01c8\u01cb\u01f2\u1f88-\u1f8f\u1f98-\u1f9f\u1fa8-\u1faf\u1fbc\u1fcc\u1ffc'
 
-Lu = u'A-Z\xc0-\xd6\xd8-\xde\u0100\u0102\u0104\u0106\u0108\u010a\u010c\u010e\u0110\u0112\u0114\u0116\u0118\u011a\u011c\u011e\u0120\u0122\u0124\u0126\u0128\u012a\u012c\u012e\u0130\u0132\u0134\u0136\u0139\u013b\u013d\u013f\u0141\u0143\u0145\u0147\u014a\u014c\u014e\u0150\u0152\u0154\u0156\u0158\u015a\u015c\u015e\u0160\u0162\u0164\u0166\u0168\u016a\u016c\u016e\u0170\u0172\u0174\u0176\u0178-\u0179\u017b\u017d\u0181-\u0182\u0184\u0186-\u0187\u0189-\u018b\u018e-\u0191\u0193-\u0194\u0196-\u0198\u019c-\u019d\u019f-\u01a0\u01a2\u01a4\u01a6-\u01a7\u01a9\u01ac\u01ae-\u01af\u01b1-\u01b3\u01b5\u01b7-\u01b8\u01bc\u01c4\u01c7\u01ca\u01cd\u01cf\u01d1\u01d3\u01d5\u01d7\u01d9\u01db\u01de\u01e0\u01e2\u01e4\u01e6\u01e8\u01ea\u01ec\u01ee\u01f1\u01f4\u01f6-\u01f8\u01fa\u01fc\u01fe\u0200\u0202\u0204\u0206\u0208\u020a\u020c\u020e\u0210\u0212\u0214\u0216\u0218\u021a\u021c\u021e\u0220\u0222\u0224\u0226\u0228\u022a\u022c\u022e\u0230\u0232\u023a-\u023b\u023d-\u023e\u0241\u0243-\u0246\u0248\u024a\u024c\u024e\u0370\u0372\u0376\u0386\u0388-\u038a\u038c\u038e-\u038f\u0391-\u03a1\u03a3-\u03ab\u03cf\u03d2-\u03d4\u03d8\u03da\u03dc\u03de\u03e0\u03e2\u03e4\u03e6\u03e8\u03ea\u03ec\u03ee\u03f4\u03f7\u03f9-\u03fa\u03fd-\u042f\u0460\u0462\u0464\u0466\u0468\u046a\u046c\u046e\u0470\u0472\u0474\u0476\u0478\u047a\u047c\u047e\u0480\u048a\u048c\u048e\u0490\u0492\u0494\u0496\u0498\u049a\u049c\u049e\u04a0\u04a2\u04a4\u04a6\u04a8\u04aa\u04ac\u04ae\u04b0\u04b2\u04b4\u04b6\u04b8\u04ba\u04bc\u04be\u04c0-\u04c1\u04c3\u04c5\u04c7\u04c9\u04cb\u04cd\u04d0\u04d2\u04d4\u04d6\u04d8\u04da\u04dc\u04de\u04e0\u04e2\u04e4\u04e6\u04e8\u04ea\u04ec\u04ee\u04f0\u04f2\u04f4\u04f6\u04f8\u04fa\u04fc\u04fe\u0500\u0502\u0504\u0506\u0508\u050a\u050c\u050e\u0510\u0512\u0514\u0516\u0518\u051a\u051c\u051e\u0520\u0522\u0524\u0526\u0531-\u0556\u10a0-\u10c5\u10c7\u10cd\u1e00\u1e02\u1e04\u1e06\u1e08\u1e0a\u1e0c\u1e0e\u1e10\u1e12\u1e14\u1e16\u1e18\u1e1a\u1e1c\u1e1e\u1e20\u1e22\u1e24\u1e26\u1e28\u1e2a\u1e2c\u1e2e\u1e30\u1e32\u1e34\u1e36\u1e38\u1e3a\u1e3c\u1e3e\u1e40\u1e42\u1e44\u1e46\u1e48\u1e4a\u1e4c\u1e4e\u1e50\u1e52\u1e54\u1e56\u1e58\u1e5a\u1e5c\u1e5e\u1e60\u1e62\u1e64\u1e66\u1e68\u1e6a\u1e6c\u1e6e\u1e70\u1e72\u1e74\u1e76\u1e78\u1e7a\u1e7c\u1e7e\u1e80\u1e82\u1e84\u1e86\u1e88\u1e8a\u1e8c\u1e8e\u1e90\u1e92\u1e94\u1e9e\u1ea0\u1ea2\u1ea4\u1ea6\u1ea8\u1eaa\u1eac\u1eae\u1eb0\u1eb2\u1eb4\u1eb6\u1eb8\u1eba\u1ebc\u1ebe\u1ec0\u1ec2\u1ec4\u1ec6\u1ec8\u1eca\u1ecc\u1ece\u1ed0\u1ed2\u1ed4\u1ed6\u1ed8\u1eda\u1edc\u1ede\u1ee0\u1ee2\u1ee4\u1ee6\u1ee8\u1eea\u1eec\u1eee\u1ef0\u1ef2\u1ef4\u1ef6\u1ef8\u1efa\u1efc\u1efe\u1f08-\u1f0f\u1f18-\u1f1d\u1f28-\u1f2f\u1f38-\u1f3f\u1f48-\u1f4d\u1f59\u1f5b\u1f5d\u1f5f\u1f68-\u1f6f\u1fb8-\u1fbb\u1fc8-\u1fcb\u1fd8-\u1fdb\u1fe8-\u1fec\u1ff8-\u1ffb\u2102\u2107\u210b-\u210d\u2110-\u2112\u2115\u2119-\u211d\u2124\u2126\u2128\u212a-\u212d\u2130-\u2133\u213e-\u213f\u2145\u2183\u2c00-\u2c2e\u2c60\u2c62-\u2c64\u2c67\u2c69\u2c6b\u2c6d-\u2c70\u2c72\u2c75\u2c7e-\u2c80\u2c82\u2c84\u2c86\u2c88\u2c8a\u2c8c\u2c8e\u2c90\u2c92\u2c94\u2c96\u2c98\u2c9a\u2c9c\u2c9e\u2ca0\u2ca2\u2ca4\u2ca6\u2ca8\u2caa\u2cac\u2cae\u2cb0\u2cb2\u2cb4\u2cb6\u2cb8\u2cba\u2cbc\u2cbe\u2cc0\u2cc2\u2cc4\u2cc6\u2cc8\u2cca\u2ccc\u2cce\u2cd0\u2cd2\u2cd4\u2cd6\u2cd8\u2cda\u2cdc\u2cde\u2ce0\u2ce2\u2ceb\u2ced\u2cf2\ua640\ua642\ua644\ua646\ua648\ua64a\ua64c\ua64e\ua650\ua652\ua654\ua656\ua658\ua65a\ua65c\ua65e\ua660\ua662\ua664\ua666\ua668\ua66a\ua66c\ua680\ua682\ua684\ua686\ua688\ua68a\ua68c\ua68e\ua690\ua692\ua694\ua696\ua722\ua724\ua726\ua728\ua72a\ua72c\ua72e\ua732\ua734\ua736\ua738\ua73a\ua73c\ua73e\ua740\ua742\ua744\ua746\ua748\ua74a\ua74c\ua74e\ua750\ua752\ua754\ua756\ua758\ua75a\ua75c\ua75e\ua760\ua762\ua764\ua766\ua768\ua76a\ua76c\ua76e\ua779\ua77b\ua77d-\ua77e\ua780\ua782\ua784\ua786\ua78b\ua78d\ua790\ua792\ua7a0\ua7a2\ua7a4\ua7a6\ua7a8\ua7aa\uff21-\uff3a'
+Lu = u'A-Z\xc0-\xd6\xd8-\xde\u0100\u0102\u0104\u0106\u0108\u010a\u010c\u010e\u0110\u0112\u0114\u0116\u0118\u011a\u011c\u011e\u0120\u0122\u0124\u0126\u0128\u012a\u012c\u012e\u0130\u0132\u0134\u0136\u0139\u013b\u013d\u013f\u0141\u0143\u0145\u0147\u014a\u014c\u014e\u0150\u0152\u0154\u0156\u0158\u015a\u015c\u015e\u0160\u0162\u0164\u0166\u0168\u016a\u016c\u016e\u0170\u0172\u0174\u0176\u0178-\u0179\u017b\u017d\u0181-\u0182\u0184\u0186-\u0187\u0189-\u018b\u018e-\u0191\u0193-\u0194\u0196-\u0198\u019c-\u019d\u019f-\u01a0\u01a2\u01a4\u01a6-\u01a7\u01a9\u01ac\u01ae-\u01af\u01b1-\u01b3\u01b5\u01b7-\u01b8\u01bc\u01c4\u01c7\u01ca\u01cd\u01cf\u01d1\u01d3\u01d5\u01d7\u01d9\u01db\u01de\u01e0\u01e2\u01e4\u01e6\u01e8\u01ea\u01ec\u01ee\u01f1\u01f4\u01f6-\u01f8\u01fa\u01fc\u01fe\u0200\u0202\u0204\u0206\u0208\u020a\u020c\u020e\u0210\u0212\u0214\u0216\u0218\u021a\u021c\u021e\u0220\u0222\u0224\u0226\u0228\u022a\u022c\u022e\u0230\u0232\u023a-\u023b\u023d-\u023e\u0241\u0243-\u0246\u0248\u024a\u024c\u024e\u0370\u0372\u0376\u037f\u0386\u0388-\u038a\u038c\u038e-\u038f\u0391-\u03a1\u03a3-\u03ab\u03cf\u03d2-\u03d4\u03d8\u03da\u03dc\u03de\u03e0\u03e2\u03e4\u03e6\u03e8\u03ea\u03ec\u03ee\u03f4\u03f7\u03f9-\u03fa\u03fd-\u042f\u0460\u0462\u0464\u0466\u0468\u046a\u046c\u046e\u0470\u0472\u0474\u0476\u0478\u047a\u047c\u047e\u0480\u048a\u048c\u048e\u0490\u0492\u0494\u0496\u0498\u049a\u049c\u049e\u04a0\u04a2\u04a4\u04a6\u04a8\u04aa\u04ac\u04ae\u04b0\u04b2\u04b4\u04b6\u04b8\u04ba\u04bc\u04be\u04c0-\u04c1\u04c3\u04c5\u04c7\u04c9\u04cb\u04cd\u04d0\u04d2\u04d4\u04d6\u04d8\u04da\u04dc\u04de\u04e0\u04e2\u04e4\u04e6\u04e8\u04ea\u04ec\u04ee\u04f0\u04f2\u04f4\u04f6\u04f8\u04fa\u04fc\u04fe\u0500\u0502\u0504\u0506\u0508\u050a\u050c\u050e\u0510\u0512\u0514\u0516\u0518\u051a\u051c\u051e\u0520\u0522\u0524\u0526\u0528\u052a\u052c\u052e\u0531-\u0556\u10a0-\u10c5\u10c7\u10cd\u13a0-\u13f5\u1c90-\u1cba\u1cbd-\u1cbf\u1e00\u1e02\u1e04\u1e06\u1e08\u1e0a\u1e0c\u1e0e\u1e10\u1e12\u1e14\u1e16\u1e18\u1e1a\u1e1c\u1e1e\u1e20\u1e22\u1e24\u1e26\u1e28\u1e2a\u1e2c\u1e2e\u1e30\u1e32\u1e34\u1e36\u1e38\u1e3a\u1e3c\u1e3e\u1e40\u1e42\u1e44\u1e46\u1e48\u1e4a\u1e4c\u1e4e\u1e50\u1e52\u1e54\u1e56\u1e58\u1e5a\u1e5c\u1e5e\u1e60\u1e62\u1e64\u1e66\u1e68\u1e6a\u1e6c\u1e6e\u1e70\u1e72\u1e74\u1e76\u1e78\u1e7a\u1e7c\u1e7e\u1e80\u1e82\u1e84\u1e86\u1e88\u1e8a\u1e8c\u1e8e\u1e90\u1e92\u1e94\u1e9e\u1ea0\u1ea2\u1ea4\u1ea6\u1ea8\u1eaa\u1eac\u1eae\u1eb0\u1eb2\u1eb4\u1eb6\u1eb8\u1eba\u1ebc\u1ebe\u1ec0\u1ec2\u1ec4\u1ec6\u1ec8\u1eca\u1ecc\u1ece\u1ed0\u1ed2\u1ed4\u1ed6\u1ed8\u1eda\u1edc\u1ede\u1ee0\u1ee2\u1ee4\u1ee6\u1ee8\u1eea\u1eec\u1eee\u1ef0\u1ef2\u1ef4\u1ef6\u1ef8\u1efa\u1efc\u1efe\u1f08-\u1f0f\u1f18-\u1f1d\u1f28-\u1f2f\u1f38-\u1f3f\u1f48-\u1f4d\u1f59\u1f5b\u1f5d\u1f5f\u1f68-\u1f6f\u1fb8-\u1fbb\u1fc8-\u1fcb\u1fd8-\u1fdb\u1fe8-\u1fec\u1ff8-\u1ffb\u2102\u2107\u210b-\u210d\u2110-\u2112\u2115\u2119-\u211d\u2124\u2126\u2128\u212a-\u212d\u2130-\u2133\u213e-\u213f\u2145\u2183\u2c00-\u2c2e\u2c60\u2c62-\u2c64\u2c67\u2c69\u2c6b\u2c6d-\u2c70\u2c72\u2c75\u2c7e-\u2c80\u2c82\u2c84\u2c86\u2c88\u2c8a\u2c8c\u2c8e\u2c90\u2c92\u2c94\u2c96\u2c98\u2c9a\u2c9c\u2c9e\u2ca0\u2ca2\u2ca4\u2ca6\u2ca8\u2caa\u2cac\u2cae\u2cb0\u2cb2\u2cb4\u2cb6\u2cb8\u2cba\u2cbc\u2cbe\u2cc0\u2cc2\u2cc4\u2cc6\u2cc8\u2cca\u2ccc\u2cce\u2cd0\u2cd2\u2cd4\u2cd6\u2cd8\u2cda\u2cdc\u2cde\u2ce0\u2ce2\u2ceb\u2ced\u2cf2\ua640\ua642\ua644\ua646\ua648\ua64a\ua64c\ua64e\ua650\ua652\ua654\ua656\ua658\ua65a\ua65c\ua65e\ua660\ua662\ua664\ua666\ua668\ua66a\ua66c\ua680\ua682\ua684\ua686\ua688\ua68a\ua68c\ua68e\ua690\ua692\ua694\ua696\ua698\ua69a\ua722\ua724\ua726\ua728\ua72a\ua72c\ua72e\ua732\ua734\ua736\ua738\ua73a\ua73c\ua73e\ua740\ua742\ua744\ua746\ua748\ua74a\ua74c\ua74e\ua750\ua752\ua754\ua756\ua758\ua75a\ua75c\ua75e\ua760\ua762\ua764\ua766\ua768\ua76a\ua76c\ua76e\ua779\ua77b\ua77d-\ua77e\ua780\ua782\ua784\ua786\ua78b\ua78d\ua790\ua792\ua796\ua798\ua79a\ua79c\ua79e\ua7a0\ua7a2\ua7a4\ua7a6\ua7a8\ua7aa-\ua7ae\ua7b0-\ua7b4\ua7b6\ua7b8\uff21-\uff3a'
 
-Mc = u'\u0903\u093b\u093e-\u0940\u0949-\u094c\u094e-\u094f\u0982-\u0983\u09be-\u09c0\u09c7-\u09c8\u09cb-\u09cc\u09d7\u0a03\u0a3e-\u0a40\u0a83\u0abe-\u0ac0\u0ac9\u0acb-\u0acc\u0b02-\u0b03\u0b3e\u0b40\u0b47-\u0b48\u0b4b-\u0b4c\u0b57\u0bbe-\u0bbf\u0bc1-\u0bc2\u0bc6-\u0bc8\u0bca-\u0bcc\u0bd7\u0c01-\u0c03\u0c41-\u0c44\u0c82-\u0c83\u0cbe\u0cc0-\u0cc4\u0cc7-\u0cc8\u0cca-\u0ccb\u0cd5-\u0cd6\u0d02-\u0d03\u0d3e-\u0d40\u0d46-\u0d48\u0d4a-\u0d4c\u0d57\u0d82-\u0d83\u0dcf-\u0dd1\u0dd8-\u0ddf\u0df2-\u0df3\u0f3e-\u0f3f\u0f7f\u102b-\u102c\u1031\u1038\u103b-\u103c\u1056-\u1057\u1062-\u1064\u1067-\u106d\u1083-\u1084\u1087-\u108c\u108f\u109a-\u109c\u17b6\u17be-\u17c5\u17c7-\u17c8\u1923-\u1926\u1929-\u192b\u1930-\u1931\u1933-\u1938\u19b0-\u19c0\u19c8-\u19c9\u1a19-\u1a1a\u1a55\u1a57\u1a61\u1a63-\u1a64\u1a6d-\u1a72\u1b04\u1b35\u1b3b\u1b3d-\u1b41\u1b43-\u1b44\u1b82\u1ba1\u1ba6-\u1ba7\u1baa\u1bac-\u1bad\u1be7\u1bea-\u1bec\u1bee\u1bf2-\u1bf3\u1c24-\u1c2b\u1c34-\u1c35\u1ce1\u1cf2-\u1cf3\u302e-\u302f\ua823-\ua824\ua827\ua880-\ua881\ua8b4-\ua8c3\ua952-\ua953\ua983\ua9b4-\ua9b5\ua9ba-\ua9bb\ua9bd-\ua9c0\uaa2f-\uaa30\uaa33-\uaa34\uaa4d\uaa7b\uaaeb\uaaee-\uaaef\uaaf5\uabe3-\uabe4\uabe6-\uabe7\uabe9-\uabea\uabec'
+Mc = u'\u0903\u093b\u093e-\u0940\u0949-\u094c\u094e-\u094f\u0982-\u0983\u09be-\u09c0\u09c7-\u09c8\u09cb-\u09cc\u09d7\u0a03\u0a3e-\u0a40\u0a83\u0abe-\u0ac0\u0ac9\u0acb-\u0acc\u0b02-\u0b03\u0b3e\u0b40\u0b47-\u0b48\u0b4b-\u0b4c\u0b57\u0bbe-\u0bbf\u0bc1-\u0bc2\u0bc6-\u0bc8\u0bca-\u0bcc\u0bd7\u0c01-\u0c03\u0c41-\u0c44\u0c82-\u0c83\u0cbe\u0cc0-\u0cc4\u0cc7-\u0cc8\u0cca-\u0ccb\u0cd5-\u0cd6\u0d02-\u0d03\u0d3e-\u0d40\u0d46-\u0d48\u0d4a-\u0d4c\u0d57\u0d82-\u0d83\u0dcf-\u0dd1\u0dd8-\u0ddf\u0df2-\u0df3\u0f3e-\u0f3f\u0f7f\u102b-\u102c\u1031\u1038\u103b-\u103c\u1056-\u1057\u1062-\u1064\u1067-\u106d\u1083-\u1084\u1087-\u108c\u108f\u109a-\u109c\u17b6\u17be-\u17c5\u17c7-\u17c8\u1923-\u1926\u1929-\u192b\u1930-\u1931\u1933-\u1938\u1a19-\u1a1a\u1a55\u1a57\u1a61\u1a63-\u1a64\u1a6d-\u1a72\u1b04\u1b35\u1b3b\u1b3d-\u1b41\u1b43-\u1b44\u1b82\u1ba1\u1ba6-\u1ba7\u1baa\u1be7\u1bea-\u1bec\u1bee\u1bf2-\u1bf3\u1c24-\u1c2b\u1c34-\u1c35\u1ce1\u1cf2-\u1cf3\u1cf7\u302e-\u302f\ua823-\ua824\ua827\ua880-\ua881\ua8b4-\ua8c3\ua952-\ua953\ua983\ua9b4-\ua9b5\ua9ba-\ua9bb\ua9bd-\ua9c0\uaa2f-\uaa30\uaa33-\uaa34\uaa4d\uaa7b\uaa7d\uaaeb\uaaee-\uaaef\uaaf5\uabe3-\uabe4\uabe6-\uabe7\uabe9-\uabea\uabec'
 
-Me = u'\u0488-\u0489\u20dd-\u20e0\u20e2-\u20e4\ua670-\ua672'
+Me = u'\u0488-\u0489\u1abe\u20dd-\u20e0\u20e2-\u20e4\ua670-\ua672'
 
-Mn = u'\u0300-\u036f\u0483-\u0487\u0591-\u05bd\u05bf\u05c1-\u05c2\u05c4-\u05c5\u05c7\u0610-\u061a\u064b-\u065f\u0670\u06d6-\u06dc\u06df-\u06e4\u06e7-\u06e8\u06ea-\u06ed\u0711\u0730-\u074a\u07a6-\u07b0\u07eb-\u07f3\u0816-\u0819\u081b-\u0823\u0825-\u0827\u0829-\u082d\u0859-\u085b\u08e4-\u08fe\u0900-\u0902\u093a\u093c\u0941-\u0948\u094d\u0951-\u0957\u0962-\u0963\u0981\u09bc\u09c1-\u09c4\u09cd\u09e2-\u09e3\u0a01-\u0a02\u0a3c\u0a41-\u0a42\u0a47-\u0a48\u0a4b-\u0a4d\u0a51\u0a70-\u0a71\u0a75\u0a81-\u0a82\u0abc\u0ac1-\u0ac5\u0ac7-\u0ac8\u0acd\u0ae2-\u0ae3\u0b01\u0b3c\u0b3f\u0b41-\u0b44\u0b4d\u0b56\u0b62-\u0b63\u0b82\u0bc0\u0bcd\u0c3e-\u0c40\u0c46-\u0c48\u0c4a-\u0c4d\u0c55-\u0c56\u0c62-\u0c63\u0cbc\u0cbf\u0cc6\u0ccc-\u0ccd\u0ce2-\u0ce3\u0d41-\u0d44\u0d4d\u0d62-\u0d63\u0dca\u0dd2-\u0dd4\u0dd6\u0e31\u0e34-\u0e3a\u0e47-\u0e4e\u0eb1\u0eb4-\u0eb9\u0ebb-\u0ebc\u0ec8-\u0ecd\u0f18-\u0f19\u0f35\u0f37\u0f39\u0f71-\u0f7e\u0f80-\u0f84\u0f86-\u0f87\u0f8d-\u0f97\u0f99-\u0fbc\u0fc6\u102d-\u1030\u1032-\u1037\u1039-\u103a\u103d-\u103e\u1058-\u1059\u105e-\u1060\u1071-\u1074\u1082\u1085-\u1086\u108d\u109d\u135d-\u135f\u1712-\u1714\u1732-\u1734\u1752-\u1753\u1772-\u1773\u17b4-\u17b5\u17b7-\u17bd\u17c6\u17c9-\u17d3\u17dd\u180b-\u180d\u18a9\u1920-\u1922\u1927-\u1928\u1932\u1939-\u193b\u1a17-\u1a18\u1a1b\u1a56\u1a58-\u1a5e\u1a60\u1a62\u1a65-\u1a6c\u1a73-\u1a7c\u1a7f\u1b00-\u1b03\u1b34\u1b36-\u1b3a\u1b3c\u1b42\u1b6b-\u1b73\u1b80-\u1b81\u1ba2-\u1ba5\u1ba8-\u1ba9\u1bab\u1be6\u1be8-\u1be9\u1bed\u1bef-\u1bf1\u1c2c-\u1c33\u1c36-\u1c37\u1cd0-\u1cd2\u1cd4-\u1ce0\u1ce2-\u1ce8\u1ced\u1cf4\u1dc0-\u1de6\u1dfc-\u1dff\u20d0-\u20dc\u20e1\u20e5-\u20f0\u2cef-\u2cf1\u2d7f\u2de0-\u2dff\u302a-\u302d\u3099-\u309a\ua66f\ua674-\ua67d\ua69f\ua6f0-\ua6f1\ua802\ua806\ua80b\ua825-\ua826\ua8c4\ua8e0-\ua8f1\ua926-\ua92d\ua947-\ua951\ua980-\ua982\ua9b3\ua9b6-\ua9b9\ua9bc\uaa29-\uaa2e\uaa31-\uaa32\uaa35-\uaa36\uaa43\uaa4c\uaab0\uaab2-\uaab4\uaab7-\uaab8\uaabe-\uaabf\uaac1\uaaec-\uaaed\uaaf6\uabe5\uabe8\uabed\ufb1e\ufe00-\ufe0f\ufe20-\ufe26'
+Mn = u'\u0300-\u036f\u0483-\u0487\u0591-\u05bd\u05bf\u05c1-\u05c2\u05c4-\u05c5\u05c7\u0610-\u061a\u064b-\u065f\u0670\u06d6-\u06dc\u06df-\u06e4\u06e7-\u06e8\u06ea-\u06ed\u0711\u0730-\u074a\u07a6-\u07b0\u07eb-\u07f3\u07fd\u0816-\u0819\u081b-\u0823\u0825-\u0827\u0829-\u082d\u0859-\u085b\u08d3-\u08e1\u08e3-\u0902\u093a\u093c\u0941-\u0948\u094d\u0951-\u0957\u0962-\u0963\u0981\u09bc\u09c1-\u09c4\u09cd\u09e2-\u09e3\u09fe\u0a01-\u0a02\u0a3c\u0a41-\u0a42\u0a47-\u0a48\u0a4b-\u0a4d\u0a51\u0a70-\u0a71\u0a75\u0a81-\u0a82\u0abc\u0ac1-\u0ac5\u0ac7-\u0ac8\u0acd\u0ae2-\u0ae3\u0afa-\u0aff\u0b01\u0b3c\u0b3f\u0b41-\u0b44\u0b4d\u0b56\u0b62-\u0b63\u0b82\u0bc0\u0bcd\u0c00\u0c04\u0c3e-\u0c40\u0c46-\u0c48\u0c4a-\u0c4d\u0c55-\u0c56\u0c62-\u0c63\u0c81\u0cbc\u0cbf\u0cc6\u0ccc-\u0ccd\u0ce2-\u0ce3\u0d00-\u0d01\u0d3b-\u0d3c\u0d41-\u0d44\u0d4d\u0d62-\u0d63\u0dca\u0dd2-\u0dd4\u0dd6\u0e31\u0e34-\u0e3a\u0e47-\u0e4e\u0eb1\u0eb4-\u0eb9\u0ebb-\u0ebc\u0ec8-\u0ecd\u0f18-\u0f19\u0f35\u0f37\u0f39\u0f71-\u0f7e\u0f80-\u0f84\u0f86-\u0f87\u0f8d-\u0f97\u0f99-\u0fbc\u0fc6\u102d-\u1030\u1032-\u1037\u1039-\u103a\u103d-\u103e\u1058-\u1059\u105e-\u1060\u1071-\u1074\u1082\u1085-\u1086\u108d\u109d\u135d-\u135f\u1712-\u1714\u1732-\u1734\u1752-\u1753\u1772-\u1773\u17b4-\u17b5\u17b7-\u17bd\u17c6\u17c9-\u17d3\u17dd\u180b-\u180d\u1885-\u1886\u18a9\u1920-\u1922\u1927-\u1928\u1932\u1939-\u193b\u1a17-\u1a18\u1a1b\u1a56\u1a58-\u1a5e\u1a60\u1a62\u1a65-\u1a6c\u1a73-\u1a7c\u1a7f\u1ab0-\u1abd\u1b00-\u1b03\u1b34\u1b36-\u1b3a\u1b3c\u1b42\u1b6b-\u1b73\u1b80-\u1b81\u1ba2-\u1ba5\u1ba8-\u1ba9\u1bab-\u1bad\u1be6\u1be8-\u1be9\u1bed\u1bef-\u1bf1\u1c2c-\u1c33\u1c36-\u1c37\u1cd0-\u1cd2\u1cd4-\u1ce0\u1ce2-\u1ce8\u1ced\u1cf4\u1cf8-\u1cf9\u1dc0-\u1df9\u1dfb-\u1dff\u20d0-\u20dc\u20e1\u20e5-\u20f0\u2cef-\u2cf1\u2d7f\u2de0-\u2dff\u302a-\u302d\u3099-\u309a\ua66f\ua674-\ua67d\ua69e-\ua69f\ua6f0-\ua6f1\ua802\ua806\ua80b\ua825-\ua826\ua8c4-\ua8c5\ua8e0-\ua8f1\ua8ff\ua926-\ua92d\ua947-\ua951\ua980-\ua982\ua9b3\ua9b6-\ua9b9\ua9bc\ua9e5\uaa29-\uaa2e\uaa31-\uaa32\uaa35-\uaa36\uaa43\uaa4c\uaa7c\uaab0\uaab2-\uaab4\uaab7-\uaab8\uaabe-\uaabf\uaac1\uaaec-\uaaed\uaaf6\uabe5\uabe8\uabed\ufb1e\ufe00-\ufe0f\ufe20-\ufe2f'
 
-Nd = u'0-9\u0660-\u0669\u06f0-\u06f9\u07c0-\u07c9\u0966-\u096f\u09e6-\u09ef\u0a66-\u0a6f\u0ae6-\u0aef\u0b66-\u0b6f\u0be6-\u0bef\u0c66-\u0c6f\u0ce6-\u0cef\u0d66-\u0d6f\u0e50-\u0e59\u0ed0-\u0ed9\u0f20-\u0f29\u1040-\u1049\u1090-\u1099\u17e0-\u17e9\u1810-\u1819\u1946-\u194f\u19d0-\u19d9\u1a80-\u1a89\u1a90-\u1a99\u1b50-\u1b59\u1bb0-\u1bb9\u1c40-\u1c49\u1c50-\u1c59\ua620-\ua629\ua8d0-\ua8d9\ua900-\ua909\ua9d0-\ua9d9\uaa50-\uaa59\uabf0-\uabf9\uff10-\uff19'
+Nd = u'0-9\u0660-\u0669\u06f0-\u06f9\u07c0-\u07c9\u0966-\u096f\u09e6-\u09ef\u0a66-\u0a6f\u0ae6-\u0aef\u0b66-\u0b6f\u0be6-\u0bef\u0c66-\u0c6f\u0ce6-\u0cef\u0d66-\u0d6f\u0de6-\u0def\u0e50-\u0e59\u0ed0-\u0ed9\u0f20-\u0f29\u1040-\u1049\u1090-\u1099\u17e0-\u17e9\u1810-\u1819\u1946-\u194f\u19d0-\u19d9\u1a80-\u1a89\u1a90-\u1a99\u1b50-\u1b59\u1bb0-\u1bb9\u1c40-\u1c49\u1c50-\u1c59\ua620-\ua629\ua8d0-\ua8d9\ua900-\ua909\ua9d0-\ua9d9\ua9f0-\ua9f9\uaa50-\uaa59\uabf0-\uabf9\uff10-\uff19'
 
 Nl = u'\u16ee-\u16f0\u2160-\u2182\u2185-\u2188\u3007\u3021-\u3029\u3038-\u303a\ua6e6-\ua6ef'
 
-No = u'\xb2-\xb3\xb9\xbc-\xbe\u09f4-\u09f9\u0b72-\u0b77\u0bf0-\u0bf2\u0c78-\u0c7e\u0d70-\u0d75\u0f2a-\u0f33\u1369-\u137c\u17f0-\u17f9\u19da\u2070\u2074-\u2079\u2080-\u2089\u2150-\u215f\u2189\u2460-\u249b\u24ea-\u24ff\u2776-\u2793\u2cfd\u3192-\u3195\u3220-\u3229\u3248-\u324f\u3251-\u325f\u3280-\u3289\u32b1-\u32bf\ua830-\ua835'
+No = u'\xb2-\xb3\xb9\xbc-\xbe\u09f4-\u09f9\u0b72-\u0b77\u0bf0-\u0bf2\u0c78-\u0c7e\u0d58-\u0d5e\u0d70-\u0d78\u0f2a-\u0f33\u1369-\u137c\u17f0-\u17f9\u19da\u2070\u2074-\u2079\u2080-\u2089\u2150-\u215f\u2189\u2460-\u249b\u24ea-\u24ff\u2776-\u2793\u2cfd\u3192-\u3195\u3220-\u3229\u3248-\u324f\u3251-\u325f\u3280-\u3289\u32b1-\u32bf\ua830-\ua835'
 
 Pc = u'_\u203f-\u2040\u2054\ufe33-\ufe34\ufe4d-\ufe4f\uff3f'
 
-Pd = u'\\-\u058a\u05be\u1400\u1806\u2010-\u2015\u2e17\u2e1a\u2e3a-\u2e3b\u301c\u3030\u30a0\ufe31-\ufe32\ufe58\ufe63\uff0d'
+Pd = u'\\-\u058a\u05be\u1400\u1806\u2010-\u2015\u2e17\u2e1a\u2e3a-\u2e3b\u2e40\u301c\u3030\u30a0\ufe31-\ufe32\ufe58\ufe63\uff0d'
 
-Pe = u')\\]}\u0f3b\u0f3d\u169c\u2046\u207e\u208e\u2309\u230b\u232a\u2769\u276b\u276d\u276f\u2771\u2773\u2775\u27c6\u27e7\u27e9\u27eb\u27ed\u27ef\u2984\u2986\u2988\u298a\u298c\u298e\u2990\u2992\u2994\u2996\u2998\u29d9\u29db\u29fd\u2e23\u2e25\u2e27\u2e29\u3009\u300b\u300d\u300f\u3011\u3015\u3017\u3019\u301b\u301e-\u301f\ufd3f\ufe18\ufe36\ufe38\ufe3a\ufe3c\ufe3e\ufe40\ufe42\ufe44\ufe48\ufe5a\ufe5c\ufe5e\uff09\uff3d\uff5d\uff60\uff63'
+Pe = u')\\]}\u0f3b\u0f3d\u169c\u2046\u207e\u208e\u2309\u230b\u232a\u2769\u276b\u276d\u276f\u2771\u2773\u2775\u27c6\u27e7\u27e9\u27eb\u27ed\u27ef\u2984\u2986\u2988\u298a\u298c\u298e\u2990\u2992\u2994\u2996\u2998\u29d9\u29db\u29fd\u2e23\u2e25\u2e27\u2e29\u3009\u300b\u300d\u300f\u3011\u3015\u3017\u3019\u301b\u301e-\u301f\ufd3e\ufe18\ufe36\ufe38\ufe3a\ufe3c\ufe3e\ufe40\ufe42\ufe44\ufe48\ufe5a\ufe5c\ufe5e\uff09\uff3d\uff5d\uff60\uff63'
 
 Pf = u'\xbb\u2019\u201d\u203a\u2e03\u2e05\u2e0a\u2e0d\u2e1d\u2e21'
 
 Pi = u'\xab\u2018\u201b-\u201c\u201f\u2039\u2e02\u2e04\u2e09\u2e0c\u2e1c\u2e20'
 
-Po = u"!-#%-'*,.-/:-;?-@\\\\\xa1\xa7\xb6-\xb7\xbf\u037e\u0387\u055a-\u055f\u0589\u05c0\u05c3\u05c6\u05f3-\u05f4\u0609-\u060a\u060c-\u060d\u061b\u061e-\u061f\u066a-\u066d\u06d4\u0700-\u070d\u07f7-\u07f9\u0830-\u083e\u085e\u0964-\u0965\u0970\u0af0\u0df4\u0e4f\u0e5a-\u0e5b\u0f04-\u0f12\u0f14\u0f85\u0fd0-\u0fd4\u0fd9-\u0fda\u104a-\u104f\u10fb\u1360-\u1368\u166d-\u166e\u16eb-\u16ed\u1735-\u1736\u17d4-\u17d6\u17d8-\u17da\u1800-\u1805\u1807-\u180a\u1944-\u1945\u1a1e-\u1a1f\u1aa0-\u1aa6\u1aa8-\u1aad\u1b5a-\u1b60\u1bfc-\u1bff\u1c3b-\u1c3f\u1c7e-\u1c7f\u1cc0-\u1cc7\u1cd3\u2016-\u2017\u2020-\u2027\u2030-\u2038\u203b-\u203e\u2041-\u2043\u2047-\u2051\u2053\u2055-\u205e\u2cf9-\u2cfc\u2cfe-\u2cff\u2d70\u2e00-\u2e01\u2e06-\u2e08\u2e0b\u2e0e-\u2e16\u2e18-\u2e19\u2e1b\u2e1e-\u2e1f\u2e2a-\u2e2e\u2e30-\u2e39\u3001-\u3003\u303d\u30fb\ua4fe-\ua4ff\ua60d-\ua60f\ua673\ua67e\ua6f2-\ua6f7\ua874-\ua877\ua8ce-\ua8cf\ua8f8-\ua8fa\ua92e-\ua92f\ua95f\ua9c1-\ua9cd\ua9de-\ua9df\uaa5c-\uaa5f\uaade-\uaadf\uaaf0-\uaaf1\uabeb\ufe10-\ufe16\ufe19\ufe30\ufe45-\ufe46\ufe49-\ufe4c\ufe50-\ufe52\ufe54-\ufe57\ufe5f-\ufe61\ufe68\ufe6a-\ufe6b\uff01-\uff03\uff05-\uff07\uff0a\uff0c\uff0e-\uff0f\uff1a-\uff1b\uff1f-\uff20\uff3c\uff61\uff64-\uff65"
+Po = u"!-#%-'*,.-/:-;?-@\\\\\xa1\xa7\xb6-\xb7\xbf\u037e\u0387\u055a-\u055f\u0589\u05c0\u05c3\u05c6\u05f3-\u05f4\u0609-\u060a\u060c-\u060d\u061b\u061e-\u061f\u066a-\u066d\u06d4\u0700-\u070d\u07f7-\u07f9\u0830-\u083e\u085e\u0964-\u0965\u0970\u09fd\u0a76\u0af0\u0c84\u0df4\u0e4f\u0e5a-\u0e5b\u0f04-\u0f12\u0f14\u0f85\u0fd0-\u0fd4\u0fd9-\u0fda\u104a-\u104f\u10fb\u1360-\u1368\u166d-\u166e\u16eb-\u16ed\u1735-\u1736\u17d4-\u17d6\u17d8-\u17da\u1800-\u1805\u1807-\u180a\u1944-\u1945\u1a1e-\u1a1f\u1aa0-\u1aa6\u1aa8-\u1aad\u1b5a-\u1b60\u1bfc-\u1bff\u1c3b-\u1c3f\u1c7e-\u1c7f\u1cc0-\u1cc7\u1cd3\u2016-\u2017\u2020-\u2027\u2030-\u2038\u203b-\u203e\u2041-\u2043\u2047-\u2051\u2053\u2055-\u205e\u2cf9-\u2cfc\u2cfe-\u2cff\u2d70\u2e00-\u2e01\u2e06-\u2e08\u2e0b\u2e0e-\u2e16\u2e18-\u2e19\u2e1b\u2e1e-\u2e1f\u2e2a-\u2e2e\u2e30-\u2e39\u2e3c-\u2e3f\u2e41\u2e43-\u2e4e\u3001-\u3003\u303d\u30fb\ua4fe-\ua4ff\ua60d-\ua60f\ua673\ua67e\ua6f2-\ua6f7\ua874-\ua877\ua8ce-\ua8cf\ua8f8-\ua8fa\ua8fc\ua92e-\ua92f\ua95f\ua9c1-\ua9cd\ua9de-\ua9df\uaa5c-\uaa5f\uaade-\uaadf\uaaf0-\uaaf1\uabeb\ufe10-\ufe16\ufe19\ufe30\ufe45-\ufe46\ufe49-\ufe4c\ufe50-\ufe52\ufe54-\ufe57\ufe5f-\ufe61\ufe68\ufe6a-\ufe6b\uff01-\uff03\uff05-\uff07\uff0a\uff0c\uff0e-\uff0f\uff1a-\uff1b\uff1f-\uff20\uff3c\uff61\uff64-\uff65"
 
-Ps = u'(\\[{\u0f3a\u0f3c\u169b\u201a\u201e\u2045\u207d\u208d\u2308\u230a\u2329\u2768\u276a\u276c\u276e\u2770\u2772\u2774\u27c5\u27e6\u27e8\u27ea\u27ec\u27ee\u2983\u2985\u2987\u2989\u298b\u298d\u298f\u2991\u2993\u2995\u2997\u29d8\u29da\u29fc\u2e22\u2e24\u2e26\u2e28\u3008\u300a\u300c\u300e\u3010\u3014\u3016\u3018\u301a\u301d\ufd3e\ufe17\ufe35\ufe37\ufe39\ufe3b\ufe3d\ufe3f\ufe41\ufe43\ufe47\ufe59\ufe5b\ufe5d\uff08\uff3b\uff5b\uff5f\uff62'
+Ps = u'(\\[{\u0f3a\u0f3c\u169b\u201a\u201e\u2045\u207d\u208d\u2308\u230a\u2329\u2768\u276a\u276c\u276e\u2770\u2772\u2774\u27c5\u27e6\u27e8\u27ea\u27ec\u27ee\u2983\u2985\u2987\u2989\u298b\u298d\u298f\u2991\u2993\u2995\u2997\u29d8\u29da\u29fc\u2e22\u2e24\u2e26\u2e28\u2e42\u3008\u300a\u300c\u300e\u3010\u3014\u3016\u3018\u301a\u301d\ufd3f\ufe17\ufe35\ufe37\ufe39\ufe3b\ufe3d\ufe3f\ufe41\ufe43\ufe47\ufe59\ufe5b\ufe5d\uff08\uff3b\uff5b\uff5f\uff62'
 
-Sc = u'$\xa2-\xa5\u058f\u060b\u09f2-\u09f3\u09fb\u0af1\u0bf9\u0e3f\u17db\u20a0-\u20ba\ua838\ufdfc\ufe69\uff04\uffe0-\uffe1\uffe5-\uffe6'
+Sc = u'$\xa2-\xa5\u058f\u060b\u07fe-\u07ff\u09f2-\u09f3\u09fb\u0af1\u0bf9\u0e3f\u17db\u20a0-\u20bf\ua838\ufdfc\ufe69\uff04\uffe0-\uffe1\uffe5-\uffe6'
 
-Sk = u'\\^`\xa8\xaf\xb4\xb8\u02c2-\u02c5\u02d2-\u02df\u02e5-\u02eb\u02ed\u02ef-\u02ff\u0375\u0384-\u0385\u1fbd\u1fbf-\u1fc1\u1fcd-\u1fcf\u1fdd-\u1fdf\u1fed-\u1fef\u1ffd-\u1ffe\u309b-\u309c\ua700-\ua716\ua720-\ua721\ua789-\ua78a\ufbb2-\ufbc1\uff3e\uff40\uffe3'
+Sk = u'\\^`\xa8\xaf\xb4\xb8\u02c2-\u02c5\u02d2-\u02df\u02e5-\u02eb\u02ed\u02ef-\u02ff\u0375\u0384-\u0385\u1fbd\u1fbf-\u1fc1\u1fcd-\u1fcf\u1fdd-\u1fdf\u1fed-\u1fef\u1ffd-\u1ffe\u309b-\u309c\ua700-\ua716\ua720-\ua721\ua789-\ua78a\uab5b\ufbb2-\ufbc1\uff3e\uff40\uffe3'
 
 Sm = u'+<->|~\xac\xb1\xd7\xf7\u03f6\u0606-\u0608\u2044\u2052\u207a-\u207c\u208a-\u208c\u2118\u2140-\u2144\u214b\u2190-\u2194\u219a-\u219b\u21a0\u21a3\u21a6\u21ae\u21ce-\u21cf\u21d2\u21d4\u21f4-\u22ff\u2320-\u2321\u237c\u239b-\u23b3\u23dc-\u23e1\u25b7\u25c1\u25f8-\u25ff\u266f\u27c0-\u27c4\u27c7-\u27e5\u27f0-\u27ff\u2900-\u2982\u2999-\u29d7\u29dc-\u29fb\u29fe-\u2aff\u2b30-\u2b44\u2b47-\u2b4c\ufb29\ufe62\ufe64-\ufe66\uff0b\uff1c-\uff1e\uff5c\uff5e\uffe2\uffe9-\uffec'
 
-So = u'\xa6\xa9\xae\xb0\u0482\u060e-\u060f\u06de\u06e9\u06fd-\u06fe\u07f6\u09fa\u0b70\u0bf3-\u0bf8\u0bfa\u0c7f\u0d79\u0f01-\u0f03\u0f13\u0f15-\u0f17\u0f1a-\u0f1f\u0f34\u0f36\u0f38\u0fbe-\u0fc5\u0fc7-\u0fcc\u0fce-\u0fcf\u0fd5-\u0fd8\u109e-\u109f\u1390-\u1399\u1940\u19de-\u19ff\u1b61-\u1b6a\u1b74-\u1b7c\u2100-\u2101\u2103-\u2106\u2108-\u2109\u2114\u2116-\u2117\u211e-\u2123\u2125\u2127\u2129\u212e\u213a-\u213b\u214a\u214c-\u214d\u214f\u2195-\u2199\u219c-\u219f\u21a1-\u21a2\u21a4-\u21a5\u21a7-\u21ad\u21af-\u21cd\u21d0-\u21d1\u21d3\u21d5-\u21f3\u2300-\u2307\u230c-\u231f\u2322-\u2328\u232b-\u237b\u237d-\u239a\u23b4-\u23db\u23e2-\u23f3\u2400-\u2426\u2440-\u244a\u249c-\u24e9\u2500-\u25b6\u25b8-\u25c0\u25c2-\u25f7\u2600-\u266e\u2670-\u26ff\u2701-\u2767\u2794-\u27bf\u2800-\u28ff\u2b00-\u2b2f\u2b45-\u2b46\u2b50-\u2b59\u2ce5-\u2cea\u2e80-\u2e99\u2e9b-\u2ef3\u2f00-\u2fd5\u2ff0-\u2ffb\u3004\u3012-\u3013\u3020\u3036-\u3037\u303e-\u303f\u3190-\u3191\u3196-\u319f\u31c0-\u31e3\u3200-\u321e\u322a-\u3247\u3250\u3260-\u327f\u328a-\u32b0\u32c0-\u32fe\u3300-\u33ff\u4dc0-\u4dff\ua490-\ua4c6\ua828-\ua82b\ua836-\ua837\ua839\uaa77-\uaa79\ufdfd\uffe4\uffe8\uffed-\uffee\ufffc-\ufffd'
+So = u'\xa6\xa9\xae\xb0\u0482\u058d-\u058e\u060e-\u060f\u06de\u06e9\u06fd-\u06fe\u07f6\u09fa\u0b70\u0bf3-\u0bf8\u0bfa\u0c7f\u0d4f\u0d79\u0f01-\u0f03\u0f13\u0f15-\u0f17\u0f1a-\u0f1f\u0f34\u0f36\u0f38\u0fbe-\u0fc5\u0fc7-\u0fcc\u0fce-\u0fcf\u0fd5-\u0fd8\u109e-\u109f\u1390-\u1399\u1940\u19de-\u19ff\u1b61-\u1b6a\u1b74-\u1b7c\u2100-\u2101\u2103-\u2106\u2108-\u2109\u2114\u2116-\u2117\u211e-\u2123\u2125\u2127\u2129\u212e\u213a-\u213b\u214a\u214c-\u214d\u214f\u218a-\u218b\u2195-\u2199\u219c-\u219f\u21a1-\u21a2\u21a4-\u21a5\u21a7-\u21ad\u21af-\u21cd\u21d0-\u21d1\u21d3\u21d5-\u21f3\u2300-\u2307\u230c-\u231f\u2322-\u2328\u232b-\u237b\u237d-\u239a\u23b4-\u23db\u23e2-\u2426\u2440-\u244a\u249c-\u24e9\u2500-\u25b6\u25b8-\u25c0\u25c2-\u25f7\u2600-\u266e\u2670-\u2767\u2794-\u27bf\u2800-\u28ff\u2b00-\u2b2f\u2b45-\u2b46\u2b4d-\u2b73\u2b76-\u2b95\u2b98-\u2bc8\u2bca-\u2bfe\u2ce5-\u2cea\u2e80-\u2e99\u2e9b-\u2ef3\u2f00-\u2fd5\u2ff0-\u2ffb\u3004\u3012-\u3013\u3020\u3036-\u3037\u303e-\u303f\u3190-\u3191\u3196-\u319f\u31c0-\u31e3\u3200-\u321e\u322a-\u3247\u3250\u3260-\u327f\u328a-\u32b0\u32c0-\u32fe\u3300-\u33ff\u4dc0-\u4dff\ua490-\ua4c6\ua828-\ua82b\ua836-\ua837\ua839\uaa77-\uaa79\ufdfd\uffe4\uffe8\uffed-\uffee\ufffc-\ufffd'
 
 Zl = u'\u2028'
 
@@ -77,49 +77,53 @@ Zp = u'\u2029'
 
 Zs = u' \xa0\u1680\u2000-\u200a\u202f\u205f\u3000'
 
-xid_continue = u'0-9A-Z_a-z\xaa\xb5\xb7\xba\xc0-\xd6\xd8-\xf6\xf8-\u02c1\u02c6-\u02d1\u02e0-\u02e4\u02ec\u02ee\u0300-\u0374\u0376-\u0377\u037b-\u037d\u0386-\u038a\u038c\u038e-\u03a1\u03a3-\u03f5\u03f7-\u0481\u0483-\u0487\u048a-\u0527\u0531-\u0556\u0559\u0561-\u0587\u0591-\u05bd\u05bf\u05c1-\u05c2\u05c4-\u05c5\u05c7\u05d0-\u05ea\u05f0-\u05f2\u0610-\u061a\u0620-\u0669\u066e-\u06d3\u06d5-\u06dc\u06df-\u06e8\u06ea-\u06fc\u06ff\u0710-\u074a\u074d-\u07b1\u07c0-\u07f5\u07fa\u0800-\u082d\u0840-\u085b\u08a0\u08a2-\u08ac\u08e4-\u08fe\u0900-\u0963\u0966-\u096f\u0971-\u0977\u0979-\u097f\u0981-\u0983\u0985-\u098c\u098f-\u0990\u0993-\u09a8\u09aa-\u09b0\u09b2\u09b6-\u09b9\u09bc-\u09c4\u09c7-\u09c8\u09cb-\u09ce\u09d7\u09dc-\u09dd\u09df-\u09e3\u09e6-\u09f1\u0a01-\u0a03\u0a05-\u0a0a\u0a0f-\u0a10\u0a13-\u0a28\u0a2a-\u0a30\u0a32-\u0a33\u0a35-\u0a36\u0a38-\u0a39\u0a3c\u0a3e-\u0a42\u0a47-\u0a48\u0a4b-\u0a4d\u0a51\u0a59-\u0a5c\u0a5e\u0a66-\u0a75\u0a81-\u0a83\u0a85-\u0a8d\u0a8f-\u0a91\u0a93-\u0aa8\u0aaa-\u0ab0\u0ab2-\u0ab3\u0ab5-\u0ab9\u0abc-\u0ac5\u0ac7-\u0ac9\u0acb-\u0acd\u0ad0\u0ae0-\u0ae3\u0ae6-\u0aef\u0b01-\u0b03\u0b05-\u0b0c\u0b0f-\u0b10\u0b13-\u0b28\u0b2a-\u0b30\u0b32-\u0b33\u0b35-\u0b39\u0b3c-\u0b44\u0b47-\u0b48\u0b4b-\u0b4d\u0b56-\u0b57\u0b5c-\u0b5d\u0b5f-\u0b63\u0b66-\u0b6f\u0b71\u0b82-\u0b83\u0b85-\u0b8a\u0b8e-\u0b90\u0b92-\u0b95\u0b99-\u0b9a\u0b9c\u0b9e-\u0b9f\u0ba3-\u0ba4\u0ba8-\u0baa\u0bae-\u0bb9\u0bbe-\u0bc2\u0bc6-\u0bc8\u0bca-\u0bcd\u0bd0\u0bd7\u0be6-\u0bef\u0c01-\u0c03\u0c05-\u0c0c\u0c0e-\u0c10\u0c12-\u0c28\u0c2a-\u0c33\u0c35-\u0c39\u0c3d-\u0c44\u0c46-\u0c48\u0c4a-\u0c4d\u0c55-\u0c56\u0c58-\u0c59\u0c60-\u0c63\u0c66-\u0c6f\u0c82-\u0c83\u0c85-\u0c8c\u0c8e-\u0c90\u0c92-\u0ca8\u0caa-\u0cb3\u0cb5-\u0cb9\u0cbc-\u0cc4\u0cc6-\u0cc8\u0cca-\u0ccd\u0cd5-\u0cd6\u0cde\u0ce0-\u0ce3\u0ce6-\u0cef\u0cf1-\u0cf2\u0d02-\u0d03\u0d05-\u0d0c\u0d0e-\u0d10\u0d12-\u0d3a\u0d3d-\u0d44\u0d46-\u0d48\u0d4a-\u0d4e\u0d57\u0d60-\u0d63\u0d66-\u0d6f\u0d7a-\u0d7f\u0d82-\u0d83\u0d85-\u0d96\u0d9a-\u0db1\u0db3-\u0dbb\u0dbd\u0dc0-\u0dc6\u0dca\u0dcf-\u0dd4\u0dd6\u0dd8-\u0ddf\u0df2-\u0df3\u0e01-\u0e3a\u0e40-\u0e4e\u0e50-\u0e59\u0e81-\u0e82\u0e84\u0e87-\u0e88\u0e8a\u0e8d\u0e94-\u0e97\u0e99-\u0e9f\u0ea1-\u0ea3\u0ea5\u0ea7\u0eaa-\u0eab\u0ead-\u0eb9\u0ebb-\u0ebd\u0ec0-\u0ec4\u0ec6\u0ec8-\u0ecd\u0ed0-\u0ed9\u0edc-\u0edf\u0f00\u0f18-\u0f19\u0f20-\u0f29\u0f35\u0f37\u0f39\u0f3e-\u0f47\u0f49-\u0f6c\u0f71-\u0f84\u0f86-\u0f97\u0f99-\u0fbc\u0fc6\u1000-\u1049\u1050-\u109d\u10a0-\u10c5\u10c7\u10cd\u10d0-\u10fa\u10fc-\u1248\u124a-\u124d\u1250-\u1256\u1258\u125a-\u125d\u1260-\u1288\u128a-\u128d\u1290-\u12b0\u12b2-\u12b5\u12b8-\u12be\u12c0\u12c2-\u12c5\u12c8-\u12d6\u12d8-\u1310\u1312-\u1315\u1318-\u135a\u135d-\u135f\u1369-\u1371\u1380-\u138f\u13a0-\u13f4\u1401-\u166c\u166f-\u167f\u1681-\u169a\u16a0-\u16ea\u16ee-\u16f0\u1700-\u170c\u170e-\u1714\u1720-\u1734\u1740-\u1753\u1760-\u176c\u176e-\u1770\u1772-\u1773\u1780-\u17d3\u17d7\u17dc-\u17dd\u17e0-\u17e9\u180b-\u180d\u1810-\u1819\u1820-\u1877\u1880-\u18aa\u18b0-\u18f5\u1900-\u191c\u1920-\u192b\u1930-\u193b\u1946-\u196d\u1970-\u1974\u1980-\u19ab\u19b0-\u19c9\u19d0-\u19da\u1a00-\u1a1b\u1a20-\u1a5e\u1a60-\u1a7c\u1a7f-\u1a89\u1a90-\u1a99\u1aa7\u1b00-\u1b4b\u1b50-\u1b59\u1b6b-\u1b73\u1b80-\u1bf3\u1c00-\u1c37\u1c40-\u1c49\u1c4d-\u1c7d\u1cd0-\u1cd2\u1cd4-\u1cf6\u1d00-\u1de6\u1dfc-\u1f15\u1f18-\u1f1d\u1f20-\u1f45\u1f48-\u1f4d\u1f50-\u1f57\u1f59\u1f5b\u1f5d\u1f5f-\u1f7d\u1f80-\u1fb4\u1fb6-\u1fbc\u1fbe\u1fc2-\u1fc4\u1fc6-\u1fcc\u1fd0-\u1fd3\u1fd6-\u1fdb\u1fe0-\u1fec\u1ff2-\u1ff4\u1ff6-\u1ffc\u203f-\u2040\u2054\u2071\u207f\u2090-\u209c\u20d0-\u20dc\u20e1\u20e5-\u20f0\u2102\u2107\u210a-\u2113\u2115\u2118-\u211d\u2124\u2126\u2128\u212a-\u2139\u213c-\u213f\u2145-\u2149\u214e\u2160-\u2188\u2c00-\u2c2e\u2c30-\u2c5e\u2c60-\u2ce4\u2ceb-\u2cf3\u2d00-\u2d25\u2d27\u2d2d\u2d30-\u2d67\u2d6f\u2d7f-\u2d96\u2da0-\u2da6\u2da8-\u2dae\u2db0-\u2db6\u2db8-\u2dbe\u2dc0-\u2dc6\u2dc8-\u2dce\u2dd0-\u2dd6\u2dd8-\u2dde\u2de0-\u2dff\u3005-\u3007\u3021-\u302f\u3031-\u3035\u3038-\u303c\u3041-\u3096\u3099-\u309a\u309d-\u309f\u30a1-\u30fa\u30fc-\u30ff\u3105-\u312d\u3131-\u318e\u31a0-\u31ba\u31f0-\u31ff\u3400-\u4db5\u4e00-\u9fcc\ua000-\ua48c\ua4d0-\ua4fd\ua500-\ua60c\ua610-\ua62b\ua640-\ua66f\ua674-\ua67d\ua67f-\ua697\ua69f-\ua6f1\ua717-\ua71f\ua722-\ua788\ua78b-\ua78e\ua790-\ua793\ua7a0-\ua7aa\ua7f8-\ua827\ua840-\ua873\ua880-\ua8c4\ua8d0-\ua8d9\ua8e0-\ua8f7\ua8fb\ua900-\ua92d\ua930-\ua953\ua960-\ua97c\ua980-\ua9c0\ua9cf-\ua9d9\uaa00-\uaa36\uaa40-\uaa4d\uaa50-\uaa59\uaa60-\uaa76\uaa7a-\uaa7b\uaa80-\uaac2\uaadb-\uaadd\uaae0-\uaaef\uaaf2-\uaaf6\uab01-\uab06\uab09-\uab0e\uab11-\uab16\uab20-\uab26\uab28-\uab2e\uabc0-\uabea\uabec-\uabed\uabf0-\uabf9\uac00-\ud7a3\ud7b0-\ud7c6\ud7cb-\ud7fb\uf900-\ufa6d\ufa70-\ufad9\ufb00-\ufb06\ufb13-\ufb17\ufb1d-\ufb28\ufb2a-\ufb36\ufb38-\ufb3c\ufb3e\ufb40-\ufb41\ufb43-\ufb44\ufb46-\ufbb1\ufbd3-\ufc5d\ufc64-\ufd3d\ufd50-\ufd8f\ufd92-\ufdc7\ufdf0-\ufdf9\ufe00-\ufe0f\ufe20-\ufe26\ufe33-\ufe34\ufe4d-\ufe4f\ufe71\ufe73\ufe77\ufe79\ufe7b\ufe7d\ufe7f-\ufefc\uff10-\uff19\uff21-\uff3a\uff3f\uff41-\uff5a\uff66-\uffbe\uffc2-\uffc7\uffca-\uffcf\uffd2-\uffd7\uffda-\uffdc'
+xid_continue = u'0-9A-Z_a-z\xaa\xb5\xb7\xba\xc0-\xd6\xd8-\xf6\xf8-\u02c1\u02c6-\u02d1\u02e0-\u02e4\u02ec\u02ee\u0300-\u0374\u0376-\u0377\u037b-\u037d\u037f\u0386-\u038a\u038c\u038e-\u03a1\u03a3-\u03f5\u03f7-\u0481\u0483-\u0487\u048a-\u052f\u0531-\u0556\u0559\u0560-\u0588\u0591-\u05bd\u05bf\u05c1-\u05c2\u05c4-\u05c5\u05c7\u05d0-\u05ea\u05ef-\u05f2\u0610-\u061a\u0620-\u0669\u066e-\u06d3\u06d5-\u06dc\u06df-\u06e8\u06ea-\u06fc\u06ff\u0710-\u074a\u074d-\u07b1\u07c0-\u07f5\u07fa\u07fd\u0800-\u082d\u0840-\u085b\u0860-\u086a\u08a0-\u08b4\u08b6-\u08bd\u08d3-\u08e1\u08e3-\u0963\u0966-\u096f\u0971-\u0983\u0985-\u098c\u098f-\u0990\u0993-\u09a8\u09aa-\u09b0\u09b2\u09b6-\u09b9\u09bc-\u09c4\u09c7-\u09c8\u09cb-\u09ce\u09d7\u09dc-\u09dd\u09df-\u09e3\u09e6-\u09f1\u09fc\u09fe\u0a01-\u0a03\u0a05-\u0a0a\u0a0f-\u0a10\u0a13-\u0a28\u0a2a-\u0a30\u0a32-\u0a33\u0a35-\u0a36\u0a38-\u0a39\u0a3c\u0a3e-\u0a42\u0a47-\u0a48\u0a4b-\u0a4d\u0a51\u0a59-\u0a5c\u0a5e\u0a66-\u0a75\u0a81-\u0a83\u0a85-\u0a8d\u0a8f-\u0a91\u0a93-\u0aa8\u0aaa-\u0ab0\u0ab2-\u0ab3\u0ab5-\u0ab9\u0abc-\u0ac5\u0ac7-\u0ac9\u0acb-\u0acd\u0ad0\u0ae0-\u0ae3\u0ae6-\u0aef\u0af9-\u0aff\u0b01-\u0b03\u0b05-\u0b0c\u0b0f-\u0b10\u0b13-\u0b28\u0b2a-\u0b30\u0b32-\u0b33\u0b35-\u0b39\u0b3c-\u0b44\u0b47-\u0b48\u0b4b-\u0b4d\u0b56-\u0b57\u0b5c-\u0b5d\u0b5f-\u0b63\u0b66-\u0b6f\u0b71\u0b82-\u0b83\u0b85-\u0b8a\u0b8e-\u0b90\u0b92-\u0b95\u0b99-\u0b9a\u0b9c\u0b9e-\u0b9f\u0ba3-\u0ba4\u0ba8-\u0baa\u0bae-\u0bb9\u0bbe-\u0bc2\u0bc6-\u0bc8\u0bca-\u0bcd\u0bd0\u0bd7\u0be6-\u0bef\u0c00-\u0c0c\u0c0e-\u0c10\u0c12-\u0c28\u0c2a-\u0c39\u0c3d-\u0c44\u0c46-\u0c48\u0c4a-\u0c4d\u0c55-\u0c56\u0c58-\u0c5a\u0c60-\u0c63\u0c66-\u0c6f\u0c80-\u0c83\u0c85-\u0c8c\u0c8e-\u0c90\u0c92-\u0ca8\u0caa-\u0cb3\u0cb5-\u0cb9\u0cbc-\u0cc4\u0cc6-\u0cc8\u0cca-\u0ccd\u0cd5-\u0cd6\u0cde\u0ce0-\u0ce3\u0ce6-\u0cef\u0cf1-\u0cf2\u0d00-\u0d03\u0d05-\u0d0c\u0d0e-\u0d10\u0d12-\u0d44\u0d46-\u0d48\u0d4a-\u0d4e\u0d54-\u0d57\u0d5f-\u0d63\u0d66-\u0d6f\u0d7a-\u0d7f\u0d82-\u0d83\u0d85-\u0d96\u0d9a-\u0db1\u0db3-\u0dbb\u0dbd\u0dc0-\u0dc6\u0dca\u0dcf-\u0dd4\u0dd6\u0dd8-\u0ddf\u0de6-\u0def\u0df2-\u0df3\u0e01-\u0e3a\u0e40-\u0e4e\u0e50-\u0e59\u0e81-\u0e82\u0e84\u0e87-\u0e88\u0e8a\u0e8d\u0e94-\u0e97\u0e99-\u0e9f\u0ea1-\u0ea3\u0ea5\u0ea7\u0eaa-\u0eab\u0ead-\u0eb9\u0ebb-\u0ebd\u0ec0-\u0ec4\u0ec6\u0ec8-\u0ecd\u0ed0-\u0ed9\u0edc-\u0edf\u0f00\u0f18-\u0f19\u0f20-\u0f29\u0f35\u0f37\u0f39\u0f3e-\u0f47\u0f49-\u0f6c\u0f71-\u0f84\u0f86-\u0f97\u0f99-\u0fbc\u0fc6\u1000-\u1049\u1050-\u109d\u10a0-\u10c5\u10c7\u10cd\u10d0-\u10fa\u10fc-\u1248\u124a-\u124d\u1250-\u1256\u1258\u125a-\u125d\u1260-\u1288\u128a-\u128d\u1290-\u12b0\u12b2-\u12b5\u12b8-\u12be\u12c0\u12c2-\u12c5\u12c8-\u12d6\u12d8-\u1310\u1312-\u1315\u1318-\u135a\u135d-\u135f\u1369-\u1371\u1380-\u138f\u13a0-\u13f5\u13f8-\u13fd\u1401-\u166c\u166f-\u167f\u1681-\u169a\u16a0-\u16ea\u16ee-\u16f8\u1700-\u170c\u170e-\u1714\u1720-\u1734\u1740-\u1753\u1760-\u176c\u176e-\u1770\u1772-\u1773\u1780-\u17d3\u17d7\u17dc-\u17dd\u17e0-\u17e9\u180b-\u180d\u1810-\u1819\u1820-\u1878\u1880-\u18aa\u18b0-\u18f5\u1900-\u191e\u1920-\u192b\u1930-\u193b\u1946-\u196d\u1970-\u1974\u1980-\u19ab\u19b0-\u19c9\u19d0-\u19da\u1a00-\u1a1b\u1a20-\u1a5e\u1a60-\u1a7c\u1a7f-\u1a89\u1a90-\u1a99\u1aa7\u1ab0-\u1abd\u1b00-\u1b4b\u1b50-\u1b59\u1b6b-\u1b73\u1b80-\u1bf3\u1c00-\u1c37\u1c40-\u1c49\u1c4d-\u1c7d\u1c80-\u1c88\u1c90-\u1cba\u1cbd-\u1cbf\u1cd0-\u1cd2\u1cd4-\u1cf9\u1d00-\u1df9\u1dfb-\u1f15\u1f18-\u1f1d\u1f20-\u1f45\u1f48-\u1f4d\u1f50-\u1f57\u1f59\u1f5b\u1f5d\u1f5f-\u1f7d\u1f80-\u1fb4\u1fb6-\u1fbc\u1fbe\u1fc2-\u1fc4\u1fc6-\u1fcc\u1fd0-\u1fd3\u1fd6-\u1fdb\u1fe0-\u1fec\u1ff2-\u1ff4\u1ff6-\u1ffc\u203f-\u2040\u2054\u2071\u207f\u2090-\u209c\u20d0-\u20dc\u20e1\u20e5-\u20f0\u2102\u2107\u210a-\u2113\u2115\u2118-\u211d\u2124\u2126\u2128\u212a-\u2139\u213c-\u213f\u2145-\u2149\u214e\u2160-\u2188\u2c00-\u2c2e\u2c30-\u2c5e\u2c60-\u2ce4\u2ceb-\u2cf3\u2d00-\u2d25\u2d27\u2d2d\u2d30-\u2d67\u2d6f\u2d7f-\u2d96\u2da0-\u2da6\u2da8-\u2dae\u2db0-\u2db6\u2db8-\u2dbe\u2dc0-\u2dc6\u2dc8-\u2dce\u2dd0-\u2dd6\u2dd8-\u2dde\u2de0-\u2dff\u3005-\u3007\u3021-\u302f\u3031-\u3035\u3038-\u303c\u3041-\u3096\u3099-\u309a\u309d-\u309f\u30a1-\u30fa\u30fc-\u30ff\u3105-\u312f\u3131-\u318e\u31a0-\u31ba\u31f0-\u31ff\u3400-\u4db5\u4e00-\u9fef\ua000-\ua48c\ua4d0-\ua4fd\ua500-\ua60c\ua610-\ua62b\ua640-\ua66f\ua674-\ua67d\ua67f-\ua6f1\ua717-\ua71f\ua722-\ua788\ua78b-\ua7b9\ua7f7-\ua827\ua840-\ua873\ua880-\ua8c5\ua8d0-\ua8d9\ua8e0-\ua8f7\ua8fb\ua8fd-\ua92d\ua930-\ua953\ua960-\ua97c\ua980-\ua9c0\ua9cf-\ua9d9\ua9e0-\ua9fe\uaa00-\uaa36\uaa40-\uaa4d\uaa50-\uaa59\uaa60-\uaa76\uaa7a-\uaac2\uaadb-\uaadd\uaae0-\uaaef\uaaf2-\uaaf6\uab01-\uab06\uab09-\uab0e\uab11-\uab16\uab20-\uab26\uab28-\uab2e\uab30-\uab5a\uab5c-\uab65\uab70-\uabea\uabec-\uabed\uabf0-\uabf9\uac00-\ud7a3\ud7b0-\ud7c6\ud7cb-\ud7fb\uf900-\ufa6d\ufa70-\ufad9\ufb00-\ufb06\ufb13-\ufb17\ufb1d-\ufb28\ufb2a-\ufb36\ufb38-\ufb3c\ufb3e\ufb40-\ufb41\ufb43-\ufb44\ufb46-\ufbb1\ufbd3-\ufc5d\ufc64-\ufd3d\ufd50-\ufd8f\ufd92-\ufdc7\ufdf0-\ufdf9\ufe00-\ufe0f\ufe20-\ufe2f\ufe33-\ufe34\ufe4d-\ufe4f\ufe71\ufe73\ufe77\ufe79\ufe7b\ufe7d\ufe7f-\ufefc\uff10-\uff19\uff21-\uff3a\uff3f\uff41-\uff5a\uff66-\uffbe\uffc2-\uffc7\uffca-\uffcf\uffd2-\uffd7\uffda-\uffdc'
 
-xid_start = u'A-Z_a-z\xaa\xb5\xba\xc0-\xd6\xd8-\xf6\xf8-\u02c1\u02c6-\u02d1\u02e0-\u02e4\u02ec\u02ee\u0370-\u0374\u0376-\u0377\u037b-\u037d\u0386\u0388-\u038a\u038c\u038e-\u03a1\u03a3-\u03f5\u03f7-\u0481\u048a-\u0527\u0531-\u0556\u0559\u0561-\u0587\u05d0-\u05ea\u05f0-\u05f2\u0620-\u064a\u066e-\u066f\u0671-\u06d3\u06d5\u06e5-\u06e6\u06ee-\u06ef\u06fa-\u06fc\u06ff\u0710\u0712-\u072f\u074d-\u07a5\u07b1\u07ca-\u07ea\u07f4-\u07f5\u07fa\u0800-\u0815\u081a\u0824\u0828\u0840-\u0858\u08a0\u08a2-\u08ac\u0904-\u0939\u093d\u0950\u0958-\u0961\u0971-\u0977\u0979-\u097f\u0985-\u098c\u098f-\u0990\u0993-\u09a8\u09aa-\u09b0\u09b2\u09b6-\u09b9\u09bd\u09ce\u09dc-\u09dd\u09df-\u09e1\u09f0-\u09f1\u0a05-\u0a0a\u0a0f-\u0a10\u0a13-\u0a28\u0a2a-\u0a30\u0a32-\u0a33\u0a35-\u0a36\u0a38-\u0a39\u0a59-\u0a5c\u0a5e\u0a72-\u0a74\u0a85-\u0a8d\u0a8f-\u0a91\u0a93-\u0aa8\u0aaa-\u0ab0\u0ab2-\u0ab3\u0ab5-\u0ab9\u0abd\u0ad0\u0ae0-\u0ae1\u0b05-\u0b0c\u0b0f-\u0b10\u0b13-\u0b28\u0b2a-\u0b30\u0b32-\u0b33\u0b35-\u0b39\u0b3d\u0b5c-\u0b5d\u0b5f-\u0b61\u0b71\u0b83\u0b85-\u0b8a\u0b8e-\u0b90\u0b92-\u0b95\u0b99-\u0b9a\u0b9c\u0b9e-\u0b9f\u0ba3-\u0ba4\u0ba8-\u0baa\u0bae-\u0bb9\u0bd0\u0c05-\u0c0c\u0c0e-\u0c10\u0c12-\u0c28\u0c2a-\u0c33\u0c35-\u0c39\u0c3d\u0c58-\u0c59\u0c60-\u0c61\u0c85-\u0c8c\u0c8e-\u0c90\u0c92-\u0ca8\u0caa-\u0cb3\u0cb5-\u0cb9\u0cbd\u0cde\u0ce0-\u0ce1\u0cf1-\u0cf2\u0d05-\u0d0c\u0d0e-\u0d10\u0d12-\u0d3a\u0d3d\u0d4e\u0d60-\u0d61\u0d7a-\u0d7f\u0d85-\u0d96\u0d9a-\u0db1\u0db3-\u0dbb\u0dbd\u0dc0-\u0dc6\u0e01-\u0e30\u0e32\u0e40-\u0e46\u0e81-\u0e82\u0e84\u0e87-\u0e88\u0e8a\u0e8d\u0e94-\u0e97\u0e99-\u0e9f\u0ea1-\u0ea3\u0ea5\u0ea7\u0eaa-\u0eab\u0ead-\u0eb0\u0eb2\u0ebd\u0ec0-\u0ec4\u0ec6\u0edc-\u0edf\u0f00\u0f40-\u0f47\u0f49-\u0f6c\u0f88-\u0f8c\u1000-\u102a\u103f\u1050-\u1055\u105a-\u105d\u1061\u1065-\u1066\u106e-\u1070\u1075-\u1081\u108e\u10a0-\u10c5\u10c7\u10cd\u10d0-\u10fa\u10fc-\u1248\u124a-\u124d\u1250-\u1256\u1258\u125a-\u125d\u1260-\u1288\u128a-\u128d\u1290-\u12b0\u12b2-\u12b5\u12b8-\u12be\u12c0\u12c2-\u12c5\u12c8-\u12d6\u12d8-\u1310\u1312-\u1315\u1318-\u135a\u1380-\u138f\u13a0-\u13f4\u1401-\u166c\u166f-\u167f\u1681-\u169a\u16a0-\u16ea\u16ee-\u16f0\u1700-\u170c\u170e-\u1711\u1720-\u1731\u1740-\u1751\u1760-\u176c\u176e-\u1770\u1780-\u17b3\u17d7\u17dc\u1820-\u1877\u1880-\u18a8\u18aa\u18b0-\u18f5\u1900-\u191c\u1950-\u196d\u1970-\u1974\u1980-\u19ab\u19c1-\u19c7\u1a00-\u1a16\u1a20-\u1a54\u1aa7\u1b05-\u1b33\u1b45-\u1b4b\u1b83-\u1ba0\u1bae-\u1baf\u1bba-\u1be5\u1c00-\u1c23\u1c4d-\u1c4f\u1c5a-\u1c7d\u1ce9-\u1cec\u1cee-\u1cf1\u1cf5-\u1cf6\u1d00-\u1dbf\u1e00-\u1f15\u1f18-\u1f1d\u1f20-\u1f45\u1f48-\u1f4d\u1f50-\u1f57\u1f59\u1f5b\u1f5d\u1f5f-\u1f7d\u1f80-\u1fb4\u1fb6-\u1fbc\u1fbe\u1fc2-\u1fc4\u1fc6-\u1fcc\u1fd0-\u1fd3\u1fd6-\u1fdb\u1fe0-\u1fec\u1ff2-\u1ff4\u1ff6-\u1ffc\u2071\u207f\u2090-\u209c\u2102\u2107\u210a-\u2113\u2115\u2118-\u211d\u2124\u2126\u2128\u212a-\u2139\u213c-\u213f\u2145-\u2149\u214e\u2160-\u2188\u2c00-\u2c2e\u2c30-\u2c5e\u2c60-\u2ce4\u2ceb-\u2cee\u2cf2-\u2cf3\u2d00-\u2d25\u2d27\u2d2d\u2d30-\u2d67\u2d6f\u2d80-\u2d96\u2da0-\u2da6\u2da8-\u2dae\u2db0-\u2db6\u2db8-\u2dbe\u2dc0-\u2dc6\u2dc8-\u2dce\u2dd0-\u2dd6\u2dd8-\u2dde\u3005-\u3007\u3021-\u3029\u3031-\u3035\u3038-\u303c\u3041-\u3096\u309d-\u309f\u30a1-\u30fa\u30fc-\u30ff\u3105-\u312d\u3131-\u318e\u31a0-\u31ba\u31f0-\u31ff\u3400-\u4db5\u4e00-\u9fcc\ua000-\ua48c\ua4d0-\ua4fd\ua500-\ua60c\ua610-\ua61f\ua62a-\ua62b\ua640-\ua66e\ua67f-\ua697\ua6a0-\ua6ef\ua717-\ua71f\ua722-\ua788\ua78b-\ua78e\ua790-\ua793\ua7a0-\ua7aa\ua7f8-\ua801\ua803-\ua805\ua807-\ua80a\ua80c-\ua822\ua840-\ua873\ua882-\ua8b3\ua8f2-\ua8f7\ua8fb\ua90a-\ua925\ua930-\ua946\ua960-\ua97c\ua984-\ua9b2\ua9cf\uaa00-\uaa28\uaa40-\uaa42\uaa44-\uaa4b\uaa60-\uaa76\uaa7a\uaa80-\uaaaf\uaab1\uaab5-\uaab6\uaab9-\uaabd\uaac0\uaac2\uaadb-\uaadd\uaae0-\uaaea\uaaf2-\uaaf4\uab01-\uab06\uab09-\uab0e\uab11-\uab16\uab20-\uab26\uab28-\uab2e\uabc0-\uabe2\uac00-\ud7a3\ud7b0-\ud7c6\ud7cb-\ud7fb\uf900-\ufa6d\ufa70-\ufad9\ufb00-\ufb06\ufb13-\ufb17\ufb1d\ufb1f-\ufb28\ufb2a-\ufb36\ufb38-\ufb3c\ufb3e\ufb40-\ufb41\ufb43-\ufb44\ufb46-\ufbb1\ufbd3-\ufc5d\ufc64-\ufd3d\ufd50-\ufd8f\ufd92-\ufdc7\ufdf0-\ufdf9\ufe71\ufe73\ufe77\ufe79\ufe7b\ufe7d\ufe7f-\ufefc\uff21-\uff3a\uff41-\uff5a\uff66-\uff9d\uffa0-\uffbe\uffc2-\uffc7\uffca-\uffcf\uffd2-\uffd7\uffda-\uffdc'
+xid_start = u'A-Z_a-z\xaa\xb5\xba\xc0-\xd6\xd8-\xf6\xf8-\u02c1\u02c6-\u02d1\u02e0-\u02e4\u02ec\u02ee\u0370-\u0374\u0376-\u0377\u037b-\u037d\u037f\u0386\u0388-\u038a\u038c\u038e-\u03a1\u03a3-\u03f5\u03f7-\u0481\u048a-\u052f\u0531-\u0556\u0559\u0560-\u0588\u05d0-\u05ea\u05ef-\u05f2\u0620-\u064a\u066e-\u066f\u0671-\u06d3\u06d5\u06e5-\u06e6\u06ee-\u06ef\u06fa-\u06fc\u06ff\u0710\u0712-\u072f\u074d-\u07a5\u07b1\u07ca-\u07ea\u07f4-\u07f5\u07fa\u0800-\u0815\u081a\u0824\u0828\u0840-\u0858\u0860-\u086a\u08a0-\u08b4\u08b6-\u08bd\u0904-\u0939\u093d\u0950\u0958-\u0961\u0971-\u0980\u0985-\u098c\u098f-\u0990\u0993-\u09a8\u09aa-\u09b0\u09b2\u09b6-\u09b9\u09bd\u09ce\u09dc-\u09dd\u09df-\u09e1\u09f0-\u09f1\u09fc\u0a05-\u0a0a\u0a0f-\u0a10\u0a13-\u0a28\u0a2a-\u0a30\u0a32-\u0a33\u0a35-\u0a36\u0a38-\u0a39\u0a59-\u0a5c\u0a5e\u0a72-\u0a74\u0a85-\u0a8d\u0a8f-\u0a91\u0a93-\u0aa8\u0aaa-\u0ab0\u0ab2-\u0ab3\u0ab5-\u0ab9\u0abd\u0ad0\u0ae0-\u0ae1\u0af9\u0b05-\u0b0c\u0b0f-\u0b10\u0b13-\u0b28\u0b2a-\u0b30\u0b32-\u0b33\u0b35-\u0b39\u0b3d\u0b5c-\u0b5d\u0b5f-\u0b61\u0b71\u0b83\u0b85-\u0b8a\u0b8e-\u0b90\u0b92-\u0b95\u0b99-\u0b9a\u0b9c\u0b9e-\u0b9f\u0ba3-\u0ba4\u0ba8-\u0baa\u0bae-\u0bb9\u0bd0\u0c05-\u0c0c\u0c0e-\u0c10\u0c12-\u0c28\u0c2a-\u0c39\u0c3d\u0c58-\u0c5a\u0c60-\u0c61\u0c80\u0c85-\u0c8c\u0c8e-\u0c90\u0c92-\u0ca8\u0caa-\u0cb3\u0cb5-\u0cb9\u0cbd\u0cde\u0ce0-\u0ce1\u0cf1-\u0cf2\u0d05-\u0d0c\u0d0e-\u0d10\u0d12-\u0d3a\u0d3d\u0d4e\u0d54-\u0d56\u0d5f-\u0d61\u0d7a-\u0d7f\u0d85-\u0d96\u0d9a-\u0db1\u0db3-\u0dbb\u0dbd\u0dc0-\u0dc6\u0e01-\u0e30\u0e32\u0e40-\u0e46\u0e81-\u0e82\u0e84\u0e87-\u0e88\u0e8a\u0e8d\u0e94-\u0e97\u0e99-\u0e9f\u0ea1-\u0ea3\u0ea5\u0ea7\u0eaa-\u0eab\u0ead-\u0eb0\u0eb2\u0ebd\u0ec0-\u0ec4\u0ec6\u0edc-\u0edf\u0f00\u0f40-\u0f47\u0f49-\u0f6c\u0f88-\u0f8c\u1000-\u102a\u103f\u1050-\u1055\u105a-\u105d\u1061\u1065-\u1066\u106e-\u1070\u1075-\u1081\u108e\u10a0-\u10c5\u10c7\u10cd\u10d0-\u10fa\u10fc-\u1248\u124a-\u124d\u1250-\u1256\u1258\u125a-\u125d\u1260-\u1288\u128a-\u128d\u1290-\u12b0\u12b2-\u12b5\u12b8-\u12be\u12c0\u12c2-\u12c5\u12c8-\u12d6\u12d8-\u1310\u1312-\u1315\u1318-\u135a\u1380-\u138f\u13a0-\u13f5\u13f8-\u13fd\u1401-\u166c\u166f-\u167f\u1681-\u169a\u16a0-\u16ea\u16ee-\u16f8\u1700-\u170c\u170e-\u1711\u1720-\u1731\u1740-\u1751\u1760-\u176c\u176e-\u1770\u1780-\u17b3\u17d7\u17dc\u1820-\u1878\u1880-\u18a8\u18aa\u18b0-\u18f5\u1900-\u191e\u1950-\u196d\u1970-\u1974\u1980-\u19ab\u19b0-\u19c9\u1a00-\u1a16\u1a20-\u1a54\u1aa7\u1b05-\u1b33\u1b45-\u1b4b\u1b83-\u1ba0\u1bae-\u1baf\u1bba-\u1be5\u1c00-\u1c23\u1c4d-\u1c4f\u1c5a-\u1c7d\u1c80-\u1c88\u1c90-\u1cba\u1cbd-\u1cbf\u1ce9-\u1cec\u1cee-\u1cf1\u1cf5-\u1cf6\u1d00-\u1dbf\u1e00-\u1f15\u1f18-\u1f1d\u1f20-\u1f45\u1f48-\u1f4d\u1f50-\u1f57\u1f59\u1f5b\u1f5d\u1f5f-\u1f7d\u1f80-\u1fb4\u1fb6-\u1fbc\u1fbe\u1fc2-\u1fc4\u1fc6-\u1fcc\u1fd0-\u1fd3\u1fd6-\u1fdb\u1fe0-\u1fec\u1ff2-\u1ff4\u1ff6-\u1ffc\u2071\u207f\u2090-\u209c\u2102\u2107\u210a-\u2113\u2115\u2118-\u211d\u2124\u2126\u2128\u212a-\u2139\u213c-\u213f\u2145-\u2149\u214e\u2160-\u2188\u2c00-\u2c2e\u2c30-\u2c5e\u2c60-\u2ce4\u2ceb-\u2cee\u2cf2-\u2cf3\u2d00-\u2d25\u2d27\u2d2d\u2d30-\u2d67\u2d6f\u2d80-\u2d96\u2da0-\u2da6\u2da8-\u2dae\u2db0-\u2db6\u2db8-\u2dbe\u2dc0-\u2dc6\u2dc8-\u2dce\u2dd0-\u2dd6\u2dd8-\u2dde\u3005-\u3007\u3021-\u3029\u3031-\u3035\u3038-\u303c\u3041-\u3096\u309d-\u309f\u30a1-\u30fa\u30fc-\u30ff\u3105-\u312f\u3131-\u318e\u31a0-\u31ba\u31f0-\u31ff\u3400-\u4db5\u4e00-\u9fef\ua000-\ua48c\ua4d0-\ua4fd\ua500-\ua60c\ua610-\ua61f\ua62a-\ua62b\ua640-\ua66e\ua67f-\ua69d\ua6a0-\ua6ef\ua717-\ua71f\ua722-\ua788\ua78b-\ua7b9\ua7f7-\ua801\ua803-\ua805\ua807-\ua80a\ua80c-\ua822\ua840-\ua873\ua882-\ua8b3\ua8f2-\ua8f7\ua8fb\ua8fd-\ua8fe\ua90a-\ua925\ua930-\ua946\ua960-\ua97c\ua984-\ua9b2\ua9cf\ua9e0-\ua9e4\ua9e6-\ua9ef\ua9fa-\ua9fe\uaa00-\uaa28\uaa40-\uaa42\uaa44-\uaa4b\uaa60-\uaa76\uaa7a\uaa7e-\uaaaf\uaab1\uaab5-\uaab6\uaab9-\uaabd\uaac0\uaac2\uaadb-\uaadd\uaae0-\uaaea\uaaf2-\uaaf4\uab01-\uab06\uab09-\uab0e\uab11-\uab16\uab20-\uab26\uab28-\uab2e\uab30-\uab5a\uab5c-\uab65\uab70-\uabe2\uac00-\ud7a3\ud7b0-\ud7c6\ud7cb-\ud7fb\uf900-\ufa6d\ufa70-\ufad9\ufb00-\ufb06\ufb13-\ufb17\ufb1d\ufb1f-\ufb28\ufb2a-\ufb36\ufb38-\ufb3c\ufb3e\ufb40-\ufb41\ufb43-\ufb44\ufb46-\ufbb1\ufbd3-\ufc5d\ufc64-\ufd3d\ufd50-\ufd8f\ufd92-\ufdc7\ufdf0-\ufdf9\ufe71\ufe73\ufe77\ufe79\ufe7b\ufe7d\ufe7f-\ufefc\uff21-\uff3a\uff41-\uff5a\uff66-\uff9d\uffa0-\uffbe\uffc2-\uffc7\uffca-\uffcf\uffd2-\uffd7\uffda-\uffdc'
 
 if sys.maxunicode > 0xFFFF:
     # non-BMP characters, use only on wide Unicode builds
-    Cf += u'\U000110bd\U0001d173-\U0001d17a\U000e0001\U000e0020-\U000e007f'
+    Cf += u'\U000110bd\U000110cd\U0001bca0-\U0001bca3\U0001d173-\U0001d17a\U000e0001\U000e0020-\U000e007f'
 
-    Cn += u'\U0001000c\U00010027\U0001003b\U0001003e\U0001004e-\U0001004f\U0001005e-\U0001007f\U000100fb-\U000100ff\U00010103-\U00010106\U00010134-\U00010136\U0001018b-\U0001018f\U0001019c-\U000101cf\U000101fe-\U0001027f\U0001029d-\U0001029f\U000102d1-\U000102ff\U0001031f\U00010324-\U0001032f\U0001034b-\U0001037f\U0001039e\U000103c4-\U000103c7\U000103d6-\U000103ff\U0001049e-\U0001049f\U000104aa-\U000107ff\U00010806-\U00010807\U00010809\U00010836\U00010839-\U0001083b\U0001083d-\U0001083e\U00010856\U00010860-\U000108ff\U0001091c-\U0001091e\U0001093a-\U0001093e\U00010940-\U0001097f\U000109b8-\U000109bd\U000109c0-\U000109ff\U00010a04\U00010a07-\U00010a0b\U00010a14\U00010a18\U00010a34-\U00010a37\U00010a3b-\U00010a3e\U00010a48-\U00010a4f\U00010a59-\U00010a5f\U00010a80-\U00010aff\U00010b36-\U00010b38\U00010b56-\U00010b57\U00010b73-\U00010b77\U00010b80-\U00010bff\U00010c49-\U00010e5f\U00010e7f-\U00010fff\U0001104e-\U00011051\U00011070-\U0001107f\U000110c2-\U000110cf\U000110e9-\U000110ef\U000110fa-\U000110ff\U00011135\U00011144-\U0001117f\U000111c9-\U000111cf\U000111da-\U0001167f\U000116b8-\U000116bf\U000116ca-\U00011fff\U0001236f-\U000123ff\U00012463-\U0001246f\U00012474-\U00012fff\U0001342f-\U000167ff\U00016a39-\U00016eff\U00016f45-\U00016f4f\U00016f7f-\U00016f8e\U00016fa0-\U0001afff\U0001b002-\U0001cfff\U0001d0f6-\U0001d0ff\U0001d127-\U0001d128\U0001d1de-\U0001d1ff\U0001d246-\U0001d2ff\U0001d357-\U0001d35f\U0001d372-\U0001d3ff\U0001d455\U0001d49d\U0001d4a0-\U0001d4a1\U0001d4a3-\U0001d4a4\U0001d4a7-\U0001d4a8\U0001d4ad\U0001d4ba\U0001d4bc\U0001d4c4\U0001d506\U0001d50b-\U0001d50c\U0001d515\U0001d51d\U0001d53a\U0001d53f\U0001d545\U0001d547-\U0001d549\U0001d551\U0001d6a6-\U0001d6a7\U0001d7cc-\U0001d7cd\U0001d800-\U0001edff\U0001ee04\U0001ee20\U0001ee23\U0001ee25-\U0001ee26\U0001ee28\U0001ee33\U0001ee38\U0001ee3a\U0001ee3c-\U0001ee41\U0001ee43-\U0001ee46\U0001ee48\U0001ee4a\U0001ee4c\U0001ee50\U0001ee53\U0001ee55-\U0001ee56\U0001ee58\U0001ee5a\U0001ee5c\U0001ee5e\U0001ee60\U0001ee63\U0001ee65-\U0001ee66\U0001ee6b\U0001ee73\U0001ee78\U0001ee7d\U0001ee7f\U0001ee8a\U0001ee9c-\U0001eea0\U0001eea4\U0001eeaa\U0001eebc-\U0001eeef\U0001eef2-\U0001efff\U0001f02c-\U0001f02f\U0001f094-\U0001f09f\U0001f0af-\U0001f0b0\U0001f0bf-\U0001f0c0\U0001f0d0\U0001f0e0-\U0001f0ff\U0001f10b-\U0001f10f\U0001f12f\U0001f16c-\U0001f16f\U0001f19b-\U0001f1e5\U0001f203-\U0001f20f\U0001f23b-\U0001f23f\U0001f249-\U0001f24f\U0001f252-\U0001f2ff\U0001f321-\U0001f32f\U0001f336\U0001f37d-\U0001f37f\U0001f394-\U0001f39f\U0001f3c5\U0001f3cb-\U0001f3df\U0001f3f1-\U0001f3ff\U0001f43f\U0001f441\U0001f4f8\U0001f4fd-\U0001f4ff\U0001f53e-\U0001f53f\U0001f544-\U0001f54f\U0001f568-\U0001f5fa\U0001f641-\U0001f644\U0001f650-\U0001f67f\U0001f6c6-\U0001f6ff\U0001f774-\U0001ffff\U0002a6d7-\U0002a6ff\U0002b735-\U0002b73f\U0002b81e-\U0002f7ff\U0002fa1e-\U000e0000\U000e0002-\U000e001f\U000e0080-\U000e00ff\U000e01f0-\U000effff\U000ffffe-\U000fffff\U0010fffe-\U0010ffff'
+    Cn += u'\U0001000c\U00010027\U0001003b\U0001003e\U0001004e-\U0001004f\U0001005e-\U0001007f\U000100fb-\U000100ff\U00010103-\U00010106\U00010134-\U00010136\U0001018f\U0001019c-\U0001019f\U000101a1-\U000101cf\U000101fe-\U0001027f\U0001029d-\U0001029f\U000102d1-\U000102df\U000102fc-\U000102ff\U00010324-\U0001032c\U0001034b-\U0001034f\U0001037b-\U0001037f\U0001039e\U000103c4-\U000103c7\U000103d6-\U000103ff\U0001049e-\U0001049f\U000104aa-\U000104af\U000104d4-\U000104d7\U000104fc-\U000104ff\U00010528-\U0001052f\U00010564-\U0001056e\U00010570-\U000105ff\U00010737-\U0001073f\U00010756-\U0001075f\U00010768-\U000107ff\U00010806-\U00010807\U00010809\U00010836\U00010839-\U0001083b\U0001083d-\U0001083e\U00010856\U0001089f-\U000108a6\U000108b0-\U000108df\U000108f3\U000108f6-\U000108fa\U0001091c-\U0001091e\U0001093a-\U0001093e\U00010940-\U0001097f\U000109b8-\U000109bb\U000109d0-\U000109d1\U00010a04\U00010a07-\U00010a0b\U00010a14\U00010a18\U00010a36-\U00010a37\U00010a3b-\U00010a3e\U00010a49-\U00010a4f\U00010a59-\U00010a5f\U00010aa0-\U00010abf\U00010ae7-\U00010aea\U00010af7-\U00010aff\U00010b36-\U00010b38\U00010b56-\U00010b57\U00010b73-\U00010b77\U00010b92-\U00010b98\U00010b9d-\U00010ba8\U00010bb0-\U00010bff\U00010c49-\U00010c7f\U00010cb3-\U00010cbf\U00010cf3-\U00010cf9\U00010d28-\U00010d2f\U00010d3a-\U00010e5f\U00010e7f-\U00010eff\U00010f28-\U00010f2f\U00010f5a-\U00010fff\U0001104e-\U00011051\U00011070-\U0001107e\U000110c2-\U000110cc\U000110ce-\U000110cf\U000110e9-\U000110ef\U000110fa-\U000110ff\U00011135\U00011147-\U0001114f\U00011177-\U0001117f\U000111ce-\U000111cf\U000111e0\U000111f5-\U000111ff\U00011212\U0001123f-\U0001127f\U00011287\U00011289\U0001128e\U0001129e\U000112aa-\U000112af\U000112eb-\U000112ef\U000112fa-\U000112ff\U00011304\U0001130d-\U0001130e\U00011311-\U00011312\U00011329\U00011331\U00011334\U0001133a\U00011345-\U00011346\U00011349-\U0001134a\U0001134e-\U0001134f\U00011351-\U00011356\U00011358-\U0001135c\U00011364-\U00011365\U0001136d-\U0001136f\U00011375-\U000113ff\U0001145a\U0001145c\U0001145f-\U0001147f\U000114c8-\U000114cf\U000114da-\U0001157f\U000115b6-\U000115b7\U000115de-\U000115ff\U00011645-\U0001164f\U0001165a-\U0001165f\U0001166d-\U0001167f\U000116b8-\U000116bf\U000116ca-\U000116ff\U0001171b-\U0001171c\U0001172c-\U0001172f\U00011740-\U000117ff\U0001183c-\U0001189f\U000118f3-\U000118fe\U00011900-\U000119ff\U00011a48-\U00011a4f\U00011a84-\U00011a85\U00011aa3-\U00011abf\U00011af9-\U00011bff\U00011c09\U00011c37\U00011c46-\U00011c4f\U00011c6d-\U00011c6f\U00011c90-\U00011c91\U00011ca8\U00011cb7-\U00011cff\U00011d07\U00011d0a\U00011d37-\U00011d39\U00011d3b\U00011d3e\U00011d48-\U00011d4f\U00011d5a-\U00011d5f\U00011d66\U00011d69\U00011d8f\U00011d92\U00011d99-\U00011d9f\U00011daa-\U00011edf\U00011ef9-\U00011fff\U0001239a-\U000123ff\U0001246f\U00012475-\U0001247f\U00012544-\U00012fff\U0001342f-\U000143ff\U00014647-\U000167ff\U00016a39-\U00016a3f\U00016a5f\U00016a6a-\U00016a6d\U00016a70-\U00016acf\U00016aee-\U00016aef\U00016af6-\U00016aff\U00016b46-\U00016b4f\U00016b5a\U00016b62\U00016b78-\U00016b7c\U00016b90-\U00016e3f\U00016e9b-\U00016eff\U00016f45-\U00016f4f\U00016f7f-\U00016f8e\U00016fa0-\U00016fdf\U00016fe2-\U00016fff\U000187f2-\U000187ff\U00018af3-\U0001afff\U0001b11f-\U0001b16f\U0001b2fc-\U0001bbff\U0001bc6b-\U0001bc6f\U0001bc7d-\U0001bc7f\U0001bc89-\U0001bc8f\U0001bc9a-\U0001bc9b\U0001bca4-\U0001cfff\U0001d0f6-\U0001d0ff\U0001d127-\U0001d128\U0001d1e9-\U0001d1ff\U0001d246-\U0001d2df\U0001d2f4-\U0001d2ff\U0001d357-\U0001d35f\U0001d379-\U0001d3ff\U0001d455\U0001d49d\U0001d4a0-\U0001d4a1\U0001d4a3-\U0001d4a4\U0001d4a7-\U0001d4a8\U0001d4ad\U0001d4ba\U0001d4bc\U0001d4c4\U0001d506\U0001d50b-\U0001d50c\U0001d515\U0001d51d\U0001d53a\U0001d53f\U0001d545\U0001d547-\U0001d549\U0001d551\U0001d6a6-\U0001d6a7\U0001d7cc-\U0001d7cd\U0001da8c-\U0001da9a\U0001daa0\U0001dab0-\U0001dfff\U0001e007\U0001e019-\U0001e01a\U0001e022\U0001e025\U0001e02b-\U0001e7ff\U0001e8c5-\U0001e8c6\U0001e8d7-\U0001e8ff\U0001e94b-\U0001e94f\U0001e95a-\U0001e95d\U0001e960-\U0001ec70\U0001ecb5-\U0001edff\U0001ee04\U0001ee20\U0001ee23\U0001ee25-\U0001ee26\U0001ee28\U0001ee33\U0001ee38\U0001ee3a\U0001ee3c-\U0001ee41\U0001ee43-\U0001ee46\U0001ee48\U0001ee4a\U0001ee4c\U0001ee50\U0001ee53\U0001ee55-\U0001ee56\U0001ee58\U0001ee5a\U0001ee5c\U0001ee5e\U0001ee60\U0001ee63\U0001ee65-\U0001ee66\U0001ee6b\U0001ee73\U0001ee78\U0001ee7d\U0001ee7f\U0001ee8a\U0001ee9c-\U0001eea0\U0001eea4\U0001eeaa\U0001eebc-\U0001eeef\U0001eef2-\U0001efff\U0001f02c-\U0001f02f\U0001f094-\U0001f09f\U0001f0af-\U0001f0b0\U0001f0c0\U0001f0d0\U0001f0f6-\U0001f0ff\U0001f10d-\U0001f10f\U0001f16c-\U0001f16f\U0001f1ad-\U0001f1e5\U0001f203-\U0001f20f\U0001f23c-\U0001f23f\U0001f249-\U0001f24f\U0001f252-\U0001f25f\U0001f266-\U0001f2ff\U0001f6d5-\U0001f6df\U0001f6ed-\U0001f6ef\U0001f6fa-\U0001f6ff\U0001f774-\U0001f77f\U0001f7d9-\U0001f7ff\U0001f80c-\U0001f80f\U0001f848-\U0001f84f\U0001f85a-\U0001f85f\U0001f888-\U0001f88f\U0001f8ae-\U0001f8ff\U0001f90c-\U0001f90f\U0001f93f\U0001f971-\U0001f972\U0001f977-\U0001f979\U0001f97b\U0001f9a3-\U0001f9af\U0001f9ba-\U0001f9bf\U0001f9c3-\U0001f9cf\U0001fa00-\U0001fa5f\U0001fa6e-\U0001ffff\U0002a6d7-\U0002a6ff\U0002b735-\U0002b73f\U0002b81e-\U0002b81f\U0002cea2-\U0002ceaf\U0002ebe1-\U0002f7ff\U0002fa1e-\U000e0000\U000e0002-\U000e001f\U000e0080-\U000e00ff\U000e01f0-\U000effff\U000ffffe-\U000fffff\U0010fffe-\U0010ffff'
 
     Co += u'\U000f0000-\U000ffffd\U00100000-\U0010fffd'
 
-    Ll += u'\U00010428-\U0001044f\U0001d41a-\U0001d433\U0001d44e-\U0001d454\U0001d456-\U0001d467\U0001d482-\U0001d49b\U0001d4b6-\U0001d4b9\U0001d4bb\U0001d4bd-\U0001d4c3\U0001d4c5-\U0001d4cf\U0001d4ea-\U0001d503\U0001d51e-\U0001d537\U0001d552-\U0001d56b\U0001d586-\U0001d59f\U0001d5ba-\U0001d5d3\U0001d5ee-\U0001d607\U0001d622-\U0001d63b\U0001d656-\U0001d66f\U0001d68a-\U0001d6a5\U0001d6c2-\U0001d6da\U0001d6dc-\U0001d6e1\U0001d6fc-\U0001d714\U0001d716-\U0001d71b\U0001d736-\U0001d74e\U0001d750-\U0001d755\U0001d770-\U0001d788\U0001d78a-\U0001d78f\U0001d7aa-\U0001d7c2\U0001d7c4-\U0001d7c9\U0001d7cb'
+    Ll += u'\U00010428-\U0001044f\U000104d8-\U000104fb\U00010cc0-\U00010cf2\U000118c0-\U000118df\U00016e60-\U00016e7f\U0001d41a-\U0001d433\U0001d44e-\U0001d454\U0001d456-\U0001d467\U0001d482-\U0001d49b\U0001d4b6-\U0001d4b9\U0001d4bb\U0001d4bd-\U0001d4c3\U0001d4c5-\U0001d4cf\U0001d4ea-\U0001d503\U0001d51e-\U0001d537\U0001d552-\U0001d56b\U0001d586-\U0001d59f\U0001d5ba-\U0001d5d3\U0001d5ee-\U0001d607\U0001d622-\U0001d63b\U0001d656-\U0001d66f\U0001d68a-\U0001d6a5\U0001d6c2-\U0001d6da\U0001d6dc-\U0001d6e1\U0001d6fc-\U0001d714\U0001d716-\U0001d71b\U0001d736-\U0001d74e\U0001d750-\U0001d755\U0001d770-\U0001d788\U0001d78a-\U0001d78f\U0001d7aa-\U0001d7c2\U0001d7c4-\U0001d7c9\U0001d7cb\U0001e922-\U0001e943'
 
-    Lm += u'\U00016f93-\U00016f9f'
+    Lm += u'\U00016b40-\U00016b43\U00016f93-\U00016f9f\U00016fe0-\U00016fe1'
 
-    Lo += u'\U00010000-\U0001000b\U0001000d-\U00010026\U00010028-\U0001003a\U0001003c-\U0001003d\U0001003f-\U0001004d\U00010050-\U0001005d\U00010080-\U000100fa\U00010280-\U0001029c\U000102a0-\U000102d0\U00010300-\U0001031e\U00010330-\U00010340\U00010342-\U00010349\U00010380-\U0001039d\U000103a0-\U000103c3\U000103c8-\U000103cf\U00010450-\U0001049d\U00010800-\U00010805\U00010808\U0001080a-\U00010835\U00010837-\U00010838\U0001083c\U0001083f-\U00010855\U00010900-\U00010915\U00010920-\U00010939\U00010980-\U000109b7\U000109be-\U000109bf\U00010a00\U00010a10-\U00010a13\U00010a15-\U00010a17\U00010a19-\U00010a33\U00010a60-\U00010a7c\U00010b00-\U00010b35\U00010b40-\U00010b55\U00010b60-\U00010b72\U00010c00-\U00010c48\U00011003-\U00011037\U00011083-\U000110af\U000110d0-\U000110e8\U00011103-\U00011126\U00011183-\U000111b2\U000111c1-\U000111c4\U00011680-\U000116aa\U00012000-\U0001236e\U00013000-\U0001342e\U00016800-\U00016a38\U00016f00-\U00016f44\U00016f50\U0001b000-\U0001b001\U0001ee00-\U0001ee03\U0001ee05-\U0001ee1f\U0001ee21-\U0001ee22\U0001ee24\U0001ee27\U0001ee29-\U0001ee32\U0001ee34-\U0001ee37\U0001ee39\U0001ee3b\U0001ee42\U0001ee47\U0001ee49\U0001ee4b\U0001ee4d-\U0001ee4f\U0001ee51-\U0001ee52\U0001ee54\U0001ee57\U0001ee59\U0001ee5b\U0001ee5d\U0001ee5f\U0001ee61-\U0001ee62\U0001ee64\U0001ee67-\U0001ee6a\U0001ee6c-\U0001ee72\U0001ee74-\U0001ee77\U0001ee79-\U0001ee7c\U0001ee7e\U0001ee80-\U0001ee89\U0001ee8b-\U0001ee9b\U0001eea1-\U0001eea3\U0001eea5-\U0001eea9\U0001eeab-\U0001eebb\U00020000-\U0002a6d6\U0002a700-\U0002b734\U0002b740-\U0002b81d\U0002f800-\U0002fa1d'
+    Lo += u'\U00010000-\U0001000b\U0001000d-\U00010026\U00010028-\U0001003a\U0001003c-\U0001003d\U0001003f-\U0001004d\U00010050-\U0001005d\U00010080-\U000100fa\U00010280-\U0001029c\U000102a0-\U000102d0\U00010300-\U0001031f\U0001032d-\U00010340\U00010342-\U00010349\U00010350-\U00010375\U00010380-\U0001039d\U000103a0-\U000103c3\U000103c8-\U000103cf\U00010450-\U0001049d\U00010500-\U00010527\U00010530-\U00010563\U00010600-\U00010736\U00010740-\U00010755\U00010760-\U00010767\U00010800-\U00010805\U00010808\U0001080a-\U00010835\U00010837-\U00010838\U0001083c\U0001083f-\U00010855\U00010860-\U00010876\U00010880-\U0001089e\U000108e0-\U000108f2\U000108f4-\U000108f5\U00010900-\U00010915\U00010920-\U00010939\U00010980-\U000109b7\U000109be-\U000109bf\U00010a00\U00010a10-\U00010a13\U00010a15-\U00010a17\U00010a19-\U00010a35\U00010a60-\U00010a7c\U00010a80-\U00010a9c\U00010ac0-\U00010ac7\U00010ac9-\U00010ae4\U00010b00-\U00010b35\U00010b40-\U00010b55\U00010b60-\U00010b72\U00010b80-\U00010b91\U00010c00-\U00010c48\U00010d00-\U00010d23\U00010f00-\U00010f1c\U00010f27\U00010f30-\U00010f45\U00011003-\U00011037\U00011083-\U000110af\U000110d0-\U000110e8\U00011103-\U00011126\U00011144\U00011150-\U00011172\U00011176\U00011183-\U000111b2\U000111c1-\U000111c4\U000111da\U000111dc\U00011200-\U00011211\U00011213-\U0001122b\U00011280-\U00011286\U00011288\U0001128a-\U0001128d\U0001128f-\U0001129d\U0001129f-\U000112a8\U000112b0-\U000112de\U00011305-\U0001130c\U0001130f-\U00011310\U00011313-\U00011328\U0001132a-\U00011330\U00011332-\U00011333\U00011335-\U00011339\U0001133d\U00011350\U0001135d-\U00011361\U00011400-\U00011434\U00011447-\U0001144a\U00011480-\U000114af\U000114c4-\U000114c5\U000114c7\U00011580-\U000115ae\U000115d8-\U000115db\U00011600-\U0001162f\U00011644\U00011680-\U000116aa\U00011700-\U0001171a\U00011800-\U0001182b\U000118ff\U00011a00\U00011a0b-\U00011a32\U00011a3a\U00011a50\U00011a5c-\U00011a83\U00011a86-\U00011a89\U00011a9d\U00011ac0-\U00011af8\U00011c00-\U00011c08\U00011c0a-\U00011c2e\U00011c40\U00011c72-\U00011c8f\U00011d00-\U00011d06\U00011d08-\U00011d09\U00011d0b-\U00011d30\U00011d46\U00011d60-\U00011d65\U00011d67-\U00011d68\U00011d6a-\U00011d89\U00011d98\U00011ee0-\U00011ef2\U00012000-\U00012399\U00012480-\U00012543\U00013000-\U0001342e\U00014400-\U00014646\U00016800-\U00016a38\U00016a40-\U00016a5e\U00016ad0-\U00016aed\U00016b00-\U00016b2f\U00016b63-\U00016b77\U00016b7d-\U00016b8f\U00016f00-\U00016f44\U00016f50\U00017000-\U000187f1\U00018800-\U00018af2\U0001b000-\U0001b11e\U0001b170-\U0001b2fb\U0001bc00-\U0001bc6a\U0001bc70-\U0001bc7c\U0001bc80-\U0001bc88\U0001bc90-\U0001bc99\U0001e800-\U0001e8c4\U0001ee00-\U0001ee03\U0001ee05-\U0001ee1f\U0001ee21-\U0001ee22\U0001ee24\U0001ee27\U0001ee29-\U0001ee32\U0001ee34-\U0001ee37\U0001ee39\U0001ee3b\U0001ee42\U0001ee47\U0001ee49\U0001ee4b\U0001ee4d-\U0001ee4f\U0001ee51-\U0001ee52\U0001ee54\U0001ee57\U0001ee59\U0001ee5b\U0001ee5d\U0001ee5f\U0001ee61-\U0001ee62\U0001ee64\U0001ee67-\U0001ee6a\U0001ee6c-\U0001ee72\U0001ee74-\U0001ee77\U0001ee79-\U0001ee7c\U0001ee7e\U0001ee80-\U0001ee89\U0001ee8b-\U0001ee9b\U0001eea1-\U0001eea3\U0001eea5-\U0001eea9\U0001eeab-\U0001eebb\U00020000-\U0002a6d6\U0002a700-\U0002b734\U0002b740-\U0002b81d\U0002b820-\U0002cea1\U0002ceb0-\U0002ebe0\U0002f800-\U0002fa1d'
 
-    Lu += u'\U00010400-\U00010427\U0001d400-\U0001d419\U0001d434-\U0001d44d\U0001d468-\U0001d481\U0001d49c\U0001d49e-\U0001d49f\U0001d4a2\U0001d4a5-\U0001d4a6\U0001d4a9-\U0001d4ac\U0001d4ae-\U0001d4b5\U0001d4d0-\U0001d4e9\U0001d504-\U0001d505\U0001d507-\U0001d50a\U0001d50d-\U0001d514\U0001d516-\U0001d51c\U0001d538-\U0001d539\U0001d53b-\U0001d53e\U0001d540-\U0001d544\U0001d546\U0001d54a-\U0001d550\U0001d56c-\U0001d585\U0001d5a0-\U0001d5b9\U0001d5d4-\U0001d5ed\U0001d608-\U0001d621\U0001d63c-\U0001d655\U0001d670-\U0001d689\U0001d6a8-\U0001d6c0\U0001d6e2-\U0001d6fa\U0001d71c-\U0001d734\U0001d756-\U0001d76e\U0001d790-\U0001d7a8\U0001d7ca'
+    Lu += u'\U00010400-\U00010427\U000104b0-\U000104d3\U00010c80-\U00010cb2\U000118a0-\U000118bf\U00016e40-\U00016e5f\U0001d400-\U0001d419\U0001d434-\U0001d44d\U0001d468-\U0001d481\U0001d49c\U0001d49e-\U0001d49f\U0001d4a2\U0001d4a5-\U0001d4a6\U0001d4a9-\U0001d4ac\U0001d4ae-\U0001d4b5\U0001d4d0-\U0001d4e9\U0001d504-\U0001d505\U0001d507-\U0001d50a\U0001d50d-\U0001d514\U0001d516-\U0001d51c\U0001d538-\U0001d539\U0001d53b-\U0001d53e\U0001d540-\U0001d544\U0001d546\U0001d54a-\U0001d550\U0001d56c-\U0001d585\U0001d5a0-\U0001d5b9\U0001d5d4-\U0001d5ed\U0001d608-\U0001d621\U0001d63c-\U0001d655\U0001d670-\U0001d689\U0001d6a8-\U0001d6c0\U0001d6e2-\U0001d6fa\U0001d71c-\U0001d734\U0001d756-\U0001d76e\U0001d790-\U0001d7a8\U0001d7ca\U0001e900-\U0001e921'
 
-    Mc += u'\U00011000\U00011002\U00011082\U000110b0-\U000110b2\U000110b7-\U000110b8\U0001112c\U00011182\U000111b3-\U000111b5\U000111bf-\U000111c0\U000116ac\U000116ae-\U000116af\U000116b6\U00016f51-\U00016f7e\U0001d165-\U0001d166\U0001d16d-\U0001d172'
+    Mc += u'\U00011000\U00011002\U00011082\U000110b0-\U000110b2\U000110b7-\U000110b8\U0001112c\U00011145-\U00011146\U00011182\U000111b3-\U000111b5\U000111bf-\U000111c0\U0001122c-\U0001122e\U00011232-\U00011233\U00011235\U000112e0-\U000112e2\U00011302-\U00011303\U0001133e-\U0001133f\U00011341-\U00011344\U00011347-\U00011348\U0001134b-\U0001134d\U00011357\U00011362-\U00011363\U00011435-\U00011437\U00011440-\U00011441\U00011445\U000114b0-\U000114b2\U000114b9\U000114bb-\U000114be\U000114c1\U000115af-\U000115b1\U000115b8-\U000115bb\U000115be\U00011630-\U00011632\U0001163b-\U0001163c\U0001163e\U000116ac\U000116ae-\U000116af\U000116b6\U00011720-\U00011721\U00011726\U0001182c-\U0001182e\U00011838\U00011a39\U00011a57-\U00011a58\U00011a97\U00011c2f\U00011c3e\U00011ca9\U00011cb1\U00011cb4\U00011d8a-\U00011d8e\U00011d93-\U00011d94\U00011d96\U00011ef5-\U00011ef6\U00016f51-\U00016f7e\U0001d165-\U0001d166\U0001d16d-\U0001d172'
 
-    Mn += u'\U000101fd\U00010a01-\U00010a03\U00010a05-\U00010a06\U00010a0c-\U00010a0f\U00010a38-\U00010a3a\U00010a3f\U00011001\U00011038-\U00011046\U00011080-\U00011081\U000110b3-\U000110b6\U000110b9-\U000110ba\U00011100-\U00011102\U00011127-\U0001112b\U0001112d-\U00011134\U00011180-\U00011181\U000111b6-\U000111be\U000116ab\U000116ad\U000116b0-\U000116b5\U000116b7\U00016f8f-\U00016f92\U0001d167-\U0001d169\U0001d17b-\U0001d182\U0001d185-\U0001d18b\U0001d1aa-\U0001d1ad\U0001d242-\U0001d244\U000e0100-\U000e01ef'
+    Mn += u'\U000101fd\U000102e0\U00010376-\U0001037a\U00010a01-\U00010a03\U00010a05-\U00010a06\U00010a0c-\U00010a0f\U00010a38-\U00010a3a\U00010a3f\U00010ae5-\U00010ae6\U00010d24-\U00010d27\U00010f46-\U00010f50\U00011001\U00011038-\U00011046\U0001107f-\U00011081\U000110b3-\U000110b6\U000110b9-\U000110ba\U00011100-\U00011102\U00011127-\U0001112b\U0001112d-\U00011134\U00011173\U00011180-\U00011181\U000111b6-\U000111be\U000111c9-\U000111cc\U0001122f-\U00011231\U00011234\U00011236-\U00011237\U0001123e\U000112df\U000112e3-\U000112ea\U00011300-\U00011301\U0001133b-\U0001133c\U00011340\U00011366-\U0001136c\U00011370-\U00011374\U00011438-\U0001143f\U00011442-\U00011444\U00011446\U0001145e\U000114b3-\U000114b8\U000114ba\U000114bf-\U000114c0\U000114c2-\U000114c3\U000115b2-\U000115b5\U000115bc-\U000115bd\U000115bf-\U000115c0\U000115dc-\U000115dd\U00011633-\U0001163a\U0001163d\U0001163f-\U00011640\U000116ab\U000116ad\U000116b0-\U000116b5\U000116b7\U0001171d-\U0001171f\U00011722-\U00011725\U00011727-\U0001172b\U0001182f-\U00011837\U00011839-\U0001183a\U00011a01-\U00011a0a\U00011a33-\U00011a38\U00011a3b-\U00011a3e\U00011a47\U00011a51-\U00011a56\U00011a59-\U00011a5b\U00011a8a-\U00011a96\U00011a98-\U00011a99\U00011c30-\U00011c36\U00011c38-\U00011c3d\U00011c3f\U00011c92-\U00011ca7\U00011caa-\U00011cb0\U00011cb2-\U00011cb3\U00011cb5-\U00011cb6\U00011d31-\U00011d36\U00011d3a\U00011d3c-\U00011d3d\U00011d3f-\U00011d45\U00011d47\U00011d90-\U00011d91\U00011d95\U00011d97\U00011ef3-\U00011ef4\U00016af0-\U00016af4\U00016b30-\U00016b36\U00016f8f-\U00016f92\U0001bc9d-\U0001bc9e\U0001d167-\U0001d169\U0001d17b-\U0001d182\U0001d185-\U0001d18b\U0001d1aa-\U0001d1ad\U0001d242-\U0001d244\U0001da00-\U0001da36\U0001da3b-\U0001da6c\U0001da75\U0001da84\U0001da9b-\U0001da9f\U0001daa1-\U0001daaf\U0001e000-\U0001e006\U0001e008-\U0001e018\U0001e01b-\U0001e021\U0001e023-\U0001e024\U0001e026-\U0001e02a\U0001e8d0-\U0001e8d6\U0001e944-\U0001e94a\U000e0100-\U000e01ef'
 
-    Nd += u'\U000104a0-\U000104a9\U00011066-\U0001106f\U000110f0-\U000110f9\U00011136-\U0001113f\U000111d0-\U000111d9\U000116c0-\U000116c9\U0001d7ce-\U0001d7ff'
+    Nd += u'\U000104a0-\U000104a9\U00010d30-\U00010d39\U00011066-\U0001106f\U000110f0-\U000110f9\U00011136-\U0001113f\U000111d0-\U000111d9\U000112f0-\U000112f9\U00011450-\U00011459\U000114d0-\U000114d9\U00011650-\U00011659\U000116c0-\U000116c9\U00011730-\U00011739\U000118e0-\U000118e9\U00011c50-\U00011c59\U00011d50-\U00011d59\U00011da0-\U00011da9\U00016a60-\U00016a69\U00016b50-\U00016b59\U0001d7ce-\U0001d7ff\U0001e950-\U0001e959'
 
-    Nl += u'\U00010140-\U00010174\U00010341\U0001034a\U000103d1-\U000103d5\U00012400-\U00012462'
+    Nl += u'\U00010140-\U00010174\U00010341\U0001034a\U000103d1-\U000103d5\U00012400-\U0001246e'
 
-    No += u'\U00010107-\U00010133\U00010175-\U00010178\U0001018a\U00010320-\U00010323\U00010858-\U0001085f\U00010916-\U0001091b\U00010a40-\U00010a47\U00010a7d-\U00010a7e\U00010b58-\U00010b5f\U00010b78-\U00010b7f\U00010e60-\U00010e7e\U00011052-\U00011065\U0001d360-\U0001d371\U0001f100-\U0001f10a'
+    No += u'\U00010107-\U00010133\U00010175-\U00010178\U0001018a-\U0001018b\U000102e1-\U000102fb\U00010320-\U00010323\U00010858-\U0001085f\U00010879-\U0001087f\U000108a7-\U000108af\U000108fb-\U000108ff\U00010916-\U0001091b\U000109bc-\U000109bd\U000109c0-\U000109cf\U000109d2-\U000109ff\U00010a40-\U00010a48\U00010a7d-\U00010a7e\U00010a9d-\U00010a9f\U00010aeb-\U00010aef\U00010b58-\U00010b5f\U00010b78-\U00010b7f\U00010ba9-\U00010baf\U00010cfa-\U00010cff\U00010e60-\U00010e7e\U00010f1d-\U00010f26\U00010f51-\U00010f54\U00011052-\U00011065\U000111e1-\U000111f4\U0001173a-\U0001173b\U000118ea-\U000118f2\U00011c5a-\U00011c6c\U00016b5b-\U00016b61\U00016e80-\U00016e96\U0001d2e0-\U0001d2f3\U0001d360-\U0001d378\U0001e8c7-\U0001e8cf\U0001ec71-\U0001ecab\U0001ecad-\U0001ecaf\U0001ecb1-\U0001ecb4\U0001f100-\U0001f10c'
 
-    Po += u'\U00010100-\U00010102\U0001039f\U000103d0\U00010857\U0001091f\U0001093f\U00010a50-\U00010a58\U00010a7f\U00010b39-\U00010b3f\U00011047-\U0001104d\U000110bb-\U000110bc\U000110be-\U000110c1\U00011140-\U00011143\U000111c5-\U000111c8\U00012470-\U00012473'
+    Po += u'\U00010100-\U00010102\U0001039f\U000103d0\U0001056f\U00010857\U0001091f\U0001093f\U00010a50-\U00010a58\U00010a7f\U00010af0-\U00010af6\U00010b39-\U00010b3f\U00010b99-\U00010b9c\U00010f55-\U00010f59\U00011047-\U0001104d\U000110bb-\U000110bc\U000110be-\U000110c1\U00011140-\U00011143\U00011174-\U00011175\U000111c5-\U000111c8\U000111cd\U000111db\U000111dd-\U000111df\U00011238-\U0001123d\U000112a9\U0001144b-\U0001144f\U0001145b\U0001145d\U000114c6\U000115c1-\U000115d7\U00011641-\U00011643\U00011660-\U0001166c\U0001173c-\U0001173e\U0001183b\U00011a3f-\U00011a46\U00011a9a-\U00011a9c\U00011a9e-\U00011aa2\U00011c41-\U00011c45\U00011c70-\U00011c71\U00011ef7-\U00011ef8\U00012470-\U00012474\U00016a6e-\U00016a6f\U00016af5\U00016b37-\U00016b3b\U00016b44\U00016e97-\U00016e9a\U0001bc9f\U0001da87-\U0001da8b\U0001e95e-\U0001e95f'
+
+    Sc += u'\U0001ecb0'
+
+    Sk += u'\U0001f3fb-\U0001f3ff'
 
     Sm += u'\U0001d6c1\U0001d6db\U0001d6fb\U0001d715\U0001d735\U0001d74f\U0001d76f\U0001d789\U0001d7a9\U0001d7c3\U0001eef0-\U0001eef1'
 
-    So += u'\U00010137-\U0001013f\U00010179-\U00010189\U00010190-\U0001019b\U000101d0-\U000101fc\U0001d000-\U0001d0f5\U0001d100-\U0001d126\U0001d129-\U0001d164\U0001d16a-\U0001d16c\U0001d183-\U0001d184\U0001d18c-\U0001d1a9\U0001d1ae-\U0001d1dd\U0001d200-\U0001d241\U0001d245\U0001d300-\U0001d356\U0001f000-\U0001f02b\U0001f030-\U0001f093\U0001f0a0-\U0001f0ae\U0001f0b1-\U0001f0be\U0001f0c1-\U0001f0cf\U0001f0d1-\U0001f0df\U0001f110-\U0001f12e\U0001f130-\U0001f16b\U0001f170-\U0001f19a\U0001f1e6-\U0001f202\U0001f210-\U0001f23a\U0001f240-\U0001f248\U0001f250-\U0001f251\U0001f300-\U0001f320\U0001f330-\U0001f335\U0001f337-\U0001f37c\U0001f380-\U0001f393\U0001f3a0-\U0001f3c4\U0001f3c6-\U0001f3ca\U0001f3e0-\U0001f3f0\U0001f400-\U0001f43e\U0001f440\U0001f442-\U0001f4f7\U0001f4f9-\U0001f4fc\U0001f500-\U0001f53d\U0001f540-\U0001f543\U0001f550-\U0001f567\U0001f5fb-\U0001f640\U0001f645-\U0001f64f\U0001f680-\U0001f6c5\U0001f700-\U0001f773'
+    So += u'\U00010137-\U0001013f\U00010179-\U00010189\U0001018c-\U0001018e\U00010190-\U0001019b\U000101a0\U000101d0-\U000101fc\U00010877-\U00010878\U00010ac8\U0001173f\U00016b3c-\U00016b3f\U00016b45\U0001bc9c\U0001d000-\U0001d0f5\U0001d100-\U0001d126\U0001d129-\U0001d164\U0001d16a-\U0001d16c\U0001d183-\U0001d184\U0001d18c-\U0001d1a9\U0001d1ae-\U0001d1e8\U0001d200-\U0001d241\U0001d245\U0001d300-\U0001d356\U0001d800-\U0001d9ff\U0001da37-\U0001da3a\U0001da6d-\U0001da74\U0001da76-\U0001da83\U0001da85-\U0001da86\U0001ecac\U0001f000-\U0001f02b\U0001f030-\U0001f093\U0001f0a0-\U0001f0ae\U0001f0b1-\U0001f0bf\U0001f0c1-\U0001f0cf\U0001f0d1-\U0001f0f5\U0001f110-\U0001f16b\U0001f170-\U0001f1ac\U0001f1e6-\U0001f202\U0001f210-\U0001f23b\U0001f240-\U0001f248\U0001f250-\U0001f251\U0001f260-\U0001f265\U0001f300-\U0001f3fa\U0001f400-\U0001f6d4\U0001f6e0-\U0001f6ec\U0001f6f0-\U0001f6f9\U0001f700-\U0001f773\U0001f780-\U0001f7d8\U0001f800-\U0001f80b\U0001f810-\U0001f847\U0001f850-\U0001f859\U0001f860-\U0001f887\U0001f890-\U0001f8ad\U0001f900-\U0001f90b\U0001f910-\U0001f93e\U0001f940-\U0001f970\U0001f973-\U0001f976\U0001f97a\U0001f97c-\U0001f9a2\U0001f9b0-\U0001f9b9\U0001f9c0-\U0001f9c2\U0001f9d0-\U0001f9ff\U0001fa60-\U0001fa6d'
 
-    xid_continue += u'\U00010000-\U0001000b\U0001000d-\U00010026\U00010028-\U0001003a\U0001003c-\U0001003d\U0001003f-\U0001004d\U00010050-\U0001005d\U00010080-\U000100fa\U00010140-\U00010174\U000101fd\U00010280-\U0001029c\U000102a0-\U000102d0\U00010300-\U0001031e\U00010330-\U0001034a\U00010380-\U0001039d\U000103a0-\U000103c3\U000103c8-\U000103cf\U000103d1-\U000103d5\U00010400-\U0001049d\U000104a0-\U000104a9\U00010800-\U00010805\U00010808\U0001080a-\U00010835\U00010837-\U00010838\U0001083c\U0001083f-\U00010855\U00010900-\U00010915\U00010920-\U00010939\U00010980-\U000109b7\U000109be-\U000109bf\U00010a00-\U00010a03\U00010a05-\U00010a06\U00010a0c-\U00010a13\U00010a15-\U00010a17\U00010a19-\U00010a33\U00010a38-\U00010a3a\U00010a3f\U00010a60-\U00010a7c\U00010b00-\U00010b35\U00010b40-\U00010b55\U00010b60-\U00010b72\U00010c00-\U00010c48\U00011000-\U00011046\U00011066-\U0001106f\U00011080-\U000110ba\U000110d0-\U000110e8\U000110f0-\U000110f9\U00011100-\U00011134\U00011136-\U0001113f\U00011180-\U000111c4\U000111d0-\U000111d9\U00011680-\U000116b7\U000116c0-\U000116c9\U00012000-\U0001236e\U00012400-\U00012462\U00013000-\U0001342e\U00016800-\U00016a38\U00016f00-\U00016f44\U00016f50-\U00016f7e\U00016f8f-\U00016f9f\U0001b000-\U0001b001\U0001d165-\U0001d169\U0001d16d-\U0001d172\U0001d17b-\U0001d182\U0001d185-\U0001d18b\U0001d1aa-\U0001d1ad\U0001d242-\U0001d244\U0001d400-\U0001d454\U0001d456-\U0001d49c\U0001d49e-\U0001d49f\U0001d4a2\U0001d4a5-\U0001d4a6\U0001d4a9-\U0001d4ac\U0001d4ae-\U0001d4b9\U0001d4bb\U0001d4bd-\U0001d4c3\U0001d4c5-\U0001d505\U0001d507-\U0001d50a\U0001d50d-\U0001d514\U0001d516-\U0001d51c\U0001d51e-\U0001d539\U0001d53b-\U0001d53e\U0001d540-\U0001d544\U0001d546\U0001d54a-\U0001d550\U0001d552-\U0001d6a5\U0001d6a8-\U0001d6c0\U0001d6c2-\U0001d6da\U0001d6dc-\U0001d6fa\U0001d6fc-\U0001d714\U0001d716-\U0001d734\U0001d736-\U0001d74e\U0001d750-\U0001d76e\U0001d770-\U0001d788\U0001d78a-\U0001d7a8\U0001d7aa-\U0001d7c2\U0001d7c4-\U0001d7cb\U0001d7ce-\U0001d7ff\U0001ee00-\U0001ee03\U0001ee05-\U0001ee1f\U0001ee21-\U0001ee22\U0001ee24\U0001ee27\U0001ee29-\U0001ee32\U0001ee34-\U0001ee37\U0001ee39\U0001ee3b\U0001ee42\U0001ee47\U0001ee49\U0001ee4b\U0001ee4d-\U0001ee4f\U0001ee51-\U0001ee52\U0001ee54\U0001ee57\U0001ee59\U0001ee5b\U0001ee5d\U0001ee5f\U0001ee61-\U0001ee62\U0001ee64\U0001ee67-\U0001ee6a\U0001ee6c-\U0001ee72\U0001ee74-\U0001ee77\U0001ee79-\U0001ee7c\U0001ee7e\U0001ee80-\U0001ee89\U0001ee8b-\U0001ee9b\U0001eea1-\U0001eea3\U0001eea5-\U0001eea9\U0001eeab-\U0001eebb\U00020000-\U0002a6d6\U0002a700-\U0002b734\U0002b740-\U0002b81d\U0002f800-\U0002fa1d\U000e0100-\U000e01ef'
+    xid_continue += u'\U00010000-\U0001000b\U0001000d-\U00010026\U00010028-\U0001003a\U0001003c-\U0001003d\U0001003f-\U0001004d\U00010050-\U0001005d\U00010080-\U000100fa\U00010140-\U00010174\U000101fd\U00010280-\U0001029c\U000102a0-\U000102d0\U000102e0\U00010300-\U0001031f\U0001032d-\U0001034a\U00010350-\U0001037a\U00010380-\U0001039d\U000103a0-\U000103c3\U000103c8-\U000103cf\U000103d1-\U000103d5\U00010400-\U0001049d\U000104a0-\U000104a9\U000104b0-\U000104d3\U000104d8-\U000104fb\U00010500-\U00010527\U00010530-\U00010563\U00010600-\U00010736\U00010740-\U00010755\U00010760-\U00010767\U00010800-\U00010805\U00010808\U0001080a-\U00010835\U00010837-\U00010838\U0001083c\U0001083f-\U00010855\U00010860-\U00010876\U00010880-\U0001089e\U000108e0-\U000108f2\U000108f4-\U000108f5\U00010900-\U00010915\U00010920-\U00010939\U00010980-\U000109b7\U000109be-\U000109bf\U00010a00-\U00010a03\U00010a05-\U00010a06\U00010a0c-\U00010a13\U00010a15-\U00010a17\U00010a19-\U00010a35\U00010a38-\U00010a3a\U00010a3f\U00010a60-\U00010a7c\U00010a80-\U00010a9c\U00010ac0-\U00010ac7\U00010ac9-\U00010ae6\U00010b00-\U00010b35\U00010b40-\U00010b55\U00010b60-\U00010b72\U00010b80-\U00010b91\U00010c00-\U00010c48\U00010c80-\U00010cb2\U00010cc0-\U00010cf2\U00010d00-\U00010d27\U00010d30-\U00010d39\U00010f00-\U00010f1c\U00010f27\U00010f30-\U00010f50\U00011000-\U00011046\U00011066-\U0001106f\U0001107f-\U000110ba\U000110d0-\U000110e8\U000110f0-\U000110f9\U00011100-\U00011134\U00011136-\U0001113f\U00011144-\U00011146\U00011150-\U00011173\U00011176\U00011180-\U000111c4\U000111c9-\U000111cc\U000111d0-\U000111da\U000111dc\U00011200-\U00011211\U00011213-\U00011237\U0001123e\U00011280-\U00011286\U00011288\U0001128a-\U0001128d\U0001128f-\U0001129d\U0001129f-\U000112a8\U000112b0-\U000112ea\U000112f0-\U000112f9\U00011300-\U00011303\U00011305-\U0001130c\U0001130f-\U00011310\U00011313-\U00011328\U0001132a-\U00011330\U00011332-\U00011333\U00011335-\U00011339\U0001133b-\U00011344\U00011347-\U00011348\U0001134b-\U0001134d\U00011350\U00011357\U0001135d-\U00011363\U00011366-\U0001136c\U00011370-\U00011374\U00011400-\U0001144a\U00011450-\U00011459\U0001145e\U00011480-\U000114c5\U000114c7\U000114d0-\U000114d9\U00011580-\U000115b5\U000115b8-\U000115c0\U000115d8-\U000115dd\U00011600-\U00011640\U00011644\U00011650-\U00011659\U00011680-\U000116b7\U000116c0-\U000116c9\U00011700-\U0001171a\U0001171d-\U0001172b\U00011730-\U00011739\U00011800-\U0001183a\U000118a0-\U000118e9\U000118ff\U00011a00-\U00011a3e\U00011a47\U00011a50-\U00011a83\U00011a86-\U00011a99\U00011a9d\U00011ac0-\U00011af8\U00011c00-\U00011c08\U00011c0a-\U00011c36\U00011c38-\U00011c40\U00011c50-\U00011c59\U00011c72-\U00011c8f\U00011c92-\U00011ca7\U00011ca9-\U00011cb6\U00011d00-\U00011d06\U00011d08-\U00011d09\U00011d0b-\U00011d36\U00011d3a\U00011d3c-\U00011d3d\U00011d3f-\U00011d47\U00011d50-\U00011d59\U00011d60-\U00011d65\U00011d67-\U00011d68\U00011d6a-\U00011d8e\U00011d90-\U00011d91\U00011d93-\U00011d98\U00011da0-\U00011da9\U00011ee0-\U00011ef6\U00012000-\U00012399\U00012400-\U0001246e\U00012480-\U00012543\U00013000-\U0001342e\U00014400-\U00014646\U00016800-\U00016a38\U00016a40-\U00016a5e\U00016a60-\U00016a69\U00016ad0-\U00016aed\U00016af0-\U00016af4\U00016b00-\U00016b36\U00016b40-\U00016b43\U00016b50-\U00016b59\U00016b63-\U00016b77\U00016b7d-\U00016b8f\U00016e40-\U00016e7f\U00016f00-\U00016f44\U00016f50-\U00016f7e\U00016f8f-\U00016f9f\U00016fe0-\U00016fe1\U00017000-\U000187f1\U00018800-\U00018af2\U0001b000-\U0001b11e\U0001b170-\U0001b2fb\U0001bc00-\U0001bc6a\U0001bc70-\U0001bc7c\U0001bc80-\U0001bc88\U0001bc90-\U0001bc99\U0001bc9d-\U0001bc9e\U0001d165-\U0001d169\U0001d16d-\U0001d172\U0001d17b-\U0001d182\U0001d185-\U0001d18b\U0001d1aa-\U0001d1ad\U0001d242-\U0001d244\U0001d400-\U0001d454\U0001d456-\U0001d49c\U0001d49e-\U0001d49f\U0001d4a2\U0001d4a5-\U0001d4a6\U0001d4a9-\U0001d4ac\U0001d4ae-\U0001d4b9\U0001d4bb\U0001d4bd-\U0001d4c3\U0001d4c5-\U0001d505\U0001d507-\U0001d50a\U0001d50d-\U0001d514\U0001d516-\U0001d51c\U0001d51e-\U0001d539\U0001d53b-\U0001d53e\U0001d540-\U0001d544\U0001d546\U0001d54a-\U0001d550\U0001d552-\U0001d6a5\U0001d6a8-\U0001d6c0\U0001d6c2-\U0001d6da\U0001d6dc-\U0001d6fa\U0001d6fc-\U0001d714\U0001d716-\U0001d734\U0001d736-\U0001d74e\U0001d750-\U0001d76e\U0001d770-\U0001d788\U0001d78a-\U0001d7a8\U0001d7aa-\U0001d7c2\U0001d7c4-\U0001d7cb\U0001d7ce-\U0001d7ff\U0001da00-\U0001da36\U0001da3b-\U0001da6c\U0001da75\U0001da84\U0001da9b-\U0001da9f\U0001daa1-\U0001daaf\U0001e000-\U0001e006\U0001e008-\U0001e018\U0001e01b-\U0001e021\U0001e023-\U0001e024\U0001e026-\U0001e02a\U0001e800-\U0001e8c4\U0001e8d0-\U0001e8d6\U0001e900-\U0001e94a\U0001e950-\U0001e959\U0001ee00-\U0001ee03\U0001ee05-\U0001ee1f\U0001ee21-\U0001ee22\U0001ee24\U0001ee27\U0001ee29-\U0001ee32\U0001ee34-\U0001ee37\U0001ee39\U0001ee3b\U0001ee42\U0001ee47\U0001ee49\U0001ee4b\U0001ee4d-\U0001ee4f\U0001ee51-\U0001ee52\U0001ee54\U0001ee57\U0001ee59\U0001ee5b\U0001ee5d\U0001ee5f\U0001ee61-\U0001ee62\U0001ee64\U0001ee67-\U0001ee6a\U0001ee6c-\U0001ee72\U0001ee74-\U0001ee77\U0001ee79-\U0001ee7c\U0001ee7e\U0001ee80-\U0001ee89\U0001ee8b-\U0001ee9b\U0001eea1-\U0001eea3\U0001eea5-\U0001eea9\U0001eeab-\U0001eebb\U00020000-\U0002a6d6\U0002a700-\U0002b734\U0002b740-\U0002b81d\U0002b820-\U0002cea1\U0002ceb0-\U0002ebe0\U0002f800-\U0002fa1d\U000e0100-\U000e01ef'
 
-    xid_start += u'\U00010000-\U0001000b\U0001000d-\U00010026\U00010028-\U0001003a\U0001003c-\U0001003d\U0001003f-\U0001004d\U00010050-\U0001005d\U00010080-\U000100fa\U00010140-\U00010174\U00010280-\U0001029c\U000102a0-\U000102d0\U00010300-\U0001031e\U00010330-\U0001034a\U00010380-\U0001039d\U000103a0-\U000103c3\U000103c8-\U000103cf\U000103d1-\U000103d5\U00010400-\U0001049d\U00010800-\U00010805\U00010808\U0001080a-\U00010835\U00010837-\U00010838\U0001083c\U0001083f-\U00010855\U00010900-\U00010915\U00010920-\U00010939\U00010980-\U000109b7\U000109be-\U000109bf\U00010a00\U00010a10-\U00010a13\U00010a15-\U00010a17\U00010a19-\U00010a33\U00010a60-\U00010a7c\U00010b00-\U00010b35\U00010b40-\U00010b55\U00010b60-\U00010b72\U00010c00-\U00010c48\U00011003-\U00011037\U00011083-\U000110af\U000110d0-\U000110e8\U00011103-\U00011126\U00011183-\U000111b2\U000111c1-\U000111c4\U00011680-\U000116aa\U00012000-\U0001236e\U00012400-\U00012462\U00013000-\U0001342e\U00016800-\U00016a38\U00016f00-\U00016f44\U00016f50\U00016f93-\U00016f9f\U0001b000-\U0001b001\U0001d400-\U0001d454\U0001d456-\U0001d49c\U0001d49e-\U0001d49f\U0001d4a2\U0001d4a5-\U0001d4a6\U0001d4a9-\U0001d4ac\U0001d4ae-\U0001d4b9\U0001d4bb\U0001d4bd-\U0001d4c3\U0001d4c5-\U0001d505\U0001d507-\U0001d50a\U0001d50d-\U0001d514\U0001d516-\U0001d51c\U0001d51e-\U0001d539\U0001d53b-\U0001d53e\U0001d540-\U0001d544\U0001d546\U0001d54a-\U0001d550\U0001d552-\U0001d6a5\U0001d6a8-\U0001d6c0\U0001d6c2-\U0001d6da\U0001d6dc-\U0001d6fa\U0001d6fc-\U0001d714\U0001d716-\U0001d734\U0001d736-\U0001d74e\U0001d750-\U0001d76e\U0001d770-\U0001d788\U0001d78a-\U0001d7a8\U0001d7aa-\U0001d7c2\U0001d7c4-\U0001d7cb\U0001ee00-\U0001ee03\U0001ee05-\U0001ee1f\U0001ee21-\U0001ee22\U0001ee24\U0001ee27\U0001ee29-\U0001ee32\U0001ee34-\U0001ee37\U0001ee39\U0001ee3b\U0001ee42\U0001ee47\U0001ee49\U0001ee4b\U0001ee4d-\U0001ee4f\U0001ee51-\U0001ee52\U0001ee54\U0001ee57\U0001ee59\U0001ee5b\U0001ee5d\U0001ee5f\U0001ee61-\U0001ee62\U0001ee64\U0001ee67-\U0001ee6a\U0001ee6c-\U0001ee72\U0001ee74-\U0001ee77\U0001ee79-\U0001ee7c\U0001ee7e\U0001ee80-\U0001ee89\U0001ee8b-\U0001ee9b\U0001eea1-\U0001eea3\U0001eea5-\U0001eea9\U0001eeab-\U0001eebb\U00020000-\U0002a6d6\U0002a700-\U0002b734\U0002b740-\U0002b81d\U0002f800-\U0002fa1d'
+    xid_start += u'\U00010000-\U0001000b\U0001000d-\U00010026\U00010028-\U0001003a\U0001003c-\U0001003d\U0001003f-\U0001004d\U00010050-\U0001005d\U00010080-\U000100fa\U00010140-\U00010174\U00010280-\U0001029c\U000102a0-\U000102d0\U00010300-\U0001031f\U0001032d-\U0001034a\U00010350-\U00010375\U00010380-\U0001039d\U000103a0-\U000103c3\U000103c8-\U000103cf\U000103d1-\U000103d5\U00010400-\U0001049d\U000104b0-\U000104d3\U000104d8-\U000104fb\U00010500-\U00010527\U00010530-\U00010563\U00010600-\U00010736\U00010740-\U00010755\U00010760-\U00010767\U00010800-\U00010805\U00010808\U0001080a-\U00010835\U00010837-\U00010838\U0001083c\U0001083f-\U00010855\U00010860-\U00010876\U00010880-\U0001089e\U000108e0-\U000108f2\U000108f4-\U000108f5\U00010900-\U00010915\U00010920-\U00010939\U00010980-\U000109b7\U000109be-\U000109bf\U00010a00\U00010a10-\U00010a13\U00010a15-\U00010a17\U00010a19-\U00010a35\U00010a60-\U00010a7c\U00010a80-\U00010a9c\U00010ac0-\U00010ac7\U00010ac9-\U00010ae4\U00010b00-\U00010b35\U00010b40-\U00010b55\U00010b60-\U00010b72\U00010b80-\U00010b91\U00010c00-\U00010c48\U00010c80-\U00010cb2\U00010cc0-\U00010cf2\U00010d00-\U00010d23\U00010f00-\U00010f1c\U00010f27\U00010f30-\U00010f45\U00011003-\U00011037\U00011083-\U000110af\U000110d0-\U000110e8\U00011103-\U00011126\U00011144\U00011150-\U00011172\U00011176\U00011183-\U000111b2\U000111c1-\U000111c4\U000111da\U000111dc\U00011200-\U00011211\U00011213-\U0001122b\U00011280-\U00011286\U00011288\U0001128a-\U0001128d\U0001128f-\U0001129d\U0001129f-\U000112a8\U000112b0-\U000112de\U00011305-\U0001130c\U0001130f-\U00011310\U00011313-\U00011328\U0001132a-\U00011330\U00011332-\U00011333\U00011335-\U00011339\U0001133d\U00011350\U0001135d-\U00011361\U00011400-\U00011434\U00011447-\U0001144a\U00011480-\U000114af\U000114c4-\U000114c5\U000114c7\U00011580-\U000115ae\U000115d8-\U000115db\U00011600-\U0001162f\U00011644\U00011680-\U000116aa\U00011700-\U0001171a\U00011800-\U0001182b\U000118a0-\U000118df\U000118ff\U00011a00\U00011a0b-\U00011a32\U00011a3a\U00011a50\U00011a5c-\U00011a83\U00011a86-\U00011a89\U00011a9d\U00011ac0-\U00011af8\U00011c00-\U00011c08\U00011c0a-\U00011c2e\U00011c40\U00011c72-\U00011c8f\U00011d00-\U00011d06\U00011d08-\U00011d09\U00011d0b-\U00011d30\U00011d46\U00011d60-\U00011d65\U00011d67-\U00011d68\U00011d6a-\U00011d89\U00011d98\U00011ee0-\U00011ef2\U00012000-\U00012399\U00012400-\U0001246e\U00012480-\U00012543\U00013000-\U0001342e\U00014400-\U00014646\U00016800-\U00016a38\U00016a40-\U00016a5e\U00016ad0-\U00016aed\U00016b00-\U00016b2f\U00016b40-\U00016b43\U00016b63-\U00016b77\U00016b7d-\U00016b8f\U00016e40-\U00016e7f\U00016f00-\U00016f44\U00016f50\U00016f93-\U00016f9f\U00016fe0-\U00016fe1\U00017000-\U000187f1\U00018800-\U00018af2\U0001b000-\U0001b11e\U0001b170-\U0001b2fb\U0001bc00-\U0001bc6a\U0001bc70-\U0001bc7c\U0001bc80-\U0001bc88\U0001bc90-\U0001bc99\U0001d400-\U0001d454\U0001d456-\U0001d49c\U0001d49e-\U0001d49f\U0001d4a2\U0001d4a5-\U0001d4a6\U0001d4a9-\U0001d4ac\U0001d4ae-\U0001d4b9\U0001d4bb\U0001d4bd-\U0001d4c3\U0001d4c5-\U0001d505\U0001d507-\U0001d50a\U0001d50d-\U0001d514\U0001d516-\U0001d51c\U0001d51e-\U0001d539\U0001d53b-\U0001d53e\U0001d540-\U0001d544\U0001d546\U0001d54a-\U0001d550\U0001d552-\U0001d6a5\U0001d6a8-\U0001d6c0\U0001d6c2-\U0001d6da\U0001d6dc-\U0001d6fa\U0001d6fc-\U0001d714\U0001d716-\U0001d734\U0001d736-\U0001d74e\U0001d750-\U0001d76e\U0001d770-\U0001d788\U0001d78a-\U0001d7a8\U0001d7aa-\U0001d7c2\U0001d7c4-\U0001d7cb\U0001e800-\U0001e8c4\U0001e900-\U0001e943\U0001ee00-\U0001ee03\U0001ee05-\U0001ee1f\U0001ee21-\U0001ee22\U0001ee24\U0001ee27\U0001ee29-\U0001ee32\U0001ee34-\U0001ee37\U0001ee39\U0001ee3b\U0001ee42\U0001ee47\U0001ee49\U0001ee4b\U0001ee4d-\U0001ee4f\U0001ee51-\U0001ee52\U0001ee54\U0001ee57\U0001ee59\U0001ee5b\U0001ee5d\U0001ee5f\U0001ee61-\U0001ee62\U0001ee64\U0001ee67-\U0001ee6a\U0001ee6c-\U0001ee72\U0001ee74-\U0001ee77\U0001ee79-\U0001ee7c\U0001ee7e\U0001ee80-\U0001ee89\U0001ee8b-\U0001ee9b\U0001eea1-\U0001eea3\U0001eea5-\U0001eea9\U0001eeab-\U0001eebb\U00020000-\U0002a6d6\U0002a700-\U0002b734\U0002b740-\U0002b81d\U0002b820-\U0002cea1\U0002ceb0-\U0002ebe0\U0002f800-\U0002fa1d'
 
 cats = ['Cc', 'Cf', 'Cn', 'Co', 'Cs', 'Ll', 'Lm', 'Lo', 'Lt', 'Lu', 'Mc', 'Me', 'Mn', 'Nd', 'Nl', 'No', 'Pc', 'Pd', 'Pe', 'Pf', 'Pi', 'Po', 'Ps', 'Sc', 'Sk', 'Sm', 'So', 'Zl', 'Zp', 'Zs']
 
-# Generated from unidata 6.3.0
+# Generated from unidata 11.0.0
 
 def combine(*args):
     return u''.join(globals()[cat] for cat in args)
-- 
cgit v1.2.1


From cd850106ed356cd56c2bf19e0eebd75c42780556 Mon Sep 17 00:00:00 2001
From: Georg Brandl 
Date: Wed, 28 Nov 2018 16:42:15 +0100
Subject: Fix more instances of invalid string escapes

Also, raise on warnings from Pygments only.
---
 pygments/formatters/img.py          |  2 +-
 pygments/lexers/_cocoa_builtins.py  | 10 +++++-----
 pygments/lexers/_php_builtins.py    |  2 +-
 pygments/lexers/apl.py              |  8 ++++----
 pygments/lexers/asm.py              |  6 +++---
 pygments/lexers/bibtex.py           |  6 +++---
 pygments/lexers/c_cpp.py            |  8 ++++----
 pygments/lexers/c_like.py           |  4 ++--
 pygments/lexers/dotnet.py           | 20 ++++++++++----------
 pygments/lexers/dylan.py            |  6 +++---
 pygments/lexers/erlang.py           |  4 ++--
 pygments/lexers/grammar_notation.py |  2 +-
 pygments/lexers/haskell.py          |  4 ++--
 pygments/lexers/haxe.py             | 10 +++++-----
 pygments/lexers/idl.py              | 12 ++++++------
 pygments/lexers/inferno.py          |  2 +-
 pygments/lexers/j.py                |  4 ++--
 pygments/lexers/matlab.py           |  4 ++--
 pygments/lexers/ml.py               |  2 +-
 pygments/lexers/objective.py        | 28 ++++++++++++++--------------
 pygments/lexers/parsers.py          | 12 ++++++------
 pygments/lexers/pascal.py           | 10 +++++-----
 pygments/lexers/pawn.py             |  8 ++++----
 pygments/lexers/prolog.py           |  8 ++++----
 pygments/lexers/qvt.py              | 10 +++++-----
 pygments/lexers/rebol.py            | 12 ++++++------
 pygments/lexers/robotframework.py   |  2 +-
 pygments/lexers/testing.py          |  2 +-
 pygments/lexers/textfmts.py         | 10 +++++-----
 pygments/lexers/varnish.py          |  4 ++--
 tests/run.py                        | 13 +++++++------
 31 files changed, 118 insertions(+), 117 deletions(-)

diff --git a/pygments/formatters/img.py b/pygments/formatters/img.py
index 2fb0dea5..6fafc476 100644
--- a/pygments/formatters/img.py
+++ b/pygments/formatters/img.py
@@ -237,7 +237,7 @@ class ImageFormatter(Formatter):
         bold and italic fonts will be generated.  This really should be a
         monospace font to look sane.
 
-        Default: "Bitstream Vera Sans Mono" on Windows, Courier New on \*nix
+        Default: "Bitstream Vera Sans Mono" on Windows, Courier New on \\*nix
 
     `font_size`
         The font size in points to be used.
diff --git a/pygments/lexers/_cocoa_builtins.py b/pygments/lexers/_cocoa_builtins.py
index 064167ff..f55e9dd7 100644
--- a/pygments/lexers/_cocoa_builtins.py
+++ b/pygments/lexers/_cocoa_builtins.py
@@ -41,23 +41,23 @@ if __name__ == '__main__':  # pragma: no cover
 
             headerFilePath = frameworkHeadersDir + f
             content = open(headerFilePath).read()
-            res = re.findall('(?<=@interface )\w+', content)
+            res = re.findall(r'(?<=@interface )\w+', content)
             for r in res:
                 all_interfaces.add(r)
 
-            res = re.findall('(?<=@protocol )\w+', content)
+            res = re.findall(r'(?<=@protocol )\w+', content)
             for r in res:
                 all_protocols.add(r)
 
-            res = re.findall('(?<=typedef enum )\w+', content)
+            res = re.findall(r'(?<=typedef enum )\w+', content)
             for r in res:
                 all_primitives.add(r)
 
-            res = re.findall('(?<=typedef struct )\w+', content)
+            res = re.findall(r'(?<=typedef struct )\w+', content)
             for r in res:
                 all_primitives.add(r)
 
-            res = re.findall('(?<=typedef const struct )\w+', content)
+            res = re.findall(r'(?<=typedef const struct )\w+', content)
             for r in res:
                 all_primitives.add(r)
 
diff --git a/pygments/lexers/_php_builtins.py b/pygments/lexers/_php_builtins.py
index fec3286a..bd4b7d99 100644
--- a/pygments/lexers/_php_builtins.py
+++ b/pygments/lexers/_php_builtins.py
@@ -4688,7 +4688,7 @@ if __name__ == '__main__':  # pragma: no cover
     PHP_MANUAL_URL     = 'http://us3.php.net/distributions/manual/php_manual_en.tar.gz'
     PHP_MANUAL_DIR     = './php-chunked-xhtml/'
     PHP_REFERENCE_GLOB = 'ref.*'
-    PHP_FUNCTION_RE    = '(.*?)'
+    PHP_FUNCTION_RE    = r'(.*?)'
     PHP_MODULE_RE      = '(.*?) Functions'
 
     def get_php_functions():
diff --git a/pygments/lexers/apl.py b/pygments/lexers/apl.py
index b3414cc0..4bb88ae3 100644
--- a/pygments/lexers/apl.py
+++ b/pygments/lexers/apl.py
@@ -71,14 +71,14 @@ class APLLexer(RegexLexer):
             #
             # Numbers
             # =======
-            (u'¯?(0[Xx][0-9A-Fa-f]+|[0-9]*\.?[0-9]+([Ee][+¯]?[0-9]+)?|¯|∞)'
-             u'([Jj]¯?(0[Xx][0-9A-Fa-f]+|[0-9]*\.?[0-9]+([Ee][+¯]?[0-9]+)?|¯|∞))?',
+            (u'¯?(0[Xx][0-9A-Fa-f]+|[0-9]*\\.?[0-9]+([Ee][+¯]?[0-9]+)?|¯|∞)'
+             u'([Jj]¯?(0[Xx][0-9A-Fa-f]+|[0-9]*\\.?[0-9]+([Ee][+¯]?[0-9]+)?|¯|∞))?',
              Number),
             #
             # Operators
             # ==========
-            (u'[\.\\\/⌿⍀¨⍣⍨⍠⍤∘]', Name.Attribute),  # closest token type
-            (u'[+\-×÷⌈⌊∣|⍳?*⍟○!⌹<≤=>≥≠≡≢∊⍷∪∩~∨∧⍱⍲⍴,⍪⌽⊖⍉↑↓⊂⊃⌷⍋⍒⊤⊥⍕⍎⊣⊢⍁⍂≈⌸⍯↗]',
+            (u'[\\.\\\\\\/⌿⍀¨⍣⍨⍠⍤∘]', Name.Attribute),  # closest token type
+            (u'[+\\-×÷⌈⌊∣|⍳?*⍟○!⌹<≤=>≥≠≡≢∊⍷∪∩~∨∧⍱⍲⍴,⍪⌽⊖⍉↑↓⊂⊃⌷⍋⍒⊤⊥⍕⍎⊣⊢⍁⍂≈⌸⍯↗]',
              Operator),
             #
             # Constant
diff --git a/pygments/lexers/asm.py b/pygments/lexers/asm.py
index 5b8cab80..9e854581 100644
--- a/pygments/lexers/asm.py
+++ b/pygments/lexers/asm.py
@@ -35,7 +35,7 @@ class GasLexer(RegexLexer):
     #: optional Comment or Whitespace
     string = r'"(\\"|[^"])*"'
     char = r'[\w$.@-]'
-    identifier = r'(?:[a-zA-Z$_]' + char + '*|\.' + char + '+)'
+    identifier = r'(?:[a-zA-Z$_]' + char + r'*|\.' + char + '+)'
     number = r'(?:0[xX][a-zA-Z0-9]+|\d+)'
 
     tokens = {
@@ -256,7 +256,7 @@ class HsailLexer(RegexLexer):
             (r'0[xX][a-fA-F0-9]+', Number.Hex),
             (ieeefloat, Number.Float),
             (float, Number.Float),
-            ('\d+', Number.Integer),
+            (r'\d+', Number.Integer),
 
             (r'[=<>{}\[\]()*.,:;!]|x\b', Punctuation)
         ],
@@ -350,7 +350,7 @@ class LlvmLexer(RegexLexer):
             include('whitespace'),
 
             # Before keywords, because keywords are valid label names :(...
-            (identifier + '\s*:', Name.Label),
+            (identifier + r'\s*:', Name.Label),
 
             include('keyword'),
 
diff --git a/pygments/lexers/bibtex.py b/pygments/lexers/bibtex.py
index a6159f81..7244ef2f 100644
--- a/pygments/lexers/bibtex.py
+++ b/pygments/lexers/bibtex.py
@@ -101,12 +101,12 @@ class BibTeXLexer(ExtendedRegexLexer):
         'quoted-string': [
             (r'\{', String, 'braced-string'),
             ('"', String, '#pop'),
-            ('[^\{\"]+', String),
+            (r'[^\{\"]+', String),
         ],
         'braced-string': [
             (r'\{', String, '#push'),
             (r'\}', String, '#pop'),
-            ('[^\{\}]+', String),
+            (r'[^\{\}]+', String),
         ],
         'whitespace': [
             (r'\s+', Text),
@@ -154,7 +154,7 @@ class BSTLexer(RegexLexer):
             default('#pop'),
         ],
         'whitespace': [
-            ('\s+', Text),
+            (r'\s+', Text),
             ('%.*?$', Comment.SingleLine),
         ],
     }
diff --git a/pygments/lexers/c_cpp.py b/pygments/lexers/c_cpp.py
index 691f5ab4..38f425db 100644
--- a/pygments/lexers/c_cpp.py
+++ b/pygments/lexers/c_cpp.py
@@ -36,7 +36,7 @@ class CFamilyLexer(RegexLexer):
     tokens = {
         'whitespace': [
             # preprocessor directives: without whitespace
-            ('^#if\s+0', Comment.Preproc, 'if0'),
+            (r'^#if\s+0', Comment.Preproc, 'if0'),
             ('^#', Comment.Preproc, 'macro'),
             # or with whitespace
             ('^(' + _ws1 + r')(#if\s+0)',
@@ -84,7 +84,7 @@ class CFamilyLexer(RegexLexer):
                 prefix=r'__', suffix=r'\b'), Keyword.Reserved),
             (r'(true|false|NULL)\b', Name.Builtin),
             (r'([a-zA-Z_]\w*)(\s*)(:)(?!:)', bygroups(Name.Label, Text, Punctuation)),
-            ('[a-zA-Z_]\w*', Name),
+            (r'[a-zA-Z_]\w*', Name),
         ],
         'root': [
             include('whitespace'),
@@ -190,9 +190,9 @@ class CLexer(CFamilyLexer):
     priority = 0.1
 
     def analyse_text(text):
-        if re.search('^\s*#include [<"]', text, re.MULTILINE):
+        if re.search(r'^\s*#include [<"]', text, re.MULTILINE):
             return 0.1
-        if re.search('^\s*#ifn?def ', text, re.MULTILINE):
+        if re.search(r'^\s*#ifn?def ', text, re.MULTILINE):
             return 0.1
 
 
diff --git a/pygments/lexers/c_like.py b/pygments/lexers/c_like.py
index 38827219..fd147a8a 100644
--- a/pygments/lexers/c_like.py
+++ b/pygments/lexers/c_like.py
@@ -245,7 +245,7 @@ class ValaLexer(RegexLexer):
                 'ulong', 'unichar', 'ushort'), suffix=r'\b'),
              Keyword.Type),
             (r'(true|false|null)\b', Name.Builtin),
-            ('[a-zA-Z_]\w*', Name),
+            (r'[a-zA-Z_]\w*', Name),
         ],
         'root': [
             include('whitespace'),
@@ -344,7 +344,7 @@ class SwigLexer(CppLexer):
             # SWIG directives
             (r'(%[a-z_][a-z0-9_]*)', Name.Function),
             # Special variables
-            ('\$\**\&?\w+', Name),
+            (r'\$\**\&?\w+', Name),
             # Stringification / additional preprocessor directives
             (r'##*[a-zA-Z_]\w*', Comment.Preproc),
             inherit,
diff --git a/pygments/lexers/dotnet.py b/pygments/lexers/dotnet.py
index 4e2bc8ab..27ae77c5 100644
--- a/pygments/lexers/dotnet.py
+++ b/pygments/lexers/dotnet.py
@@ -58,7 +58,7 @@ class CSharpLexer(RegexLexer):
     # http://www.ecma-international.org/publications/files/ECMA-ST/Ecma-334.pdf
 
     levels = {
-        'none': '@?[_a-zA-Z]\w*',
+        'none': r'@?[_a-zA-Z]\w*',
         'basic': ('@?[_' + uni.combine('Lu', 'Ll', 'Lt', 'Lm', 'Nl') + ']' +
                   '[' + uni.combine('Lu', 'Ll', 'Lt', 'Lm', 'Nl', 'Nd', 'Pc',
                                     'Cf', 'Mn', 'Mc') + ']*'),
@@ -171,7 +171,7 @@ class NemerleLexer(RegexLexer):
     # http://www.ecma-international.org/publications/files/ECMA-ST/Ecma-334.pdf
 
     levels = {
-        'none': '@?[_a-zA-Z]\w*',
+        'none': r'@?[_a-zA-Z]\w*',
         'basic': ('@?[_' + uni.combine('Lu', 'Ll', 'Lt', 'Lm', 'Nl') + ']' +
                   '[' + uni.combine('Lu', 'Ll', 'Lt', 'Lm', 'Nl', 'Nd', 'Pc',
                                     'Cf', 'Mn', 'Mc') + ']*'),
@@ -352,13 +352,13 @@ class BooLexer(RegexLexer):
             ('[*/]', Comment.Multiline)
         ],
         'funcname': [
-            ('[a-zA-Z_]\w*', Name.Function, '#pop')
+            (r'[a-zA-Z_]\w*', Name.Function, '#pop')
         ],
         'classname': [
-            ('[a-zA-Z_]\w*', Name.Class, '#pop')
+            (r'[a-zA-Z_]\w*', Name.Class, '#pop')
         ],
         'namespace': [
-            ('[a-zA-Z_][\w.]*', Name.Namespace, '#pop')
+            (r'[a-zA-Z_][\w.]*', Name.Namespace, '#pop')
         ]
     }
 
@@ -413,7 +413,7 @@ class VbNetLexer(RegexLexer):
                 'Static', 'Step', 'Stop', 'SyncLock', 'Then', 'Throw', 'To',
                 'True', 'Try', 'TryCast', 'Wend', 'Using', 'When', 'While',
                 'Widening', 'With', 'WithEvents', 'WriteOnly'),
-                   prefix='(?', '-', '\.\.', '\.', '::', ':=', ':>', ':', ';;', ';', '<-',
-        '<\]', '<', '>\]', '>', '\?\?', '\?', '\[<', '\[\|', '\[', '\]',
-        '_', '`', '\{', '\|\]', '\|', '\}', '~', '<@@', '<@', '=', '@>', '@@>',
+        '!=', '#', '&&', '&', r'\(', r'\)', r'\*', r'\+', ',', r'-\.',
+        '->', '-', r'\.\.', r'\.', '::', ':=', ':>', ':', ';;', ';', '<-',
+        r'<\]', '<', r'>\]', '>', r'\?\?', r'\?', r'\[<', r'\[\|', r'\[', r'\]',
+        '_', '`', r'\{', r'\|\]', r'\|', r'\}', '~', '<@@', '<@', '=', '@>', '@@>',
     ]
 
     operators = r'[!$%&*+\./:<=>?@^|~-]'
diff --git a/pygments/lexers/dylan.py b/pygments/lexers/dylan.py
index f61bb60d..30318f38 100644
--- a/pygments/lexers/dylan.py
+++ b/pygments/lexers/dylan.py
@@ -179,10 +179,10 @@ class DylanLexer(RegexLexer):
             (valid_name + ':', Keyword),
 
             # class names
-            (r'<' + valid_name + '>', Name.Class),
+            ('<' + valid_name + '>', Name.Class),
 
             # define variable forms.
-            (r'\*' + valid_name + '\*', Name.Variable.Global),
+            (r'\*' + valid_name + r'\*', Name.Variable.Global),
 
             # define constant forms.
             (r'\$' + valid_name, Name.Constant),
@@ -260,7 +260,7 @@ class DylanConsoleLexer(Lexer):
     mimetypes = ['text/x-dylan-console']
 
     _line_re = re.compile('.*?\n')
-    _prompt_re = re.compile('\?| ')
+    _prompt_re = re.compile(r'\?| ')
 
     def get_tokens_unprocessed(self, text):
         dylexer = DylanLexer(**self.options)
diff --git a/pygments/lexers/erlang.py b/pygments/lexers/erlang.py
index 9e7f85c1..0d0d0798 100644
--- a/pygments/lexers/erlang.py
+++ b/pygments/lexers/erlang.py
@@ -344,7 +344,7 @@ class ElixirLexer(RegexLexer):
     op1_re = "|".join(re.escape(s) for s in OPERATORS1)
     ops_re = r'(?:%s|%s|%s)' % (op3_re, op2_re, op1_re)
     punctuation_re = "|".join(re.escape(s) for s in PUNCTUATION)
-    alnum = '\w'
+    alnum = r'\w'
     name_re = r'(?:\.\.\.|[a-z_]%s*[!?]?)' % alnum
     modname_re = r'[A-Z]%(alnum)s*(?:\.[A-Z]%(alnum)s*)*' % {'alnum': alnum}
     complex_name_re = r'(?:%s|%s|%s)' % (name_re, modname_re, ops_re)
@@ -495,7 +495,7 @@ class ElixirConsoleLexer(Lexer):
     aliases = ['iex']
     mimetypes = ['text/x-elixir-shellsession']
 
-    _prompt_re = re.compile('(iex|\.{3})(\(\d+\))?> ')
+    _prompt_re = re.compile(r'(iex|\.{3})(\(\d+\))?> ')
 
     def get_tokens_unprocessed(self, text):
         exlexer = ElixirLexer(**self.options)
diff --git a/pygments/lexers/grammar_notation.py b/pygments/lexers/grammar_notation.py
index 076249d3..bc715ffa 100644
--- a/pygments/lexers/grammar_notation.py
+++ b/pygments/lexers/grammar_notation.py
@@ -158,7 +158,7 @@ class JsgfLexer(RegexLexer):
             (r'//.*', Comment.Single),
         ],
         'non-comments': [
-            ('\A#JSGF[^;]*', Comment.Preproc),
+            (r'\A#JSGF[^;]*', Comment.Preproc),
             (r'\s+', Text),
             (r';', Punctuation),
             (r'[=|()\[\]*+]', Operator),
diff --git a/pygments/lexers/haskell.py b/pygments/lexers/haskell.py
index 1a2f2217..e3604ed8 100644
--- a/pygments/lexers/haskell.py
+++ b/pygments/lexers/haskell.py
@@ -677,10 +677,10 @@ class KokaLexer(RegexLexer):
     symbols = r'[$%&*+@!/\\^~=.:\-?|<>]+'
 
     # symbol boundary: an operator keyword should not be followed by any of these
-    sboundary = '(?!'+symbols+')'
+    sboundary = '(?!' + symbols + ')'
 
     # name boundary: a keyword should not be followed by any of these
-    boundary = '(?![\w/])'
+    boundary = r'(?![\w/])'
 
     # koka token abstractions
     tokenType = Name.Attribute
diff --git a/pygments/lexers/haxe.py b/pygments/lexers/haxe.py
index 6f5c3599..364ad344 100644
--- a/pygments/lexers/haxe.py
+++ b/pygments/lexers/haxe.py
@@ -43,7 +43,7 @@ class HaxeLexer(ExtendedRegexLexer):
     typeid = r'_*[A-Z]\w*'
 
     # combined ident and dollar and idtype
-    ident = r'(?:_*[a-z]\w*|_+[0-9]\w*|' + typeid + '|_+|\$\w+)'
+    ident = r'(?:_*[a-z]\w*|_+[0-9]\w*|' + typeid + r'|_+|\$\w+)'
 
     binop = (r'(?:%=|&=|\|=|\^=|\+=|\-=|\*=|/=|<<=|>\s*>\s*=|>\s*>\s*>\s*=|==|'
              r'!=|<=|>\s*=|&&|\|\||<<|>>>|>\s*>|\.\.\.|<|>|%|&|\||\^|\+|\*|'
@@ -182,7 +182,7 @@ class HaxeLexer(ExtendedRegexLexer):
             (r'[0-9]+[eE][+\-]?[0-9]+', Number.Float),
             (r'[0-9]+\.[0-9]*[eE][+\-]?[0-9]+', Number.Float),
             (r'[0-9]+\.[0-9]+', Number.Float),
-            (r'[0-9]+\.(?!' + ident + '|\.\.)', Number.Float),
+            (r'[0-9]+\.(?!' + ident + r'|\.\.)', Number.Float),
 
             # Int
             (r'0x[0-9a-fA-F]+', Number.Hex),
@@ -219,7 +219,7 @@ class HaxeLexer(ExtendedRegexLexer):
             (r'[0-9]+[eE][+\-]?[0-9]+', Number.Float, ('#pop', 'preproc-expr-chain')),
             (r'[0-9]+\.[0-9]*[eE][+\-]?[0-9]+', Number.Float, ('#pop', 'preproc-expr-chain')),
             (r'[0-9]+\.[0-9]+', Number.Float, ('#pop', 'preproc-expr-chain')),
-            (r'[0-9]+\.(?!' + ident + '|\.\.)', Number.Float, ('#pop', 'preproc-expr-chain')),
+            (r'[0-9]+\.(?!' + ident + r'|\.\.)', Number.Float, ('#pop', 'preproc-expr-chain')),
 
             # Int
             (r'0x[0-9a-fA-F]+', Number.Hex, ('#pop', 'preproc-expr-chain')),
@@ -456,7 +456,7 @@ class HaxeLexer(ExtendedRegexLexer):
             (r'[0-9]+[eE][+\-]?[0-9]+', Number.Float, ('#pop', 'expr-chain')),
             (r'[0-9]+\.[0-9]*[eE][+\-]?[0-9]+', Number.Float, ('#pop', 'expr-chain')),
             (r'[0-9]+\.[0-9]+', Number.Float, ('#pop', 'expr-chain')),
-            (r'[0-9]+\.(?!' + ident + '|\.\.)', Number.Float, ('#pop', 'expr-chain')),
+            (r'[0-9]+\.(?!' + ident + r'|\.\.)', Number.Float, ('#pop', 'expr-chain')),
 
             # Int
             (r'0x[0-9a-fA-F]+', Number.Hex, ('#pop', 'expr-chain')),
@@ -711,7 +711,7 @@ class HaxeLexer(ExtendedRegexLexer):
             (r'[0-9]+[eE][+\-]?[0-9]+', Number.Float, '#pop'),
             (r'[0-9]+\.[0-9]*[eE][+\-]?[0-9]+', Number.Float, '#pop'),
             (r'[0-9]+\.[0-9]+', Number.Float, '#pop'),
-            (r'[0-9]+\.(?!' + ident + '|\.\.)', Number.Float, '#pop'),
+            (r'[0-9]+\.(?!' + ident + r'|\.\.)', Number.Float, '#pop'),
 
             # Int
             (r'0x[0-9a-fA-F]+', Number.Hex, '#pop'),
diff --git a/pygments/lexers/idl.py b/pygments/lexers/idl.py
index 2fc39318..87cafe6a 100644
--- a/pygments/lexers/idl.py
+++ b/pygments/lexers/idl.py
@@ -53,7 +53,7 @@ class IDLLexer(RegexLexer):
         'broyden', 'butterworth', 'bytarr', 'byte', 'byteorder',
         'bytscl', 'caldat', 'calendar', 'call_external',
         'call_function', 'call_method', 'call_procedure', 'canny',
-        'catch', 'cd', 'cdf_\w*', 'ceil', 'chebyshev',
+        'catch', 'cd', r'cdf_\w*', 'ceil', 'chebyshev',
         'check_math',
         'chisqr_cvf', 'chisqr_pdf', 'choldc', 'cholsol', 'cindgen',
         'cir_3pnt', 'close', 'cluster', 'cluster_tree', 'clust_wts',
@@ -87,7 +87,7 @@ class IDLLexer(RegexLexer):
         'dlm_load', 'dlm_register', 'doc_library', 'double',
         'draw_roi', 'edge_dog', 'efont', 'eigenql', 'eigenvec',
         'ellipse', 'elmhes', 'emboss', 'empty', 'enable_sysrtn',
-        'eof', 'eos_\w*', 'erase', 'erf', 'erfc', 'erfcx',
+        'eof', r'eos_\w*', 'erase', 'erf', 'erfc', 'erfcx',
         'erode', 'errorplot', 'errplot', 'estimator_filter',
         'execute', 'exit', 'exp', 'expand', 'expand_path', 'expint',
         'extrac', 'extract_slice', 'factorial', 'fft', 'filepath',
@@ -104,11 +104,11 @@ class IDLLexer(RegexLexer):
         'gauss_cvf', 'gauss_pdf', 'gauss_smooth', 'getenv',
         'getwindows', 'get_drive_list', 'get_dxf_objects',
         'get_kbrd', 'get_login_info', 'get_lun', 'get_screen_size',
-        'greg2jul', 'grib_\w*', 'grid3', 'griddata',
+        'greg2jul', r'grib_\w*', 'grid3', 'griddata',
         'grid_input', 'grid_tps', 'gs_iter',
-        'h5[adfgirst]_\w*', 'h5_browser', 'h5_close',
+        r'h5[adfgirst]_\w*', 'h5_browser', 'h5_close',
         'h5_create', 'h5_get_libversion', 'h5_open', 'h5_parse',
-        'hanning', 'hash', 'hdf_\w*', 'heap_free',
+        'hanning', 'hash', r'hdf_\w*', 'heap_free',
         'heap_gc', 'heap_nosave', 'heap_refcount', 'heap_save',
         'help', 'hilbert', 'histogram', 'hist_2d', 'hist_equal',
         'hls', 'hough', 'hqr', 'hsv', 'h_eq_ct', 'h_eq_int',
@@ -156,7 +156,7 @@ class IDLLexer(RegexLexer):
         'modifyct', 'moment', 'morph_close', 'morph_distance',
         'morph_gradient', 'morph_hitormiss', 'morph_open',
         'morph_thin', 'morph_tophat', 'multi', 'm_correlate',
-        'ncdf_\w*', 'newton', 'noise_hurl', 'noise_pick',
+        r'ncdf_\w*', 'newton', 'noise_hurl', 'noise_pick',
         'noise_scatter', 'noise_slur', 'norm', 'n_elements',
         'n_params', 'n_tags', 'objarr', 'obj_class', 'obj_destroy',
         'obj_hasmethod', 'obj_isa', 'obj_new', 'obj_valid',
diff --git a/pygments/lexers/inferno.py b/pygments/lexers/inferno.py
index 5fc5a0ba..0d68856d 100644
--- a/pygments/lexers/inferno.py
+++ b/pygments/lexers/inferno.py
@@ -64,7 +64,7 @@ class LimboLexer(RegexLexer):
             (r'(byte|int|big|real|string|array|chan|list|adt'
              r'|fn|ref|of|module|self|type)\b', Keyword.Type),
             (r'(con|iota|nil)\b', Keyword.Constant),
-            ('[a-zA-Z_]\w*', Name),
+            (r'[a-zA-Z_]\w*', Name),
         ],
         'statement' : [
             include('whitespace'),
diff --git a/pygments/lexers/j.py b/pygments/lexers/j.py
index 434964fe..46037820 100644
--- a/pygments/lexers/j.py
+++ b/pygments/lexers/j.py
@@ -52,13 +52,13 @@ class JLexer(RegexLexer):
              Name.Function, 'explicitDefinition'),
 
             # Flow Control
-            (words(('for_', 'goto_', 'label_'), suffix=validName+'\.'), Name.Label),
+            (words(('for_', 'goto_', 'label_'), suffix=validName+r'\.'), Name.Label),
             (words((
                 'assert', 'break', 'case', 'catch', 'catchd',
                 'catcht', 'continue', 'do', 'else', 'elseif',
                 'end', 'fcase', 'for', 'if', 'return',
                 'select', 'throw', 'try', 'while', 'whilst',
-                ), suffix='\.'), Name.Label),
+                ), suffix=r'\.'), Name.Label),
 
             # Variable Names
             (validName, Name.Variable),
diff --git a/pygments/lexers/matlab.py b/pygments/lexers/matlab.py
index 56a0f6d6..1c77b60c 100644
--- a/pygments/lexers/matlab.py
+++ b/pygments/lexers/matlab.py
@@ -134,9 +134,9 @@ class MatlabLexer(RegexLexer):
     }
 
     def analyse_text(text):
-        if re.match('^\s*%', text, re.M):  # comment
+        if re.match(r'^\s*%', text, re.M):  # comment
             return 0.2
-        elif re.match('^!\w+', text, re.M):  # system cmd
+        elif re.match(r'^!\w+', text, re.M):  # system cmd
             return 0.2
 
 
diff --git a/pygments/lexers/ml.py b/pygments/lexers/ml.py
index f80d5bfa..0bff9816 100644
--- a/pygments/lexers/ml.py
+++ b/pygments/lexers/ml.py
@@ -43,7 +43,7 @@ class SMLLexer(RegexLexer):
 
     symbolicid_reserved = set((
         # Core
-        ':', '\|', '=', '=>', '->', '#',
+        ':', r'\|', '=', '=>', '->', '#',
         # Modules
         ':>',
     ))
diff --git a/pygments/lexers/objective.py b/pygments/lexers/objective.py
index 7807255e..179928e9 100644
--- a/pygments/lexers/objective.py
+++ b/pygments/lexers/objective.py
@@ -87,26 +87,26 @@ def objective(baselexer):
             ],
             'oc_classname': [
                 # interface definition that inherits
-                ('([a-zA-Z$_][\w$]*)(\s*:\s*)([a-zA-Z$_][\w$]*)?(\s*)(\{)',
+                (r'([a-zA-Z$_][\w$]*)(\s*:\s*)([a-zA-Z$_][\w$]*)?(\s*)(\{)',
                  bygroups(Name.Class, Text, Name.Class, Text, Punctuation),
                  ('#pop', 'oc_ivars')),
-                ('([a-zA-Z$_][\w$]*)(\s*:\s*)([a-zA-Z$_][\w$]*)?',
+                (r'([a-zA-Z$_][\w$]*)(\s*:\s*)([a-zA-Z$_][\w$]*)?',
                  bygroups(Name.Class, Text, Name.Class), '#pop'),
                 # interface definition for a category
-                ('([a-zA-Z$_][\w$]*)(\s*)(\([a-zA-Z$_][\w$]*\))(\s*)(\{)',
+                (r'([a-zA-Z$_][\w$]*)(\s*)(\([a-zA-Z$_][\w$]*\))(\s*)(\{)',
                  bygroups(Name.Class, Text, Name.Label, Text, Punctuation),
                  ('#pop', 'oc_ivars')),
-                ('([a-zA-Z$_][\w$]*)(\s*)(\([a-zA-Z$_][\w$]*\))',
+                (r'([a-zA-Z$_][\w$]*)(\s*)(\([a-zA-Z$_][\w$]*\))',
                  bygroups(Name.Class, Text, Name.Label), '#pop'),
                 # simple interface / implementation
-                ('([a-zA-Z$_][\w$]*)(\s*)(\{)',
+                (r'([a-zA-Z$_][\w$]*)(\s*)(\{)',
                  bygroups(Name.Class, Text, Punctuation), ('#pop', 'oc_ivars')),
-                ('([a-zA-Z$_][\w$]*)', Name.Class, '#pop')
+                (r'([a-zA-Z$_][\w$]*)', Name.Class, '#pop')
             ],
             'oc_forward_classname': [
-                ('([a-zA-Z$_][\w$]*)(\s*,\s*)',
+                (r'([a-zA-Z$_][\w$]*)(\s*,\s*)',
                  bygroups(Name.Class, Text), 'oc_forward_classname'),
-                ('([a-zA-Z$_][\w$]*)(\s*;?)',
+                (r'([a-zA-Z$_][\w$]*)(\s*;?)',
                  bygroups(Name.Class, Text), '#pop')
             ],
             'oc_ivars': [
@@ -244,17 +244,17 @@ class LogosLexer(ObjectiveCppLexer):
             inherit,
         ],
         'logos_init_directive': [
-            ('\s+', Text),
+            (r'\s+', Text),
             (',', Punctuation, ('logos_init_directive', '#pop')),
-            ('([a-zA-Z$_][\w$]*)(\s*)(=)(\s*)([^);]*)',
+            (r'([a-zA-Z$_][\w$]*)(\s*)(=)(\s*)([^);]*)',
              bygroups(Name.Class, Text, Punctuation, Text, Text)),
-            ('([a-zA-Z$_][\w$]*)', Name.Class),
-            ('\)', Punctuation, '#pop'),
+            (r'([a-zA-Z$_][\w$]*)', Name.Class),
+            (r'\)', Punctuation, '#pop'),
         ],
         'logos_classname': [
-            ('([a-zA-Z$_][\w$]*)(\s*:\s*)([a-zA-Z$_][\w$]*)?',
+            (r'([a-zA-Z$_][\w$]*)(\s*:\s*)([a-zA-Z$_][\w$]*)?',
              bygroups(Name.Class, Text, Name.Class), '#pop'),
-            ('([a-zA-Z$_][\w$]*)', Name.Class, '#pop')
+            (r'([a-zA-Z$_][\w$]*)', Name.Class, '#pop')
         ],
         'root': [
             (r'(%subclass)(\s+)', bygroups(Keyword, Text),
diff --git a/pygments/lexers/parsers.py b/pygments/lexers/parsers.py
index 1f3c9b4d..43eb6c1f 100644
--- a/pygments/lexers/parsers.py
+++ b/pygments/lexers/parsers.py
@@ -364,13 +364,13 @@ class AntlrLexer(RegexLexer):
             # tokensSpec
             (r'tokens\b', Keyword, 'tokens'),
             # attrScope
-            (r'(scope)(\s*)(' + _id + ')(\s*)(\{)',
+            (r'(scope)(\s*)(' + _id + r')(\s*)(\{)',
              bygroups(Keyword, Whitespace, Name.Variable, Whitespace,
                       Punctuation), 'action'),
             # exception
             (r'(catch|finally)\b', Keyword, 'exception'),
             # action
-            (r'(@' + _id + ')(\s*)(::)?(\s*)(' + _id + ')(\s*)(\{)',
+            (r'(@' + _id + r')(\s*)(::)?(\s*)(' + _id + r')(\s*)(\{)',
              bygroups(Name.Label, Whitespace, Punctuation, Whitespace,
                       Name.Label, Whitespace, Punctuation), 'action'),
             # rule
@@ -405,10 +405,10 @@ class AntlrLexer(RegexLexer):
             # L173 ANTLRv3.g from ANTLR book
             (r'(scope)(\s+)(\{)', bygroups(Keyword, Whitespace, Punctuation),
              'action'),
-            (r'(scope)(\s+)(' + _id + ')(\s*)(;)',
+            (r'(scope)(\s+)(' + _id + r')(\s*)(;)',
              bygroups(Keyword, Whitespace, Name.Label, Whitespace, Punctuation)),
             # ruleAction
-            (r'(@' + _id + ')(\s*)(\{)',
+            (r'(@' + _id + r')(\s*)(\{)',
              bygroups(Name.Label, Whitespace, Punctuation), 'action'),
             # finished prelims, go to rule alts!
             (r':', Punctuation, '#pop')
@@ -442,7 +442,7 @@ class AntlrLexer(RegexLexer):
             include('comments'),
             (r'\{', Punctuation),
             (r'(' + _TOKEN_REF + r')(\s*)(=)?(\s*)(' + _STRING_LITERAL
-             + ')?(\s*)(;)',
+             + r')?(\s*)(;)',
              bygroups(Name.Label, Whitespace, Punctuation, Whitespace,
                       String, Whitespace, Punctuation)),
             (r'\}', Punctuation, '#pop'),
@@ -452,7 +452,7 @@ class AntlrLexer(RegexLexer):
             include('comments'),
             (r'\{', Punctuation),
             (r'(' + _id + r')(\s*)(=)(\s*)(' +
-             '|'.join((_id, _STRING_LITERAL, _INT, '\*')) + ')(\s*)(;)',
+             '|'.join((_id, _STRING_LITERAL, _INT, r'\*')) + r')(\s*)(;)',
              bygroups(Name.Variable, Whitespace, Punctuation, Whitespace,
                       Text, Whitespace, Punctuation)),
             (r'\}', Punctuation, '#pop'),
diff --git a/pygments/lexers/pascal.py b/pygments/lexers/pascal.py
index 9aa1ac8f..467a0b2c 100644
--- a/pygments/lexers/pascal.py
+++ b/pygments/lexers/pascal.py
@@ -593,8 +593,8 @@ class AdaLexer(RegexLexer):
         ],
         'end': [
             ('(if|case|record|loop|select)', Keyword.Reserved),
-            ('"[^"]+"|[\w.]+', Name.Function),
-            ('\s+', Text),
+            (r'"[^"]+"|[\w.]+', Name.Function),
+            (r'\s+', Text),
             (';', Punctuation, '#pop'),
         ],
         'type_def': [
@@ -628,11 +628,11 @@ class AdaLexer(RegexLexer):
         ],
         'package': [
             ('body', Keyword.Declaration),
-            ('is\s+new|renames', Keyword.Reserved),
+            (r'is\s+new|renames', Keyword.Reserved),
             ('is', Keyword.Reserved, '#pop'),
             (';', Punctuation, '#pop'),
-            ('\(', Punctuation, 'package_instantiation'),
-            ('([\w.]+)', Name.Class),
+            (r'\(', Punctuation, 'package_instantiation'),
+            (r'([\w.]+)', Name.Class),
             include('root'),
         ],
         'package_instantiation': [
diff --git a/pygments/lexers/pawn.py b/pygments/lexers/pawn.py
index f462a883..0ef28175 100644
--- a/pygments/lexers/pawn.py
+++ b/pygments/lexers/pawn.py
@@ -36,7 +36,7 @@ class SourcePawnLexer(RegexLexer):
     tokens = {
         'root': [
             # preprocessor directives: without whitespace
-            ('^#if\s+0', Comment.Preproc, 'if0'),
+            (r'^#if\s+0', Comment.Preproc, 'if0'),
             ('^#', Comment.Preproc, 'macro'),
             # or with whitespace
             ('^' + _ws1 + r'#if\s+0', Comment.Preproc, 'if0'),
@@ -62,7 +62,7 @@ class SourcePawnLexer(RegexLexer):
              r'public|return|sizeof|static|decl|struct|switch)\b', Keyword),
             (r'(bool|Float)\b', Keyword.Type),
             (r'(true|false)\b', Keyword.Constant),
-            ('[a-zA-Z_]\w*', Name),
+            (r'[a-zA-Z_]\w*', Name),
         ],
         'string': [
             (r'"', String, '#pop'),
@@ -148,7 +148,7 @@ class PawnLexer(RegexLexer):
     tokens = {
         'root': [
             # preprocessor directives: without whitespace
-            ('^#if\s+0', Comment.Preproc, 'if0'),
+            (r'^#if\s+0', Comment.Preproc, 'if0'),
             ('^#', Comment.Preproc, 'macro'),
             # or with whitespace
             ('^' + _ws1 + r'#if\s+0', Comment.Preproc, 'if0'),
@@ -174,7 +174,7 @@ class PawnLexer(RegexLexer):
              r'public|return|sizeof|tagof|state|goto)\b', Keyword),
             (r'(bool|Float)\b', Keyword.Type),
             (r'(true|false)\b', Keyword.Constant),
-            ('[a-zA-Z_]\w*', Name),
+            (r'[a-zA-Z_]\w*', Name),
         ],
         'string': [
             (r'"', String, '#pop'),
diff --git a/pygments/lexers/prolog.py b/pygments/lexers/prolog.py
index 90f9529c..58e762b0 100644
--- a/pygments/lexers/prolog.py
+++ b/pygments/lexers/prolog.py
@@ -57,15 +57,15 @@ class PrologLexer(RegexLexer):
             (r'_', Keyword),  # The don't-care variable
             (r'([a-z]+)(:)', bygroups(Name.Namespace, Punctuation)),
             (u'([a-z\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]'
-             u'[\w$\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]*)'
+             u'[\\w$\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]*)'
              u'(\\s*)(:-|-->)',
              bygroups(Name.Function, Text, Operator)),  # function defn
             (u'([a-z\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]'
-             u'[\w$\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]*)'
+             u'[\\w$\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]*)'
              u'(\\s*)(\\()',
              bygroups(Name.Function, Text, Punctuation)),
             (u'[a-z\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]'
-             u'[\w$\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]*',
+             u'[\\w$\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]*',
              String.Atom),  # atom, characters
             # This one includes !
             (u'[#&*+\\-./:<=>?@\\\\^~\u00a1-\u00bf\u2010-\u303f]+',
@@ -300,7 +300,7 @@ class LogtalkLexer(RegexLexer):
             return 1.0
         elif ':- category(' in text:
             return 1.0
-        elif re.search('^:-\s[a-z]', text, re.M):
+        elif re.search(r'^:-\s[a-z]', text, re.M):
             return 0.9
         else:
             return 0.0
diff --git a/pygments/lexers/qvt.py b/pygments/lexers/qvt.py
index f496d600..af091a65 100644
--- a/pygments/lexers/qvt.py
+++ b/pygments/lexers/qvt.py
@@ -126,7 +126,7 @@ class QVToLexer(RegexLexer):
             (r'[^\\\'"\n]+', String),
             # quotes, percents and backslashes must be parsed one at a time
             (r'[\'"\\]', String),
-            ],
+        ],
         'stringescape': [
             (r'\\([\\btnfr"\']|u[0-3][0-7]{2}|u[0-7]{1,2})', String.Escape)
         ],
@@ -134,15 +134,15 @@ class QVToLexer(RegexLexer):
             (r'"', String, '#pop'),
             (r'\\\\|\\"', String.Escape),
             include('strings')
-            ],
+        ],
         'sqs': [  # single-quoted string
             (r"'", String, '#pop'),
             (r"\\\\|\\'", String.Escape),
             include('strings')
-            ],
+        ],
         'name': [
-            ('[a-zA-Z_]\w*', Name),
-            ],
+            (r'[a-zA-Z_]\w*', Name),
+        ],
         # numbers: excerpt taken from the python lexer
         'numbers': [
             (r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?', Number.Float),
diff --git a/pygments/lexers/rebol.py b/pygments/lexers/rebol.py
index f3d00200..4d24daaa 100644
--- a/pygments/lexers/rebol.py
+++ b/pygments/lexers/rebol.py
@@ -102,12 +102,12 @@ class RebolLexer(RegexLexer):
             yield match.start(), Generic.Heading, word
         elif re.match("to-.*", word):
             yield match.start(), Keyword, word
-        elif re.match('(\+|-|\*|/|//|\*\*|and|or|xor|=\?|=|==|<>|<|>|<=|>=)$',
+        elif re.match(r'(\+|-|\*|/|//|\*\*|and|or|xor|=\?|=|==|<>|<|>|<=|>=)$',
                       word):
             yield match.start(), Operator, word
-        elif re.match(".*\?$", word):
+        elif re.match(r".*\?$", word):
             yield match.start(), Keyword, word
-        elif re.match(".*\!$", word):
+        elif re.match(r".*\!$", word):
             yield match.start(), Keyword.Type, word
         elif re.match("'.*", word):
             yield match.start(), Name.Variable.Instance, word  # lit-word
@@ -297,10 +297,10 @@ class RedLexer(RegexLexer):
             yield match.start(), Keyword.Namespace, word
         elif re.match("to-.*", word):
             yield match.start(), Keyword, word
-        elif re.match('(\+|-\*\*|-|\*\*|//|/|\*|and|or|xor|=\?|===|==|=|<>|<=|>=|'
-                      '<<<|>>>|<<|>>|<|>%)$', word):
+        elif re.match(r'(\+|-\*\*|-|\*\*|//|/|\*|and|or|xor|=\?|===|==|=|<>|<=|>=|'
+                      r'<<<|>>>|<<|>>|<|>%)$', word):
             yield match.start(), Operator, word
-        elif re.match(".*\!$", word):
+        elif re.match(r".*\!$", word):
             yield match.start(), Keyword.Type, word
         elif re.match("'.*", word):
             yield match.start(), Name.Variable.Instance, word  # lit-word
diff --git a/pygments/lexers/robotframework.py b/pygments/lexers/robotframework.py
index e868127b..5bacffa3 100644
--- a/pygments/lexers/robotframework.py
+++ b/pygments/lexers/robotframework.py
@@ -161,7 +161,7 @@ class RowTokenizer(object):
 
 class RowSplitter(object):
     _space_splitter = re.compile('( {2,})')
-    _pipe_splitter = re.compile('((?:^| +)\|(?: +|$))')
+    _pipe_splitter = re.compile(r'((?:^| +)\|(?: +|$))')
 
     def split(self, row):
         splitter = (row.startswith('| ') and self._split_from_pipes
diff --git a/pygments/lexers/testing.py b/pygments/lexers/testing.py
index 1e0795b1..86e60f25 100644
--- a/pygments/lexers/testing.py
+++ b/pygments/lexers/testing.py
@@ -29,7 +29,7 @@ class GherkinLexer(RegexLexer):
     feature_keywords = u'^(기능|機能|功能|フィーチャ|خاصية|תכונה|Функціонал|Функционалност|Функционал|Фича|Особина|Могућност|Özellik|Właściwość|Tính năng|Trajto|Savybė|Požiadavka|Požadavek|Osobina|Ominaisuus|Omadus|OH HAI|Mogućnost|Mogucnost|Jellemző|Fīča|Funzionalità|Funktionalität|Funkcionalnost|Funkcionalitāte|Funcționalitate|Functionaliteit|Functionalitate|Funcionalitat|Funcionalidade|Fonctionnalité|Fitur|Feature|Egenskap|Egenskab|Crikey|Característica|Arwedd)(:)(.*)$'
     feature_element_keywords = u'^(\\s*)(시나리오 개요|시나리오|배경|背景|場景大綱|場景|场景大纲|场景|劇本大綱|劇本|剧本大纲|剧本|テンプレ|シナリオテンプレート|シナリオテンプレ|シナリオアウトライン|シナリオ|سيناريو مخطط|سيناريو|الخلفية|תרחיש|תבנית תרחיש|רקע|Тарих|Сценарій|Сценарио|Сценарий структураси|Сценарий|Структура сценарію|Структура сценарија|Структура сценария|Скица|Рамка на сценарий|Пример|Предыстория|Предистория|Позадина|Передумова|Основа|Концепт|Контекст|Założenia|Wharrimean is|Tình huống|The thing of it is|Tausta|Taust|Tapausaihio|Tapaus|Szenariogrundriss|Szenario|Szablon scenariusza|Stsenaarium|Struktura scenarija|Skica|Skenario konsep|Skenario|Situācija|Senaryo taslağı|Senaryo|Scénář|Scénario|Schema dello scenario|Scenārijs pēc parauga|Scenārijs|Scenár|Scenaro|Scenariusz|Scenariul de şablon|Scenariul de sablon|Scenariu|Scenario Outline|Scenario Amlinellol|Scenario|Scenarijus|Scenarijaus šablonas|Scenarij|Scenarie|Rerefons|Raamstsenaarium|Primer|Pozadí|Pozadina|Pozadie|Plan du scénario|Plan du Scénario|Osnova scénáře|Osnova|Náčrt Scénáře|Náčrt Scenáru|Mate|MISHUN SRSLY|MISHUN|Kịch bản|Konturo de la scenaro|Kontext|Konteksts|Kontekstas|Kontekst|Koncept|Khung tình huống|Khung kịch bản|Háttér|Grundlage|Geçmiş|Forgatókönyv vázlat|Forgatókönyv|Fono|Esquema do Cenário|Esquema do Cenario|Esquema del escenario|Esquema de l\'escenari|Escenario|Escenari|Dis is what went down|Dasar|Contexto|Contexte|Contesto|Condiţii|Conditii|Cenário|Cenario|Cefndir|Bối cảnh|Blokes|Bakgrunn|Bakgrund|Baggrund|Background|B4|Antecedents|Antecedentes|All y\'all|Achtergrond|Abstrakt Scenario|Abstract Scenario)(:)(.*)$'
     examples_keywords = u'^(\\s*)(예|例子|例|サンプル|امثلة|דוגמאות|Сценарији|Примери|Приклади|Мисоллар|Значения|Örnekler|Voorbeelden|Variantai|Tapaukset|Scenarios|Scenariji|Scenarijai|Příklady|Példák|Príklady|Przykłady|Primjeri|Primeri|Piemēri|Pavyzdžiai|Paraugs|Juhtumid|Exemplos|Exemples|Exemplele|Exempel|Examples|Esempi|Enghreifftiau|Ekzemploj|Eksempler|Ejemplos|EXAMPLZ|Dữ liệu|Contoh|Cobber|Beispiele)(:)(.*)$'
-    step_keywords = u'^(\\s*)(하지만|조건|먼저|만일|만약|단|그리고|그러면|那麼|那么|而且|當|当|前提|假設|假设|假如|假定|但是|但し|並且|并且|同時|同时|もし|ならば|ただし|しかし|かつ|و |متى |لكن |عندما |ثم |بفرض |اذاً |כאשר |וגם |בהינתן |אזי |אז |אבל |Якщо |Унда |То |Припустимо, що |Припустимо |Онда |Но |Нехай |Лекин |Когато |Када |Кад |К тому же |И |Задато |Задати |Задате |Если |Допустим |Дадено |Ва |Бирок |Аммо |Али |Але |Агар |А |І |Și |És |Zatati |Zakładając |Zadato |Zadate |Zadano |Zadani |Zadan |Youse know when youse got |Youse know like when |Yna |Ya know how |Ya gotta |Y |Wun |Wtedy |When y\'all |When |Wenn |WEN |Và |Ve |Und |Un |Thì |Then y\'all |Then |Tapi |Tak |Tada |Tad |Så |Stel |Soit |Siis |Si |Sed |Se |Quando |Quand |Quan |Pryd |Pokud |Pokiaľ |Però |Pero |Pak |Oraz |Onda |Ond |Oletetaan |Og |Och |O zaman |Når |När |Niin |Nhưng |N |Mutta |Men |Mas |Maka |Majd |Mais |Maar |Ma |Lorsque |Lorsqu\'|Kun |Kuid |Kui |Khi |Keď |Ketika |Když |Kaj |Kai |Kada |Kad |Jeżeli |Ja |Ir |I CAN HAZ |I |Ha |Givun |Givet |Given y\'all |Given |Gitt |Gegeven |Gegeben sei |Fakat |Eğer ki |Etant donné |Et |Então |Entonces |Entao |En |Eeldades |E |Duota |Dun |Donitaĵo |Donat |Donada |Do |Diyelim ki |Dengan |Den youse gotta |De |Dato |Dar |Dann |Dan |Dado |Dacă |Daca |DEN |Când |Cuando |Cho |Cept |Cand |Cal |But y\'all |But |Buh |Biết |Bet |BUT |Atès |Atunci |Atesa |Anrhegedig a |Angenommen |And y\'all |And |An |Ama |Als |Alors |Allora |Ali |Aleshores |Ale |Akkor |Aber |AN |A také |A |\* )'
+    step_keywords = u'^(\\s*)(하지만|조건|먼저|만일|만약|단|그리고|그러면|那麼|那么|而且|當|当|前提|假設|假设|假如|假定|但是|但し|並且|并且|同時|同时|もし|ならば|ただし|しかし|かつ|و |متى |لكن |عندما |ثم |بفرض |اذاً |כאשר |וגם |בהינתן |אזי |אז |אבל |Якщо |Унда |То |Припустимо, що |Припустимо |Онда |Но |Нехай |Лекин |Когато |Када |Кад |К тому же |И |Задато |Задати |Задате |Если |Допустим |Дадено |Ва |Бирок |Аммо |Али |Але |Агар |А |І |Și |És |Zatati |Zakładając |Zadato |Zadate |Zadano |Zadani |Zadan |Youse know when youse got |Youse know like when |Yna |Ya know how |Ya gotta |Y |Wun |Wtedy |When y\'all |When |Wenn |WEN |Và |Ve |Und |Un |Thì |Then y\'all |Then |Tapi |Tak |Tada |Tad |Så |Stel |Soit |Siis |Si |Sed |Se |Quando |Quand |Quan |Pryd |Pokud |Pokiaľ |Però |Pero |Pak |Oraz |Onda |Ond |Oletetaan |Og |Och |O zaman |Når |När |Niin |Nhưng |N |Mutta |Men |Mas |Maka |Majd |Mais |Maar |Ma |Lorsque |Lorsqu\'|Kun |Kuid |Kui |Khi |Keď |Ketika |Když |Kaj |Kai |Kada |Kad |Jeżeli |Ja |Ir |I CAN HAZ |I |Ha |Givun |Givet |Given y\'all |Given |Gitt |Gegeven |Gegeben sei |Fakat |Eğer ki |Etant donné |Et |Então |Entonces |Entao |En |Eeldades |E |Duota |Dun |Donitaĵo |Donat |Donada |Do |Diyelim ki |Dengan |Den youse gotta |De |Dato |Dar |Dann |Dan |Dado |Dacă |Daca |DEN |Când |Cuando |Cho |Cept |Cand |Cal |But y\'all |But |Buh |Biết |Bet |BUT |Atès |Atunci |Atesa |Anrhegedig a |Angenommen |And y\'all |And |An |Ama |Als |Alors |Allora |Ali |Aleshores |Ale |Akkor |Aber |AN |A také |A |\\* )'
 
     tokens = {
         'comments': [
diff --git a/pygments/lexers/textfmts.py b/pygments/lexers/textfmts.py
index bb8124ef..b70c2ad6 100644
--- a/pygments/lexers/textfmts.py
+++ b/pygments/lexers/textfmts.py
@@ -266,7 +266,7 @@ class TodotxtLexer(RegexLexer):
             # 5. Leading project
             (project_regex, Project, 'incomplete'),
             # 6. Non-whitespace catch-all
-            ('\S+', IncompleteTaskText, 'incomplete'),
+            (r'\S+', IncompleteTaskText, 'incomplete'),
         ],
 
         # Parse a complete task
@@ -277,9 +277,9 @@ class TodotxtLexer(RegexLexer):
             (context_regex, Context),
             (project_regex, Project),
             # Tokenize non-whitespace text
-            ('\S+', CompleteTaskText),
+            (r'\S+', CompleteTaskText),
             # Tokenize whitespace not containing a newline
-            ('\s+', CompleteTaskText),
+            (r'\s+', CompleteTaskText),
         ],
 
         # Parse an incomplete task
@@ -290,8 +290,8 @@ class TodotxtLexer(RegexLexer):
             (context_regex, Context),
             (project_regex, Project),
             # Tokenize non-whitespace text
-            ('\S+', IncompleteTaskText),
+            (r'\S+', IncompleteTaskText),
             # Tokenize whitespace not containing a newline
-            ('\s+', IncompleteTaskText),
+            (r'\s+', IncompleteTaskText),
         ],
     }
diff --git a/pygments/lexers/varnish.py b/pygments/lexers/varnish.py
index 44521422..f3b37d60 100644
--- a/pygments/lexers/varnish.py
+++ b/pygments/lexers/varnish.py
@@ -36,7 +36,7 @@ class VCLLexer(RegexLexer):
         # Skip over comments and blank lines
         # This is accurate enough that returning 0.9 is reasonable.
         # Almost no VCL files start without some comments.
-        elif '\nvcl 4\.0;' in text[:1000]:
+        elif '\nvcl 4.0;' in text[:1000]:
             return 0.9
 
     tokens = {
@@ -120,7 +120,7 @@ class VCLLexer(RegexLexer):
              r'([a-zA-Z_]\w*)'
              r'(\s*\(.*\))',
              bygroups(Name.Function, Punctuation, Name.Function, using(this))),
-            ('[a-zA-Z_]\w*', Name),
+            (r'[a-zA-Z_]\w*', Name),
         ],
         'comment': [
             (r'[^*/]+', Comment.Multiline),
diff --git a/tests/run.py b/tests/run.py
index 161e5c04..2e962f2f 100644
--- a/tests/run.py
+++ b/tests/run.py
@@ -22,6 +22,13 @@ import warnings
 if os.path.dirname(__file__):
     os.chdir(os.path.dirname(__file__))
 
+# make FutureWarnings (coming from Regex syntax most likely) and
+# DeprecationWarnings due to non-raw strings an error
+warnings.filterwarnings("error", module=r"pygments\..*",
+                        category=FutureWarning)
+warnings.filterwarnings("error", module=r".*pygments.*",
+                        category=DeprecationWarning)
+
 
 try:
     import nose
@@ -32,12 +39,6 @@ except ImportError:
 # make sure the current source is first on sys.path
 sys.path.insert(0, '..')
 
-# make FutureWarnings (coming from Regex syntax most likely) and
-# DeprecationWarnings (coming from invalid escapes due to non-raw strings)
-# an error
-warnings.filterwarnings("error", category=FutureWarning)
-warnings.filterwarnings("error", category=DeprecationWarning)
-
 if '--with-coverage' not in sys.argv:
     # if running with coverage, pygments should not be imported before coverage
     # is started, otherwise it will count already executed lines as uncovered
-- 
cgit v1.2.1


From 86d2528cee60fd6e0e7a94bbac797eefeafec6eb Mon Sep 17 00:00:00 2001
From: James Edwards 
Date: Wed, 5 Dec 2018 00:46:05 +0000
Subject: Added newer Terraform keywords.

---
 pygments/lexers/configs.py    |  5 +++--
 tests/examplefiles/example.tf | 35 +++++++++++++++++++++++++++++++++++
 2 files changed, 38 insertions(+), 2 deletions(-)

diff --git a/pygments/lexers/configs.py b/pygments/lexers/configs.py
index c35e866d..db288c7d 100644
--- a/pygments/lexers/configs.py
+++ b/pygments/lexers/configs.py
@@ -574,6 +574,8 @@ class TerraformLexer(RegexLexer):
     filenames = ['*.tf']
     mimetypes = ['application/x-tf', 'application/x-terraform']
 
+    embedded_keywords = ('ingress', 'egress', 'listener', 'default', 'connection', 'alias', 'tags', 'lifecycle', 'timeout')
+
     tokens = {
         'root': [
              include('string'),
@@ -590,8 +592,7 @@ class TerraformLexer(RegexLexer):
              (r'(.*?)(\s*)(=)', bygroups(Name.Attribute, Text, Operator)),
              (words(('variable', 'resource', 'provider', 'provisioner', 'module'),
                     prefix=r'\b', suffix=r'\b'), Keyword.Reserved, 'function'),
-             (words(('ingress', 'egress', 'listener', 'default', 'connection', 'alias'),
-                    prefix=r'\b', suffix=r'\b'), Keyword.Declaration),
+             (words(keywords, prefix=r'\b', suffix=r'\b'), Keyword.Declaration),
              (r'\$\{', String.Interpol, 'var_builtin'),
         ],
         'function': [
diff --git a/tests/examplefiles/example.tf b/tests/examplefiles/example.tf
index 5a85dbee..50082616 100644
--- a/tests/examplefiles/example.tf
+++ b/tests/examplefiles/example.tf
@@ -23,6 +23,14 @@ variable "aws_amis" {
 }
 
 
+resource "aws_internet_gateway" "base_igw" {
+  vpc_id = "${aws_vpc.something.id}"
+  tags {
+    Name = "igw-${var.something}-${var.something}"
+  }
+}
+
+
 
 
 
@@ -170,3 +178,30 @@ resource "aws_instance" "web" {
   }
 }
 
+
+
+resource "aws_autoscaling_group" "bar" {
+  name                 = "terraform-asg-example"
+  launch_configuration = "${aws_launch_configuration.as_conf.name}"
+  min_size             = 1
+  max_size             = 2
+
+  lifecycle {
+    create_before_destroy = true
+  }
+}
+
+
+resource "aws_db_instance" "timeout_example" {
+  allocated_storage = 10
+  engine            = "mysql"
+  engine_version    = "5.6.17"
+  instance_class    = "db.t1.micro"
+  name              = "mydb"
+  timeouts {
+    create = "60m"
+    delete = "2h"
+  }
+}
+
+
-- 
cgit v1.2.1


From 78d3f931c52041d71120ba8b8b02972b2fc8b4f4 Mon Sep 17 00:00:00 2001
From: James Edwards 
Date: Wed, 5 Dec 2018 01:06:25 +0000
Subject: Fixed typos in terraform lexer

---
 pygments/lexers/configs.py    | 4 ++--
 tests/examplefiles/example.tf | 1 +
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/pygments/lexers/configs.py b/pygments/lexers/configs.py
index db288c7d..206ec360 100644
--- a/pygments/lexers/configs.py
+++ b/pygments/lexers/configs.py
@@ -574,7 +574,7 @@ class TerraformLexer(RegexLexer):
     filenames = ['*.tf']
     mimetypes = ['application/x-tf', 'application/x-terraform']
 
-    embedded_keywords = ('ingress', 'egress', 'listener', 'default', 'connection', 'alias', 'tags', 'lifecycle', 'timeout')
+    embedded_keywords = ('ingress', 'egress', 'listener', 'default', 'connection', 'alias', 'tags', 'lifecycle', 'timeouts')
 
     tokens = {
         'root': [
@@ -592,7 +592,7 @@ class TerraformLexer(RegexLexer):
              (r'(.*?)(\s*)(=)', bygroups(Name.Attribute, Text, Operator)),
              (words(('variable', 'resource', 'provider', 'provisioner', 'module'),
                     prefix=r'\b', suffix=r'\b'), Keyword.Reserved, 'function'),
-             (words(keywords, prefix=r'\b', suffix=r'\b'), Keyword.Declaration),
+             (words(embedded_keywords, prefix=r'\b', suffix=r'\b'), Keyword.Declaration),
              (r'\$\{', String.Interpol, 'var_builtin'),
         ],
         'function': [
diff --git a/tests/examplefiles/example.tf b/tests/examplefiles/example.tf
index 50082616..4cbef52c 100644
--- a/tests/examplefiles/example.tf
+++ b/tests/examplefiles/example.tf
@@ -198,6 +198,7 @@ resource "aws_db_instance" "timeout_example" {
   engine_version    = "5.6.17"
   instance_class    = "db.t1.micro"
   name              = "mydb"
+
   timeouts {
     create = "60m"
     delete = "2h"
-- 
cgit v1.2.1


From e3d894c5b82644ca248f8840638cb48017080e8e Mon Sep 17 00:00:00 2001
From: "Matth?us G. Chajdas" 
Date: Thu, 13 Dec 2018 20:02:24 +0100
Subject: Update changelog.

---
 CHANGES | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/CHANGES b/CHANGES
index c82ca1cf..3b723a08 100644
--- a/CHANGES
+++ b/CHANGES
@@ -6,6 +6,25 @@ Issue numbers refer to the tracker at
 pull request numbers to the requests at
 .
 
+Version 2.3.1
+-------------
+(released Dec 16, 2018)
+
+- Updated lexers:
+
+  * ASM (PR#784)
+  * Chapel (PR#735)
+  * Clean (PR#621)
+  * CSound (PR#684)
+  * Elm (PR#744)
+  * Fortran (PR#747)
+  * GLSL (PR#740)
+  * Hy (PR#754)
+  * PowerShell (PR#705)
+  * Python (PR#720, #1299, PR#715)
+  * SLexer (PR#680)
+  * YAML (PR#762, PR#724)
+
 Version 2.3.0
 -------------
 (released Nov 25, 2018)
-- 
cgit v1.2.1


From 816828941b69b71b0f33f7169f0001b758ec4618 Mon Sep 17 00:00:00 2001
From: "Matth?us G. Chajdas" 
Date: Sun, 16 Dec 2018 17:25:26 +0100
Subject: Prepare 2.3.1 release.

---
 CHANGES              | 2 ++
 pygments/__init__.py | 2 +-
 setup.py             | 2 +-
 3 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/CHANGES b/CHANGES
index 3b723a08..6ec1d0d4 100644
--- a/CHANGES
+++ b/CHANGES
@@ -19,7 +19,9 @@ Version 2.3.1
   * Elm (PR#744)
   * Fortran (PR#747)
   * GLSL (PR#740)
+  * Haskell (PR#745)
   * Hy (PR#754)
+  * Igor Pro (PR#764)
   * PowerShell (PR#705)
   * Python (PR#720, #1299, PR#715)
   * SLexer (PR#680)
diff --git a/pygments/__init__.py b/pygments/__init__.py
index 19aafdeb..4dd38fee 100644
--- a/pygments/__init__.py
+++ b/pygments/__init__.py
@@ -29,7 +29,7 @@ import sys
 
 from pygments.util import StringIO, BytesIO
 
-__version__ = '2.3.0'
+__version__ = '2.3.1'
 __docformat__ = 'restructuredtext'
 
 __all__ = ['lex', 'format', 'highlight']
diff --git a/setup.py b/setup.py
index 7e6eca8c..3f75a20e 100755
--- a/setup.py
+++ b/setup.py
@@ -48,7 +48,7 @@ else:
 
 setup(
     name = 'Pygments',
-    version = '2.3.0',
+    version = '2.3.1',
     url = 'http://pygments.org/',
     license = 'BSD License',
     author = 'Georg Brandl',
-- 
cgit v1.2.1


From 29ee6e7cae4150e46090b2b2a71ae3a8a0c1704e Mon Sep 17 00:00:00 2001
From: "Matth?us G. Chajdas" 
Date: Sun, 16 Dec 2018 17:29:48 +0100
Subject: Update changelog.

---
 CHANGES | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/CHANGES b/CHANGES
index 6ec1d0d4..b6362785 100644
--- a/CHANGES
+++ b/CHANGES
@@ -27,6 +27,9 @@ Version 2.3.1
   * SLexer (PR#680)
   * YAML (PR#762, PR#724)
 
+- Fix invalid string escape sequences
+- Fix `FutureWarning` introduced by regex changes in Python 3.7
+
 Version 2.3.0
 -------------
 (released Nov 25, 2018)
-- 
cgit v1.2.1

-- 
cgit v1.2.1


From 96933283e66f789e09332e7216e7e6e4519845f5 Mon Sep 17 00:00:00 2001
From: "Matth?us G. Chajdas" 
Date: Sun, 16 Dec 2018 17:41:09 +0100
Subject: Update ansi color name change version.

---
 CHANGES                            |  6 ++++++
 doc/docs/styles.rst                | 12 ++++++------
 pygments/formatters/terminal256.py |  4 ++--
 3 files changed, 14 insertions(+), 8 deletions(-)

diff --git a/CHANGES b/CHANGES
index b6362785..03471bf2 100644
--- a/CHANGES
+++ b/CHANGES
@@ -6,6 +6,12 @@ Issue numbers refer to the tracker at
 pull request numbers to the requests at
 .
 
+Version 2.4.0
+-------------
+(not released yet)
+
+- Change ANSI color names (PR#777)
+
 Version 2.3.1
 -------------
 (released Dec 16, 2018)
diff --git a/doc/docs/styles.rst b/doc/docs/styles.rst
index cd0144ab..65a2a863 100644
--- a/doc/docs/styles.rst
+++ b/doc/docs/styles.rst
@@ -203,10 +203,10 @@ settings.
 
 .. _NewAnsiColorNames:
 
-.. versionchanged:: 2.3
+.. versionchanged:: 2.4
 
 The definition of the ansi color names has changed.
-New names are easier to understand and align to the colors used in other projects. 
+New names are easier to understand and align to the colors used in other projects.
 
 
 +-------------------------+--------------------------+
@@ -218,17 +218,17 @@ New names are easier to understand and align to the colors used in other project
 |  ``ansiyellow``         |      ``#ansibrown``      |
 |  ``ansiblue``           |      ``#ansidarkblue``   |
 |  ``ansimagenta``        |      ``#ansipurple``     |
-|  ``ansicyan``           |      ``#ansiteal``       | 
-|  ``ansigray``           |      ``#ansilightgray``  | 
+|  ``ansicyan``           |      ``#ansiteal``       |
+|  ``ansigray``           |      ``#ansilightgray``  |
 |  ``ansibrightblack``    |      ``#ansidarkgray``   |
 |  ``ansibrightred``      |      ``#ansired``        |
 |  ``ansibrightgreen``    |      ``#ansigreen``      |
-|  ``ansibrightyellow``   |      ``#ansiyellow``     | 
+|  ``ansibrightyellow``   |      ``#ansiyellow``     |
 |  ``ansibrightblue``     |      ``#ansiblue``       |
 |  ``ansibrightmagenta``  |      ``#ansifuchsia``    |
 |  ``ansibrightcyan``     |      ``#ansiturquoise``  |
 |  ``ansiwhite``          |      ``#ansiwhite``      |
 +=========================+==========================+
 
-Old ansi color names are deprecated but will still work. 
+Old ansi color names are deprecated but will still work.
 
diff --git a/pygments/formatters/terminal256.py b/pygments/formatters/terminal256.py
index aec19804..b18aca65 100644
--- a/pygments/formatters/terminal256.py
+++ b/pygments/formatters/terminal256.py
@@ -110,11 +110,11 @@ class Terminal256Formatter(Formatter):
        `Terminal256Formatter` will map these to non extended foreground color.
        See :ref:`AnsiTerminalStyle` for more information.
 
-    .. versionchanged:: 2.3
+    .. versionchanged:: 2.4
        The ansi color names have been updated with names that are easier to
        understand and align with colornames of other projects and terminals.
        See :ref:`NewAnsiColorNames` for more information.
-    
+
 
     Options accepted:
 
-- 
cgit v1.2.1


From 670b662c46611d1ad8f8b15dc221026dd2b104ab Mon Sep 17 00:00:00 2001
From: "Matth?us G. Chajdas" 
Date: Sun, 16 Dec 2018 17:52:10 +0100
Subject: Fix PowerShellLexer defining built-in aliases incorrectly.

---
 pygments/lexers/shell.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pygments/lexers/shell.py b/pygments/lexers/shell.py
index 68150811..0fd73464 100644
--- a/pygments/lexers/shell.py
+++ b/pygments/lexers/shell.py
@@ -650,7 +650,7 @@ class PowerShellLexer(RegexLexer):
         'convertfrom convert connect confirm compress complete compare close '
         'clear checkpoint block backup assert approve aggregate add').split()
 
-    aliases = (
+    aliases_ = (
         'ac asnp cat cd cfs chdir clc clear clhy cli clp cls clv cnsn '
         'compare copy cp cpi cpp curl cvpa dbp del diff dir dnsn ebp echo epal '
         'epcsv epsn erase etsn exsn fc fhx fl foreach ft fw gal gbp gc gci gcm '
@@ -688,7 +688,7 @@ class PowerShellLexer(RegexLexer):
             (r'(%s)\b' % '|'.join(keywords), Keyword),
             (r'-(%s)\b' % '|'.join(operators), Operator),
             (r'(%s)-[a-z_]\w*\b' % '|'.join(verbs), Name.Builtin),
-            (r'(%s)\s' % '|'.join(aliases), Name.Builtin),
+            (r'(%s)\s' % '|'.join(aliases_), Name.Builtin),
             (r'\[[a-z_\[][\w. `,\[\]]*\]', Name.Constant),  # .net [type]s
             (r'-[a-z_]\w*', Name),
             (r'\w+', Name),
-- 
cgit v1.2.1


From 791b75c01efaf204cde2daf67087f04feeb00377 Mon Sep 17 00:00:00 2001
From: "Matth?us G. Chajdas" 
Date: Sun, 16 Dec 2018 17:52:30 +0100
Subject: Update SARL added date, regenerate mapping.

---
 pygments/lexers/_mapping.py | 3 ++-
 pygments/lexers/jvm.py      | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py
index 9ccc7615..cc029f17 100644
--- a/pygments/lexers/_mapping.py
+++ b/pygments/lexers/_mapping.py
@@ -105,7 +105,7 @@ LEXERS = {
     'CryptolLexer': ('pygments.lexers.haskell', 'Cryptol', ('cryptol', 'cry'), ('*.cry',), ('text/x-cryptol',)),
     'CrystalLexer': ('pygments.lexers.crystal', 'Crystal', ('cr', 'crystal'), ('*.cr',), ('text/x-crystal',)),
     'CsoundDocumentLexer': ('pygments.lexers.csound', 'Csound Document', ('csound-document', 'csound-csd'), ('*.csd',), ()),
-    'CsoundOrchestraLexer': ('pygments.lexers.csound', 'Csound Orchestra', ('csound', 'csound-orc'), ('*.orc',), ()),
+    'CsoundOrchestraLexer': ('pygments.lexers.csound', 'Csound Orchestra', ('csound', 'csound-orc'), ('*.orc', '*.udo'), ()),
     'CsoundScoreLexer': ('pygments.lexers.csound', 'Csound Score', ('csound-score', 'csound-sco'), ('*.sco',), ()),
     'CssDjangoLexer': ('pygments.lexers.templates', 'CSS+Django/Jinja', ('css+django', 'css+jinja'), (), ('text/css+django', 'text/css+jinja')),
     'CssErbLexer': ('pygments.lexers.templates', 'CSS+Ruby', ('css+erb', 'css+ruby'), (), ('text/css+ruby',)),
@@ -373,6 +373,7 @@ LEXERS = {
     'SASLexer': ('pygments.lexers.sas', 'SAS', ('sas',), ('*.SAS', '*.sas'), ('text/x-sas', 'text/sas', 'application/x-sas')),
     'SLexer': ('pygments.lexers.r', 'S', ('splus', 's', 'r'), ('*.S', '*.R', '.Rhistory', '.Rprofile', '.Renviron'), ('text/S-plus', 'text/S', 'text/x-r-source', 'text/x-r', 'text/x-R', 'text/x-r-history', 'text/x-r-profile')),
     'SMLLexer': ('pygments.lexers.ml', 'Standard ML', ('sml',), ('*.sml', '*.sig', '*.fun'), ('text/x-standardml', 'application/x-standardml')),
+    'SarlLexer': ('pygments.lexers.jvm', 'SARL', ('sarl',), ('*.sarl',), ('text/x-sarl',)),
     'SassLexer': ('pygments.lexers.css', 'Sass', ('sass',), ('*.sass',), ('text/x-sass',)),
     'ScalaLexer': ('pygments.lexers.jvm', 'Scala', ('scala',), ('*.scala',), ('text/x-scala',)),
     'ScamlLexer': ('pygments.lexers.html', 'Scaml', ('scaml',), ('*.scaml',), ('text/x-scaml',)),
diff --git a/pygments/lexers/jvm.py b/pygments/lexers/jvm.py
index 80d5b342..5a9a74a9 100644
--- a/pygments/lexers/jvm.py
+++ b/pygments/lexers/jvm.py
@@ -1577,7 +1577,7 @@ class SarlLexer(RegexLexer):
 	"""
 	For `SARL `_ source code.
 	
-	.. versionadded:: 0.6
+	.. versionadded:: 2.4
 	"""
 	
 	name = 'SARL'
-- 
cgit v1.2.1


From 76621e29863a35fb4381f0db9d6c84f1417930a5 Mon Sep 17 00:00:00 2001
From: "Matth?us G. Chajdas" 
Date: Mon, 17 Dec 2018 19:36:35 +0100
Subject: Add FloScript sample file, update mappings & changes.

---
 CHANGES                        |  8 ++++++++
 pygments/lexers/_mapping.py    |  4 ++--
 pygments/lexers/floscript.py   | 18 ++++++++++--------
 tests/examplefiles/example.flo | 40 ++++++++++++++++++++++++++++++++++++++++
 4 files changed, 60 insertions(+), 10 deletions(-)
 create mode 100644 tests/examplefiles/example.flo

diff --git a/CHANGES b/CHANGES
index 03471bf2..b0257335 100644
--- a/CHANGES
+++ b/CHANGES
@@ -10,6 +10,14 @@ Version 2.4.0
 -------------
 (not released yet)
 
+- Added lexers:
+
+  * FloScript (PR#750)
+
+- Updated lexers:
+
+  * Terraform (PR#787)
+
 - Change ANSI color names (PR#777)
 
 Version 2.3.1
diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py
index 1e56e262..f51d9047 100644
--- a/pygments/lexers/_mapping.py
+++ b/pygments/lexers/_mapping.py
@@ -51,7 +51,7 @@ LEXERS = {
     'BCLexer': ('pygments.lexers.algebra', 'BC', ('bc',), ('*.bc',), ()),
     'BSTLexer': ('pygments.lexers.bibtex', 'BST', ('bst', 'bst-pybtex'), ('*.bst',), ()),
     'BaseMakefileLexer': ('pygments.lexers.make', 'Base Makefile', ('basemake',), (), ()),
-    'BashLexer': ('pygments.lexers.shell', 'Bash', ('bash', 'sh', 'ksh', 'zsh', 'shell'), ('*.sh', '*.ksh', '*.bash', '*.ebuild', '*.eclass', '*.exheres-0', '*.exlib', '*.zsh', '.bashrc', 'bashrc', '.bash_*', 'bash_*', 'zshrc', '.zshrc', 'PKGBUILD'), ('application/x-sh', 'application/x-shellscript')),
+    'BashLexer': ('pygments.lexers.shell', 'Bash', ('bash', 'sh', 'ksh', 'zsh', 'shell'), ('*.sh', '*.ksh', '*.bash', '*.ebuild', '*.eclass', '*.exheres-0', '*.exlib', '*.zsh', '.bashrc', 'bashrc', '.bash_*', 'bash_*', 'zshrc', '.zshrc', 'PKGBUILD'), ('application/x-sh', 'application/x-shellscript', 'text/x-shellscript')),
     'BashSessionLexer': ('pygments.lexers.shell', 'Bash Session', ('console', 'shell-session'), ('*.sh-session', '*.shell-session'), ('application/x-shell-session', 'application/x-sh-session')),
     'BatchLexer': ('pygments.lexers.shell', 'Batchfile', ('bat', 'batch', 'dosbatch', 'winbatch'), ('*.bat', '*.cmd'), ('application/x-dos-batch',)),
     'BefungeLexer': ('pygments.lexers.esoteric', 'Befunge', ('befunge',), ('*.befunge',), ('application/x-befunge',)),
@@ -156,7 +156,7 @@ LEXERS = {
     'FennelLexer': ('pygments.lexers.lisp', 'Fennel', ('fennel', 'fnl'), ('*.fnl',), ()),
     'FishShellLexer': ('pygments.lexers.shell', 'Fish', ('fish', 'fishshell'), ('*.fish', '*.load'), ('application/x-fish',)),
     'FlatlineLexer': ('pygments.lexers.dsls', 'Flatline', ('flatline',), (), ('text/x-flatline',)),
-    'FloScriptLexer': ('pygments.lexers.floscript', 'FloScript', ('floscript|flo',), ('*.flo',), ()),
+    'FloScriptLexer': ('pygments.lexers.floscript', 'FloScript', ('floscript', 'flo'), ('*.flo',), ()),
     'ForthLexer': ('pygments.lexers.forth', 'Forth', ('forth',), ('*.frt', '*.fs'), ('application/x-forth',)),
     'FortranFixedLexer': ('pygments.lexers.fortran', 'FortranFixed', ('fortranfixed',), ('*.f', '*.F'), ()),
     'FortranLexer': ('pygments.lexers.fortran', 'Fortran', ('fortran',), ('*.f03', '*.f90', '*.F03', '*.F90'), ('text/x-fortran',)),
diff --git a/pygments/lexers/floscript.py b/pygments/lexers/floscript.py
index 9efee613..b393c1e9 100644
--- a/pygments/lexers/floscript.py
+++ b/pygments/lexers/floscript.py
@@ -3,7 +3,7 @@
     pygments.lexers.floscript
     ~~~~~~~~~~~~~~~~~~~~~~
 
-    Lexer for FloScript 
+    Lexer for FloScript
 
     :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
@@ -23,12 +23,14 @@ __all__ = ['FloScriptLexer',]
 class FloScriptLexer(RegexLexer):
     """
     For `FloScript `_ configuration language source code.
+
+    .. versionadded:: 2.4
     """
 
     name = 'FloScript'
-    aliases = ['floscript|flo']
+    aliases = ['floscript', 'flo']
     filenames = ['*.flo']
-    
+
     def innerstring_rules(ttype):
         return [
             # the old style '%s' % (...) string formatting
@@ -47,13 +49,13 @@ class FloScriptLexer(RegexLexer):
         'root': [
             (r'\n', Text),
             (r'[^\S\n]+', Text),
-            
+
             (r'[]{}:(),;[]', Punctuation),
             (r'\\\n', Text),
             (r'\\', Text),
             (r'(to|by|with|from|per|for|cum|qua|via|as|at|in|of|on|re|is|if|be|into|and|not)\b', Operator.Word),
             (r'!=|==|<<|>>|[-~+/*%=<>&^|.]', Operator),
-            (r'(load|init|server|logger|log|loggee|first|over|under|next|done|timeout|repeat|native|benter|enter|recur|exit|precur|renter|rexit|print|put|inc|copy|set|aux|rear|raze|go|let|do|bid|ready|start|stop|run|abort|use|flo|give|take)\b', Name.Builtin), 
+            (r'(load|init|server|logger|log|loggee|first|over|under|next|done|timeout|repeat|native|benter|enter|recur|exit|precur|renter|rexit|print|put|inc|copy|set|aux|rear|raze|go|let|do|bid|ready|start|stop|run|abort|use|flo|give|take)\b', Name.Builtin),
 			(r'(frame|framer|house)\b', Keyword),
             ('"', String, 'string'),
 
@@ -74,12 +76,12 @@ class FloScriptLexer(RegexLexer):
             (r'\d+L', Number.Integer.Long),
             (r'\d+j?', Number.Integer)
         ],
-        
+
         'name': [
             (r'@[\w.]+', Name.Decorator),
             (r'[a-zA-Z_]\w*', Name),
         ],
 
-        
-        
+
+
     }
diff --git a/tests/examplefiles/example.flo b/tests/examplefiles/example.flo
new file mode 100644
index 00000000..2d4ab5e7
--- /dev/null
+++ b/tests/examplefiles/example.flo
@@ -0,0 +1,40 @@
+#example mission box1.flo
+#from: https://github.com/ioflo/ioflo
+
+house box1
+
+   framer vehiclesim be active first vehicle_run
+      frame vehicle_run
+         do simulator motion uuv
+
+   framer mission be active first northleg
+      frame northleg
+         set elapsed with 20.0
+         set heading with 0.0
+         set depth with 5.0
+         set speed with 2.5
+         go next if elapsed >= goal
+
+      frame eastleg
+         set heading with 90.0
+         go next if elapsed >= goal
+
+      frame southleg
+         set heading with 180.0
+         go next if elapsed >= goal
+
+      frame westleg
+         set heading with 270.0
+         go next if elapsed >= goal
+
+      frame mission_stop
+         bid stop vehiclesim
+         bid stop autopilot
+         bid stop me
+
+   framer autopilot be active first autopilot_run
+      frame autopilot_run
+         do controller pid speed
+         do controller pid heading
+         do controller pid depth
+         do controller pid pitch
\ No newline at end of file
-- 
cgit v1.2.1


From dbef56e2176d01f7d682320ad14b928b71e43f9a Mon Sep 17 00:00:00 2001
From: Matthias Diener 
Date: Mon, 17 Dec 2018 15:25:22 -0600
Subject: add lexer for Charm++ ci files

---
 pygments/lexers/_mapping.py |  1 +
 pygments/lexers/c_like.py   | 16 +++++++++++++++-
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py
index f51d9047..c7032c96 100644
--- a/pygments/lexers/_mapping.py
+++ b/pygments/lexers/_mapping.py
@@ -80,6 +80,7 @@ LEXERS = {
     'Cfengine3Lexer': ('pygments.lexers.configs', 'CFEngine3', ('cfengine3', 'cf3'), ('*.cf',), ()),
     'ChaiscriptLexer': ('pygments.lexers.scripting', 'ChaiScript', ('chai', 'chaiscript'), ('*.chai',), ('text/x-chaiscript', 'application/x-chaiscript')),
     'ChapelLexer': ('pygments.lexers.chapel', 'Chapel', ('chapel', 'chpl'), ('*.chpl',), ()),
+    'CharmciLexer': ('pygments.lexers.c_like', 'Charmci', ('charmci',), ('*.ci',), ('text/x-c++hdr', 'text/x-c++src')),
     'CheetahHtmlLexer': ('pygments.lexers.templates', 'HTML+Cheetah', ('html+cheetah', 'html+spitfire', 'htmlcheetah'), (), ('text/html+cheetah', 'text/html+spitfire')),
     'CheetahJavascriptLexer': ('pygments.lexers.templates', 'JavaScript+Cheetah', ('js+cheetah', 'javascript+cheetah', 'js+spitfire', 'javascript+spitfire'), (), ('application/x-javascript+cheetah', 'text/x-javascript+cheetah', 'text/javascript+cheetah', 'application/x-javascript+spitfire', 'text/x-javascript+spitfire', 'text/javascript+spitfire')),
     'CheetahLexer': ('pygments.lexers.templates', 'Cheetah', ('cheetah', 'spitfire'), ('*.tmpl', '*.spt'), ('application/x-cheetah', 'application/x-spitfire')),
diff --git a/pygments/lexers/c_like.py b/pygments/lexers/c_like.py
index fd147a8a..b5ddf43c 100644
--- a/pygments/lexers/c_like.py
+++ b/pygments/lexers/c_like.py
@@ -20,7 +20,7 @@ from pygments.lexers.c_cpp import CLexer, CppLexer
 from pygments.lexers import _mql_builtins
 
 __all__ = ['PikeLexer', 'NesCLexer', 'ClayLexer', 'ECLexer', 'ValaLexer',
-           'CudaLexer', 'SwigLexer', 'MqlLexer', 'ArduinoLexer']
+           'CudaLexer', 'SwigLexer', 'MqlLexer', 'ArduinoLexer', 'CharmciLexer']
 
 
 class PikeLexer(CppLexer):
@@ -539,3 +539,17 @@ class ArduinoLexer(CppLexer):
                 yield index, Name.Function, value
             else:
                 yield index, token, value
+
+class CharmciLexer(CppLexer):
+    name = 'Charmci'
+    aliases = ['charmci']
+    filenames = ['*.ci']
+
+    tokens = {
+        'statements': [
+            (r'(module)(\s+)', bygroups(Keyword, Text), 'classname'),
+            (words(('mainmodule','mainchare','chare','array','group','nodegroup','message','conditional')), Keyword),
+            (words(('entry','aggregate','threaded','sync','exclusive','nokeep','notrace','immediate','expedited','inline','local','python','accel','readwrite','writeonly','accelblock','memcritical','packed','varsize','initproc','initnode','initcall','stacksize','createhere','createhome','reductiontarget','iget','nocopy','mutable','migratable','readonly')), Keyword),
+            inherit,
+        ],
+    }
-- 
cgit v1.2.1


From 90f099f0aed96438ee293413caffd26f1758e380 Mon Sep 17 00:00:00 2001
From: Georg Brandl 
Date: Tue, 18 Dec 2018 14:30:49 +0100
Subject: Fix a few markup problems in the docs.

---
 doc/docs/styles.rst                | 50 ++++++++++++++++++--------------------
 pygments/formatters/terminal256.py |  4 +--
 pygments/lexers/esoteric.py        |  2 +-
 pygments/lexers/floscript.py       |  2 +-
 pygments/lexers/forth.py           |  2 ++
 5 files changed, 30 insertions(+), 30 deletions(-)

diff --git a/doc/docs/styles.rst b/doc/docs/styles.rst
index 65a2a863..570293a5 100644
--- a/doc/docs/styles.rst
+++ b/doc/docs/styles.rst
@@ -201,34 +201,32 @@ The following are considered "dark" colors and will be rendered as non-bold:
 Exact behavior might depends on the terminal emulator you are using, and its
 settings.
 
-.. _NewAnsiColorNames:
+.. _new-ansi-color-names:
 
 .. versionchanged:: 2.4
 
-The definition of the ansi color names has changed.
+The definition of the ANSI color names has changed.
 New names are easier to understand and align to the colors used in other projects.
 
-
-+-------------------------+--------------------------+
-|   New names             |      Pygments 2.2        |
-+=======================+============================+
-|  ``ansiblack``          |      ``#ansiblack``      |
-|  ``ansired``            |      ``#ansidarkred``    |
-|  ``ansigreen``          |      ``#ansidarkgreen``  |
-|  ``ansiyellow``         |      ``#ansibrown``      |
-|  ``ansiblue``           |      ``#ansidarkblue``   |
-|  ``ansimagenta``        |      ``#ansipurple``     |
-|  ``ansicyan``           |      ``#ansiteal``       |
-|  ``ansigray``           |      ``#ansilightgray``  |
-|  ``ansibrightblack``    |      ``#ansidarkgray``   |
-|  ``ansibrightred``      |      ``#ansired``        |
-|  ``ansibrightgreen``    |      ``#ansigreen``      |
-|  ``ansibrightyellow``   |      ``#ansiyellow``     |
-|  ``ansibrightblue``     |      ``#ansiblue``       |
-|  ``ansibrightmagenta``  |      ``#ansifuchsia``    |
-|  ``ansibrightcyan``     |      ``#ansiturquoise``  |
-|  ``ansiwhite``          |      ``#ansiwhite``      |
-+=========================+==========================+
-
-Old ansi color names are deprecated but will still work.
-
+===================== ====================
+New names             Pygments up to 2.3
+===================== ====================
+``ansiblack``         ``#ansiblack``
+``ansired``           ``#ansidarkred``
+``ansigreen``         ``#ansidarkgreen``
+``ansiyellow``        ``#ansibrown``
+``ansiblue``          ``#ansidarkblue``
+``ansimagenta``       ``#ansipurple``
+``ansicyan``          ``#ansiteal``
+``ansigray``          ``#ansilightgray``
+``ansibrightblack``   ``#ansidarkgray``
+``ansibrightred``     ``#ansired``
+``ansibrightgreen``   ``#ansigreen``
+``ansibrightyellow``  ``#ansiyellow``
+``ansibrightblue``    ``#ansiblue``
+``ansibrightmagenta`` ``#ansifuchsia``
+``ansibrightcyan``    ``#ansiturquoise``
+``ansiwhite``         ``#ansiwhite``
+===================== ====================
+
+Old ANSI color names are deprecated but will still work.
diff --git a/pygments/formatters/terminal256.py b/pygments/formatters/terminal256.py
index b18aca65..1235e9ed 100644
--- a/pygments/formatters/terminal256.py
+++ b/pygments/formatters/terminal256.py
@@ -111,9 +111,9 @@ class Terminal256Formatter(Formatter):
        See :ref:`AnsiTerminalStyle` for more information.
 
     .. versionchanged:: 2.4
-       The ansi color names have been updated with names that are easier to
+       The ANSI color names have been updated with names that are easier to
        understand and align with colornames of other projects and terminals.
-       See :ref:`NewAnsiColorNames` for more information.
+       See :ref:`this table ` for more information.
 
 
     Options accepted:
diff --git a/pygments/lexers/esoteric.py b/pygments/lexers/esoteric.py
index 793c28be..26222c9f 100644
--- a/pygments/lexers/esoteric.py
+++ b/pygments/lexers/esoteric.py
@@ -245,7 +245,7 @@ class AheuiLexer(RegexLexer):
 
     Aheui_ is esoteric language based on Korean alphabets.
 
-    .. _Aheui:: http://aheui.github.io/
+    .. _Aheui: http://aheui.github.io/
 
     """
 
diff --git a/pygments/lexers/floscript.py b/pygments/lexers/floscript.py
index b393c1e9..4f200809 100644
--- a/pygments/lexers/floscript.py
+++ b/pygments/lexers/floscript.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 """
     pygments.lexers.floscript
-    ~~~~~~~~~~~~~~~~~~~~~~
+    ~~~~~~~~~~~~~~~~~~~~~~~~~
 
     Lexer for FloScript
 
diff --git a/pygments/lexers/forth.py b/pygments/lexers/forth.py
index a51f1b57..7fecdd52 100644
--- a/pygments/lexers/forth.py
+++ b/pygments/lexers/forth.py
@@ -3,6 +3,8 @@
     pygments.lexers.forth
     ~~~~~~~~~~~~~~~~~~~~~
 
+    Lexer for the Forth language.
+
     :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
-- 
cgit v1.2.1


From df656b31ecabd8a1b649f79de050854a2974e395 Mon Sep 17 00:00:00 2001
From: Mauricio Caceres Bravo 
Date: Tue, 18 Dec 2018 13:31:32 -0500
Subject: Stata lexer and styles improvements and bug fixes

- Nested comments correctly highlighted.
- Globals and locals correctly nested inside each other and strings.
- Extended locals and stored results correctly highlighted.
- Keywords and operators correctly highlighted.
- Parentheses and functions correctly highlighted.
---
 pygments/lexers/_stata_builtins.py | 421 +++++++++++++++++++++++++++
 pygments/lexers/stata.py           | 565 ++++++++-----------------------------
 pygments/styles/__init__.py        |   9 +-
 pygments/styles/stata.py           |  37 ---
 pygments/styles/stata_dark.py      |  40 +++
 pygments/styles/stata_light.py     |  39 +++
 6 files changed, 622 insertions(+), 489 deletions(-)
 create mode 100644 pygments/lexers/_stata_builtins.py
 delete mode 100644 pygments/styles/stata.py
 create mode 100644 pygments/styles/stata_dark.py
 create mode 100644 pygments/styles/stata_light.py

diff --git a/pygments/lexers/_stata_builtins.py b/pygments/lexers/_stata_builtins.py
new file mode 100644
index 00000000..13a3dacf
--- /dev/null
+++ b/pygments/lexers/_stata_builtins.py
@@ -0,0 +1,421 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.lexers._stata_builtins
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Builtins for Stata
+
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+
+builtins_special = (
+    "if", "in", "using", "replace", "by", "gen", "generate"
+)
+
+builtins_base = (
+    "if", "else", "else\s+if", "in", "foreach", "for", "forv", "forva",
+    "forval", "forvalu", "forvalue", "forvalues", "by", "bys",
+    "bysort", "quietly", "qui", "about", "ac",
+    "ac_7", "acprplot", "acprplot_7", "adjust", "ado", "adopath",
+    "adoupdate", "alpha", "ameans", "an", "ano", "anov", "anova",
+    "anova_estat", "anova_terms", "anovadef", "aorder", "ap", "app",
+    "appe", "appen", "append", "arch", "arch_dr", "arch_estat",
+    "arch_p", "archlm", "areg", "areg_p", "args", "arima",
+    "arima_dr", "arima_estat", "arima_p", "as", "asmprobit",
+    "asmprobit_estat", "asmprobit_lf", "asmprobit_mfx__dlg",
+    "asmprobit_p", "ass", "asse", "asser", "assert", "avplot",
+    "avplot_7", "avplots", "avplots_7", "bcskew0", "bgodfrey",
+    "binreg", "bip0_lf", "biplot", "bipp_lf", "bipr_lf",
+    "bipr_p", "biprobit", "bitest", "bitesti", "bitowt", "blogit",
+    "bmemsize", "boot", "bootsamp", "bootstrap", "bootstrap_8",
+    "boxco_l", "boxco_p", "boxcox", "boxcox_6", "boxcox_p",
+    "bprobit", "br", "break", "brier", "bro", "brow", "brows",
+    "browse", "brr", "brrstat", "bs", "bs_7", "bsampl_w",
+    "bsample", "bsample_7", "bsqreg", "bstat", "bstat_7", "bstat_8",
+    "bstrap", "bstrap_7", "ca", "ca_estat", "ca_p", "cabiplot",
+    "camat", "canon", "canon_8", "canon_8_p", "canon_estat",
+    "canon_p", "cap", "caprojection", "capt", "captu", "captur",
+    "capture", "cat", "cc", "cchart", "cchart_7", "cci",
+    "cd", "censobs_table", "centile", "cf", "char", "chdir",
+    "checkdlgfiles", "checkestimationsample", "checkhlpfiles",
+    "checksum", "chelp", "ci", "cii", "cl", "class", "classutil",
+    "clear", "cli", "clis", "clist", "clo", "clog", "clog_lf",
+    "clog_p", "clogi", "clogi_sw", "clogit", "clogit_lf",
+    "clogit_p", "clogitp", "clogl_sw", "cloglog", "clonevar",
+    "clslistarray", "cluster", "cluster_measures", "cluster_stop",
+    "cluster_tree", "cluster_tree_8", "clustermat", "cmdlog",
+    "cnr", "cnre", "cnreg", "cnreg_p", "cnreg_sw", "cnsreg",
+    "codebook", "collaps4", "collapse", "colormult_nb",
+    "colormult_nw", "compare", "compress", "conf", "confi",
+    "confir", "confirm", "conren", "cons", "const", "constr",
+    "constra", "constrai", "constrain", "constraint", "continue",
+    "contract", "copy", "copyright", "copysource", "cor", "corc",
+    "corr", "corr2data", "corr_anti", "corr_kmo", "corr_smc",
+    "corre", "correl", "correla", "correlat", "correlate",
+    "corrgram", "cou", "coun", "count", "cox", "cox_p", "cox_sw",
+    "coxbase", "coxhaz", "coxvar", "cprplot", "cprplot_7",
+    "crc", "cret", "cretu", "cretur", "creturn", "cross", "cs",
+    "cscript", "cscript_log", "csi", "ct", "ct_is", "ctset",
+    "ctst_5", "ctst_st", "cttost", "cumsp", "cumsp_7", "cumul",
+    "cusum", "cusum_7", "cutil", "d", "datasig", "datasign",
+    "datasigna", "datasignat", "datasignatu", "datasignatur",
+    "datasignature", "datetof", "db", "dbeta", "de", "dec",
+    "deco", "decod", "decode", "deff", "des", "desc", "descr",
+    "descri", "describ", "describe", "destring", "dfbeta",
+    "dfgls", "dfuller", "di", "di_g", "dir", "dirstats", "dis",
+    "discard", "disp", "disp_res", "disp_s", "displ", "displa",
+    "display", "distinct", "do", "doe", "doed", "doedi",
+    "doedit", "dotplot", "dotplot_7", "dprobit", "drawnorm",
+    "drop", "ds", "ds_util", "dstdize", "duplicates", "durbina",
+    "dwstat", "dydx", "e", "ed", "edi", "edit", "egen",
+    "eivreg", "emdef", "end", "en", "enc", "enco", "encod", "encode",
+    "eq", "erase", "ereg", "ereg_lf", "ereg_p", "ereg_sw",
+    "ereghet", "ereghet_glf", "ereghet_glf_sh", "ereghet_gp",
+    "ereghet_ilf", "ereghet_ilf_sh", "ereghet_ip", "eret",
+    "eretu", "eretur", "ereturn", "err", "erro", "error", "est",
+    "est_cfexist", "est_cfname", "est_clickable", "est_expand",
+    "est_hold", "est_table", "est_unhold", "est_unholdok",
+    "estat", "estat_default", "estat_summ", "estat_vce_only",
+    "esti", "estimates", "etodow", "etof", "etomdy", "ex",
+    "exi", "exit", "expand", "expandcl", "fac", "fact", "facto",
+    "factor", "factor_estat", "factor_p", "factor_pca_rotated",
+    "factor_rotate", "factormat", "fcast", "fcast_compute",
+    "fcast_graph", "fdades", "fdadesc", "fdadescr", "fdadescri",
+    "fdadescrib", "fdadescribe", "fdasav", "fdasave", "fdause",
+    "fh_st", "open", "read", "close",
+    "file", "filefilter", "fillin", "find_hlp_file", "findfile",
+    "findit", "findit_7", "fit", "fl", "fli", "flis", "flist",
+    "for5_0", "form", "forma", "format", "fpredict", "frac_154",
+    "frac_adj", "frac_chk", "frac_cox", "frac_ddp", "frac_dis",
+    "frac_dv", "frac_in", "frac_mun", "frac_pp", "frac_pq",
+    "frac_pv", "frac_wgt", "frac_xo", "fracgen", "fracplot",
+    "fracplot_7", "fracpoly", "fracpred", "fron_ex", "fron_hn",
+    "fron_p", "fron_tn", "fron_tn2", "frontier", "ftodate", "ftoe",
+    "ftomdy", "ftowdate", "g", "gamhet_glf", "gamhet_gp",
+    "gamhet_ilf", "gamhet_ip", "gamma", "gamma_d2", "gamma_p",
+    "gamma_sw", "gammahet", "gdi_hexagon", "gdi_spokes", "ge",
+    "gen", "gene", "gener", "genera", "generat", "generate",
+    "genrank", "genstd", "genvmean", "gettoken", "gl", "gladder",
+    "gladder_7", "glim_l01", "glim_l02", "glim_l03", "glim_l04",
+    "glim_l05", "glim_l06", "glim_l07", "glim_l08", "glim_l09",
+    "glim_l10", "glim_l11", "glim_l12", "glim_lf", "glim_mu",
+    "glim_nw1", "glim_nw2", "glim_nw3", "glim_p", "glim_v1",
+    "glim_v2", "glim_v3", "glim_v4", "glim_v5", "glim_v6",
+    "glim_v7", "glm", "glm_6", "glm_p", "glm_sw", "glmpred", "glo",
+    "glob", "globa", "global", "glogit", "glogit_8", "glogit_p",
+    "gmeans", "gnbre_lf", "gnbreg", "gnbreg_5", "gnbreg_p",
+    "gomp_lf", "gompe_sw", "gomper_p", "gompertz", "gompertzhet",
+    "gomphet_glf", "gomphet_glf_sh", "gomphet_gp", "gomphet_ilf",
+    "gomphet_ilf_sh", "gomphet_ip", "gphdot", "gphpen",
+    "gphprint", "gprefs", "gprobi_p", "gprobit", "gprobit_8", "gr",
+    "gr7", "gr_copy", "gr_current", "gr_db", "gr_describe",
+    "gr_dir", "gr_draw", "gr_draw_replay", "gr_drop", "gr_edit",
+    "gr_editviewopts", "gr_example", "gr_example2", "gr_export",
+    "gr_print", "gr_qscheme", "gr_query", "gr_read", "gr_rename",
+    "gr_replay", "gr_save", "gr_set", "gr_setscheme", "gr_table",
+    "gr_undo", "gr_use", "graph", "graph7", "grebar", "greigen",
+    "greigen_7", "greigen_8", "grmeanby", "grmeanby_7",
+    "gs_fileinfo", "gs_filetype", "gs_graphinfo", "gs_stat",
+    "gsort", "gwood", "h", "hadimvo", "hareg", "hausman",
+    "haver", "he", "heck_d2", "heckma_p", "heckman", "heckp_lf",
+    "heckpr_p", "heckprob", "hel", "help", "hereg", "hetpr_lf",
+    "hetpr_p", "hetprob", "hettest", "hexdump", "hilite",
+    "hist", "hist_7", "histogram", "hlogit", "hlu", "hmeans",
+    "hotel", "hotelling", "hprobit", "hreg", "hsearch", "icd9",
+    "icd9_ff", "icd9p", "iis", "impute", "imtest", "inbase",
+    "include", "inf", "infi", "infil", "infile", "infix", "inp",
+    "inpu", "input", "ins", "insheet", "insp", "inspe",
+    "inspec", "inspect", "integ", "inten", "intreg", "intreg_7",
+    "intreg_p", "intrg2_ll", "intrg_ll", "intrg_ll2", "ipolate",
+    "iqreg", "ir", "irf", "irf_create", "irfm", "iri", "is_svy",
+    "is_svysum", "isid", "istdize", "ivprob_1_lf", "ivprob_lf",
+    "ivprobit", "ivprobit_p", "ivreg", "ivreg_footnote",
+    "ivtob_1_lf", "ivtob_lf", "ivtobit", "ivtobit_p", "jackknife",
+    "jacknife", "jknife", "jknife_6", "jknife_8", "jkstat",
+    "joinby", "kalarma1", "kap", "kap_3", "kapmeier", "kappa",
+    "kapwgt", "kdensity", "kdensity_7", "keep", "ksm", "ksmirnov",
+    "ktau", "kwallis", "l", "la", "lab", "labe", "label",
+    "labelbook", "ladder", "levels", "levelsof", "leverage",
+    "lfit", "lfit_p", "li", "lincom", "line", "linktest",
+    "lis", "list", "lloghet_glf", "lloghet_glf_sh", "lloghet_gp",
+    "lloghet_ilf", "lloghet_ilf_sh", "lloghet_ip", "llogi_sw",
+    "llogis_p", "llogist", "llogistic", "llogistichet",
+    "lnorm_lf", "lnorm_sw", "lnorma_p", "lnormal", "lnormalhet",
+    "lnormhet_glf", "lnormhet_glf_sh", "lnormhet_gp",
+    "lnormhet_ilf", "lnormhet_ilf_sh", "lnormhet_ip", "lnskew0",
+    "loadingplot", "loc", "loca", "local", "log", "logi",
+    "logis_lf", "logistic", "logistic_p", "logit", "logit_estat",
+    "logit_p", "loglogs", "logrank", "loneway", "lookfor",
+    "lookup", "lowess", "lowess_7", "lpredict", "lrecomp", "lroc",
+    "lroc_7", "lrtest", "ls", "lsens", "lsens_7", "lsens_x",
+    "lstat", "ltable", "ltable_7", "ltriang", "lv", "lvr2plot",
+    "lvr2plot_7", "m", "ma", "mac", "macr", "macro", "makecns",
+    "man", "manova", "manova_estat", "manova_p", "manovatest",
+    "mantel", "mark", "markin", "markout", "marksample", "mat",
+    "mat_capp", "mat_order", "mat_put_rr", "mat_rapp", "mata",
+    "mata_clear", "mata_describe", "mata_drop", "mata_matdescribe",
+    "mata_matsave", "mata_matuse", "mata_memory", "mata_mlib",
+    "mata_mosave", "mata_rename", "mata_which", "matalabel",
+    "matcproc", "matlist", "matname", "matr", "matri",
+    "matrix", "matrix_input__dlg", "matstrik", "mcc", "mcci",
+    "md0_", "md1_", "md1debug_", "md2_", "md2debug_", "mds",
+    "mds_estat", "mds_p", "mdsconfig", "mdslong", "mdsmat",
+    "mdsshepard", "mdytoe", "mdytof", "me_derd", "mean",
+    "means", "median", "memory", "memsize", "meqparse", "mer",
+    "merg", "merge", "mfp", "mfx", "mhelp", "mhodds", "minbound",
+    "mixed_ll", "mixed_ll_reparm", "mkassert", "mkdir",
+    "mkmat", "mkspline", "ml", "ml_5", "ml_adjs", "ml_bhhhs",
+    "ml_c_d", "ml_check", "ml_clear", "ml_cnt", "ml_debug",
+    "ml_defd", "ml_e0", "ml_e0_bfgs", "ml_e0_cycle", "ml_e0_dfp",
+    "ml_e0i", "ml_e1", "ml_e1_bfgs", "ml_e1_bhhh", "ml_e1_cycle",
+    "ml_e1_dfp", "ml_e2", "ml_e2_cycle", "ml_ebfg0", "ml_ebfr0",
+    "ml_ebfr1", "ml_ebh0q", "ml_ebhh0", "ml_ebhr0", "ml_ebr0i",
+    "ml_ecr0i", "ml_edfp0", "ml_edfr0", "ml_edfr1", "ml_edr0i",
+    "ml_eds", "ml_eer0i", "ml_egr0i", "ml_elf", "ml_elf_bfgs",
+    "ml_elf_bhhh", "ml_elf_cycle", "ml_elf_dfp", "ml_elfi",
+    "ml_elfs", "ml_enr0i", "ml_enrr0", "ml_erdu0", "ml_erdu0_bfgs",
+    "ml_erdu0_bhhh", "ml_erdu0_bhhhq", "ml_erdu0_cycle",
+    "ml_erdu0_dfp", "ml_erdu0_nrbfgs", "ml_exde", "ml_footnote",
+    "ml_geqnr", "ml_grad0", "ml_graph", "ml_hbhhh", "ml_hd0",
+    "ml_hold", "ml_init", "ml_inv", "ml_log", "ml_max",
+    "ml_mlout", "ml_mlout_8", "ml_model", "ml_nb0", "ml_opt",
+    "ml_p", "ml_plot", "ml_query", "ml_rdgrd", "ml_repor",
+    "ml_s_e", "ml_score", "ml_searc", "ml_technique", "ml_unhold",
+    "mleval", "mlf_", "mlmatbysum", "mlmatsum", "mlog", "mlogi",
+    "mlogit", "mlogit_footnote", "mlogit_p", "mlopts", "mlsum",
+    "mlvecsum", "mnl0_", "mor", "more", "mov", "move", "mprobit",
+    "mprobit_lf", "mprobit_p", "mrdu0_", "mrdu1_", "mvdecode",
+    "mvencode", "mvreg", "mvreg_estat", "n", "nbreg",
+    "nbreg_al", "nbreg_lf", "nbreg_p", "nbreg_sw", "nestreg", "net",
+    "newey", "newey_7", "newey_p", "news", "nl", "nl_7", "nl_9",
+    "nl_9_p", "nl_p", "nl_p_7", "nlcom", "nlcom_p", "nlexp2",
+    "nlexp2_7", "nlexp2a", "nlexp2a_7", "nlexp3", "nlexp3_7",
+    "nlgom3", "nlgom3_7", "nlgom4", "nlgom4_7", "nlinit", "nllog3",
+    "nllog3_7", "nllog4", "nllog4_7", "nlog_rd", "nlogit",
+    "nlogit_p", "nlogitgen", "nlogittree", "nlpred", "no",
+    "nobreak", "noi", "nois", "noisi", "noisil", "noisily", "note",
+    "notes", "notes_dlg", "nptrend", "numlabel", "numlist", "odbc",
+    "old_ver", "olo", "olog", "ologi", "ologi_sw", "ologit",
+    "ologit_p", "ologitp", "on", "one", "onew", "onewa", "oneway",
+    "op_colnm", "op_comp", "op_diff", "op_inv", "op_str", "opr",
+    "opro", "oprob", "oprob_sw", "oprobi", "oprobi_p", "oprobit",
+    "oprobitp", "opts_exclusive", "order", "orthog", "orthpoly",
+    "ou", "out", "outf", "outfi", "outfil", "outfile", "outs",
+    "outsh", "outshe", "outshee", "outsheet", "ovtest", "pac",
+    "pac_7", "palette", "parse", "parse_dissim", "pause", "pca",
+    "pca_8", "pca_display", "pca_estat", "pca_p", "pca_rotate",
+    "pcamat", "pchart", "pchart_7", "pchi", "pchi_7", "pcorr",
+    "pctile", "pentium", "pergram", "pergram_7", "permute",
+    "permute_8", "personal", "peto_st", "pkcollapse", "pkcross",
+    "pkequiv", "pkexamine", "pkexamine_7", "pkshape", "pksumm",
+    "pksumm_7", "pl", "plo", "plot", "plugin", "pnorm",
+    "pnorm_7", "poisgof", "poiss_lf", "poiss_sw", "poisso_p",
+    "poisson", "poisson_estat", "post", "postclose", "postfile",
+    "postutil", "pperron", "pr", "prais", "prais_e", "prais_e2",
+    "prais_p", "predict", "predictnl", "preserve", "print",
+    "pro", "prob", "probi", "probit", "probit_estat", "probit_p",
+    "proc_time", "procoverlay", "procrustes", "procrustes_estat",
+    "procrustes_p", "profiler", "prog", "progr", "progra",
+    "program", "prop", "proportion", "prtest", "prtesti", "pwcorr",
+    "pwd", "q", "s", "qby", "qbys", "qchi", "qchi_7", "qladder",
+    "qladder_7", "qnorm", "qnorm_7", "qqplot", "qqplot_7", "qreg",
+    "qreg_c", "qreg_p", "qreg_sw", "qu", "quadchk", "quantile",
+    "quantile_7", "que", "quer", "query", "range", "ranksum",
+    "ratio", "rchart", "rchart_7", "rcof", "recast", "reclink",
+    "recode", "reg", "reg3", "reg3_p", "regdw", "regr", "regre",
+    "regre_p2", "regres", "regres_p", "regress", "regress_estat",
+    "regriv_p", "remap", "ren", "rena", "renam", "rename",
+    "renpfix", "repeat", "replace", "report", "reshape",
+    "restore", "ret", "retu", "retur", "return", "rm", "rmdir",
+    "robvar", "roccomp", "roccomp_7", "roccomp_8", "rocf_lf",
+    "rocfit", "rocfit_8", "rocgold", "rocplot", "rocplot_7",
+    "roctab", "roctab_7", "rolling", "rologit", "rologit_p",
+    "rot", "rota", "rotat", "rotate", "rotatemat", "rreg",
+    "rreg_p", "ru", "run", "runtest", "rvfplot", "rvfplot_7",
+    "rvpplot", "rvpplot_7", "sa", "safesum", "sample",
+    "sampsi", "sav", "save", "savedresults", "saveold", "sc",
+    "sca", "scal", "scala", "scalar", "scatter", "scm_mine",
+    "sco", "scob_lf", "scob_p", "scobi_sw", "scobit", "scor",
+    "score", "scoreplot", "scoreplot_help", "scree", "screeplot",
+    "screeplot_help", "sdtest", "sdtesti", "se", "search",
+    "separate", "seperate", "serrbar", "serrbar_7", "serset", "set",
+    "set_defaults", "sfrancia", "sh", "she", "shel", "shell",
+    "shewhart", "shewhart_7", "signestimationsample", "signrank",
+    "signtest", "simul", "simul_7", "simulate", "simulate_8",
+    "sktest", "sleep", "slogit", "slogit_d2", "slogit_p", "smooth",
+    "snapspan", "so", "sor", "sort", "spearman", "spikeplot",
+    "spikeplot_7", "spikeplt", "spline_x", "split", "sqreg",
+    "sqreg_p", "sret", "sretu", "sretur", "sreturn", "ssc", "st",
+    "st_ct", "st_hc", "st_hcd", "st_hcd_sh", "st_is", "st_issys",
+    "st_note", "st_promo", "st_set", "st_show", "st_smpl",
+    "st_subid", "stack", "statsby", "statsby_8", "stbase", "stci",
+    "stci_7", "stcox", "stcox_estat", "stcox_fr", "stcox_fr_ll",
+    "stcox_p", "stcox_sw", "stcoxkm", "stcoxkm_7", "stcstat",
+    "stcurv", "stcurve", "stcurve_7", "stdes", "stem", "stepwise",
+    "stereg", "stfill", "stgen", "stir", "stjoin", "stmc", "stmh",
+    "stphplot", "stphplot_7", "stphtest", "stphtest_7",
+    "stptime", "strate", "strate_7", "streg", "streg_sw", "streset",
+    "sts", "sts_7", "stset", "stsplit", "stsum", "sttocc",
+    "sttoct", "stvary", "stweib", "su", "suest", "suest_8",
+    "sum", "summ", "summa", "summar", "summari", "summariz",
+    "summarize", "sunflower", "sureg", "survcurv", "survsum",
+    "svar", "svar_p", "svmat", "svy", "svy_disp", "svy_dreg",
+    "svy_est", "svy_est_7", "svy_estat", "svy_get", "svy_gnbreg_p",
+    "svy_head", "svy_header", "svy_heckman_p", "svy_heckprob_p",
+    "svy_intreg_p", "svy_ivreg_p", "svy_logistic_p", "svy_logit_p",
+    "svy_mlogit_p", "svy_nbreg_p", "svy_ologit_p", "svy_oprobit_p",
+    "svy_poisson_p", "svy_probit_p", "svy_regress_p", "svy_sub",
+    "svy_sub_7", "svy_x", "svy_x_7", "svy_x_p", "svydes",
+    "svydes_8", "svygen", "svygnbreg", "svyheckman", "svyheckprob",
+    "svyintreg", "svyintreg_7", "svyintrg", "svyivreg", "svylc",
+    "svylog_p", "svylogit", "svymarkout", "svymarkout_8",
+    "svymean", "svymlog", "svymlogit", "svynbreg", "svyolog",
+    "svyologit", "svyoprob", "svyoprobit", "svyopts",
+    "svypois", "svypois_7", "svypoisson", "svyprobit", "svyprobt",
+    "svyprop", "svyprop_7", "svyratio", "svyreg", "svyreg_p",
+    "svyregress", "svyset", "svyset_7", "svyset_8", "svytab",
+    "svytab_7", "svytest", "svytotal", "sw", "sw_8", "swcnreg",
+    "swcox", "swereg", "swilk", "swlogis", "swlogit",
+    "swologit", "swoprbt", "swpois", "swprobit", "swqreg",
+    "swtobit", "swweib", "symmetry", "symmi", "symplot",
+    "symplot_7", "syntax", "sysdescribe", "sysdir", "sysuse",
+    "szroeter", "ta", "tab", "tab1", "tab2", "tab_or", "tabd",
+    "tabdi", "tabdis", "tabdisp", "tabi", "table", "tabodds",
+    "tabodds_7", "tabstat", "tabu", "tabul", "tabula", "tabulat",
+    "tabulate", "te", "tempfile", "tempname", "tempvar", "tes",
+    "test", "testnl", "testparm", "teststd", "tetrachoric",
+    "time_it", "timer", "tis", "tob", "tobi", "tobit", "tobit_p",
+    "tobit_sw", "token", "tokeni", "tokeniz", "tokenize",
+    "tostring", "total", "translate", "translator", "transmap",
+    "treat_ll", "treatr_p", "treatreg", "trim", "trnb_cons",
+    "trnb_mean", "trpoiss_d2", "trunc_ll", "truncr_p", "truncreg",
+    "tsappend", "tset", "tsfill", "tsline", "tsline_ex",
+    "tsreport", "tsrevar", "tsrline", "tsset", "tssmooth",
+    "tsunab", "ttest", "ttesti", "tut_chk", "tut_wait", "tutorial",
+    "tw", "tware_st", "two", "twoway", "twoway__fpfit_serset",
+    "twoway__function_gen", "twoway__histogram_gen",
+    "twoway__ipoint_serset", "twoway__ipoints_serset",
+    "twoway__kdensity_gen", "twoway__lfit_serset",
+    "twoway__normgen_gen", "twoway__pci_serset",
+    "twoway__qfit_serset", "twoway__scatteri_serset",
+    "twoway__sunflower_gen", "twoway_ksm_serset", "ty", "typ",
+    "type", "typeof", "u", "unab", "unabbrev", "unabcmd",
+    "update", "us", "use", "uselabel", "var", "var_mkcompanion",
+    "var_p", "varbasic", "varfcast", "vargranger", "varirf",
+    "varirf_add", "varirf_cgraph", "varirf_create", "varirf_ctable",
+    "varirf_describe", "varirf_dir", "varirf_drop", "varirf_erase",
+    "varirf_graph", "varirf_ograph", "varirf_rename", "varirf_set",
+    "varirf_table", "varlist", "varlmar", "varnorm", "varsoc",
+    "varstable", "varstable_w", "varstable_w2", "varwle",
+    "vce", "vec", "vec_fevd", "vec_mkphi", "vec_p", "vec_p_w",
+    "vecirf_create", "veclmar", "veclmar_w", "vecnorm",
+    "vecnorm_w", "vecrank", "vecstable", "verinst", "vers",
+    "versi", "versio", "version", "view", "viewsource", "vif",
+    "vwls", "wdatetof", "webdescribe", "webseek", "webuse",
+    "weib1_lf", "weib2_lf", "weib_lf", "weib_lf0", "weibhet_glf",
+    "weibhet_glf_sh", "weibhet_glfa", "weibhet_glfa_sh",
+    "weibhet_gp", "weibhet_ilf", "weibhet_ilf_sh", "weibhet_ilfa",
+    "weibhet_ilfa_sh", "weibhet_ip", "weibu_sw", "weibul_p",
+    "weibull", "weibull_c", "weibull_s", "weibullhet",
+    "wh", "whelp", "whi", "which", "whil", "while", "wilc_st",
+    "wilcoxon", "win", "wind", "windo", "window", "winexec",
+    "wntestb", "wntestb_7", "wntestq", "xchart", "xchart_7",
+    "xcorr", "xcorr_7", "xi", "xi_6", "xmlsav", "xmlsave",
+    "xmluse", "xpose", "xsh", "xshe", "xshel", "xshell",
+    "xt_iis", "xt_tis", "xtab_p", "xtabond", "xtbin_p",
+    "xtclog", "xtcloglog", "xtcloglog_8", "xtcloglog_d2",
+    "xtcloglog_pa_p", "xtcloglog_re_p", "xtcnt_p", "xtcorr",
+    "xtdata", "xtdes", "xtfront_p", "xtfrontier", "xtgee",
+    "xtgee_elink", "xtgee_estat", "xtgee_makeivar", "xtgee_p",
+    "xtgee_plink", "xtgls", "xtgls_p", "xthaus", "xthausman",
+    "xtht_p", "xthtaylor", "xtile", "xtint_p", "xtintreg",
+    "xtintreg_8", "xtintreg_d2", "xtintreg_p", "xtivp_1",
+    "xtivp_2", "xtivreg", "xtline", "xtline_ex", "xtlogit",
+    "xtlogit_8", "xtlogit_d2", "xtlogit_fe_p", "xtlogit_pa_p",
+    "xtlogit_re_p", "xtmixed", "xtmixed_estat", "xtmixed_p",
+    "xtnb_fe", "xtnb_lf", "xtnbreg", "xtnbreg_pa_p",
+    "xtnbreg_refe_p", "xtpcse", "xtpcse_p", "xtpois", "xtpoisson",
+    "xtpoisson_d2", "xtpoisson_pa_p", "xtpoisson_refe_p", "xtpred",
+    "xtprobit", "xtprobit_8", "xtprobit_d2", "xtprobit_re_p",
+    "xtps_fe", "xtps_lf", "xtps_ren", "xtps_ren_8", "xtrar_p",
+    "xtrc", "xtrc_p", "xtrchh", "xtrefe_p", "xtreg", "xtreg_be",
+    "xtreg_fe", "xtreg_ml", "xtreg_pa_p", "xtreg_re",
+    "xtregar", "xtrere_p", "xtset", "xtsf_ll", "xtsf_llti",
+    "xtsum", "xttab", "xttest0", "xttobit", "xttobit_8",
+    "xttobit_p", "xttrans", "yx", "yxview__barlike_draw",
+    "yxview_area_draw", "yxview_bar_draw", "yxview_dot_draw",
+    "yxview_dropline_draw", "yxview_function_draw",
+    "yxview_iarrow_draw", "yxview_ilabels_draw",
+    "yxview_normal_draw", "yxview_pcarrow_draw",
+    "yxview_pcbarrow_draw", "yxview_pccapsym_draw",
+    "yxview_pcscatter_draw", "yxview_pcspike_draw",
+    "yxview_rarea_draw", "yxview_rbar_draw", "yxview_rbarm_draw",
+    "yxview_rcap_draw", "yxview_rcapsym_draw",
+    "yxview_rconnected_draw", "yxview_rline_draw",
+    "yxview_rscatter_draw", "yxview_rspike_draw",
+    "yxview_spike_draw", "yxview_sunflower_draw", "zap_s", "zinb",
+    "zinb_llf", "zinb_plf", "zip", "zip_llf", "zip_p", "zip_plf",
+    "zt_ct_5", "zt_hc_5", "zt_hcd_5", "zt_is_5", "zt_iss_5",
+    "zt_sho_5", "zt_smp_5", "ztbase_5", "ztcox_5", "ztdes_5",
+    "ztereg_5", "ztfill_5", "ztgen_5", "ztir_5", "ztjoin_5", "ztnb",
+    "ztnb_p", "ztp", "ztp_p", "zts_5", "ztset_5", "ztspli_5",
+    "ztsum_5", "zttoct_5", "ztvary_5", "ztweib_5"
+)
+
+builtins_functions = (
+    "Cdhms", "Chms", "Clock", "Cmdyhms", "Cofc", "Cofd", "F",
+    "Fden", "Ftail", "I", "J", "_caller", "abbrev", "abs", "acos",
+    "acosh", "asin", "asinh", "atan", "atan2", "atanh",
+    "autocode", "betaden", "binomial", "binomialp", "binomialtail",
+    "binormal", "bofd", "byteorder", "c", "ceil", "char",
+    "chi2", "chi2den", "chi2tail", "cholesky", "chop", "clip",
+    "clock", "cloglog", "cofC", "cofd", "colnumb", "colsof", "comb",
+    "cond", "corr", "cos", "cosh", "d", "daily", "date", "day",
+    "det", "dgammapda", "dgammapdada", "dgammapdadx", "dgammapdx",
+    "dgammapdxdx", "dhms", "diag", "diag0cnt", "digamma",
+    "dofC", "dofb", "dofc", "dofh", "dofm", "dofq", "dofw",
+    "dofy", "dow", "doy", "dunnettprob", "e", "el", "epsdouble",
+    "epsfloat", "exp", "fileexists", "fileread", "filereaderror",
+    "filewrite", "float", "floor", "fmtwidth", "gammaden",
+    "gammap", "gammaptail", "get", "group", "h", "hadamard",
+    "halfyear", "halfyearly", "has_eprop", "hh", "hhC", "hms",
+    "hofd", "hours", "hypergeometric", "hypergeometricp", "ibeta",
+    "ibetatail", "index", "indexnot", "inlist", "inrange", "int",
+    "inv", "invF", "invFtail", "invbinomial", "invbinomialtail",
+    "invchi2", "invchi2tail", "invcloglog", "invdunnettprob",
+    "invgammap", "invgammaptail", "invibeta", "invibetatail",
+    "invlogit", "invnFtail", "invnbinomial", "invnbinomialtail",
+    "invnchi2", "invnchi2tail", "invnibeta", "invnorm", "invnormal",
+    "invnttail", "invpoisson", "invpoissontail", "invsym", "invt",
+    "invttail", "invtukeyprob", "irecode", "issym", "issymmetric",
+    "itrim", "length", "ln", "lnfact", "lnfactorial", "lngamma",
+    "lnnormal", "lnnormalden", "log", "log10", "logit", "lower",
+    "ltrim", "m", "match", "matmissing", "matrix", "matuniform",
+    "max", "maxbyte", "maxdouble", "maxfloat", "maxint", "maxlong",
+    "mdy", "mdyhms", "mi", "min", "minbyte", "mindouble",
+    "minfloat", "minint", "minlong", "minutes", "missing", "mm",
+    "mmC", "mod", "mofd", "month", "monthly", "mreldif",
+    "msofhours", "msofminutes", "msofseconds", "nF", "nFden",
+    "nFtail", "nbetaden", "nbinomial", "nbinomialp", "nbinomialtail",
+    "nchi2", "nchi2den", "nchi2tail", "nibeta", "norm", "normal",
+    "normalden", "normd", "npnF", "npnchi2", "npnt", "nt", "ntden",
+    "nttail", "nullmat", "plural", "poisson", "poissonp",
+    "poissontail", "proper", "q", "qofd", "quarter", "quarterly",
+    "r", "rbeta", "rbinomial", "rchi2", "real", "recode", "regexm",
+    "regexr", "regexs", "reldif", "replay", "return", "reverse",
+    "rgamma", "rhypergeometric", "rnbinomial", "rnormal", "round",
+    "rownumb", "rowsof", "rpoisson", "rt", "rtrim", "runiform", "s",
+    "scalar", "seconds", "sign", "sin", "sinh", "smallestdouble",
+    "soundex", "soundex_nara", "sqrt", "ss", "ssC", "strcat",
+    "strdup", "string", "strlen", "strlower", "strltrim", "strmatch",
+    "strofreal", "strpos", "strproper", "strreverse", "strrtrim",
+    "strtoname", "strtrim", "strupper", "subinstr", "subinword",
+    "substr", "sum", "sweep", "syminv", "t", "tC", "tan", "tanh",
+    "tc", "td", "tden", "th", "tin", "tm", "tq", "trace",
+    "trigamma", "trim", "trunc", "ttail", "tukeyprob", "tw",
+    "twithin", "uniform", "upper", "vec", "vecdiag", "w", "week",
+    "weekly", "wofd", "word", "wordcount", "year", "yearly",
+    "yh", "ym", "yofd", "yq", "yw"
+)
diff --git a/pygments/lexers/stata.py b/pygments/lexers/stata.py
index fd63db4a..3aad1ea9 100644
--- a/pygments/lexers/stata.py
+++ b/pygments/lexers/stata.py
@@ -1,487 +1,152 @@
 # -*- coding: utf-8 -*-
-
+# yapf: disable
 """
     pygments.lexers.stata
     ~~~~~~~~~~~~~~~~~~~~~
 
     Lexer for Stata
+
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
 """
 
+import re
 from pygments.lexer import RegexLexer, include, words
 from pygments.token import Comment, Keyword, Name, Number, \
     String, Text, Operator
 
+from pygments.lexers._stata_builtins import builtins_base, builtins_functions
+
 __all__ = ['StataLexer']
 
+
 class StataLexer(RegexLexer):
     """
-    For `Stata ` do files.
+    For `Stata `_ do files.
 
-    Syntax from
-    - http://fmwww.bc.edu/RePEc/bocode/s/synlightlist.ado
-    - github.com/isagalaev/highlight.js/blob/master/src/languages/stata.js
-    - github.com/jpitblado/vim-stata/blob/master/syntax/stata.vim
+    .. versionadded:: 2.2
     """
+    # Syntax based on
+    # - http://fmwww.bc.edu/RePEc/bocode/s/synlightlist.ado
+    # - http://github.com/isagalaev/highlight.js/blob/master/src/languages/stata.js
+    # - http://github.com/jpitblado/vim-stata/blob/master/syntax/stata.vim
 
-    name      = 'Stata'
-    aliases   = ['stata', 'do', 'Stata']
+    name = 'Stata'
+    aliases = ['stata', 'do']
     filenames = ['*.do', '*.ado']
     mimetypes = ['text/x-stata', 'text/stata', 'application/x-stata']
-
-    builtins_base = (
-        "if", "else", "in", "foreach", "for", "forv", "forva",
-        "forval", "forvalu", "forvalue", "forvalues", "by", "bys",
-        "bysort", "xi", "quietly", "qui", "capture", "about", "ac",
-        "ac_7", "acprplot", "acprplot_7 adjust", "ado", "adopath",
-        "adoupdate", "alpha", "ameans", "an", "ano", "anov", "anova",
-        "anova_estat", "anova_terms", "anovadef", "aorder", "ap", "app",
-        "appe", "appen", "append", "arch", "arch_dr", "arch_estat",
-        "arch_p", "archlm", "areg", "areg_p", "args", "arima",
-        "arima_dr", "arima_estat", "arima_p", "as", "asmprobit",
-        "asmprobit_estat", "asmprobit_lf", "asmprobit_mfx__dlg",
-        "asmprobit_p", "ass", "asse", "asser", "assert", "avplot",
-        "avplot_7", "avplots", "avplots_7 bcskew0", "bgodfrey",
-        "binreg", "bip0_lf", "biplot", "bipp_lf", "bipr_lf",
-        "bipr_p", "biprobit", "bitest", "bitesti", "bitowt", "blogit",
-        "bmemsize", "boot", "bootsamp", "bootstrap", "bootstrap_8",
-        "boxco_l", "boxco_p", "boxcox", "boxcox_6", "boxcox_p",
-        "bprobit", "br", "break", "brier", "bro", "brow", "brows",
-        "browse", "brr", "brrstat", "bs", "bs_7", "bsampl_w",
-        "bsample", "bsample_7", "bsqreg", "bstat", "bstat_7", "bstat_8",
-        "bstrap", "bstrap_7", "ca", "ca_estat", "ca_p", "cabiplot",
-        "camat", "canon", "canon_8", "canon_8_p", "canon_estat",
-        "canon_p", "cap", "caprojection", "capt", "captu", "captur",
-        "capture", "cat", "cc", "cchart", "cchart_7", "cci",
-        "cd", "censobs_table", "centile", "cf", "char", "chdir",
-        "checkdlgfiles", "checkestimationsample", "checkhlpfiles",
-        "checksum", "chelp", "ci", "cii", "cl", "class", "classutil",
-        "clear", "cli", "clis", "clist", "clo", "clog", "clog_lf",
-        "clog_p", "clogi", "clogi_sw", "clogit", "clogit_lf",
-        "clogit_p", "clogitp", "clogl_sw", "cloglog", "clonevar",
-        "clslistarray", "cluster", "cluster_measures", "cluster_stop",
-        "cluster_tree", "cluster_tree_8", "clustermat", "cmdlog",
-        "cnr", "cnre", "cnreg", "cnreg_p", "cnreg_sw", "cnsreg",
-        "codebook", "collaps4", "collapse", "colormult_nb",
-        "colormult_nw", "compare", "compress", "conf", "confi",
-        "confir", "confirm", "conren", "cons", "const", "constr",
-        "constra", "constrai", "constrain", "constraint", "continue",
-        "contract", "copy", "copyright", "copysource", "cor", "corc",
-        "corr", "corr2data", "corr_anti", "corr_kmo", "corr_smc",
-        "corre", "correl", "correla", "correlat", "correlate",
-        "corrgram", "cou", "coun", "count", "cox", "cox_p", "cox_sw",
-        "coxbase", "coxhaz", "coxvar", "cprplot", "cprplot_7",
-        "crc", "cret", "cretu", "cretur", "creturn", "cross", "cs",
-        "cscript", "cscript_log", "csi", "ct", "ct_is", "ctset",
-        "ctst_5", "ctst_st", "cttost", "cumsp", "cumsp_7", "cumul",
-        "cusum", "cusum_7", "cutil", "d", "datasig", "datasign",
-        "datasigna", "datasignat", "datasignatu", "datasignatur",
-        "datasignature", "datetof", "db", "dbeta", "de", "dec",
-        "deco", "decod", "decode", "deff", "des", "desc", "descr",
-        "descri", "describ", "describe", "destring", "dfbeta",
-        "dfgls", "dfuller", "di", "di_g", "dir", "dirstats", "dis",
-        "discard", "disp", "disp_res", "disp_s", "displ", "displa",
-        "display", "distinct", "do", "doe", "doed", "doedi",
-        "doedit", "dotplot", "dotplot_7", "dprobit", "drawnorm",
-        "drop", "ds", "ds_util", "dstdize", "duplicates", "durbina",
-        "dwstat", "dydx", "e", "ed", "edi", "edit", "egen",
-        "eivreg", "emdef", "en", "enc", "enco", "encod", "encode",
-        "eq", "erase", "ereg", "ereg_lf", "ereg_p", "ereg_sw",
-        "ereghet", "ereghet_glf", "ereghet_glf_sh", "ereghet_gp",
-        "ereghet_ilf", "ereghet_ilf_sh", "ereghet_ip", "eret",
-        "eretu", "eretur", "ereturn", "err", "erro", "error", "est",
-        "est_cfexist", "est_cfname", "est_clickable", "est_expand",
-        "est_hold", "est_table", "est_unhold", "est_unholdok",
-        "estat", "estat_default", "estat_summ", "estat_vce_only",
-        "esti", "estimates", "etodow", "etof", "etomdy", "ex",
-        "exi", "exit", "expand", "expandcl", "fac", "fact", "facto",
-        "factor", "factor_estat", "factor_p", "factor_pca_rotated",
-        "factor_rotate", "factormat", "fcast", "fcast_compute",
-        "fcast_graph", "fdades", "fdadesc", "fdadescr", "fdadescri",
-        "fdadescrib", "fdadescribe", "fdasav", "fdasave", "fdause",
-        "fh_st", "file", "open", "file", "read", "file", "close",
-        "file", "filefilter", "fillin", "find_hlp_file", "findfile",
-        "findit", "findit_7", "fit", "fl", "fli", "flis", "flist",
-        "for5_0", "form", "forma", "format", "fpredict", "frac_154",
-        "frac_adj", "frac_chk", "frac_cox", "frac_ddp", "frac_dis",
-        "frac_dv", "frac_in", "frac_mun", "frac_pp", "frac_pq",
-        "frac_pv", "frac_wgt", "frac_xo", "fracgen", "fracplot",
-        "fracplot_7", "fracpoly", "fracpred", "fron_ex", "fron_hn",
-        "fron_p", "fron_tn", "fron_tn2", "frontier", "ftodate", "ftoe",
-        "ftomdy", "ftowdate", "g", "gamhet_glf", "gamhet_gp",
-        "gamhet_ilf", "gamhet_ip", "gamma", "gamma_d2", "gamma_p",
-        "gamma_sw", "gammahet", "gdi_hexagon", "gdi_spokes", "ge",
-        "gen", "gene", "gener", "genera", "generat", "generate",
-        "genrank", "genstd", "genvmean", "gettoken", "gl", "gladder",
-        "gladder_7", "glim_l01", "glim_l02 glim_l03", "glim_l04",
-        "glim_l05", "glim_l06", "glim_l07", "glim_l08", "glim_l09",
-        "glim_l10 glim_l11", "glim_l12", "glim_lf", "glim_mu",
-        "glim_nw1", "glim_nw2", "glim_nw3", "glim_p", "glim_v1",
-        "glim_v2", "glim_v3", "glim_v4", "glim_v5", "glim_v6",
-        "glim_v7", "glm", "glm_6 glm_p", "glm_sw", "glmpred", "glo",
-        "glob", "globa", "global", "glogit", "glogit_8", "glogit_p",
-        "gmeans", "gnbre_lf", "gnbreg", "gnbreg_5", "gnbreg_p",
-        "gomp_lf", "gompe_sw", "gomper_p", "gompertz", "gompertzhet",
-        "gomphet_glf", "gomphet_glf_sh", "gomphet_gp", "gomphet_ilf",
-        "gomphet_ilf_sh", "gomphet_ip", "gphdot", "gphpen",
-        "gphprint", "gprefs", "gprobi_p", "gprobit", "gprobit_8", "gr",
-        "gr7", "gr_copy", "gr_current", "gr_db", "gr_describe",
-        "gr_dir", "gr_draw", "gr_draw_replay", "gr_drop", "gr_edit",
-        "gr_editviewopts", "gr_example", "gr_example2 gr_export",
-        "gr_print", "gr_qscheme", "gr_query", "gr_read", "gr_rename",
-        "gr_replay", "gr_save", "gr_set", "gr_setscheme", "gr_table",
-        "gr_undo", "gr_use", "graph", "graph7 grebar", "greigen",
-        "greigen_7", "greigen_8", "grmeanby", "grmeanby_7",
-        "gs_fileinfo", "gs_filetype", "gs_graphinfo", "gs_stat",
-        "gsort", "gwood", "h", "hadimvo", "hareg", "hausman",
-        "haver", "he", "heck_d2", "heckma_p", "heckman", "heckp_lf",
-        "heckpr_p", "heckprob", "hel", "help", "hereg", "hetpr_lf",
-        "hetpr_p", "hetprob", "hettest", "hexdump", "hilite",
-        "hist", "hist_7 histogram", "hlogit", "hlu", "hmeans",
-        "hotel", "hotelling", "hprobit", "hreg", "hsearch", "icd9",
-        "icd9_ff", "icd9p", "iis", "impute", "imtest", "inbase",
-        "include", "inf", "infi", "infil", "infile", "infix", "inp",
-        "inpu", "input", "ins", "insheet", "insp", "inspe",
-        "inspec", "inspect", "integ", "inten", "intreg", "intreg_7",
-        "intreg_p", "intrg2_ll", "intrg_ll", "intrg_ll2", "ipolate",
-        "iqreg", "ir", "irf", "irf_create", "irfm", "iri", "is_svy",
-        "is_svysum", "isid", "istdize", "ivprob_1_lf", "ivprob_lf",
-        "ivprobit", "ivprobit_p", "ivreg", "ivreg_footnote",
-        "ivtob_1_lf", "ivtob_lf", "ivtobit", "ivtobit_p", "jackknife",
-        "jacknife", "jknife", "jknife_6", "jknife_8", "jkstat",
-        "joinby", "kalarma1", "kap", "kap_3", "kapmeier", "kappa",
-        "kapwgt", "kdensity", "kdensity_7 keep", "ksm", "ksmirnov",
-        "ktau", "kwallis", "l", "la", "lab", "labe", "label",
-        "labelbook", "ladder", "levels", "levelsof", "leverage",
-        "lfit", "lfit_p", "li", "lincom", "line", "linktest",
-        "lis", "list", "lloghet_glf", "lloghet_glf_sh", "lloghet_gp",
-        "lloghet_ilf", "lloghet_ilf_sh", "lloghet_ip", "llogi_sw",
-        "llogis_p", "llogist", "llogistic", "llogistichet",
-        "lnorm_lf", "lnorm_sw", "lnorma_p", "lnormal", "lnormalhet",
-        "lnormhet_glf", "lnormhet_glf_sh", "lnormhet_gp",
-        "lnormhet_ilf", "lnormhet_ilf_sh", "lnormhet_ip", "lnskew0",
-        "loadingplot", "loc", "loca", "local", "log", "logi",
-        "logis_lf", "logistic", "logistic_p", "logit", "logit_estat",
-        "logit_p", "loglogs", "logrank", "loneway", "lookfor",
-        "lookup", "lowess", "lowess_7", "lpredict", "lrecomp", "lroc",
-        "lroc_7", "lrtest", "ls", "lsens", "lsens_7", "lsens_x",
-        "lstat", "ltable", "ltable_7", "ltriang", "lv", "lvr2plot",
-        "lvr2plot_7", "m", "ma", "mac", "macr", "macro", "makecns",
-        "man", "manova", "manova_estat", "manova_p", "manovatest",
-        "mantel", "mark", "markin", "markout", "marksample", "mat",
-        "mat_capp", "mat_order", "mat_put_rr", "mat_rapp", "mata",
-        "mata_clear", "mata_describe", "mata_drop", "mata_matdescribe",
-        "mata_matsave", "mata_matuse", "mata_memory", "mata_mlib",
-        "mata_mosave", "mata_rename", "mata_which", "matalabel",
-        "matcproc", "matlist", "matname", "matr", "matri",
-        "matrix", "matrix_input__dlg", "matstrik", "mcc", "mcci",
-        "md0_", "md1_", "md1debug_", "md2_", "md2debug_", "mds",
-        "mds_estat", "mds_p", "mdsconfig", "mdslong", "mdsmat",
-        "mdsshepard", "mdytoe", "mdytof", "me_derd", "mean",
-        "means", "median", "memory", "memsize", "meqparse", "mer",
-        "merg", "merge", "mfp", "mfx", "mhelp", "mhodds", "minbound",
-        "mixed_ll", "mixed_ll_reparm", "mkassert", "mkdir",
-        "mkmat", "mkspline", "ml", "ml_5 ml_adjs", "ml_bhhhs",
-        "ml_c_d", "ml_check", "ml_clear", "ml_cnt", "ml_debug",
-        "ml_defd", "ml_e0 ml_e0_bfgs", "ml_e0_cycle", "ml_e0_dfp",
-        "ml_e0i", "ml_e1", "ml_e1_bfgs", "ml_e1_bhhh", "ml_e1_cycle",
-        "ml_e1_dfp", "ml_e2", "ml_e2_cycle", "ml_ebfg0", "ml_ebfr0",
-        "ml_ebfr1 ml_ebh0q", "ml_ebhh0", "ml_ebhr0", "ml_ebr0i",
-        "ml_ecr0i", "ml_edfp0", "ml_edfr0", "ml_edfr1 ml_edr0i",
-        "ml_eds", "ml_eer0i", "ml_egr0i", "ml_elf", "ml_elf_bfgs",
-        "ml_elf_bhhh", "ml_elf_cycle", "ml_elf_dfp", "ml_elfi",
-        "ml_elfs", "ml_enr0i", "ml_enrr0", "ml_erdu0 ml_erdu0_bfgs",
-        "ml_erdu0_bhhh", "ml_erdu0_bhhhq", "ml_erdu0_cycle",
-        "ml_erdu0_dfp", "ml_erdu0_nrbfgs", "ml_exde", "ml_footnote",
-        "ml_geqnr", "ml_grad0", "ml_graph", "ml_hbhhh", "ml_hd0",
-        "ml_hold", "ml_init", "ml_inv", "ml_log", "ml_max",
-        "ml_mlout", "ml_mlout_8", "ml_model", "ml_nb0", "ml_opt",
-        "ml_p", "ml_plot", "ml_query", "ml_rdgrd", "ml_repor",
-        "ml_s_e", "ml_score", "ml_searc", "ml_technique", "ml_unhold",
-        "mleval", "mlf_", "mlmatbysum", "mlmatsum", "mlog", "mlogi",
-        "mlogit", "mlogit_footnote", "mlogit_p", "mlopts", "mlsum",
-        "mlvecsum", "mnl0_", "mor", "more", "mov", "move", "mprobit",
-        "mprobit_lf", "mprobit_p", "mrdu0_", "mrdu1_", "mvdecode",
-        "mvencode", "mvreg", "mvreg_estat", "n", "nbreg",
-        "nbreg_al", "nbreg_lf", "nbreg_p", "nbreg_sw", "nestreg", "net",
-        "newey", "newey_7", "newey_p", "news", "nl", "nl_7", "nl_9",
-        "nl_9_p", "nl_p", "nl_p_7 nlcom", "nlcom_p", "nlexp2",
-        "nlexp2_7", "nlexp2a", "nlexp2a_7", "nlexp3", "nlexp3_7",
-        "nlgom3 nlgom3_7", "nlgom4", "nlgom4_7", "nlinit", "nllog3",
-        "nllog3_7", "nllog4", "nllog4_7", "nlog_rd", "nlogit",
-        "nlogit_p", "nlogitgen", "nlogittree", "nlpred", "no",
-        "nobreak", "noi", "nois", "noisi", "noisil", "noisily", "note",
-        "notes", "notes_dlg", "nptrend", "numlabel", "numlist", "odbc",
-        "old_ver", "olo", "olog", "ologi", "ologi_sw", "ologit",
-        "ologit_p", "ologitp", "on", "one", "onew", "onewa", "oneway",
-        "op_colnm", "op_comp", "op_diff", "op_inv", "op_str", "opr",
-        "opro", "oprob", "oprob_sw", "oprobi", "oprobi_p", "oprobit",
-        "oprobitp", "opts_exclusive", "order", "orthog", "orthpoly",
-        "ou", "out", "outf", "outfi", "outfil", "outfile", "outs",
-        "outsh", "outshe", "outshee", "outsheet", "ovtest", "pac",
-        "pac_7", "palette", "parse", "parse_dissim", "pause", "pca",
-        "pca_8 pca_display", "pca_estat", "pca_p", "pca_rotate",
-        "pcamat", "pchart", "pchart_7", "pchi", "pchi_7", "pcorr",
-        "pctile", "pentium", "pergram", "pergram_7", "permute",
-        "permute_8", "personal", "peto_st", "pkcollapse", "pkcross",
-        "pkequiv", "pkexamine", "pkexamine_7", "pkshape", "pksumm",
-        "pksumm_7", "pl", "plo", "plot", "plugin", "pnorm",
-        "pnorm_7", "poisgof", "poiss_lf", "poiss_sw", "poisso_p",
-        "poisson", "poisson_estat", "post", "postclose", "postfile",
-        "postutil", "pperron", "pr", "prais", "prais_e", "prais_e2",
-        "prais_p", "predict", "predictnl", "preserve", "print",
-        "pro", "prob", "probi", "probit", "probit_estat", "probit_p",
-        "proc_time", "procoverlay", "procrustes", "procrustes_estat",
-        "procrustes_p", "profiler", "prog", "progr", "progra",
-        "program", "prop", "proportion", "prtest", "prtesti", "pwcorr",
-        "pwd", "q", "s", "qby", "qbys", "qchi", "qchi_7", "qladder",
-        "qladder_7", "qnorm", "qnorm_7", "qqplot", "qqplot_7", "qreg",
-        "qreg_c", "qreg_p", "qreg_sw", "qu", "quadchk", "quantile",
-        "quantile_7", "que", "quer", "query", "range", "ranksum",
-        "ratio", "rchart", "rchart_7", "rcof", "recast", "reclink",
-        "recode", "reg", "reg3", "reg3_p", "regdw", "regr", "regre",
-        "regre_p2", "regres", "regres_p", "regress", "regress_estat",
-        "regriv_p", "remap", "ren", "rena", "renam", "rename",
-        "renpfix", "repeat", "replace", "report", "reshape",
-        "restore", "ret", "retu", "retur", "return", "rm", "rmdir",
-        "robvar", "roccomp", "roccomp_7", "roccomp_8", "rocf_lf",
-        "rocfit", "rocfit_8", "rocgold", "rocplot", "rocplot_7",
-        "roctab", "roctab_7", "rolling", "rologit", "rologit_p",
-        "rot", "rota", "rotat", "rotate", "rotatemat", "rreg",
-        "rreg_p", "ru", "run", "runtest", "rvfplot", "rvfplot_7",
-        "rvpplot", "rvpplot_7", "sa", "safesum", "sample",
-        "sampsi", "sav", "save", "savedresults", "saveold", "sc",
-        "sca", "scal", "scala", "scalar", "scatter", "scm_mine",
-        "sco", "scob_lf", "scob_p", "scobi_sw", "scobit", "scor",
-        "score", "scoreplot", "scoreplot_help", "scree", "screeplot",
-        "screeplot_help", "sdtest", "sdtesti", "se", "search",
-        "separate", "seperate", "serrbar", "serrbar_7", "serset", "set",
-        "set_defaults", "sfrancia", "sh", "she", "shel", "shell",
-        "shewhart", "shewhart_7", "signestimationsample", "signrank",
-        "signtest", "simul", "simul_7 simulate", "simulate_8",
-        "sktest", "sleep", "slogit", "slogit_d2", "slogit_p", "smooth",
-        "snapspan", "so", "sor", "sort", "spearman", "spikeplot",
-        "spikeplot_7", "spikeplt", "spline_x", "split", "sqreg",
-        "sqreg_p", "sret", "sretu", "sretur", "sreturn", "ssc", "st",
-        "st_ct", "st_hc", "st_hcd", "st_hcd_sh", "st_is", "st_issys",
-        "st_note", "st_promo", "st_set", "st_show", "st_smpl",
-        "st_subid", "stack", "statsby", "statsby_8", "stbase", "stci",
-        "stci_7", "stcox", "stcox_estat", "stcox_fr", "stcox_fr_ll",
-        "stcox_p", "stcox_sw", "stcoxkm", "stcoxkm_7", "stcstat",
-        "stcurv", "stcurve", "stcurve_7", "stdes", "stem", "stepwise",
-        "stereg", "stfill", "stgen", "stir", "stjoin", "stmc", "stmh",
-        "stphplot", "stphplot_7", "stphtest", "stphtest_7",
-        "stptime", "strate", "strate_7", "streg", "streg_sw", "streset",
-        "sts", "sts_7", "stset", "stsplit", "stsum", "sttocc",
-        "sttoct", "stvary", "stweib", "su", "suest", "suest_8",
-        "sum", "summ", "summa", "summar", "summari", "summariz",
-        "summarize", "sunflower", "sureg", "survcurv", "survsum",
-        "svar", "svar_p", "svmat", "svy", "svy_disp", "svy_dreg",
-        "svy_est", "svy_est_7", "svy_estat", "svy_get", "svy_gnbreg_p",
-        "svy_head", "svy_header", "svy_heckman_p", "svy_heckprob_p",
-        "svy_intreg_p", "svy_ivreg_p", "svy_logistic_p", "svy_logit_p",
-        "svy_mlogit_p", "svy_nbreg_p", "svy_ologit_p", "svy_oprobit_p",
-        "svy_poisson_p", "svy_probit_p", "svy_regress_p", "svy_sub",
-        "svy_sub_7", "svy_x", "svy_x_7", "svy_x_p", "svydes",
-        "svydes_8", "svygen", "svygnbreg", "svyheckman", "svyheckprob",
-        "svyintreg", "svyintreg_7", "svyintrg", "svyivreg", "svylc",
-        "svylog_p", "svylogit", "svymarkout", "svymarkout_8",
-        "svymean", "svymlog", "svymlogit", "svynbreg", "svyolog",
-        "svyologit", "svyoprob", "svyoprobit", "svyopts",
-        "svypois", "svypois_7 svypoisson", "svyprobit", "svyprobt",
-        "svyprop", "svyprop_7", "svyratio", "svyreg", "svyreg_p",
-        "svyregress", "svyset", "svyset_7", "svyset_8", "svytab",
-        "svytab_7", "svytest", "svytotal", "sw", "sw_8", "swcnreg",
-        "swcox", "swereg", "swilk", "swlogis", "swlogit",
-        "swologit", "swoprbt", "swpois", "swprobit", "swqreg",
-        "swtobit", "swweib", "symmetry", "symmi", "symplot",
-        "symplot_7 syntax", "sysdescribe", "sysdir", "sysuse",
-        "szroeter", "ta", "tab", "tab1", "tab2", "tab_or", "tabd",
-        "tabdi", "tabdis", "tabdisp", "tabi", "table", "tabodds",
-        "tabodds_7", "tabstat", "tabu", "tabul", "tabula", "tabulat",
-        "tabulate", "te", "tempfile", "tempname", "tempvar", "tes",
-        "test", "testnl", "testparm", "teststd", "tetrachoric",
-        "time_it", "timer", "tis", "tob", "tobi", "tobit", "tobit_p",
-        "tobit_sw", "token", "tokeni", "tokeniz", "tokenize",
-        "tostring", "total", "translate", "translator", "transmap",
-        "treat_ll", "treatr_p", "treatreg", "trim", "trnb_cons",
-        "trnb_mean", "trpoiss_d2", "trunc_ll", "truncr_p", "truncreg",
-        "tsappend", "tset", "tsfill", "tsline", "tsline_ex",
-        "tsreport", "tsrevar", "tsrline", "tsset", "tssmooth",
-        "tsunab", "ttest", "ttesti", "tut_chk", "tut_wait", "tutorial",
-        "tw", "tware_st", "two", "twoway", "twoway__fpfit_serset",
-        "twoway__function_gen", "twoway__histogram_gen",
-        "twoway__ipoint_serset", "twoway__ipoints_serset",
-        "twoway__kdensity_gen", "twoway__lfit_serset",
-        "twoway__normgen_gen", "twoway__pci_serset",
-        "twoway__qfit_serset", "twoway__scatteri_serset",
-        "twoway__sunflower_gen", "twoway_ksm_serset", "ty", "typ",
-        "type", "typeof", "u", "unab", "unabbrev", "unabcmd",
-        "update", "us", "use", "uselabel", "var", "var_mkcompanion",
-        "var_p", "varbasic", "varfcast", "vargranger", "varirf",
-        "varirf_add", "varirf_cgraph", "varirf_create", "varirf_ctable",
-        "varirf_describe", "varirf_dir", "varirf_drop", "varirf_erase",
-        "varirf_graph", "varirf_ograph", "varirf_rename", "varirf_set",
-        "varirf_table", "varlist", "varlmar", "varnorm", "varsoc",
-        "varstable", "varstable_w", "varstable_w2", "varwle",
-        "vce", "vec", "vec_fevd", "vec_mkphi", "vec_p", "vec_p_w",
-        "vecirf_create", "veclmar", "veclmar_w", "vecnorm",
-        "vecnorm_w", "vecrank", "vecstable", "verinst", "vers",
-        "versi", "versio", "version", "view", "viewsource", "vif",
-        "vwls", "wdatetof", "webdescribe", "webseek", "webuse",
-        "weib1_lf", "weib2_lf", "weib_lf", "weib_lf0 weibhet_glf",
-        "weibhet_glf_sh", "weibhet_glfa", "weibhet_glfa_sh",
-        "weibhet_gp", "weibhet_ilf", "weibhet_ilf_sh", "weibhet_ilfa",
-        "weibhet_ilfa_sh", "weibhet_ip", "weibu_sw", "weibul_p",
-        "weibull", "weibull_c", "weibull_s", "weibullhet",
-        "wh", "whelp", "whi", "which", "whil", "while", "wilc_st",
-        "wilcoxon", "win", "wind", "windo", "window", "winexec",
-        "wntestb", "wntestb_7", "wntestq", "xchart", "xchart_7",
-        "xcorr", "xcorr_7", "xi", "xi_6", "xmlsav", "xmlsave",
-        "xmluse", "xpose", "xsh", "xshe", "xshel", "xshell",
-        "xt_iis", "xt_tis", "xtab_p", "xtabond", "xtbin_p",
-        "xtclog", "xtcloglog", "xtcloglog_8", "xtcloglog_d2",
-        "xtcloglog_pa_p", "xtcloglog_re_p", "xtcnt_p", "xtcorr",
-        "xtdata", "xtdes", "xtfront_p", "xtfrontier", "xtgee",
-        "xtgee_elink", "xtgee_estat", "xtgee_makeivar", "xtgee_p",
-        "xtgee_plink", "xtgls", "xtgls_p", "xthaus", "xthausman",
-        "xtht_p", "xthtaylor", "xtile", "xtint_p", "xtintreg",
-        "xtintreg_8", "xtintreg_d2 xtintreg_p", "xtivp_1",
-        "xtivp_2", "xtivreg", "xtline", "xtline_ex", "xtlogit",
-        "xtlogit_8 xtlogit_d2", "xtlogit_fe_p", "xtlogit_pa_p",
-        "xtlogit_re_p", "xtmixed", "xtmixed_estat", "xtmixed_p",
-        "xtnb_fe", "xtnb_lf", "xtnbreg", "xtnbreg_pa_p",
-        "xtnbreg_refe_p", "xtpcse", "xtpcse_p", "xtpois", "xtpoisson",
-        "xtpoisson_d2", "xtpoisson_pa_p", "xtpoisson_refe_p", "xtpred",
-        "xtprobit", "xtprobit_8", "xtprobit_d2", "xtprobit_re_p",
-        "xtps_fe", "xtps_lf", "xtps_ren", "xtps_ren_8", "xtrar_p",
-        "xtrc", "xtrc_p", "xtrchh", "xtrefe_p", "xtreg", "xtreg_be",
-        "xtreg_fe", "xtreg_ml", "xtreg_pa_p", "xtreg_re",
-        "xtregar", "xtrere_p", "xtset", "xtsf_ll", "xtsf_llti",
-        "xtsum", "xttab", "xttest0", "xttobit", "xttobit_8",
-        "xttobit_p", "xttrans", "yx", "yxview__barlike_draw",
-        "yxview_area_draw", "yxview_bar_draw", "yxview_dot_draw",
-        "yxview_dropline_draw", "yxview_function_draw",
-        "yxview_iarrow_draw", "yxview_ilabels_draw",
-        "yxview_normal_draw", "yxview_pcarrow_draw",
-        "yxview_pcbarrow_draw", "yxview_pccapsym_draw",
-        "yxview_pcscatter_draw", "yxview_pcspike_draw",
-        "yxview_rarea_draw", "yxview_rbar_draw", "yxview_rbarm_draw",
-        "yxview_rcap_draw", "yxview_rcapsym_draw",
-        "yxview_rconnected_draw", "yxview_rline_draw",
-        "yxview_rscatter_draw", "yxview_rspike_draw",
-        "yxview_spike_draw", "yxview_sunflower_draw", "zap_s", "zinb",
-        "zinb_llf", "zinb_plf", "zip", "zip_llf", "zip_p", "zip_plf",
-        "zt_ct_5", "zt_hc_5", "zt_hcd_5", "zt_is_5", "zt_iss_5",
-        "zt_sho_5 zt_smp_5", "ztbase_5", "ztcox_5", "ztdes_5",
-        "ztereg_5", "ztfill_5", "ztgen_5", "ztir_5 ztjoin_5", "ztnb",
-        "ztnb_p", "ztp", "ztp_p", "zts_5", "ztset_5", "ztspli_5",
-        "ztsum_5", "zttoct_5 ztvary_5", "ztweib_5"
-    )
-
-    builtins_functions = (
-        "Cdhms", "Chms", "Clock", "Cmdyhms", "Cofc", "Cofd", "F",
-        "Fden", "Ftail", "I", "J", "_caller", "abbrev", "abs", "acos",
-        "acosh", "asin", "asinh", "atan", "atan2", "atanh",
-        "autocode", "betaden", "binomial", "binomialp", "binomialtail",
-        "binormal", "bofd", "byteorder", "c", "ceil", "char",
-        "chi2", "chi2den", "chi2tail", "cholesky", "chop", "clip",
-        "clock", "cloglog", "cofC", "cofd", "colnumb", "colsof", "comb",
-        "cond", "corr", "cos", "cosh", "d", "daily", "date", "day",
-        "det", "dgammapda", "dgammapdada", "dgammapdadx", "dgammapdx",
-        "dgammapdxdx", "dhms", "diag", "diag0cnt", "digamma",
-        "dofC", "dofb", "dofc", "dofh", "dofm", "dofq", "dofw",
-        "dofy", "dow", "doy", "dunnettprob", "e", "el", "epsdouble",
-        "epsfloat", "exp", "fileexists", "fileread", "filereaderror",
-        "filewrite", "float", "floor", "fmtwidth", "gammaden",
-        "gammap", "gammaptail", "get", "group", "h", "hadamard",
-        "halfyear", "halfyearly", "has_eprop", "hh", "hhC", "hms",
-        "hofd", "hours", "hypergeometric", "hypergeometricp", "ibeta",
-        "ibetatail", "index", "indexnot", "inlist", "inrange", "int",
-        "inv", "invF", "invFtail", "invbinomial", "invbinomialtail",
-        "invchi2", "invchi2tail", "invcloglog", "invdunnettprob",
-        "invgammap", "invgammaptail", "invibeta", "invibetatail",
-        "invlogit", "invnFtail", "invnbinomial", "invnbinomialtail",
-        "invnchi2", "invnchi2tail", "invnibeta", "invnorm", "invnormal",
-        "invnttail", "invpoisson", "invpoissontail", "invsym", "invt",
-        "invttail", "invtukeyprob", "irecode", "issym", "issymmetric",
-        "itrim", "length", "ln", "lnfact", "lnfactorial", "lngamma",
-        "lnnormal", "lnnormalden", "log", "log10", "logit", "lower",
-        "ltrim", "m", "match", "matmissing", "matrix", "matuniform",
-        "max", "maxbyte", "maxdouble", "maxfloat", "maxint", "maxlong",
-        "mdy", "mdyhms", "mi", "mi", "min", "minbyte", "mindouble",
-        "minfloat", "minint", "minlong", "minutes", "missing", "mm",
-        "mmC", "mod", "mofd", "month", "monthly", "mreldif",
-        "msofhours", "msofminutes", "msofseconds", "nF", "nFden",
-        "nFtail", "nbetaden", "nbinomial", "nbinomialp", "nbinomialtail",
-        "nchi2", "nchi2den", "nchi2tail", "nibeta", "norm", "normal",
-        "normalden", "normd", "npnF", "npnchi2", "npnt", "nt", "ntden",
-        "nttail", "nullmat", "plural", "poisson", "poissonp",
-        "poissontail", "proper", "q", "qofd", "quarter", "quarterly",
-        "r", "rbeta", "rbinomial", "rchi2 real", "recode", "regexm",
-        "regexr", "regexs", "reldif", "replay", "return", "reverse",
-        "rgamma", "rhypergeometric", "rnbinomial", "rnormal", "round",
-        "rownumb", "rowsof", "rpoisson", "rt", "rtrim", "runiform", "s",
-        "scalar", "seconds", "sign", "sin", "sinh", "smallestdouble",
-        "soundex", "soundex_nara", "sqrt", "ss", "ssC", "strcat",
-        "strdup", "string", "strlen", "strlower", "strltrim", "strmatch",
-        "strofreal", "strpos", "strproper", "strreverse", "strrtrim",
-        "strtoname", "strtrim", "strupper", "subinstr", "subinword",
-        "substr", "sum", "sweep", "syminv", "t", "tC", "tan", "tanh",
-        "tc", "td", "tden", "th", "tin", "tm", "tq", "trace",
-        "trigamma", "trim", "trunc", "ttail", "tukeyprob", "tw",
-        "twithin", "uniform", "upper", "vec", "vecdiag", "w", "week",
-        "weekly", "wofd", "word", "wordcount", "year", "yearly",
-        "yh", "ym", "yofd", "yq", "yw"
-    )
+    flags = re.MULTILINE | re.DOTALL
 
     tokens = {
         'root': [
             include('comments'),
-            include('vars-strings'),
+            include('strings'),
+            include('macros'),
             include('numbers'),
             include('keywords'),
+            include('operators'),
+            include('format'),
             (r'.', Text),
         ],
-        # Global and local macros; regular and special strings
-        'vars-strings': [
-            (r'\$[a-zA-Z_0-9\{]', Name.Variable.Global, 'var_validglobal'),
-            (r'`[a-zA-Z_0-9]{0,31}\'', Name.Variable),
-            (r'"', String, 'string_dquote'),
-            (r'`"', String, 'string_mquote'),
+        # Comments are a complicated beast in Stata because they can be
+        # nested and there are a few corner cases with that. See:
+        # - github.com/kylebarron/language-stata/issues/90
+        # - statalist.org/forums/forum/general-stata-discussion/general/1448244
+        'comments': [
+            (r'(^//|(?<=\s)//)(?!/)', Comment.Single, 'comments-double-slash'),
+            (r'^\s*\*', Comment.Single, 'comments-star'),
+            (r'/\*', Comment.Multiline, 'comments-block'),
+            (r'(^///|(?<=\s)///)', Comment.Special, 'comments-triple-slash')
+        ],
+        'comments-block': [
+            (r'/\*', Comment.Multiline, '#push'),
+            # this ends and restarts a comment block. but need to catch this so
+            # that it doesn\'t start _another_ level of comment blocks
+            (r'\*/\*', Comment.Multiline),
+            (r'(\*/\s+\*(?!/)[^\n]*)|(\*/)', Comment.Multiline, '#pop'),
+            # Match anything else as a character inside the comment
+            (r'.', Comment.Multiline),
+        ],
+        'comments-star': [
+            (r'///.*?\n', Comment.Single,
+                ('#pop', 'comments-triple-slash')),
+            (r'(^//|(?<=\s)//)(?!/)', Comment.Single,
+                ('#pop', 'comments-double-slash')),
+            (r'/\*', Comment.Multiline, 'comments-block'),
+            (r'.(?=\n)', Comment.Single, '#pop'),
+            (r'.', Comment.Single),
+        ],
+        'comments-triple-slash': [
+            (r'\n', Comment.Special, '#pop'),
+            # A // breaks out of a comment for the rest of the line
+            (r'//.*?(?=\n)', Comment.Single, '#pop'),
+            (r'.', Comment.Special),
+        ],
+        'comments-double-slash': [
+            (r'\n', Text, '#pop'),
+            (r'.', Comment.Single),
         ],
-        # For either string type, highlight macros as macros
-        'string_dquote': [
-            (r'"', String, '#pop'),
-            (r'\\\\|\\"|\\\n', String.Escape),
-            (r'\$', Name.Variable.Global, 'var_validglobal'),
-            (r'`', Name.Variable, 'var_validlocal'),
-            (r'[^$\$`"\\]+', String),
-            (r'[$"\\]', String),
+        # `"compound string"' and regular "string"; note the former are
+        # nested.
+        'strings': [
+            (r'`"', String, 'string-compound'),
+            (r'(?
Date: Tue, 18 Dec 2018 14:50:36 -0600
Subject: add comment

---
 pygments/lexers/c_like.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/pygments/lexers/c_like.py b/pygments/lexers/c_like.py
index b5ddf43c..ba735a6b 100644
--- a/pygments/lexers/c_like.py
+++ b/pygments/lexers/c_like.py
@@ -541,6 +541,10 @@ class ArduinoLexer(CppLexer):
                 yield index, token, value
 
 class CharmciLexer(CppLexer):
+    """
+    For `Charm++ `_ interface files (.ci).
+    """
+
     name = 'Charmci'
     aliases = ['charmci']
     filenames = ['*.ci']
-- 
cgit v1.2.1


From 374c774f204b1999f164df485adde6d33aeed19c Mon Sep 17 00:00:00 2001
From: Morten Enemark Lund 
Date: Tue, 18 Dec 2018 23:24:28 +0100
Subject: Update remaining color names

---
 pygments/formatters/irc.py      | 76 ++++++++++++++++++++---------------------
 pygments/formatters/terminal.py | 48 +++++++++++++-------------
 2 files changed, 62 insertions(+), 62 deletions(-)

diff --git a/pygments/formatters/irc.py b/pygments/formatters/irc.py
index eb744d74..d55584f8 100644
--- a/pygments/formatters/irc.py
+++ b/pygments/formatters/irc.py
@@ -25,55 +25,55 @@ __all__ = ['IRCFormatter']
 IRC_COLORS = {
     Token:              ('',            ''),
 
-    Whitespace:         ('lightgray',   'darkgray'),
-    Comment:            ('lightgray',   'darkgray'),
-    Comment.Preproc:    ('teal',        'turquoise'),
-    Keyword:            ('darkblue',    'blue'),
-    Keyword.Type:       ('teal',        'turquoise'),
-    Operator.Word:      ('purple',      'fuchsia'),
-    Name.Builtin:       ('teal',        'turquoise'),
-    Name.Function:      ('darkgreen',   'green'),
-    Name.Namespace:     ('_teal_',      '_turquoise_'),
-    Name.Class:         ('_darkgreen_', '_green_'),
-    Name.Exception:     ('teal',        'turquoise'),
-    Name.Decorator:     ('darkgray',    'lightgray'),
-    Name.Variable:      ('darkred',     'red'),
-    Name.Constant:      ('darkred',     'red'),
-    Name.Attribute:     ('teal',        'turquoise'),
-    Name.Tag:           ('blue',        'blue'),
-    String:             ('brown',       'brown'),
-    Number:             ('darkblue',    'blue'),
-
-    Generic.Deleted:    ('red',        'red'),
-    Generic.Inserted:   ('darkgreen',  'green'),
+    Whitespace:         ('gray',   'brightblack'),
+    Comment:            ('gray',   'brightblack'),
+    Comment.Preproc:    ('cyan',        'brightcyan'),
+    Keyword:            ('blue',    'brightblue'),
+    Keyword.Type:       ('cyan',        'brightcyan'),
+    Operator.Word:      ('magenta',      'brightcyan'),
+    Name.Builtin:       ('cyan',        'brightcyan'),
+    Name.Function:      ('green',   'brightgreen'),
+    Name.Namespace:     ('_cyan_',      '_brightcyan_'),
+    Name.Class:         ('_green_', '_brightgreen_'),
+    Name.Exception:     ('cyan',        'brightcyan'),
+    Name.Decorator:     ('brightblack',    'gray'),
+    Name.Variable:      ('red',     'brightred'),
+    Name.Constant:      ('red',     'brightred'),
+    Name.Attribute:     ('cyan',        'brightcyan'),
+    Name.Tag:           ('brightblue',        'brightblue'),
+    String:             ('yellow',       'yellow'),
+    Number:             ('blue',    'brightblue'),
+
+    Generic.Deleted:    ('brightred',        'brightred'),
+    Generic.Inserted:   ('green',  'brightgreen'),
     Generic.Heading:    ('**',         '**'),
-    Generic.Subheading: ('*purple*',   '*fuchsia*'),
-    Generic.Error:      ('red',        'red'),
+    Generic.Subheading: ('*magenta*',   '*brightmagenta*'),
+    Generic.Error:      ('brightred',        'brightred'),
 
-    Error:              ('_red_',      '_red_'),
+    Error:              ('_brightred_',      '_brightred_'),
 }
 
 
 IRC_COLOR_MAP = {
     'white': 0,
     'black': 1,
-    'darkblue': 2,
-    'green': 3,
-    'red': 4,
-    'brown': 5,
-    'purple': 6,
+    'blue': 2,
+    'brightgreen': 3,
+    'brightred': 4,
+    'yellow': 5,
+    'magenta': 6,
     'orange': 7,
-    'darkgreen': 7, #compat w/ ansi
-    'yellow': 8,
+    'green': 7, #compat w/ ansi
+    'brightyellow': 8,
     'lightgreen': 9,
-    'turquoise': 9, # compat w/ ansi
-    'teal': 10,
+    'brightcyan': 9, # compat w/ ansi
+    'cyan': 10,
     'lightblue': 11,
-    'darkred': 11, # compat w/ ansi
-    'blue': 12,
-    'fuchsia': 13,
-    'darkgray': 14,
-    'lightgray': 15,
+    'red': 11, # compat w/ ansi
+    'brightblue': 12,
+    'brightmagenta': 13,
+    'brightblack': 14,
+    'gray': 15,
 }
 
 def ircformat(color, text):
diff --git a/pygments/formatters/terminal.py b/pygments/formatters/terminal.py
index b8fec52e..fcb52d94 100644
--- a/pygments/formatters/terminal.py
+++ b/pygments/formatters/terminal.py
@@ -26,33 +26,33 @@ __all__ = ['TerminalFormatter']
 TERMINAL_COLORS = {
     Token:              ('',            ''),
 
-    Whitespace:         ('lightgray',   'darkgray'),
-    Comment:            ('lightgray',   'darkgray'),
-    Comment.Preproc:    ('teal',        'turquoise'),
-    Keyword:            ('darkblue',    'blue'),
-    Keyword.Type:       ('teal',        'turquoise'),
-    Operator.Word:      ('purple',      'fuchsia'),
-    Name.Builtin:       ('teal',        'turquoise'),
-    Name.Function:      ('darkgreen',   'green'),
-    Name.Namespace:     ('_teal_',      '_turquoise_'),
-    Name.Class:         ('_darkgreen_', '_green_'),
-    Name.Exception:     ('teal',        'turquoise'),
-    Name.Decorator:     ('darkgray',    'lightgray'),
-    Name.Variable:      ('darkred',     'red'),
-    Name.Constant:      ('darkred',     'red'),
-    Name.Attribute:     ('teal',        'turquoise'),
-    Name.Tag:           ('blue',        'blue'),
-    String:             ('brown',       'brown'),
-    Number:             ('darkblue',    'blue'),
-
-    Generic.Deleted:    ('red',        'red'),
-    Generic.Inserted:   ('darkgreen',  'green'),
+    Whitespace:         ('gray',   'brightblack'),
+    Comment:            ('gray',   'brightblack'),
+    Comment.Preproc:    ('cyan',        'brightcyan'),
+    Keyword:            ('blue',    'brightblue'),
+    Keyword.Type:       ('cyan',        'brightcyan'),
+    Operator.Word:      ('magenta',      'brightmagenta'),
+    Name.Builtin:       ('cyan',        'brightcyan'),
+    Name.Function:      ('green',   'brightgreen'),
+    Name.Namespace:     ('_cyan_',      '_brightcyan_'),
+    Name.Class:         ('_green_', '_brightgreen_'),
+    Name.Exception:     ('cyan',        'brightcyan'),
+    Name.Decorator:     ('brightblack',    'gray'),
+    Name.Variable:      ('red',     'brightred'),
+    Name.Constant:      ('red',     'brightred'),
+    Name.Attribute:     ('cyan',        'brightcyan'),
+    Name.Tag:           ('brightblue',        'brightblue'),
+    String:             ('yellow',       'yellow'),
+    Number:             ('blue',    'brightblue'),
+
+    Generic.Deleted:    ('brightred',        'brightred'),
+    Generic.Inserted:   ('green',  'brightgreen'),
     Generic.Heading:    ('**',         '**'),
-    Generic.Subheading: ('*purple*',   '*fuchsia*'),
+    Generic.Subheading: ('*magenta*',   '*brightmagenta*'),
     Generic.Prompt:     ('**',         '**'),
-    Generic.Error:      ('red',        'red'),
+    Generic.Error:      ('brightred',        'brightred'),
 
-    Error:              ('_red_',      '_red_'),
+    Error:              ('_brightred_',      '_brightred_'),
 }
 
 
-- 
cgit v1.2.1


From 07e9d8a0b09e40c8ac2a265e17aa47c3205a78d3 Mon Sep 17 00:00:00 2001
From: Simon Hengel 
Date: Wed, 19 Dec 2018 23:06:04 +0800
Subject: Add Hspec lexer

---
 CHANGES                     |  1 +
 pygments/lexers/_mapping.py |  1 +
 pygments/lexers/haskell.py  | 26 ++++++++++++++++++++++++--
 3 files changed, 26 insertions(+), 2 deletions(-)

diff --git a/CHANGES b/CHANGES
index b0257335..21eeb609 100644
--- a/CHANGES
+++ b/CHANGES
@@ -13,6 +13,7 @@ Version 2.4.0
 - Added lexers:
 
   * FloScript (PR#750)
+  * Hspec (PR#710)
 
 - Updated lexers:
 
diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py
index f51d9047..a0573174 100644
--- a/pygments/lexers/_mapping.py
+++ b/pygments/lexers/_mapping.py
@@ -184,6 +184,7 @@ LEXERS = {
     'HaxeLexer': ('pygments.lexers.haxe', 'Haxe', ('hx', 'haxe', 'hxsl'), ('*.hx', '*.hxsl'), ('text/haxe', 'text/x-haxe', 'text/x-hx')),
     'HexdumpLexer': ('pygments.lexers.hexdump', 'Hexdump', ('hexdump',), (), ()),
     'HsailLexer': ('pygments.lexers.asm', 'HSAIL', ('hsail', 'hsa'), ('*.hsail',), ('text/x-hsail',)),
+    'HspecLexer': ('pygments.lexers.haskell', 'Hspec', ('hspec',), (), ()),
     'HtmlDjangoLexer': ('pygments.lexers.templates', 'HTML+Django/Jinja', ('html+django', 'html+jinja', 'htmldjango'), (), ('text/html+django', 'text/html+jinja')),
     'HtmlGenshiLexer': ('pygments.lexers.templates', 'HTML+Genshi', ('html+genshi', 'html+kid'), (), ('text/html+genshi',)),
     'HtmlLexer': ('pygments.lexers.html', 'HTML', ('html',), ('*.html', '*.htm', '*.xhtml', '*.xslt'), ('text/html', 'application/xhtml+xml')),
diff --git a/pygments/lexers/haskell.py b/pygments/lexers/haskell.py
index 88d4a4df..b3884f5c 100644
--- a/pygments/lexers/haskell.py
+++ b/pygments/lexers/haskell.py
@@ -12,12 +12,12 @@
 import re
 
 from pygments.lexer import Lexer, RegexLexer, bygroups, do_insertions, \
-    default, include
+    default, include, inherit
 from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
     Number, Punctuation, Generic
 from pygments import unistring as uni
 
-__all__ = ['HaskellLexer', 'IdrisLexer', 'AgdaLexer', 'CryptolLexer',
+__all__ = ['HaskellLexer', 'HspecLexer', 'IdrisLexer', 'AgdaLexer', 'CryptolLexer',
            'LiterateHaskellLexer', 'LiterateIdrisLexer', 'LiterateAgdaLexer',
            'LiterateCryptolLexer', 'KokaLexer']
 
@@ -157,6 +157,28 @@ class HaskellLexer(RegexLexer):
     }
 
 
+class HspecLexer(HaskellLexer):
+    """
+    A Haskell lexer with support for Hspec constructs.
+
+    .. versionadded:: 2.4.0
+    """
+
+    name = 'Hspec'
+    aliases = ['hspec']
+    filenames = []
+    mimetypes = []
+
+    tokens = {
+        'root': [
+            (r'(it\s*)("[^"]*")', bygroups(Text, String.Doc)),
+            (r'(describe\s*)("[^"]*")', bygroups(Text, String.Doc)),
+            (r'(context\s*)("[^"]*")', bygroups(Text, String.Doc)),
+            inherit,
+        ],
+    }
+
+
 class IdrisLexer(RegexLexer):
     """
     A lexer for the dependently typed programming language Idris.
-- 
cgit v1.2.1


From 5736613862da472e79e81cd094a4039ec5cc84f2 Mon Sep 17 00:00:00 2001
From: Simon Hengel 
Date: Wed, 19 Dec 2018 23:16:05 +0800
Subject: Update CHANGES

---
 CHANGES | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGES b/CHANGES
index 21eeb609..e868c096 100644
--- a/CHANGES
+++ b/CHANGES
@@ -13,7 +13,7 @@ Version 2.4.0
 - Added lexers:
 
   * FloScript (PR#750)
-  * Hspec (PR#710)
+  * Hspec (PR#790)
 
 - Updated lexers:
 
-- 
cgit v1.2.1


From 6c9c585cc788aceb6798e059cfe1a0820577a3d6 Mon Sep 17 00:00:00 2001
From: "Matth?us G. Chajdas" 
Date: Fri, 21 Dec 2018 17:34:15 +0100
Subject: Small cleanups to the Slurm lexer.

Remove debug output, add a test file for the Slurm lexer.
---
 CHANGES                       | 1 +
 pygments/lexers/_mapping.py   | 2 +-
 pygments/lexers/shell.py      | 6 ++++--
 tests/examplefiles/example.sl | 6 ++++++
 4 files changed, 12 insertions(+), 3 deletions(-)
 create mode 100644 tests/examplefiles/example.sl

diff --git a/CHANGES b/CHANGES
index e868c096..6d6f463a 100644
--- a/CHANGES
+++ b/CHANGES
@@ -14,6 +14,7 @@ Version 2.4.0
 
   * FloScript (PR#750)
   * Hspec (PR#790)
+  * Slurm (PR#760)
 
 - Updated lexers:
 
diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py
index 494f2d6b..d9894a33 100644
--- a/pygments/lexers/_mapping.py
+++ b/pygments/lexers/_mapping.py
@@ -385,7 +385,7 @@ LEXERS = {
     'ShenLexer': ('pygments.lexers.lisp', 'Shen', ('shen',), ('*.shen',), ('text/x-shen', 'application/x-shen')),
     'SilverLexer': ('pygments.lexers.verification', 'Silver', ('silver',), ('*.sil', '*.vpr'), ()),
     'SlimLexer': ('pygments.lexers.webmisc', 'Slim', ('slim',), ('*.slim',), ('text/x-slim',)),
-    'SlurmBashLexer': ('pygments.lexers.shell', 'Slurm', ('slurm', 'sbatch'), ('*.sl',), ('application/x-sh', 'application/x-shellscript')),
+    'SlurmBashLexer': ('pygments.lexers.shell', 'Slurm', ('slurm', 'sbatch'), ('*.sl',), ()),
     'SmaliLexer': ('pygments.lexers.dalvik', 'Smali', ('smali',), ('*.smali',), ('text/smali',)),
     'SmalltalkLexer': ('pygments.lexers.smalltalk', 'Smalltalk', ('smalltalk', 'squeak', 'st'), ('*.st',), ('text/x-smalltalk',)),
     'SmartyLexer': ('pygments.lexers.templates', 'Smarty', ('smarty',), ('*.tpl',), ('application/x-smarty',)),
diff --git a/pygments/lexers/shell.py b/pygments/lexers/shell.py
index 5e9fcd6a..86d8c37a 100644
--- a/pygments/lexers/shell.py
+++ b/pygments/lexers/shell.py
@@ -129,23 +129,25 @@ class BashLexer(RegexLexer):
 class SlurmBashLexer(BashLexer):
     """
     Lexer for (ba|k|z|)sh Slurm scripts.
+
+    .. versionadded:: 2.4
     """
 
     name = 'Slurm'
     aliases = ['slurm', 'sbatch']
     filenames = ['*.sl']
+    mimetypes = []
     EXTRA_KEYWORDS = {'srun'}
 
     def get_tokens_unprocessed(self, text):
         for index, token, value in BashLexer.get_tokens_unprocessed(self, text):
-            print(index, token, value)
             if token is Text and value in self.EXTRA_KEYWORDS:
                 yield index, Name.Builtin, value
             elif token is Comment.Single and 'SBATCH' in value:
                 yield index, Keyword.Pseudo, value
             else:
                 yield index, token, value
-        
+
 class ShellSessionBaseLexer(Lexer):
     """
     Base lexer for simplistic shell sessions.
diff --git a/tests/examplefiles/example.sl b/tests/examplefiles/example.sl
new file mode 100644
index 00000000..5fb430de
--- /dev/null
+++ b/tests/examplefiles/example.sl
@@ -0,0 +1,6 @@
+#!/bin/bash
+#SBATCH --partition=part
+#SBATCH --job-name=job
+#SBATCH --mem=1G
+#SBATCH --cpus-per-task=8
+srun /usr/bin/sleep
\ No newline at end of file
-- 
cgit v1.2.1


From 899884637800cbe1b2c913647016bf98a54cbb20 Mon Sep 17 00:00:00 2001
From: "Matth?us G. Chajdas" 
Date: Sun, 23 Dec 2018 13:37:14 +0100
Subject: Fix Stan lexer changes eating whitespace, fix typos in example file.

---
 CHANGES                         |  1 +
 pygments/lexers/modeling.py     | 10 +++++-----
 tests/examplefiles/example.stan |  4 ++--
 3 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/CHANGES b/CHANGES
index 6d6f463a..59351ac7 100644
--- a/CHANGES
+++ b/CHANGES
@@ -18,6 +18,7 @@ Version 2.4.0
 
 - Updated lexers:
 
+  * Stan (PR#774)
   * Terraform (PR#787)
 
 - Change ANSI color names (PR#777)
diff --git a/pygments/lexers/modeling.py b/pygments/lexers/modeling.py
index 49d98d1b..481cce38 100644
--- a/pygments/lexers/modeling.py
+++ b/pygments/lexers/modeling.py
@@ -13,7 +13,7 @@ import re
 
 from pygments.lexer import RegexLexer, include, bygroups, using, default
 from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
-    Number, Punctuation
+    Number, Punctuation, Whitespace
 
 from pygments.lexers.html import HtmlLexer
 from pygments.lexers import _stan_builtins
@@ -326,14 +326,14 @@ class StanLexer(RegexLexer):
             (r'(%s)\b' % r'|'.join(_stan_builtins.TYPES), Keyword.Type),
              # < should be punctuation, but elsewhere I can't tell if it is in
              # a range constraint
-            (r'(<)\s*(upper|lower)\s*(=)', bygroups(Operator, Keyword, Punctuation)),
-            (r'(,)\s*(upper)\s*(=)', bygroups(Punctuation, Keyword, Punctuation)),
+            (r'(<)(\s*)(upper|lower)(\s*)(=)', bygroups(Operator, Whitespace, Keyword, Whitespace, Punctuation)),
+            (r'(,)(\s*)(upper)(\s*)(=)', bygroups(Punctuation, Whitespace, Keyword, Whitespace, Punctuation)),
             # Punctuation
             (r"[;,\[\]()]", Punctuation),
             # Builtin
             (r'(%s)(?=\s*\()' % '|'.join(_stan_builtins.FUNCTIONS), Name.Builtin),
-            (r'(~)\s*(%s)(?=\s*\()' % '|'.join(_stan_builtins.DISTRIBUTIONS),
-                bygroups(Operator, Name.Builtin)),
+            (r'(~)(\s*)(%s)(?=\s*\()' % '|'.join(_stan_builtins.DISTRIBUTIONS),
+                bygroups(Operator, Whitespace, Name.Builtin)),
             # Special names ending in __, like lp__
             (r'[A-Za-z]\w*__\b', Name.Builtin.Pseudo),
             (r'(%s)\b' % r'|'.join(_stan_builtins.RESERVED), Keyword.Reserved),
diff --git a/tests/examplefiles/example.stan b/tests/examplefiles/example.stan
index 69c9ac70..03b7b1b5 100644
--- a/tests/examplefiles/example.stan
+++ b/tests/examplefiles/example.stan
@@ -16,7 +16,7 @@ functions {
 data {
   // valid name
   int abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_abc;
-  // all types should be highlighed
+  // all types should be highlighted
   int a3;
   real foo[2];
   vector[3] bar;
@@ -48,7 +48,7 @@ transformed data {
   thud <- -12309865;
   // ./ and .* should be recognized as operators
   grault2 <- grault .* garply ./ garply;
-  // ' and \ should be regognized as operators
+  // ' and \ should be recognized as operators
   qux2 <- qux' \ bar;
   
 }
-- 
cgit v1.2.1


From 86d45fbf627c596c894fbc3a63402d797811db0e Mon Sep 17 00:00:00 2001
From: Robin Eklind 
Date: Mon, 31 Dec 2018 03:13:24 +0100
Subject: Update LLVM lexer to use keywords of LLVM 7.0.

The keywords are based extracted from the source code
of the LLVM lexer (llvm-7.0.0.src/lib/AsmParser) and
is based on the first argument to the KEYWORD,
TYPEKEYWORD and INSTKEYWORD macros. The list of keywords
were then sorted in lexicographic order.

A diff with added and removed keywords is made available
at https://gist.github.com/mewmew/508e090a361095eeb60ffa88321e878a

Specifically the following 10 keywords were removed:

	begin, dbg, end, free, getresult, linker_private,
	linker_private, malloc, singlethread and x86_64_win64cc

And 96 new keywords were added.
---
 pygments/lexers/asm.py | 105 +++++++++++++++++++++++++++----------------------
 1 file changed, 57 insertions(+), 48 deletions(-)

diff --git a/pygments/lexers/asm.py b/pygments/lexers/asm.py
index 0a40e641..2f08d510 100644
--- a/pygments/lexers/asm.py
+++ b/pygments/lexers/asm.py
@@ -377,54 +377,63 @@ class LlvmLexer(RegexLexer):
         'keyword': [
             # Regular keywords
             (words((
-                'begin', 'end', 'true', 'false', 'declare', 'define', 'global',
-                'constant', 'private', 'linker_private', 'internal',
-                'available_externally', 'linkonce', 'linkonce_odr', 'weak',
-                'weak_odr', 'appending', 'dllimport', 'dllexport', 'common',
-                'default', 'hidden', 'protected', 'extern_weak', 'external',
-                'thread_local', 'zeroinitializer', 'undef', 'null', 'to', 'tail',
-                'target', 'triple', 'datalayout', 'volatile', 'nuw', 'nsw', 'nnan',
-                'ninf', 'nsz', 'arcp', 'fast', 'exact', 'inbounds', 'align',
-                'addrspace', 'section', 'alias', 'module', 'asm', 'sideeffect',
-                'gc', 'dbg', 'linker_private_weak', 'attributes', 'blockaddress',
-                'initialexec', 'localdynamic', 'localexec', 'prefix', 'unnamed_addr',
-                'ccc', 'fastcc', 'coldcc', 'x86_stdcallcc', 'x86_fastcallcc',
-                'arm_apcscc', 'arm_aapcscc', 'arm_aapcs_vfpcc', 'ptx_device',
-                'ptx_kernel', 'intel_ocl_bicc', 'msp430_intrcc', 'spir_func',
-                'spir_kernel', 'x86_64_sysvcc', 'x86_64_win64cc', 'x86_thiscallcc',
-                'cc', 'c', 'signext', 'zeroext', 'inreg', 'sret', 'nounwind',
-                'noreturn', 'noalias', 'nocapture', 'byval', 'nest', 'readnone',
-                'readonly', 'inlinehint', 'noinline', 'alwaysinline', 'optsize', 'ssp',
-                'sspreq', 'noredzone', 'noimplicitfloat', 'naked', 'builtin', 'cold',
-                'nobuiltin', 'noduplicate', 'nonlazybind', 'optnone', 'returns_twice',
-                'sanitize_address', 'sanitize_memory', 'sanitize_thread', 'sspstrong',
-                'uwtable', 'returned', 'type', 'opaque', 'eq', 'ne', 'slt', 'sgt',
-                'sle', 'sge', 'ult', 'ugt', 'ule', 'uge', 'oeq', 'one', 'olt', 'ogt',
-                'ole', 'oge', 'ord', 'uno', 'ueq', 'une', 'x', 'acq_rel', 'acquire',
-                'alignstack', 'atomic', 'catch', 'cleanup', 'filter', 'inteldialect',
-                'max', 'min', 'monotonic', 'nand', 'personality', 'release', 'seq_cst',
-                'singlethread', 'umax', 'umin', 'unordered', 'xchg', 'add', 'fadd',
-                'sub', 'fsub', 'mul', 'fmul', 'udiv', 'sdiv', 'fdiv', 'urem', 'srem',
-                'frem', 'shl', 'lshr', 'ashr', 'and', 'or', 'xor', 'icmp', 'fcmp',
-                'phi', 'call', 'trunc', 'zext', 'sext', 'fptrunc', 'fpext', 'uitofp',
-                'sitofp', 'fptoui', 'fptosi', 'inttoptr', 'ptrtoint', 'bitcast',
-                'addrspacecast', 'select', 'va_arg', 'ret', 'br', 'switch', 'invoke',
-                'unwind', 'unreachable', 'indirectbr', 'landingpad', 'resume',
-                'malloc', 'alloca', 'free', 'load', 'store', 'getelementptr',
-                'extractelement', 'insertelement', 'shufflevector', 'getresult',
-                'extractvalue', 'insertvalue', 'atomicrmw', 'cmpxchg', 'fence',
-                'allocsize', 'amdgpu_cs', 'amdgpu_gs', 'amdgpu_kernel', 'amdgpu_ps',
-                'amdgpu_vs', 'any', 'anyregcc', 'argmemonly', 'avr_intrcc',
-                'avr_signalcc', 'caller', 'catchpad', 'catchret', 'catchswitch',
-                'cleanuppad', 'cleanupret', 'comdat', 'convergent', 'cxx_fast_tlscc',
-                'deplibs', 'dereferenceable', 'dereferenceable_or_null', 'distinct',
-                'exactmatch', 'externally_initialized', 'from', 'ghccc', 'hhvm_ccc',
-                'hhvmcc', 'ifunc', 'inaccessiblemem_or_argmemonly', 'inaccessiblememonly',
-                'inalloca', 'jumptable', 'largest', 'local_unnamed_addr', 'minsize',
-                'musttail', 'noduplicates', 'none', 'nonnull', 'norecurse', 'notail',
-                'preserve_allcc', 'preserve_mostcc', 'prologue', 'safestack', 'samesize',
-                'source_filename', 'swiftcc', 'swifterror', 'swiftself', 'webkit_jscc',
-                'within', 'writeonly', 'x86_intrcc', 'x86_vectorcallcc'),
+                'acq_rel', 'acquire', 'add', 'addrspace', 'addrspacecast', 'afn', 'alias',
+                'aliasee', 'align', 'alignLog2', 'alignstack', 'alloca', 'allocsize', 'allOnes',
+                'alwaysinline', 'amdgpu_cs', 'amdgpu_es', 'amdgpu_gs', 'amdgpu_hs',
+                'amdgpu_kernel', 'amdgpu_ls', 'amdgpu_ps', 'amdgpu_vs', 'and', 'any',
+                'anyregcc', 'appending', 'arcp', 'argmemonly', 'args', 'arm_aapcs_vfpcc',
+                'arm_aapcscc', 'arm_apcscc', 'ashr', 'asm', 'atomic', 'atomicrmw', 'attributes',
+                'available_externally', 'avr_intrcc', 'avr_signalcc', 'bit', 'bitcast',
+                'bitMask', 'blockaddress', 'br', 'branchFunnel', 'builtin', 'byArg', 'byte',
+                'byteArray', 'byval', 'c', 'call', 'callee', 'caller', 'calls', 'catch',
+                'catchpad', 'catchret', 'catchswitch', 'cc', 'ccc', 'cleanup', 'cleanuppad',
+                'cleanupret', 'cmpxchg', 'cold', 'coldcc', 'comdat', 'common', 'constant',
+                'contract', 'convergent', 'critical', 'cxx_fast_tlscc', 'datalayout', 'declare',
+                'default', 'define', 'deplibs', 'dereferenceable', 'dereferenceable_or_null',
+                'distinct', 'dllexport', 'dllimport', 'double', 'dso_local', 'dso_preemptable',
+                'dsoLocal', 'eq', 'exact', 'exactmatch', 'extern_weak', 'external',
+                'externally_initialized', 'extractelement', 'extractvalue', 'fadd', 'false',
+                'fast', 'fastcc', 'fcmp', 'fdiv', 'fence', 'filter', 'flags', 'float', 'fmul',
+                'fp128', 'fpext', 'fptosi', 'fptoui', 'fptrunc', 'frem', 'from', 'fsub',
+                'funcFlags', 'function', 'gc', 'getelementptr', 'ghccc', 'global', 'guid', 'gv',
+                'half', 'hash', 'hhvm_ccc', 'hhvmcc', 'hidden', 'hot', 'hotness', 'icmp',
+                'ifunc', 'inaccessiblemem_or_argmemonly', 'inaccessiblememonly', 'inalloca',
+                'inbounds', 'indir', 'indirectbr', 'info', 'initialexec', 'inline',
+                'inlineBits', 'inlinehint', 'inrange', 'inreg', 'insertelement', 'insertvalue',
+                'insts', 'intel_ocl_bicc', 'inteldialect', 'internal', 'inttoptr', 'invoke',
+                'jumptable', 'kind', 'label', 'landingpad', 'largest', 'linkage', 'linkonce',
+                'linkonce_odr', 'live', 'load', 'local_unnamed_addr', 'localdynamic',
+                'localexec', 'lshr', 'max', 'metadata', 'min', 'minsize', 'module', 'monotonic',
+                'msp430_intrcc', 'mul', 'musttail', 'naked', 'name', 'nand', 'ne', 'nest',
+                'ninf', 'nnan', 'noalias', 'nobuiltin', 'nocapture', 'nocf_check',
+                'noduplicate', 'noduplicates', 'noimplicitfloat', 'noinline', 'none',
+                'nonlazybind', 'nonnull', 'norecurse', 'noRecurse', 'noredzone', 'noreturn',
+                'notail', 'notEligibleToImport', 'nounwind', 'nsw', 'nsz', 'null', 'nuw', 'oeq',
+                'offset', 'oge', 'ogt', 'ole', 'olt', 'one', 'opaque', 'optforfuzzing',
+                'optnone', 'optsize', 'or', 'ord', 'path', 'personality', 'phi', 'ppc_fp128',
+                'prefix', 'preserve_allcc', 'preserve_mostcc', 'private', 'prologue',
+                'protected', 'ptrtoint', 'ptx_device', 'ptx_kernel', 'readnone', 'readNone',
+                'readonly', 'readOnly', 'reassoc', 'refs', 'relbf', 'release', 'resByArg',
+                'resume', 'ret', 'returnDoesNotAlias', 'returned', 'returns_twice', 'safestack',
+                'samesize', 'sanitize_address', 'sanitize_hwaddress', 'sanitize_memory',
+                'sanitize_thread', 'sdiv', 'section', 'select', 'seq_cst', 'sext', 'sge', 'sgt',
+                'shadowcallstack', 'shl', 'shufflevector', 'sideeffect', 'signext', 'single',
+                'singleImpl', 'singleImplName', 'sitofp', 'sizeM1', 'sizeM1BitWidth', 'sle',
+                'slt', 'source_filename', 'speculatable', 'spir_func', 'spir_kernel', 'srem',
+                'sret', 'ssp', 'sspreq', 'sspstrong', 'store', 'strictfp', 'sub', 'summaries',
+                'summary', 'swiftcc', 'swifterror', 'swiftself', 'switch', 'syncscope', 'tail',
+                'target', 'thread_local', 'to', 'token', 'triple', 'true', 'trunc', 'type',
+                'typeCheckedLoadConstVCalls', 'typeCheckedLoadVCalls', 'typeid', 'typeIdInfo',
+                'typeTestAssumeConstVCalls', 'typeTestAssumeVCalls', 'typeTestRes', 'typeTests',
+                'udiv', 'ueq', 'uge', 'ugt', 'uitofp', 'ule', 'ult', 'umax', 'umin', 'undef',
+                'une', 'uniformRetVal', 'uniqueRetVal', 'unknown', 'unnamed_addr', 'uno',
+                'unordered', 'unreachable', 'unsat', 'unwind', 'urem', 'uselistorder',
+                'uselistorder_bb', 'uwtable', 'va_arg', 'variable', 'vFuncId',
+                'virtualConstProp', 'void', 'volatile', 'weak', 'weak_odr', 'webkit_jscc',
+                'win64cc', 'within', 'wpdRes', 'wpdResolutions', 'writeonly', 'x',
+                'x86_64_sysvcc', 'x86_fastcallcc', 'x86_fp80', 'x86_intrcc', 'x86_mmx',
+                'x86_regcallcc', 'x86_stdcallcc', 'x86_thiscallcc', 'x86_vectorcallcc', 'xchg',
+                'xor', 'zeroext', 'zeroinitializer', 'zext'),
                 suffix=r'\b'), Keyword),
 
             # Types
-- 
cgit v1.2.1


From a895aac301b1f0dae17f03ac78b3e66bc6ebc2f8 Mon Sep 17 00:00:00 2001
From: "Matth?us G. Chajdas" 
Date: Fri, 4 Jan 2019 14:57:29 +0100
Subject: Update CHANGES.

---
 CHANGES | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGES b/CHANGES
index 59351ac7..a27baeec 100644
--- a/CHANGES
+++ b/CHANGES
@@ -14,6 +14,7 @@ Version 2.4.0
 
   * FloScript (PR#750)
   * Hspec (PR#790)
+  * LLVM (PR#792)
   * Slurm (PR#760)
 
 - Updated lexers:
-- 
cgit v1.2.1


From 5aa7fd79de2d137fb77cc60410b831312a7908aa Mon Sep 17 00:00:00 2001
From: "Matth?us G. Chajdas" 
Date: Fri, 4 Jan 2019 15:07:51 +0100
Subject: Fix a small bug in the SQL analysis.

Updated CHANGES as well, and moved the LLVM PR to the right list.
---
 CHANGES                |  3 ++-
 pygments/lexers/sql.py | 10 ++++++++--
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/CHANGES b/CHANGES
index a27baeec..eba251c6 100644
--- a/CHANGES
+++ b/CHANGES
@@ -14,11 +14,12 @@ Version 2.4.0
 
   * FloScript (PR#750)
   * Hspec (PR#790)
-  * LLVM (PR#792)
   * Slurm (PR#760)
 
 - Updated lexers:
 
+  * LLVM (PR#792)
+  * SQL (PR#672)
   * Stan (PR#774)
   * Terraform (PR#787)
 
diff --git a/pygments/lexers/sql.py b/pygments/lexers/sql.py
index d3bc8e23..8884db22 100644
--- a/pygments/lexers/sql.py
+++ b/pygments/lexers/sql.py
@@ -550,7 +550,11 @@ class TransactSqlLexer(RegexLexer):
                 name_between_backtick_re.findall((text)))
             name_between_bracket_count = len(
                 name_between_bracket_re.findall(text))
-            if name_between_bracket_count >= 2 * name_between_backtick_count:
+            # We need to check if there are any names using
+            # backticks or brackets, as otherwise both are 0
+            # and 0 >= 2 * 0, so we would always assume it's true
+            dialect_name_count = name_between_backtick_count + name_between_bracket_count
+            if dialect_name_count >= 1 and name_between_bracket_count >= 2 * name_between_backtick_count:
                 # Found at least twice as many [name] as `name`.
                 rating += 0.5
             elif name_between_bracket_count > name_between_backtick_count:
@@ -642,7 +646,9 @@ class MySqlLexer(RegexLexer):
             name_between_backtick_re.findall((text)))
         name_between_bracket_count = len(
             name_between_bracket_re.findall(text))
-        if name_between_backtick_count >= 2 * name_between_bracket_count:
+        # Same logic as above in the TSQL analysis
+        dialect_name_count = name_between_backtick_count + name_between_bracket_count
+        if dialect_name_count >= 1 and name_between_backtick_count >= 2 * name_between_bracket_count:
             # Found at least twice as many `name` as [name].
             rating += 0.5
         elif name_between_backtick_count > name_between_bracket_count:
-- 
cgit v1.2.1


From 867fbbd26114b70d04c5885e1447af1d8acb08df Mon Sep 17 00:00:00 2001
From: "Matth?us G. Chajdas" 
Date: Fri, 4 Jan 2019 15:16:01 +0100
Subject: Small improvements to the SGF lexer.

Update CHANGES, simplify regex, regenerate mappings.
---
 CHANGES                     | 1 +
 pygments/lexers/_mapping.py | 1 +
 pygments/lexers/sgf.py      | 6 ++++--
 3 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/CHANGES b/CHANGES
index eba251c6..d1330d93 100644
--- a/CHANGES
+++ b/CHANGES
@@ -14,6 +14,7 @@ Version 2.4.0
 
   * FloScript (PR#750)
   * Hspec (PR#790)
+  * SGF (PR#780)
   * Slurm (PR#760)
 
 - Updated lexers:
diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py
index d9894a33..fea05640 100644
--- a/pygments/lexers/_mapping.py
+++ b/pygments/lexers/_mapping.py
@@ -388,6 +388,7 @@ LEXERS = {
     'SlurmBashLexer': ('pygments.lexers.shell', 'Slurm', ('slurm', 'sbatch'), ('*.sl',), ()),
     'SmaliLexer': ('pygments.lexers.dalvik', 'Smali', ('smali',), ('*.smali',), ('text/smali',)),
     'SmalltalkLexer': ('pygments.lexers.smalltalk', 'Smalltalk', ('smalltalk', 'squeak', 'st'), ('*.st',), ('text/x-smalltalk',)),
+    'SmartGameFormatLexer': ('pygments.lexers.sgf', 'SmartGameFormat', ('sgf',), ('*.sgf',), ()),
     'SmartyLexer': ('pygments.lexers.templates', 'Smarty', ('smarty',), ('*.tpl',), ('application/x-smarty',)),
     'SnobolLexer': ('pygments.lexers.snobol', 'Snobol', ('snobol',), ('*.snobol',), ('text/x-snobol',)),
     'SnowballLexer': ('pygments.lexers.dsls', 'Snowball', ('snowball',), ('*.sbl',), ()),
diff --git a/pygments/lexers/sgf.py b/pygments/lexers/sgf.py
index 8dd9beaa..aa934b49 100644
--- a/pygments/lexers/sgf.py
+++ b/pygments/lexers/sgf.py
@@ -12,6 +12,8 @@
 
     :copyright: Copyright 2006-2018 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
+
+    .. versionadded:: 2.4
 """
 
 from pygments.lexer import RegexLexer, bygroups
@@ -27,7 +29,7 @@ class SmartGameFormatLexer(RegexLexer):
 
     tokens = {
         'root': [
-            (r'([\(\):;\s])', Punctuation),
+            (r'[\s():;]', Punctuation),
             # tokens:
             (r'(A[BW]|AE|AN|AP|AR|AS|[BW]L|BM|[BW]R|[BW]S|[BW]T|CA|CH|CP|CR|DD|DM|DO|DT|EL|EV|EX|FF|FG|G[BW]|GC|GM|GN|HA|HO|ID|IP|IT|IY|KM|KO|L|LB|LN|LT|M|MA|MN|N|OB|OM|ON|OP|OT|OV|P[BW]|PC|PL|PM|RE|RG|RO|RU|SO|SC|SE|SI|SL|SO|SQ|ST|SU|SZ|T[BW]|TC|TE|TM|TR|UC|US|V|VW|[BW]|C)',
              Name.Builtin),
@@ -44,7 +46,7 @@ class SmartGameFormatLexer(RegexLexer):
             (r'(\[)([a-z]{2})(:)([a-z]{2})(\])',
              bygroups(Punctuation, String, Punctuation, String, Punctuation)),
 
-            (r'(\[)([\w\s\(\)\.\-\:,\#\+\?]+)(\])',
+            (r'(\[)([\w\s#()+,\-.:?]+)(\])',
              bygroups(Punctuation, String, Punctuation)),
             (r'(\[)(\s.*)(\])',
              bygroups(Punctuation, Text, Punctuation)),
-- 
cgit v1.2.1


From 810e049f65b44e06dbfd7c8800750a6230f93b39 Mon Sep 17 00:00:00 2001
From: Micka?l Schoentgen 
Date: Mon, 7 Jan 2019 18:20:59 +0100
Subject: Fix ResourceWarning: unclosed file

Also uniformize usage of the 'with' contact manager to prevent resource leaks.
---
 pygments/formatters/__init__.py    |  3 ++-
 pygments/formatters/html.py        |  7 +++----
 pygments/lexers/__init__.py        |  3 ++-
 pygments/lexers/_cocoa_builtins.py |  3 ++-
 pygments/lexers/_php_builtins.py   | 28 ++++++++++++++--------------
 scripts/check_sources.py           |  3 ++-
 tests/test_html_formatter.py       | 10 ++++------
 7 files changed, 29 insertions(+), 28 deletions(-)

diff --git a/pygments/formatters/__init__.py b/pygments/formatters/__init__.py
index 965c5f1a..457d76ec 100644
--- a/pygments/formatters/__init__.py
+++ b/pygments/formatters/__init__.py
@@ -98,7 +98,8 @@ def load_formatter_from_file(filename, formattername="CustomFormatter",
     try:
         # This empty dict will contain the namespace for the exec'd file
         custom_namespace = {}
-        exec(open(filename, 'rb').read(), custom_namespace)
+        with open(filename, 'rb') as f:
+            exec(f.read(), custom_namespace)
         # Retrieve the class `formattername` from that namespace
         if formattername not in custom_namespace:
             raise ClassNotFound('no valid %s class found in %s' %
diff --git a/pygments/formatters/html.py b/pygments/formatters/html.py
index 2969d502..7d7605eb 100644
--- a/pygments/formatters/html.py
+++ b/pygments/formatters/html.py
@@ -535,10 +535,9 @@ class HtmlFormatter(Formatter):
             # write CSS file only if noclobber_cssfile isn't given as an option.
             try:
                 if not os.path.exists(cssfilename) or not self.noclobber_cssfile:
-                    cf = open(cssfilename, "w")
-                    cf.write(CSSFILE_TEMPLATE %
-                             {'styledefs': self.get_style_defs('body')})
-                    cf.close()
+                    with open(cssfilename, "w") as cf:
+                        cf.write(CSSFILE_TEMPLATE %
+                                 {'styledefs': self.get_style_defs('body')})
             except IOError as err:
                 err.strerror = 'Error writing CSS file: ' + err.strerror
                 raise
diff --git a/pygments/lexers/__init__.py b/pygments/lexers/__init__.py
index 328e072c..50f39d4e 100644
--- a/pygments/lexers/__init__.py
+++ b/pygments/lexers/__init__.py
@@ -133,7 +133,8 @@ def load_lexer_from_file(filename, lexername="CustomLexer", **options):
     try:
         # This empty dict will contain the namespace for the exec'd file
         custom_namespace = {}
-        exec(open(filename, 'rb').read(), custom_namespace)
+        with open(filename, 'rb') as f:
+            exec(f.read(), custom_namespace)
         # Retrieve the class `lexername` from that namespace
         if lexername not in custom_namespace:
             raise ClassNotFound('no valid %s class found in %s' %
diff --git a/pygments/lexers/_cocoa_builtins.py b/pygments/lexers/_cocoa_builtins.py
index f55e9dd7..f17ea876 100644
--- a/pygments/lexers/_cocoa_builtins.py
+++ b/pygments/lexers/_cocoa_builtins.py
@@ -40,7 +40,8 @@ if __name__ == '__main__':  # pragma: no cover
                 continue
 
             headerFilePath = frameworkHeadersDir + f
-            content = open(headerFilePath).read()
+            with open(headerFilePath) as f:
+                content = f.read()
             res = re.findall(r'(?<=@interface )\w+', content)
             for r in res:
                 all_interfaces.add(r)
diff --git a/pygments/lexers/_php_builtins.py b/pygments/lexers/_php_builtins.py
index bd4b7d99..c6084003 100644
--- a/pygments/lexers/_php_builtins.py
+++ b/pygments/lexers/_php_builtins.py
@@ -4698,18 +4698,19 @@ if __name__ == '__main__':  # pragma: no cover
 
         for file in get_php_references():
             module = ''
-            for line in open(file):
-                if not module:
-                    search = module_re.search(line)
-                    if search:
-                        module = search.group(1)
-                        modules[module] = []
+            with open(file) as f:
+                for line in f:
+                    if not module:
+                        search = module_re.search(line)
+                        if search:
+                            module = search.group(1)
+                            modules[module] = []
 
-                elif 'href="function.' in line:
-                    for match in function_re.finditer(line):
-                        fn = match.group(1)
-                        if '->' not in fn and '::' not in fn and fn not in modules[module]:
-                            modules[module].append(fn)
+                    elif 'href="function.' in line:
+                        for match in function_re.finditer(line):
+                            fn = match.group(1)
+                            if '->' not in fn and '::' not in fn and fn not in modules[module]:
+                                modules[module].append(fn)
 
             if module:
                 # These are dummy manual pages, not actual functions
@@ -4726,9 +4727,8 @@ if __name__ == '__main__':  # pragma: no cover
 
     def get_php_references():
         download = urlretrieve(PHP_MANUAL_URL)
-        tar = tarfile.open(download[0])
-        tar.extractall()
-        tar.close()
+        with tarfile.open(download[0]) as tar:
+            tar.extractall()
         for file in glob.glob("%s%s" % (PHP_MANUAL_DIR, PHP_REFERENCE_GLOB)):
             yield file
         os.remove(download[0])
diff --git a/scripts/check_sources.py b/scripts/check_sources.py
index db09de42..c0524b6c 100755
--- a/scripts/check_sources.py
+++ b/scripts/check_sources.py
@@ -185,7 +185,8 @@ def main(argv):
                 print("Checking %s..." % fn)
 
             try:
-                lines = open(fn, 'rb').read().decode('utf-8').splitlines()
+                with open(fn, 'rb') as f:
+                    lines = f.read().decode('utf-8').splitlines()
             except (IOError, OSError) as err:
                 print("%s: cannot open: %s" % (fn, err))
                 num += 1
diff --git a/tests/test_html_formatter.py b/tests/test_html_formatter.py
index 10450c56..670a5be9 100644
--- a/tests/test_html_formatter.py
+++ b/tests/test_html_formatter.py
@@ -132,9 +132,8 @@ class HtmlFormatterTest(unittest.TestCase):
                             outencoding='utf-8')
 
         handle, pathname = tempfile.mkstemp('.html')
-        tfile = os.fdopen(handle, 'w+b')
-        fmt.format(tokensource, tfile)
-        tfile.close()
+        with os.fdopen(handle, 'w+b') as tfile:
+            fmt.format(tokensource, tfile)
         catname = os.path.join(TESTDIR, 'dtds', 'HTML4.soc')
         try:
             import subprocess
@@ -173,9 +172,8 @@ class HtmlFormatterTest(unittest.TestCase):
                             cssstyles=u'div:before { content: \'bäz\' }',
                             encoding='utf-8')
         handle, pathname = tempfile.mkstemp('.html')
-        tfile = os.fdopen(handle, 'w+b')
-        fmt.format(tokensource, tfile)
-        tfile.close()
+        with os.fdopen(handle, 'w+b') as tfile:
+            fmt.format(tokensource, tfile)
 
     def test_ctags(self):
         try:
-- 
cgit v1.2.1


From ea47446222a927e88e1223e0218c2088baf0bec7 Mon Sep 17 00:00:00 2001
From: Felix Fontein 
Date: Sat, 12 Jan 2019 17:01:52 +0100
Subject: Improve YAML mapping lexing: fix problems with quoted colons.

---
 pygments/lexers/data.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pygments/lexers/data.py b/pygments/lexers/data.py
index a67d084e..7593b487 100644
--- a/pygments/lexers/data.py
+++ b/pygments/lexers/data.py
@@ -233,7 +233,7 @@ class YamlLexer(ExtendedRegexLexer):
             # whitespaces separating tokens
             (r'[ ]+', Text),
             # key with colon
-            (r'([^,:?\[\]{}\n]+)(:)(?=[ ]|$)',
+            (r'''([^,:?\[\]{}"'\n]+)(:)(?=[ ]|$)''',
              bygroups(Name.Tag, set_indent(Punctuation, implicit=True))),
             # tags, anchors and aliases,
             include('descriptors'),
@@ -312,7 +312,7 @@ class YamlLexer(ExtendedRegexLexer):
         # a flow mapping indicated by '{' and '}'
         'flow-mapping': [
             # key with colon
-            (r'([^,:?\[\]{}\n]+)(:)(?=[ ]|$)',
+            (r'''([^,:?\[\]{}"'\n]+)(:)(?=[ ]|$)''',
              bygroups(Name.Tag, Punctuation)),
             # include flow collection rules
             include('flow-collection'),
-- 
cgit v1.2.1


From 82300e5231c4d917c6221ab347226faf5a3bea3c Mon Sep 17 00:00:00 2001
From: Felix Fontein 
Date: Sat, 12 Jan 2019 17:13:50 +0100
Subject: Django lexer: add support for % and != operators.

---
 pygments/lexers/templates.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pygments/lexers/templates.py b/pygments/lexers/templates.py
index c184b2dd..8000deba 100644
--- a/pygments/lexers/templates.py
+++ b/pygments/lexers/templates.py
@@ -375,7 +375,7 @@ class DjangoLexer(RegexLexer):
             (r'\.\w+', Name.Variable),
             (r':?"(\\\\|\\"|[^"])*"', String.Double),
             (r":?'(\\\\|\\'|[^'])*'", String.Single),
-            (r'([{}()\[\]+\-*/,:~]|[><=]=?)', Operator),
+            (r'([{}()\[\]+\-*/%,:~]|[><=]=?|!=)', Operator),
             (r"[0-9](\.[0-9]*)?(eE[+-][0-9])?[flFLdD]?|"
              r"0[xX][0-9a-fA-F]+[Ll]?", Number),
         ],
-- 
cgit v1.2.1


From af0e2a48443e501d1eebfc69e92b35f36752a951 Mon Sep 17 00:00:00 2001
From: Nikolay Orlyuk 
Date: Thu, 17 Jan 2019 23:30:33 +0100
Subject: Use unicode literals in docstrings as well

Resolves #1492
---
 pygments/lexers/qvt.py           |  2 +-
 pygments/styles/arduino.py       |  4 ++--
 pygments/styles/paraiso_dark.py  |  2 +-
 pygments/styles/paraiso_light.py |  2 +-
 tests/test_cmdline.py            | 13 ++++++++++---
 5 files changed, 15 insertions(+), 8 deletions(-)

diff --git a/pygments/lexers/qvt.py b/pygments/lexers/qvt.py
index af091a65..9b2559b1 100644
--- a/pygments/lexers/qvt.py
+++ b/pygments/lexers/qvt.py
@@ -18,7 +18,7 @@ __all__ = ['QVToLexer']
 
 
 class QVToLexer(RegexLexer):
-    """
+    u"""
     For the `QVT Operational Mapping language `_.
 
     Reference for implementing this: «Meta Object Facility (MOF) 2.0
diff --git a/pygments/styles/arduino.py b/pygments/styles/arduino.py
index 57e3809e..b500b6d9 100644
--- a/pygments/styles/arduino.py
+++ b/pygments/styles/arduino.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-"""
+u"""
     pygments.styles.arduino
     ~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -15,7 +15,7 @@ from pygments.token import Keyword, Name, Comment, String, Error, \
 
 
 class ArduinoStyle(Style):
-    """
+    u"""
     The Arduino® language style. This style is designed to highlight the
     Arduino source code, so exepect the best results with it.
     """
diff --git a/pygments/styles/paraiso_dark.py b/pygments/styles/paraiso_dark.py
index 5f334bb9..68abb9f6 100644
--- a/pygments/styles/paraiso_dark.py
+++ b/pygments/styles/paraiso_dark.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-"""
+u"""
     pygments.styles.paraiso_dark
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
diff --git a/pygments/styles/paraiso_light.py b/pygments/styles/paraiso_light.py
index a8112819..186e4775 100644
--- a/pygments/styles/paraiso_light.py
+++ b/pygments/styles/paraiso_light.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-"""
+u"""
     pygments.styles.paraiso_light
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
diff --git a/tests/test_cmdline.py b/tests/test_cmdline.py
index 1500c875..a55e30ec 100644
--- a/tests/test_cmdline.py
+++ b/tests/test_cmdline.py
@@ -28,6 +28,13 @@ def func(args):
 '''
 
 
+def _decode_output(text):
+    try:
+        return text.decode('utf-8')
+    except UnicodeEncodeError:  # implicit encode on Python 2 with data loss
+        return text
+
+
 def run_cmdline(*args, **kwds):
     saved_stdin = sys.stdin
     saved_stdout = sys.stdout
@@ -53,9 +60,9 @@ def run_cmdline(*args, **kwds):
         sys.stderr = saved_stderr
     new_stdout.flush()
     new_stderr.flush()
-    out, err = stdout_buffer.getvalue().decode('utf-8'), \
-        stderr_buffer.getvalue().decode('utf-8')
-    return (ret, out, err)
+    out, err = stdout_buffer.getvalue(), \
+        stderr_buffer.getvalue()
+    return (ret, _decode_output(out), _decode_output(err))
 
 
 class CmdLineTest(unittest.TestCase):
-- 
cgit v1.2.1


From 564b072c8d0d3c40563fa1a416b58221d71d0bff Mon Sep 17 00:00:00 2001
From: "Matth?us G. Chajdas" 
Date: Tue, 29 Jan 2019 12:08:09 +0100
Subject: Update CHANGES.

---
 CHANGES | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/CHANGES b/CHANGES
index d1330d93..ebef0fbe 100644
--- a/CHANGES
+++ b/CHANGES
@@ -24,7 +24,9 @@ Version 2.4.0
   * Stan (PR#774)
   * Terraform (PR#787)
 
+- Add solarized style (PR#708)
 - Change ANSI color names (PR#777)
+- Fix rare unicode errors on Python 2.7 (PR#798, #1492)
 
 Version 2.3.1
 -------------
-- 
cgit v1.2.1


From 6250ccc494d88412ce7570e33fce9d8639935457 Mon Sep 17 00:00:00 2001
From: hugovk 
Date: Wed, 6 Feb 2019 20:14:54 +0000
Subject: Add python_requires and update Trove classifiers

---
 setup.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/setup.py b/setup.py
index 3f75a20e..52889227 100755
--- a/setup.py
+++ b/setup.py
@@ -60,6 +60,7 @@ setup(
     platforms = 'any',
     zip_safe = False,
     include_package_data = True,
+    python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*',
     classifiers = [
         'License :: OSI Approved :: BSD License',
         'Intended Audience :: Developers',
@@ -68,7 +69,11 @@ setup(
         'Development Status :: 6 - Mature',
         'Programming Language :: Python',
         'Programming Language :: Python :: 2',
+        'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
+        'Programming Language :: Python :: 3.5',
+        'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
         'Operating System :: OS Independent',
         'Topic :: Text Processing :: Filters',
         'Topic :: Utilities',
-- 
cgit v1.2.1


From a1467b3b8f2b889ae8dc7cea89ba3efceb17683e Mon Sep 17 00:00:00 2001
From: "Matth?us G. Chajdas" 
Date: Tue, 12 Feb 2019 15:48:58 +0100
Subject: Update CHANGES.

---
 CHANGES | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/CHANGES b/CHANGES
index ebef0fbe..a8b63c02 100644
--- a/CHANGES
+++ b/CHANGES
@@ -19,6 +19,7 @@ Version 2.4.0
 
 - Updated lexers:
 
+  * Cypher (PR#746)
   * LLVM (PR#792)
   * SQL (PR#672)
   * Stan (PR#774)
@@ -27,6 +28,7 @@ Version 2.4.0
 - Add solarized style (PR#708)
 - Change ANSI color names (PR#777)
 - Fix rare unicode errors on Python 2.7 (PR#798, #1492)
+- Updated Trove classifiers and ``pip`` requirements (PR#799)
 
 Version 2.3.1
 -------------
-- 
cgit v1.2.1


From 2920dc2735ff50fd1da7c9ff7553298f36a29d63 Mon Sep 17 00:00:00 2001
From: "Matth?us G. Chajdas" 
Date: Tue, 12 Feb 2019 16:00:40 +0100
Subject: Make TypeScript the default for .ts files.

---
 CHANGES                       | 1 +
 pygments/lexers/javascript.py | 4 ++++
 pygments/lexers/typoscript.py | 2 +-
 3 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/CHANGES b/CHANGES
index a8b63c02..3cb104fa 100644
--- a/CHANGES
+++ b/CHANGES
@@ -29,6 +29,7 @@ Version 2.4.0
 - Change ANSI color names (PR#777)
 - Fix rare unicode errors on Python 2.7 (PR#798, #1492)
 - Updated Trove classifiers and ``pip`` requirements (PR#799)
+- For ``.ts`` files, TypeScript is assumed now (was TypoScript previously)
 
 Version 2.3.1
 -------------
diff --git a/pygments/lexers/javascript.py b/pygments/lexers/javascript.py
index 840ce9ed..87c4af2d 100644
--- a/pygments/lexers/javascript.py
+++ b/pygments/lexers/javascript.py
@@ -453,6 +453,10 @@ class TypeScriptLexer(RegexLexer):
 
     flags = re.DOTALL | re.MULTILINE
 
+    # Higher priority than the TypoScriptLexer, as TypeScript is far more
+    # common these days
+    priority = 0.5
+
     tokens = {
         'commentsandwhitespace': [
             (r'\s+', Text),
diff --git a/pygments/lexers/typoscript.py b/pygments/lexers/typoscript.py
index 7da87c75..6e1c3155 100644
--- a/pygments/lexers/typoscript.py
+++ b/pygments/lexers/typoscript.py
@@ -114,7 +114,7 @@ class TypoScriptLexer(RegexLexer):
     flags = re.DOTALL | re.MULTILINE
 
     # Slightly higher than TypeScript (which is 0).
-    priority = 0.1
+    priority = 0.0
 
     tokens = {
         'root': [
-- 
cgit v1.2.1


From 77e24d5ab193f6c3fa8492f5410a2e841f9ebfad Mon Sep 17 00:00:00 2001
From: "Matth?us G. Chajdas" 
Date: Tue, 12 Feb 2019 16:04:45 +0100
Subject: Update CHANGES and language list.

---
 CHANGES           | 1 +
 doc/languages.rst | 5 +++++
 2 files changed, 6 insertions(+)

diff --git a/CHANGES b/CHANGES
index 3cb104fa..3657c3fc 100644
--- a/CHANGES
+++ b/CHANGES
@@ -12,6 +12,7 @@ Version 2.4.0
 
 - Added lexers:
 
+  * Charm++ CI (PR#788)
   * FloScript (PR#750)
   * Hspec (PR#790)
   * SGF (PR#780)
diff --git a/doc/languages.rst b/doc/languages.rst
index e5399403..31146531 100644
--- a/doc/languages.rst
+++ b/doc/languages.rst
@@ -20,6 +20,7 @@ Programming languages
 * BrainFuck
 * C, C++
 * C#
+* `Charm++ CI `_
 * Clojure
 * CoffeeScript
 * ColdFusion
@@ -38,6 +39,7 @@ Programming languages
 * Factor
 * Fancy
 * `Fennel `_
+* `FloScript `_
 * Fortran
 * F#
 * GAP
@@ -46,6 +48,7 @@ Programming languages
 * Groovy
 * `Haskell `_ (incl. Literate Haskell)
 * HLSL
+* `HSpec `_
 * IDL
 * Io
 * Java
@@ -81,6 +84,8 @@ Programming languages
 * Scala
 * Scheme
 * Scilab
+* `SGF `_
+* `Slurm `_
 * Smalltalk
 * SNOBOL
 * Tcl
-- 
cgit v1.2.1


From a1db49fc6bb671f38808e0db94fe6590ab01bd19 Mon Sep 17 00:00:00 2001
From: "Matth?us G. Chajdas" 
Date: Tue, 12 Feb 2019 16:09:53 +0100
Subject: Don't use Charm++ CI for C++ reserved MIME types.

---
 pygments/lexers/_mapping.py | 2 +-
 pygments/lexers/c_like.py   | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py
index 8085bb8a..331dadf3 100644
--- a/pygments/lexers/_mapping.py
+++ b/pygments/lexers/_mapping.py
@@ -80,7 +80,7 @@ LEXERS = {
     'Cfengine3Lexer': ('pygments.lexers.configs', 'CFEngine3', ('cfengine3', 'cf3'), ('*.cf',), ()),
     'ChaiscriptLexer': ('pygments.lexers.scripting', 'ChaiScript', ('chai', 'chaiscript'), ('*.chai',), ('text/x-chaiscript', 'application/x-chaiscript')),
     'ChapelLexer': ('pygments.lexers.chapel', 'Chapel', ('chapel', 'chpl'), ('*.chpl',), ()),
-    'CharmciLexer': ('pygments.lexers.c_like', 'Charmci', ('charmci',), ('*.ci',), ('text/x-c++hdr', 'text/x-c++src')),
+    'CharmciLexer': ('pygments.lexers.c_like', 'Charmci', ('charmci',), ('*.ci',), ()),
     'CheetahHtmlLexer': ('pygments.lexers.templates', 'HTML+Cheetah', ('html+cheetah', 'html+spitfire', 'htmlcheetah'), (), ('text/html+cheetah', 'text/html+spitfire')),
     'CheetahJavascriptLexer': ('pygments.lexers.templates', 'JavaScript+Cheetah', ('js+cheetah', 'javascript+cheetah', 'js+spitfire', 'javascript+spitfire'), (), ('application/x-javascript+cheetah', 'text/x-javascript+cheetah', 'text/javascript+cheetah', 'application/x-javascript+spitfire', 'text/x-javascript+spitfire', 'text/javascript+spitfire')),
     'CheetahLexer': ('pygments.lexers.templates', 'Cheetah', ('cheetah', 'spitfire'), ('*.tmpl', '*.spt'), ('application/x-cheetah', 'application/x-spitfire')),
diff --git a/pygments/lexers/c_like.py b/pygments/lexers/c_like.py
index ba735a6b..58372b81 100644
--- a/pygments/lexers/c_like.py
+++ b/pygments/lexers/c_like.py
@@ -549,6 +549,8 @@ class CharmciLexer(CppLexer):
     aliases = ['charmci']
     filenames = ['*.ci']
 
+    mimetypes = []
+
     tokens = {
         'statements': [
             (r'(module)(\s+)', bygroups(Keyword, Text), 'classname'),
-- 
cgit v1.2.1


From 4c3606b87d8af193003e30a2b6ad699ef9434469 Mon Sep 17 00:00:00 2001
From: "Matth?us G. Chajdas" 
Date: Tue, 12 Feb 2019 16:15:45 +0100
Subject: Fix CI minimum Python version.

---
 bitbucket-pipelines.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml
index fc745f0f..4a9f1b6d 100644
--- a/bitbucket-pipelines.yml
+++ b/bitbucket-pipelines.yml
@@ -9,13 +9,13 @@ pipelines:
           - pip install -r requirements.txt
           - tox -e py27
     - step:
-        name: Test on Python 3.4
-        image: python:3.4
+        name: Test on Python 3.5
+        image: python:3.5
         caches:
           - pip
         script:
           - pip install -r requirements.txt
-          - tox -e py34
+          - tox -e py35
     - step:
         name: Test on Python 3.6
         image: python:3.6
-- 
cgit v1.2.1


From 53a39c2b6dc9d2e7b73943c842f698ce8f60258d Mon Sep 17 00:00:00 2001
From: "Matth?us G. Chajdas" 
Date: Tue, 12 Feb 2019 16:28:16 +0100
Subject: Improve CSound name handling.

This should fix the last of the spurious errors we're seeing in CI.
---
 pygments/lexers/csound.py | 10 +++++++---
 tests/test_csound.py      | 11 +++++++++++
 2 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/pygments/lexers/csound.py b/pygments/lexers/csound.py
index c30b87cc..161e1576 100644
--- a/pygments/lexers/csound.py
+++ b/pygments/lexers/csound.py
@@ -227,9 +227,13 @@ class CsoundOrchestraLexer(CsoundLexer):
                 yield nameMatch.start(2), Name, nameMatch.group(2)
             else:
                 yield match.start(), Name, name
-                if match.group(2):
-                    yield match.start(2), Punctuation, match.group(2)
-                    yield match.start(3), Name, match.group(3)
+
+            # If there's a trailing :V, for example, we want to keep this around
+            # and emit it as well, otherwise this lexer will not pass round-trip
+            # testing
+            if match.group(2):
+                yield match.start(2), Punctuation, match.group(2)
+                yield match.start(3), Name, match.group(3)
 
     tokens = {
         'root': [
diff --git a/tests/test_csound.py b/tests/test_csound.py
index 4d10c267..d493bd04 100644
--- a/tests/test_csound.py
+++ b/tests/test_csound.py
@@ -478,3 +478,14 @@ class CsoundOrchestraTest(unittest.TestCase):
             (Text, u'\n')
         ]
         self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+    def testName(self):
+        fragment = 'kG:V'
+        tokens = [
+            (Keyword.Type, 'k'),
+            (Name, 'G'),
+            (Punctuation, ':'),
+            (Name, 'V'),
+            (Text, '\n')
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
-- 
cgit v1.2.1


From a3a6f24f62652d7eee55bd5fb557d631c5adf69d Mon Sep 17 00:00:00 2001
From: Matthias Diener 
Date: Tue, 12 Feb 2019 09:35:54 -0600
Subject: Add test for the Charmci lexer

---
 tests/examplefiles/Charmci.ci | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)
 create mode 100644 tests/examplefiles/Charmci.ci

diff --git a/tests/examplefiles/Charmci.ci b/tests/examplefiles/Charmci.ci
new file mode 100644
index 00000000..2e5cd5c6
--- /dev/null
+++ b/tests/examplefiles/Charmci.ci
@@ -0,0 +1,20 @@
+module CkCallback {
+	readonly CProxy_ckcallback_group _ckcallbackgroup;
+	message CkCcsRequestMsg {
+		char data[];
+ 	};
+	message CkDataMsg {
+		char data[];
+	};
+	
+	mainchare ckcallback_main {
+		entry ckcallback_main(CkArgMsg *m);
+	};
+	group [migratable] ckcallback_group : IrrGroup {
+		entry ckcallback_group();
+		entry void registerCcsCallback(char name[strlen(name)+1],
+			CkCallback cb);
+		entry void call(CkCallback c,CkMarshalledMessage msg);
+		entry void call(CkCallback c, int length, char data[length]);
+	};
+};
-- 
cgit v1.2.1


From 4b9174ce07c5365424ed25c223e484fc35427dec Mon Sep 17 00:00:00 2001
From: Mauricio Caceres Bravo 
Date: Tue, 19 Feb 2019 11:40:43 -0500
Subject: Fix StataLexer format highlight error (change Name.Format to
 Name.Other)

StataLexer defined Name.Format for variable and print format
highlighting; Name.Format is not a built-in token, which
caused issues downstream in certain applications (e.g.
[here](https://github.com/kylebarron/stata_kernel/issues/295)).
This commit changes Name.Format to Name.Other.
---
 pygments/lexers/stata.py       | 8 ++++----
 pygments/styles/stata_dark.py  | 2 +-
 pygments/styles/stata_light.py | 2 +-
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/pygments/lexers/stata.py b/pygments/lexers/stata.py
index 8955a05c..9566d12a 100644
--- a/pygments/lexers/stata.py
+++ b/pygments/lexers/stata.py
@@ -163,9 +163,9 @@ class StataLexer(RegexLexer):
         ],
         # Stata formats
         'format': [
-            (r'%-?\d{1,2}(\.\d{1,2})?[gfe]c?', Name.Format),
-            (r'%(21x|16H|16L|8H|8L)', Name.Format),
-            (r'%-?(tc|tC|td|tw|tm|tq|th|ty|tg).{0,32}', Name.Format),
-            (r'%[-~]?\d{1,4}s', Name.Format),
+            (r'%-?\d{1,2}(\.\d{1,2})?[gfe]c?', Name.Other),
+            (r'%(21x|16H|16L|8H|8L)', Name.Other),
+            (r'%-?(tc|tC|td|tw|tm|tq|th|ty|tg)\S{0,32}', Name.Other),
+            (r'%[-~]?\d{1,4}s', Name.Other),
         ]
     }
diff --git a/pygments/styles/stata_dark.py b/pygments/styles/stata_dark.py
index 7610ba0f..851a9a8d 100644
--- a/pygments/styles/stata_dark.py
+++ b/pygments/styles/stata_dark.py
@@ -31,7 +31,7 @@ class StataDarkStyle(Style):
         Number:                '#4FB8CC',
         Operator:              '',
         Name.Function:         '#6a6aff',
-        Name.Format:           '#e2828e',
+        Name.Other:            '#e2828e',
         Keyword:               'bold #7686bb',
         Keyword.Constant:      '',
         Comment:               'italic #777777',
diff --git a/pygments/styles/stata_light.py b/pygments/styles/stata_light.py
index 86802ef7..fcca85e5 100644
--- a/pygments/styles/stata_light.py
+++ b/pygments/styles/stata_light.py
@@ -30,7 +30,7 @@ class StataLightStyle(Style):
         Number:                '#2c2cff',
         Operator:              '',
         Name.Function:         '#2c2cff',
-        Name.Format:           '#be646c',
+        Name.Other:            '#be646c',
         Keyword:               'bold #353580',
         Keyword.Constant:      '',
         Comment:               'italic #008800',
-- 
cgit v1.2.1


From e198bea6ac1c3ca535a08f5cb1a5e21cba7faec5 Mon Sep 17 00:00:00 2001
From: Tobias Boege 
Date: Sat, 2 Mar 2019 00:46:03 +0100
Subject: Add Perl 6 to the language list

It is sufficiently different from Perl 5 from the perspective of
syntax highlighting that it deserves a separate mention.
---
 doc/languages.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/languages.rst b/doc/languages.rst
index 31146531..47e3363f 100644
--- a/doc/languages.rst
+++ b/doc/languages.rst
@@ -69,7 +69,7 @@ Programming languages
 * Octave
 * OCaml
 * PHP
-* `Perl `_
+* `Perl 5 `_ and `Perl 6 `_
 * PovRay
 * PostScript
 * PowerShell
-- 
cgit v1.2.1


From e58ac2780e96fe1b33275066891bf2dd03822d56 Mon Sep 17 00:00:00 2001
From: "Matth?us G. Chajdas" 
Date: Wed, 6 Mar 2019 19:44:06 +0100
Subject: Fix #1494.

This fixes the catastrophic backtracking which previously occured for strings
like "\057hom\145/e2\071501\057pub\154ic_\150tml\057a
---
 pygments/lexers/shell.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pygments/lexers/shell.py b/pygments/lexers/shell.py
index 86d8c37a..31bc7e94 100644
--- a/pygments/lexers/shell.py
+++ b/pygments/lexers/shell.py
@@ -76,7 +76,7 @@ class BashLexer(RegexLexer):
             (r'&&|\|\|', Operator),
         ],
         'data': [
-            (r'(?s)\$?"(\\\\|\\[0-7]+|\\.|[^"\\$])*"', String.Double),
+            (r'(?s)\$?"(\\.|[^"\\$])*"', String.Double),
             (r'"', String.Double, 'string'),
             (r"(?s)\$'(\\\\|\\[0-7]+|\\.|[^'\\])*'", String.Single),
             (r"(?s)'.*?'", String.Single),
-- 
cgit v1.2.1


From 424a3cc8c6367ed35c75cb259e199a1567e3e226 Mon Sep 17 00:00:00 2001
From: "Matth?us G. Chajdas" 
Date: Wed, 6 Mar 2019 19:47:03 +0100
Subject: Fix #1498.

.typoscript is used for TypoScript files, removing the ambiguity with
TypeScript.
---
 pygments/lexers/_mapping.py   | 2 +-
 pygments/lexers/javascript.py | 6 ------
 pygments/lexers/typoscript.py | 6 +-----
 3 files changed, 2 insertions(+), 12 deletions(-)

diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py
index 331dadf3..12427b3f 100644
--- a/pygments/lexers/_mapping.py
+++ b/pygments/lexers/_mapping.py
@@ -428,7 +428,7 @@ LEXERS = {
     'TypeScriptLexer': ('pygments.lexers.javascript', 'TypeScript', ('ts', 'typescript'), ('*.ts', '*.tsx'), ('text/x-typescript',)),
     'TypoScriptCssDataLexer': ('pygments.lexers.typoscript', 'TypoScriptCssData', ('typoscriptcssdata',), (), ()),
     'TypoScriptHtmlDataLexer': ('pygments.lexers.typoscript', 'TypoScriptHtmlData', ('typoscripthtmldata',), (), ()),
-    'TypoScriptLexer': ('pygments.lexers.typoscript', 'TypoScript', ('typoscript',), ('*.ts', '*.txt'), ('text/x-typoscript',)),
+    'TypoScriptLexer': ('pygments.lexers.typoscript', 'TypoScript', ('typoscript',), ('*.typoscript',), ('text/x-typoscript',)),
     'UrbiscriptLexer': ('pygments.lexers.urbi', 'UrbiScript', ('urbiscript',), ('*.u',), ('application/x-urbiscript',)),
     'VCLLexer': ('pygments.lexers.varnish', 'VCL', ('vcl',), ('*.vcl',), ('text/x-vclsrc',)),
     'VCLSnippetLexer': ('pygments.lexers.varnish', 'VCLSnippets', ('vclsnippets', 'vclsnippet'), (), ('text/x-vclsnippet',)),
diff --git a/pygments/lexers/javascript.py b/pygments/lexers/javascript.py
index 87c4af2d..0507375f 100644
--- a/pygments/lexers/javascript.py
+++ b/pygments/lexers/javascript.py
@@ -538,12 +538,6 @@ class TypeScriptLexer(RegexLexer):
         ],
     }
 
-    def analyse_text(text):
-        if re.search(r'^(import.+(from\s+)?["\']|'
-                     r'(export\s*)?(interface|class|function)\s+)',
-                     text, re.MULTILINE):
-            return 1.0
-
 
 class LassoLexer(RegexLexer):
     """
diff --git a/pygments/lexers/typoscript.py b/pygments/lexers/typoscript.py
index 6e1c3155..f75a6f02 100644
--- a/pygments/lexers/typoscript.py
+++ b/pygments/lexers/typoscript.py
@@ -108,7 +108,7 @@ class TypoScriptLexer(RegexLexer):
 
     name = 'TypoScript'
     aliases = ['typoscript']
-    filenames = ['*.ts', '*.txt']
+    filenames = ['*.typoscript']
     mimetypes = ['text/x-typoscript']
 
     flags = re.DOTALL | re.MULTILINE
@@ -220,7 +220,3 @@ class TypoScriptLexer(RegexLexer):
             (r'[\w"\-!/&;]+', Text),
         ],
     }
-
-    def analyse_text(text):
-        if '
Date: Wed, 6 Mar 2019 19:51:32 +0100
Subject: Fix #1482.

Add a shebang_match for php.
---
 pygments/lexers/php.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/pygments/lexers/php.py b/pygments/lexers/php.py
index f959fb1f..440d9d81 100644
--- a/pygments/lexers/php.py
+++ b/pygments/lexers/php.py
@@ -15,7 +15,8 @@ from pygments.lexer import RegexLexer, include, bygroups, default, using, \
     this, words
 from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
     Number, Punctuation, Other
-from pygments.util import get_bool_opt, get_list_opt, iteritems
+from pygments.util import get_bool_opt, get_list_opt, iteritems, \
+    shebang_matches
 
 __all__ = ['ZephirLexer', 'PhpLexer']
 
@@ -261,6 +262,8 @@ class PhpLexer(RegexLexer):
             yield index, token, value
 
     def analyse_text(text):
+        if shebang_matches(text, r'php'):
+            return True
         rv = 0.0
         if re.search(r'<\?(?!xml)', text):
             rv += 0.3
-- 
cgit v1.2.1


From 1f404da40668543d4b3b4493029f75fe732eca9d Mon Sep 17 00:00:00 2001
From: "Matth?us G. Chajdas" 
Date: Wed, 6 Mar 2019 20:03:27 +0100
Subject: Update CHANGES.

---
 CHANGES                  | 6 +++++-
 pygments/lexers/basic.py | 2 +-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/CHANGES b/CHANGES
index 3657c3fc..9a7426a3 100644
--- a/CHANGES
+++ b/CHANGES
@@ -17,11 +17,13 @@ Version 2.4.0
   * Hspec (PR#790)
   * SGF (PR#780)
   * Slurm (PR#760)
+  * VBScript (PR#673)
 
 - Updated lexers:
 
   * Cypher (PR#746)
   * LLVM (PR#792)
+  * PHP (#1482)
   * SQL (PR#672)
   * Stan (PR#774)
   * Terraform (PR#787)
@@ -30,7 +32,9 @@ Version 2.4.0
 - Change ANSI color names (PR#777)
 - Fix rare unicode errors on Python 2.7 (PR#798, #1492)
 - Updated Trove classifiers and ``pip`` requirements (PR#799)
-- For ``.ts`` files, TypeScript is assumed now (was TypoScript previously)
+- TypoScript uses ``.typoscript`` now (#1498)
+- Fix catastrophic backtracking in the bash lexer (#1494)
+- Fix incorrect links in the Lisp and R lexer documentation (PR#775)
 
 Version 2.3.1
 -------------
diff --git a/pygments/lexers/basic.py b/pygments/lexers/basic.py
index e1b018d1..f5f0498e 100644
--- a/pygments/lexers/basic.py
+++ b/pygments/lexers/basic.py
@@ -506,7 +506,7 @@ class VBScriptLexer(RegexLexer):
     """
     VBScript is scripting language that is modeled on Visual Basic.
 
-    .. versionadded:: 2.2
+    .. versionadded:: 2.4
     """
     name = 'VBScript'
     aliases = []
-- 
cgit v1.2.1


From 9585006f1d3d777e1a264a6b761070f52946af68 Mon Sep 17 00:00:00 2001
From: "Matth?us G. Chajdas" 
Date: Wed, 6 Mar 2019 20:12:50 +0100
Subject: Fix invalid escape sequence in the VBScript lexer.

---
 pygments/lexers/basic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pygments/lexers/basic.py b/pygments/lexers/basic.py
index f5f0498e..b0409386 100644
--- a/pygments/lexers/basic.py
+++ b/pygments/lexers/basic.py
@@ -523,7 +523,7 @@ class VBScriptLexer(RegexLexer):
             (r'[0-9]+\.[0-9]*(e[+-]?[0-9]+)?', Number.Float),
             (r'\.[0-9]+(e[+-]?[0-9]+)?', Number.Float),  # Float variant 2, for example: .1, .1e2
             (r'[0-9]+e[+-]?[0-9]+', Number.Float),  # Float variant 3, for example: 123e45
-            ('\d+', Number.Integer),
+            (r'\d+', Number.Integer),
             ('#.+#', String),  # date or time value
             (r'(dim)(\s+)([a-z_][a-z0-9_]*)',
              bygroups(Keyword.Declaration, Whitespace, Name.Variable), 'dim_more'),
-- 
cgit v1.2.1


From 665f287849402bdf05ef636e77989104e6cf8e9e Mon Sep 17 00:00:00 2001
From: Charles Ferguson 
Date: Thu, 14 Mar 2019 22:24:08 +0000
Subject: Create a Lexer class for BBC Basic files.

The Lexer class for BBC Basic handles both the numbered lines, and
unnumbered lines, of the detokenised (text) format of BBC BASIC.
The tokeniser copes, in a naive manner, with the orignal versions,
and BASIC V. It does not handle other extensions at this time, nor
does it handle inline assembler. This should be sufficient for most
cases where code needs to be presented in a colourful manner.
---
 pygments/lexers/_mapping.py    |   1 +
 pygments/lexers/basic.py       |  99 +++++++++++++++++++++++++-
 tests/examplefiles/example.bbc | 156 +++++++++++++++++++++++++++++++++++++++++
 3 files changed, 254 insertions(+), 2 deletions(-)
 create mode 100644 tests/examplefiles/example.bbc

diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py
index f8207e70..e57124db 100644
--- a/pygments/lexers/_mapping.py
+++ b/pygments/lexers/_mapping.py
@@ -47,6 +47,7 @@ LEXERS = {
     'AutoItLexer': ('pygments.lexers.automation', 'AutoIt', ('autoit',), ('*.au3',), ('text/x-autoit',)),
     'AutohotkeyLexer': ('pygments.lexers.automation', 'autohotkey', ('ahk', 'autohotkey'), ('*.ahk', '*.ahkl'), ('text/x-autohotkey',)),
     'AwkLexer': ('pygments.lexers.textedit', 'Awk', ('awk', 'gawk', 'mawk', 'nawk'), ('*.awk',), ('application/x-awk',)),
+    'BBCBasicLexer': ('pygments.lexers.basic', 'BBC Basic', ('bbcbasic',), ('*.bbc',), ()),
     'BBCodeLexer': ('pygments.lexers.markup', 'BBCode', ('bbcode',), (), ('text/x-bbcode',)),
     'BCLexer': ('pygments.lexers.algebra', 'BC', ('bc',), ('*.bc',), ()),
     'BSTLexer': ('pygments.lexers.bibtex', 'BST', ('bst', 'bst-pybtex'), ('*.bst',), ()),
diff --git a/pygments/lexers/basic.py b/pygments/lexers/basic.py
index b0409386..4d957c2b 100644
--- a/pygments/lexers/basic.py
+++ b/pygments/lexers/basic.py
@@ -18,7 +18,8 @@ from pygments.lexers import _vbscript_builtins
 
 
 __all__ = ['BlitzBasicLexer', 'BlitzMaxLexer', 'MonkeyLexer', 'CbmBasicV2Lexer',
-           'QBasicLexer', 'VBScriptLexer']
+           'QBasicLexer', 'VBScriptLexer', 'BBCBasicLexer']
+
 
 
 class BlitzMaxLexer(RegexLexer):
@@ -561,4 +562,98 @@ class VBScriptLexer(RegexLexer):
             (r'"', String.Double, '#pop'),
             (r'\n', Error, '#pop'),  # Unterminated string
         ],
-    }
\ No newline at end of file
+    }
+
+
+class BBCBasicLexer(RegexLexer):
+    """
+    BBC Basic was supplied on the BBC Micro, and later Acorn RISC OS.
+    It is also used by BBC Basic For Windows.
+
+    .. versionadded:: 2.4????
+    """
+    base_keywords = ['OTHERWISE', 'AND', 'DIV', 'EOR', 'MOD', 'OR', 'ERROR',
+                     'LINE', 'OFF', 'STEP', 'SPC', 'TAB', 'ELSE', 'THEN',
+                     'OPENIN', 'PTR', 'PAGE', 'TIME', 'LOMEM', 'HIMEM', 'ABS',
+                     'ACS', 'ADVAL', 'ASC', 'ASN', 'ATN', 'BGET', 'COS', 'COUNT',
+                     'DEG', 'ERL', 'ERR', 'EVAL', 'EXP', 'EXT', 'FALSE', 'FN',
+                     'GET', 'INKEY', 'INSTR', 'INT', 'LEN', 'LN', 'LOG', 'NOT',
+                     'OPENUP', 'OPENOUT', 'PI', 'POINT', 'POS', 'RAD', 'RND',
+                     'SGN', 'SIN', 'SQR', 'TAN', 'TO', 'TRUE', 'USR', 'VAL',
+                     'VPOS', 'CHR$', 'GET$', 'INKEY$', 'LEFT$', 'MID$',
+                     'RIGHT$', 'STR$', 'STRING$', 'EOF', 'PTR', 'PAGE', 'TIME',
+                     'LOMEM', 'HIMEM', 'SOUND', 'BPUT', 'CALL', 'CHAIN', 'CLEAR',
+                     'CLOSE', 'CLG', 'CLS', 'DATA', 'DEF', 'DIM', 'DRAW', 'END',
+                     'ENDPROC', 'ENVELOPE', 'FOR', 'GOSUB', 'GOTO', 'GCOL', 'IF',
+                     'INPUT', 'LET', 'LOCAL', 'MODE', 'MOVE', 'NEXT', 'ON',
+                     'VDU', 'PLOT', 'PRINT', 'PROC', 'READ', 'REM', 'REPEAT',
+                     'REPORT', 'RESTORE', 'RETURN', 'RUN', 'STOP', 'COLOUR',
+                     'TRACE', 'UNTIL', 'WIDTH', 'OSCLI']
+
+    basic5_keywords = ['WHEN', 'OF', 'ENDCASE', 'ENDIF', 'ENDWHILE', 'CASE',
+                       'CIRCLE', 'FILL', 'ORIGIN', 'POINT', 'RECTANGLE', 'SWAP',
+                       'WHILE', 'WAIT', 'MOUSE', 'QUIT', 'SYS', 'INSTALL',
+                       'LIBRARY', 'TINT', 'ELLIPSE', 'BEATS', 'TEMPO', 'VOICES',
+                       'VOICE', 'STEREO', 'OVERLAY', 'APPEND', 'AUTO', 'CRUNCH',
+                       'DELETE', 'EDIT', 'HELP', 'LIST', 'LOAD', 'LVAR', 'NEW',
+                       'OLD', 'RENUMBER', 'SAVE', 'TEXTLOAD', 'TEXTSAVE',
+                       'TWIN', 'TWINO', 'INSTALL', 'SUM', 'BEAT']
+
+
+    name = 'BBC Basic'
+    aliases = ['bbcbasic']
+    filenames = ['*.bbc']
+
+    tokens = {
+        'root': [
+            (r"[0-9]+", Name.Label),
+            (r"(\*)([^\n]*)",
+             bygroups(Keyword.Pseudo, Comment.Special)),
+            (r"", Whitespace, 'code'),
+        ],
+
+        'code': [
+            (r"(REM)([^\n]*)",
+             bygroups(Keyword.Declaration, Comment.Single)),
+            (r'\n', Whitespace, 'root'),
+            (r'\s+', Whitespace),
+            (r':', Comment.Preproc),
+
+            # Some special cases to make functions come out nicer
+            (r'(DEF)(\s*)(FN|PROC)([A-Za-z_@][A-Za-z0-9_@]*)',
+             bygroups(Keyword.Declaration, Whitespace, Keyword.Declaration, Name.Function)),
+            (r'(FN|PROC)([A-Za-z_@][A-Za-z0-9_@]*)',
+             bygroups(Keyword, Name.Function)),
+
+            (r'(GOTO|GOSUB|THEN|RESTORE)(\s*)(\d+)',
+             bygroups(Keyword, Whitespace, Name.Label)),
+
+            (r'(TRUE|FALSE)', Keyword.Constant),
+            (r'(PAGE|LOMEM|HIMEM|TIME|WIDTH|ERL|ERR|REPORT\$|POS|VPOS|VOICES)', Keyword.Pseudo),
+
+            (words(base_keywords), Keyword),
+            (words(basic5_keywords), Keyword),
+
+            ('"', String.Double, 'string'),
+
+            ('%[01]{1,32}', Number.Bin),
+            ('&[0-9a-f]{1,8}', Number.Hex),
+
+            (r'[+-]?[0-9]+\.[0-9]*(E[+-]?[0-9]+)?', Number.Float),
+            (r'[+-]?\.[0-9]+(E[+-]?[0-9]+)?', Number.Float),
+            (r'[+-]?[0-9]+E[+-]?[0-9]+', Number.Float),
+            (r'[+-]?\d+', Number.Integer),
+
+            (r'([A-Za-z_@][A-Za-z0-9_@]*[%$]?)', Name.Variable),
+            (r'([+\-]=|[$!|?+\-*/%^=><();]|>=|<=|<>|<<|>>|>>>|,)', Operator),
+        ],
+        'string': [
+            (r'[^"\n]+', String.Double),
+            (r'"', String.Double, '#pop'),
+            (r'\n', Error, 'root'),  # Unterminated string
+        ],
+    }
+
+    def analyse_text(text):
+        if text.startswith('10REM >') or text.startswith('REM >'):
+            return 0.9
diff --git a/tests/examplefiles/example.bbc b/tests/examplefiles/example.bbc
new file mode 100644
index 00000000..ebdb8537
--- /dev/null
+++ b/tests/examplefiles/example.bbc
@@ -0,0 +1,156 @@
+10REM >EIRC
+20REM The simplest IRC client you can write. Maybe.
+30REM (C) Justin Fletcher, 1998
+40:
+50END=PAGE+1024*16
+60REM Change these if you wish
+70host$="irc.stealth.net"
+80port=6667
+90nick$="eirc"
+100ourchan$="#acorn"
+110:
+120REM Start connecting to a host
+130SYS "ESocket_ConnectToHost",host$,port TO handle
+140REPEAT
+150 SYS "ESocket_CheckState",handle TO state
+160 IF state<-1 THENSYS "ESocket_Forget",handle:SYS "ESocket_DecodeState",state TO a$:ERROR 1,"Failed ("+a$+")"
+170UNTIL state=4
+180:
+190REM We are now connected
+200PRINT"Connected"
+210:
+220REM Log on to the server
+230SYS "ESocket_SendLine",handle,"USER "+nick$+" x x :"+nick$
+240SYS "ESocket_SendLine",handle,"NICK "+nick$
+250SYS "ESocket_SendLine",handle,"JOIN "+ourchan$
+260REM Install a monitor so that we don't waste time
+270SYS "ESocket_Monitor",0,handle TO monitor
+280SYS "ESocket_ResetMonitor",monitor,0 TO polladdr%
+290:
+300REM If we crash, we should tidy up after ourselves
+310ON ERROR SYS "XESocket_Forget",handle:SYS "XESocket_Forget",monitor:ERROR EXT ERR,REPORT$+" at line "+STR$ERL
+320:
+330REM Memory buffer for our data
+340bufsize%=1024
+350DIM buf% bufsize%
+360:
+370input$="":REM The input line
+380REPEAT
+390 REM In a taskwindow we should yield until there is data
+400 SYS "OS_UpCall",6,polladdr%
+410 IF !polladdr%<>0 THEN
+420  REM Reset the monitor for the time being
+430  SYS "ESocket_ResetMonitor",monitor,0 TO polladdr%
+440  REPEAT
+450   REM Read lines from the connection until this buffer is empty
+460   SYS "ESocket_ReadLine",handle,buf%,bufsize%,%100 TO ,str,len
+470   IF str<>0 AND $str<>"" THEN
+480    line$=$str
+490    IF LEFT$(line$,4)="PING" THEN
+500     REM Ping's must be replied to immediately
+510     SYS "ESocket_SendLine",handle,"PONG "+MID$(line$,6)
+520    ELSE
+530     REM Extract source info
+540     from$=MID$(LEFT$(line$,INSTR(line$+" "," ")-1),2)
+550     line$=MID$(line$,INSTR(line$+" "," ")+1)
+560     uid$=LEFT$(from$,INSTR(from$+"!","!")-1)
+570     com$=LEFT$(line$,INSTR(line$+" "," ")-1)
+580     line$=MID$(line$,INSTR(line$+" "," ")+1)
+590     REM remove the input line
+600     IF input$<>"" THENFORI=1TOLEN(input$):VDU127:NEXT
+610     CASE FNupper(com$) OF
+620      WHEN "PRIVMSG"
+630       REM Extract the destination
+640       chan$=LEFT$(line$,INSTR(line$+" "," ")-1)
+650       line$=MID$(line$,INSTR(line$+" "," ")+2):REM Skip :
+660       IF LEFT$(line$,1)=CHR$1 THEN
+670        REM CTCP, so respond to it
+680        line$=MID$(line$,2,LEN(line$)-2)
+690        com$=LEFT$(line$,INSTR(line$+" "," ")-1)
+700        line$=MID$(line$,INSTR(line$+" "," ")+1)
+710        CASE FNupper(com$) OF
+720         WHEN "PING"
+730          REM Ping lag timing
+740          line$="PONG "+line$
+750          PRINTuid$;" pinged us"
+760         WHEN "VERSION"
+770          REM Version checking
+780          line$="VERSION EIRC 1.00 (c) Justin Fletcher"
+790          PRINTuid$;" wanted our version"
+800         WHEN "ACTION"
+810          PRINT"* ";uid$;" ";line$
+820          line$=""
+830         OTHERWISE
+840          REM everything else is an error
+850          line$="ERRMSG "+com$+" not understood"
+860          PRINT"CTCP '";com$;"' from ";uid$;" (";line$;")"
+870        ENDCASE
+880        IF line$<>"" THEN
+890         SYS "ESocket_SendLine",handle,"NOTICE "+uid$+" :"+CHR$1+line$+CHR$1
+900        ENDIF
+910       ELSE
+920        REM Somebody said something...
+930        PRINT"<";uid$;"> ";FNsafe(line$)
+940       ENDIF
+950      WHEN "JOIN"
+960       REM We (or someone else) has joined the channel
+970       chan$=LEFT$(line$,INSTR(line$+" "," ")):REM Skip :
+980       IF LEFT$(chan$,1)=":" THENchan$=MID$(chan$,2)
+990       PRINTuid$;" has joined ";chan$
+1000      WHEN "PART"
+1010       REM Someone else has left the channel
+1020       chan$=LEFT$(line$,INSTR(line$+" "," ")-1)
+1030       IF LEFT$(chan$,1)=":" THENchan$=MID$(chan$,2)
+1040       PRINTuid$;" has left ";chan$
+1050      WHEN "QUIT"
+1060       REM Someone else has quit IRC
+1070       PRINTuid$;" quit IRC"
+1080      OTHERWISE
+1090       REM Some unknown command
+1100       PRINTuid$;":";com$;":";FNsafe(line$)
+1110     ENDCASE
+1120     REM Re-display our input line
+1130     PRINTinput$;
+1140    ENDIF
+1150   ENDIF
+1160  UNTIL str=0
+1170 ENDIF
+1180 b$=INKEY$(0)
+1190 IF b$<>"" THEN
+1200  CASE b$ OF
+1210   WHEN CHR$13
+1220    SYS "ESocket_SendLine",handle,"PRIVMSG "+ourchan$+" :"+input$
+1230    REM Remove the line
+1240    IF input$<>"" THENFORI=1TOLEN(input$):VDU127:NEXT
+1250    REM We said it...
+1260    PRINT"<"+nick$+"> ";input$
+1270    input$=""
+1280   WHEN CHR$127,CHR$8
+1290    REM Backspace
+1300    IF input$<>"" THENVDU127
+1310    input$=LEFT$(input$)
+1320   OTHERWISE
+1330    REM Ad to current input
+1340    input$+=b$
+1350    PRINTb$;
+1360  ENDCASE
+1370 ENDIF
+1380 REM Has the socket closed
+1390 SYS "ESocket_Closed",handle,%0 TO closed
+1400UNTIL closed
+1410SYS "ESocket_Forget",handle
+1420SYS "ESocket_Forget",monitor
+1430END
+1440:
+1450DEFFNupper(a$):LOCAL c$,b$,I
+1460FORI=1TOLEN(a$)
+1470c$=MID$(a$,I,1):IF c$>="a"ANDc$<="z"THENc$=CHR$(ASC(c$)-32)
+1480b$+=c$:NEXT:=b$
+1490
+1500REM Remove control codes
+1510DEFFNsafe(line$)
+1520LOCAL I
+1530FORI=1TOLEN(line$)
+1540 IF MID$(line$,I,1)<" " THENMID$(line$,I,1)="*"
+1550NEXT
+1560=line$
-- 
cgit v1.2.1


From 2b6127732b2c638303e4f6e06fef4b3dcad05aea Mon Sep 17 00:00:00 2001
From: Brian Tiffin 
Date: Wed, 20 Mar 2019 08:57:15 +0000
Subject: _mapping.py edited online with Bitbucket to fix Typoscript mapping
 merge conflict when submitting the Unicon and Icon lexers

---
 pygments/lexers/_mapping.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py
index e0ec34af..1a4606db 100644
--- a/pygments/lexers/_mapping.py
+++ b/pygments/lexers/_mapping.py
@@ -421,7 +421,7 @@ LEXERS = {
     'TypeScriptLexer': ('pygments.lexers.javascript', 'TypeScript', ('ts', 'typescript'), ('*.ts', '*.tsx'), ('text/x-typescript',)),
     'TypoScriptCssDataLexer': ('pygments.lexers.typoscript', 'TypoScriptCssData', ('typoscriptcssdata',), (), ()),
     'TypoScriptHtmlDataLexer': ('pygments.lexers.typoscript', 'TypoScriptHtmlData', ('typoscripthtmldata',), (), ()),
-    'TypoScriptLexer': ('pygments.lexers.typoscript', 'TypoScript', ('typoscript',), ('*.ts', '*.txt'), ('text/x-typoscript',)),
+    'TypoScriptLexer': ('pygments.lexers.typoscript', 'TypoScript', ('typoscript',), ('*.typoscript',), ('text/x-typoscript',)),
     'UcodeLexer': ('pygments.lexers.unicon', 'ucode', ('ucode',), ('*.u', '*.u1', '*.u2'), ()),
     'UniconLexer': ('pygments.lexers.unicon', 'Unicon', ('unicon',), ('*.icn',), ('text/unicon',)),    
     'UrbiscriptLexer': ('pygments.lexers.urbi', 'UrbiScript', ('urbiscript',), ('*.u',), ('application/x-urbiscript',)),
-- 
cgit v1.2.1


From 1bcd973510624a5449b2b1155229b9eb4ca85f70 Mon Sep 17 00:00:00 2001
From: "Frederik ?Freso? S. Olesen" 
Date: Sun, 31 Mar 2019 17:32:01 +0200
Subject: Add lexers for DASM16, Augeas, TOML, and Slash

Lexers copied unmodified from
https://github.com/liluo/pygments-github-lexers
which is available under a 2-clause BSD license (same as pygments),
copyright 2012 to GitHub, Inc.

Fixes #1391 and #1150.
---
 AUTHORS                     |   1 +
 CHANGES                     |   4 +
 pygments/lexers/_mapping.py |   4 +
 pygments/lexers/asm.py      | 106 ++++++++++++++++++++++++-
 pygments/lexers/configs.py  |  79 ++++++++++++++++++-
 pygments/lexers/slash.py    | 184 ++++++++++++++++++++++++++++++++++++++++++++
 6 files changed, 376 insertions(+), 2 deletions(-)
 create mode 100644 pygments/lexers/slash.py

diff --git a/AUTHORS b/AUTHORS
index ed9c547c..e6f90fea 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -221,5 +221,6 @@ Other contributors, listed alphabetically, are:
 * Rob Zimmerman -- Kal lexer
 * Vincent Zurczak -- Roboconf lexer
 * Rostyslav Golda -- FloScript lexer
+* GitHub, Inc -- DASM16, Augeas, TOML, and Slash lexers
 
 Many thanks for all contributions!
diff --git a/CHANGES b/CHANGES
index 15863b27..3464ef22 100644
--- a/CHANGES
+++ b/CHANGES
@@ -19,6 +19,10 @@ Version 2.4.0
   * Slurm (PR#760)
   * Unicon (PR#731)
   * VBScript (PR#673)
+  * DASM16 (PR#807)
+  * Augeas (PR#807)
+  * TOML (PR#807)
+  * Slash (PR#807)
 
 - Updated lexers:
 
diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py
index 6e0f728c..3c05d147 100644
--- a/pygments/lexers/_mapping.py
+++ b/pygments/lexers/_mapping.py
@@ -44,6 +44,7 @@ LEXERS = {
     'ArduinoLexer': ('pygments.lexers.c_like', 'Arduino', ('arduino',), ('*.ino',), ('text/x-arduino',)),
     'AspectJLexer': ('pygments.lexers.jvm', 'AspectJ', ('aspectj',), ('*.aj',), ('text/x-aspectj',)),
     'AsymptoteLexer': ('pygments.lexers.graphics', 'Asymptote', ('asy', 'asymptote'), ('*.asy',), ('text/x-asymptote',)),
+    'AugeasLexer': ('pygments.lexers.configs', 'Augeas', ('augeas',), ('*.aug'), ()),
     'AutoItLexer': ('pygments.lexers.automation', 'AutoIt', ('autoit',), ('*.au3',), ('text/x-autoit',)),
     'AutohotkeyLexer': ('pygments.lexers.automation', 'autohotkey', ('ahk', 'autohotkey'), ('*.ahk', '*.ahkl'), ('text/x-autohotkey',)),
     'AwkLexer': ('pygments.lexers.textedit', 'Awk', ('awk', 'gawk', 'mawk', 'nawk'), ('*.awk',), ('application/x-awk',)),
@@ -121,6 +122,7 @@ LEXERS = {
     'DObjdumpLexer': ('pygments.lexers.asm', 'd-objdump', ('d-objdump',), ('*.d-objdump',), ('text/x-d-objdump',)),
     'DarcsPatchLexer': ('pygments.lexers.diff', 'Darcs Patch', ('dpatch',), ('*.dpatch', '*.darcspatch'), ()),
     'DartLexer': ('pygments.lexers.javascript', 'Dart', ('dart',), ('*.dart',), ('text/x-dart',)),
+    'Dasm16Lexer': ('pygments.lexers.asm', 'DASM16', ('dasm16',), ('*.dasm16', '*.dasm'), ('text/x-dasm16')),
     'DebianControlLexer': ('pygments.lexers.installers', 'Debian Control file', ('control', 'debcontrol'), ('control',), ()),
     'DelphiLexer': ('pygments.lexers.pascal', 'Delphi', ('delphi', 'pas', 'pascal', 'objectpascal'), ('*.pas', '*.dpr'), ('text/x-pascal',)),
     'DgLexer': ('pygments.lexers.python', 'dg', ('dg',), ('*.dg',), ('text/x-dg',)),
@@ -375,6 +377,7 @@ LEXERS = {
     'RubyLexer': ('pygments.lexers.ruby', 'Ruby', ('rb', 'ruby', 'duby'), ('*.rb', '*.rbw', 'Rakefile', '*.rake', '*.gemspec', '*.rbx', '*.duby', 'Gemfile'), ('text/x-ruby', 'application/x-ruby')),
     'RustLexer': ('pygments.lexers.rust', 'Rust', ('rust', 'rs'), ('*.rs', '*.rs.in'), ('text/rust',)),
     'SASLexer': ('pygments.lexers.sas', 'SAS', ('sas',), ('*.SAS', '*.sas'), ('text/x-sas', 'text/sas', 'application/x-sas')),
+    'SlashLexer': ('pygments.lexers.slash', 'Slash', ('slash'), ('*.sl'), ()),
     'SLexer': ('pygments.lexers.r', 'S', ('splus', 's', 'r'), ('*.S', '*.R', '.Rhistory', '.Rprofile', '.Renviron'), ('text/S-plus', 'text/S', 'text/x-r-source', 'text/x-r', 'text/x-R', 'text/x-r-history', 'text/x-r-profile')),
     'SMLLexer': ('pygments.lexers.ml', 'Standard ML', ('sml',), ('*.sml', '*.sig', '*.fun'), ('text/x-standardml', 'application/x-standardml')),
     'SarlLexer': ('pygments.lexers.jvm', 'SARL', ('sarl',), ('*.sarl',), ('text/x-sarl',)),
@@ -421,6 +424,7 @@ LEXERS = {
     'TextLexer': ('pygments.lexers.special', 'Text only', ('text',), ('*.txt',), ('text/plain',)),
     'ThriftLexer': ('pygments.lexers.dsls', 'Thrift', ('thrift',), ('*.thrift',), ('application/x-thrift',)),
     'TodotxtLexer': ('pygments.lexers.textfmts', 'Todotxt', ('todotxt',), ('todo.txt', '*.todotxt'), ('text/x-todo',)),
+    'TOMLLexer': ('pygments.lexers.configs', 'TOML', ('toml',), ('*.toml'), ()),
     'TransactSqlLexer': ('pygments.lexers.sql', 'Transact-SQL', ('tsql', 't-sql'), ('*.sql',), ('text/x-tsql',)),
     'TreetopLexer': ('pygments.lexers.parsers', 'Treetop', ('treetop',), ('*.treetop', '*.tt'), ()),
     'TurtleLexer': ('pygments.lexers.rdf', 'Turtle', ('turtle',), ('*.ttl',), ('text/turtle', 'application/x-turtle')),
diff --git a/pygments/lexers/asm.py b/pygments/lexers/asm.py
index 2f08d510..761b3315 100644
--- a/pygments/lexers/asm.py
+++ b/pygments/lexers/asm.py
@@ -20,7 +20,7 @@ from pygments.token import Text, Name, Number, String, Comment, Punctuation, \
 
 __all__ = ['GasLexer', 'ObjdumpLexer', 'DObjdumpLexer', 'CppObjdumpLexer',
            'CObjdumpLexer', 'HsailLexer', 'LlvmLexer', 'NasmLexer',
-           'NasmObjdumpLexer', 'TasmLexer', 'Ca65Lexer']
+           'NasmObjdumpLexer', 'TasmLexer', 'Ca65Lexer', 'Dasm16Lexer']
 
 
 class GasLexer(RegexLexer):
@@ -650,3 +650,107 @@ class Ca65Lexer(RegexLexer):
         # comments in GAS start with "#"
         if re.match(r'^\s*;', text, re.MULTILINE):
             return 0.9
+
+
+class Dasm16Lexer(RegexLexer):
+    """
+    Simple lexer for DCPU-16 Assembly
+
+    Check http://0x10c.com/doc/dcpu-16.txt
+    """
+    name = 'dasm16'
+    aliases = ['DASM16']
+    filenames = ['*.dasm16', '*.dasm']
+    mimetypes = ['text/x-dasm16']
+
+    INSTRUCTIONS = [
+        'SET',
+        'ADD', 'SUB',
+        'MUL', 'MLI',
+        'DIV', 'DVI',
+        'MOD', 'MDI',
+        'AND', 'BOR', 'XOR',
+        'SHR', 'ASR', 'SHL',
+        'IFB', 'IFC', 'IFE', 'IFN', 'IFG', 'IFA', 'IFL', 'IFU',
+        'ADX', 'SBX',
+        'STI', 'STD',
+        'JSR',
+        'INT', 'IAG', 'IAS', 'RFI', 'IAQ', 'HWN', 'HWQ', 'HWI',
+    ]
+
+    REGISTERS = [
+        'A', 'B', 'C',
+        'X', 'Y', 'Z',
+        'I', 'J',
+        'SP', 'PC', 'EX',
+        'POP', 'PEEK', 'PUSH'
+    ]
+
+    # Regexes yo
+    char = r'[a-zA-Z$._0-9@]'
+    identifier = r'(?:[a-zA-Z$_]' + char + '*|\.' + char + '+)'
+    number = r'[+-]?(?:0[xX][a-zA-Z0-9]+|\d+)'
+    binary_number = r'0b[01_]+'
+    instruction = r'(?i)(' + '|'.join(INSTRUCTIONS) + ')'
+    single_char = r"'\\?" + char + "'"
+    string = r'"(\\"|[^"])*"'
+
+    def guess_identifier(lexer, match):
+        ident = match.group(0)
+        klass = Name.Variable if ident.upper() in lexer.REGISTERS else Name.Label
+        yield match.start(), klass, ident
+
+    tokens = {
+        'root': [
+            include('whitespace'),
+            (':' + identifier, Name.Label),
+            (identifier + ':', Name.Label),
+            (instruction, Name.Function, 'instruction-args'),
+            (r'\.' + identifier, Name.Function, 'data-args'),
+            (r'[\r\n]+', Text)
+        ],
+
+        'numeric' : [
+            (binary_number, Number.Integer),
+            (number, Number.Integer),
+            (single_char, String),
+        ],
+
+        'arg' : [
+            (identifier, guess_identifier),
+            include('numeric')
+        ],
+
+        'deref' : [
+            (r'\+', Punctuation),
+            (r'\]', Punctuation, '#pop'),
+            include('arg'),
+            include('whitespace')
+        ],
+
+        'instruction-line' : [
+            (r'[\r\n]+', Text, '#pop'),
+            (r';.*?$', Comment, '#pop'),
+            include('whitespace')
+        ],
+
+        'instruction-args': [
+            (r',', Punctuation),
+            (r'\[', Punctuation, 'deref'),
+            include('arg'),
+            include('instruction-line')
+        ],
+
+        'data-args' : [
+            (r',', Punctuation),
+            include('numeric'),
+            (string, String),
+            include('instruction-line')
+        ],
+
+        'whitespace': [
+            (r'\n', Text),
+            (r'\s+', Text),
+            (r';.*?\n', Comment)
+        ],
+    }
diff --git a/pygments/lexers/configs.py b/pygments/lexers/configs.py
index 206ec360..7c9bd655 100644
--- a/pygments/lexers/configs.py
+++ b/pygments/lexers/configs.py
@@ -21,7 +21,7 @@ __all__ = ['IniLexer', 'RegeditLexer', 'PropertiesLexer', 'KconfigLexer',
            'Cfengine3Lexer', 'ApacheConfLexer', 'SquidConfLexer',
            'NginxConfLexer', 'LighttpdConfLexer', 'DockerLexer',
            'TerraformLexer', 'TermcapLexer', 'TerminfoLexer',
-           'PkgConfigLexer', 'PacmanConfLexer']
+           'PkgConfigLexer', 'PacmanConfLexer', 'AugeasLexer', 'TOMLLexer']
 
 
 class IniLexer(RegexLexer):
@@ -838,3 +838,80 @@ class PacmanConfLexer(RegexLexer):
             (r'.', Text),
         ],
     }
+
+
+class AugeasLexer(RegexLexer):
+    name = 'Augeas'
+    aliases = ['augeas']
+    filenames = ['*.aug']
+
+    tokens = {
+        'root': [
+            (r'(module)(\s*)([^\s=]+)', bygroups(Keyword.Namespace, Text, Name.Namespace)),
+            (r'(let)(\s*)([^\s=]+)', bygroups(Keyword.Declaration, Text, Name.Variable)),
+            (r'(del|store|value|counter|seq|key|label|autoload|incl|excl|transform|test|get|put)(\s+)', bygroups(Name.Builtin, Text)),
+            (r'(\()([^\:]+)(\:)(unit|string|regexp|lens|tree|filter)(\))', bygroups(Punctuation, Name.Variable, Punctuation, Keyword.Type, Punctuation)),
+            (r'\(\*', Comment.Multiline, 'comment'),
+            (r'[\+=\|\.\*\;\?-]', Operator),
+            (r'[\[\]\(\)\{\}]', Operator),
+            (r'"', String.Double, 'string'),
+            (r'\/', String.Regex, 'regex'),
+            (r'([A-Z]\w*)(\.)(\w+)', bygroups(Name.Namespace, Punctuation, Name.Variable)),
+            (r'.', Name.Variable),
+            (r'\s', Text),
+        ],
+        'string': [
+            (r'\\.', String.Escape),
+            (r'[^"]', String.Double),
+            (r'"', String.Double, '#pop'),
+        ],
+        'regex': [
+            (r'\\.', String.Escape),
+            (r'[^\/]', String.Regex),
+            (r'\/', String.Regex, '#pop'),
+        ],
+        'comment': [
+            (r'[^*\)]', Comment.Multiline),
+            (r'\(\*', Comment.Multiline, '#push'),
+            (r'\*\)', Comment.Multiline, '#pop'),
+            (r'[\*\)]', Comment.Multiline)
+        ],
+    }
+
+
+class TOMLLexer(RegexLexer):
+    """
+    Lexer for TOML, a simple language for config files
+    """
+
+    name = 'TOML'
+    aliases = ['toml']
+    filenames = ['*.toml']
+
+    tokens = {
+        'root': [
+
+            # Basics, comments, strings
+            (r'\s+', Text),
+            (r'#.*?$', Comment.Single),
+            (r'"(\\\\|\\"|[^"])*"', String),
+            (r'(true|false)$', Keyword.Constant),
+            ('[a-zA-Z_][a-zA-Z0-9_\-]*', Name),
+
+            # Datetime
+            (r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z', Number.Integer),
+
+            # Numbers
+            (r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?j?', Number.Float),
+            (r'\d+[eE][+-]?[0-9]+j?', Number.Float),
+            (r'\-?\d+', Number.Integer),
+
+            # Punctuation
+            (r'[]{}:(),;[]', Punctuation),
+            (r'\.', Punctuation),
+
+            # Operators
+            (r'=', Operator)
+
+        ]
+    }
diff --git a/pygments/lexers/slash.py b/pygments/lexers/slash.py
new file mode 100644
index 00000000..f18059f7
--- /dev/null
+++ b/pygments/lexers/slash.py
@@ -0,0 +1,184 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.lexers.slash
+    ~~~~~~~~~~~~~~~~~~~~~
+
+    Lexer for the Slash programming language.
+
+    :copyright: Copyright 2012 by GitHub, Inc
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import ExtendedRegexLexer, bygroups, DelegatingLexer
+from pygments.token import Name, Number, String, Comment, Punctuation, \
+     Other, Keyword, Operator, Whitespace
+
+__all__ = ['SlashLexer']
+
+
+class SlashLanguageLexer(ExtendedRegexLexer):
+    _nkw = r'(?=[^a-zA-Z_0-9])'
+
+    def move_state(new_state):
+        return ("#pop", new_state)
+
+    def right_angle_bracket(lexer, match, ctx):
+        if len(ctx.stack) > 1 and ctx.stack[-2] == "string":
+            ctx.stack.pop()
+        yield match.start(), String.Interpol, "}"
+        ctx.pos = match.end()
+        pass
+
+    tokens = {
+        "root": [
+            (r"<%=",        Comment.Preproc,    move_state("slash")),
+            (r"<%!!",       Comment.Preproc,    move_state("slash")),
+            (r"<%#.*?%>",   Comment.Multiline),
+            (r"<%",         Comment.Preproc,    move_state("slash")),
+            (r".|\n",       Other),
+        ],
+        "string": [
+            (r"\\",         String.Escape,      move_state("string_e")),
+            (r"\"",         String,             move_state("slash")),
+            (r"#\{",        String.Interpol,    "slash"),
+            (r'.|\n',       String),
+        ],
+        "string_e": [
+            (r'n',                  String.Escape,      move_state("string")),
+            (r't',                  String.Escape,      move_state("string")),
+            (r'r',                  String.Escape,      move_state("string")),
+            (r'e',                  String.Escape,      move_state("string")),
+            (r'x[a-fA-F0-9]{2}',    String.Escape,      move_state("string")),
+            (r'.',                  String.Escape,      move_state("string")),
+        ],
+        "regexp": [
+            (r'}[a-z]*',            String.Regex,       move_state("slash")),
+            (r'\\(.|\n)',           String.Regex),
+            (r'{',                  String.Regex,       "regexp_r"),
+            (r'.|\n',               String.Regex),
+        ],
+        "regexp_r": [
+            (r'}[a-z]*',            String.Regex,       "#pop"),
+            (r'\\(.|\n)',           String.Regex),
+            (r'{',                  String.Regex,       "regexp_r"),
+        ],
+        "slash": [
+            (r"%>",                     Comment.Preproc,    move_state("root")),
+            (r"\"",                     String,             move_state("string")),
+            (r"'[a-zA-Z0-9_]+",         String),
+            (r'%r{',                    String.Regex,       move_state("regexp")),
+            (r'/\*.*?\*/',              Comment.Multiline),
+            (r"(#|//).*?\n",            Comment.Single),
+            (r'-?[0-9]+e[+-]?[0-9]+',   Number.Float),
+            (r'-?[0-9]+\.[0-9]+(e[+-]?[0-9]+)?', Number.Float),
+            (r'-?[0-9]+',               Number.Integer),
+            (r'nil'+_nkw,               Name.Builtin),
+            (r'true'+_nkw,              Name.Builtin),
+            (r'false'+_nkw,             Name.Builtin),
+            (r'self'+_nkw,              Name.Builtin),
+            (r'(class)(\s+)([A-Z][a-zA-Z0-9_\']*)',
+                bygroups(Keyword, Whitespace, Name.Class)),
+            (r'class'+_nkw,             Keyword),
+            (r'extends'+_nkw,           Keyword),
+            (r'(def)(\s+)(self)(\s*)(\.)(\s*)([a-z_][a-zA-Z0-9_\']*=?|<<|>>|==|<=>|<=|<|>=|>|\+|-(self)?|~(self)?|\*|/|%|^|&&|&|\||\[\]=?)',
+                bygroups(Keyword, Whitespace, Name.Builtin, Whitespace, Punctuation, Whitespace, Name.Function)),
+            (r'(def)(\s+)([a-z_][a-zA-Z0-9_\']*=?|<<|>>|==|<=>|<=|<|>=|>|\+|-(self)?|~(self)?|\*|/|%|^|&&|&|\||\[\]=?)',
+                bygroups(Keyword, Whitespace, Name.Function)),
+            (r'def'+_nkw,               Keyword),
+            (r'if'+_nkw,                Keyword),
+            (r'elsif'+_nkw,             Keyword),
+            (r'else'+_nkw,              Keyword),
+            (r'unless'+_nkw,            Keyword),
+            (r'for'+_nkw,               Keyword),
+            (r'in'+_nkw,                Keyword),
+            (r'while'+_nkw,             Keyword),
+            (r'until'+_nkw,             Keyword),
+            (r'and'+_nkw,               Keyword),
+            (r'or'+_nkw,                Keyword),
+            (r'not'+_nkw,               Keyword),
+            (r'lambda'+_nkw,            Keyword),
+            (r'try'+_nkw,               Keyword),
+            (r'catch'+_nkw,             Keyword),
+            (r'return'+_nkw,            Keyword),
+            (r'next'+_nkw,              Keyword),
+            (r'last'+_nkw,              Keyword),
+            (r'throw'+_nkw,             Keyword),
+            (r'use'+_nkw,               Keyword),
+            (r'switch'+_nkw,            Keyword),
+            (r'\\',                     Keyword),
+            (r'λ',                      Keyword),
+            (r'__FILE__'+_nkw,          Name.Builtin.Pseudo),
+            (r'__LINE__'+_nkw,          Name.Builtin.Pseudo),
+            (r'[A-Z][a-zA-Z0-9_\']*'+_nkw, Name.Constant),
+            (r'[a-z_][a-zA-Z0-9_\']*'+_nkw, Name),
+            (r'@[a-z_][a-zA-Z0-9_\']*'+_nkw, Name.Variable.Instance),
+            (r'@@[a-z_][a-zA-Z0-9_\']*'+_nkw, Name.Variable.Class),
+            (r'\(',                     Punctuation),
+            (r'\)',                     Punctuation),
+            (r'\[',                     Punctuation),
+            (r'\]',                     Punctuation),
+            (r'\{',                     Punctuation),
+            (r'\}',                     right_angle_bracket),
+            (r';',                      Punctuation),
+            (r',',                      Punctuation),
+            (r'<<=',                    Operator),
+            (r'>>=',                    Operator),
+            (r'<<',                     Operator),
+            (r'>>',                     Operator),
+            (r'==',                     Operator),
+            (r'!=',                     Operator),
+            (r'=>',                     Operator),
+            (r'=',                      Operator),
+            (r'<=>',                    Operator),
+            (r'<=',                     Operator),
+            (r'>=',                     Operator),
+            (r'<',                      Operator),
+            (r'>',                      Operator),
+            (r'\+\+',                   Operator),
+            (r'\+=',                    Operator),
+            (r'-=',                     Operator),
+            (r'\*\*=',                  Operator),
+            (r'\*=',                    Operator),
+            (r'\*\*',                   Operator),
+            (r'\*',                     Operator),
+            (r'/=',                     Operator),
+            (r'\+',                     Operator),
+            (r'-',                      Operator),
+            (r'/',                      Operator),
+            (r'%=',                     Operator),
+            (r'%',                      Operator),
+            (r'^=',                     Operator),
+            (r'&&=',                    Operator),
+            (r'&=',                     Operator),
+            (r'&&',                     Operator),
+            (r'&',                      Operator),
+            (r'\|\|=',                  Operator),
+            (r'\|=',                    Operator),
+            (r'\|\|',                   Operator),
+            (r'\|',                     Operator),
+            (r'!',                      Operator),
+            (r'\.\.\.',                 Operator),
+            (r'\.\.',                   Operator),
+            (r'\.',                     Operator),
+            (r'::',                     Operator),
+            (r':',                      Operator),
+            (r'(\s|\n)+',               Whitespace),
+            (r'[a-z_][a-zA-Z0-9_\']*',  Name.Variable),
+        ],
+    }
+
+
+class SlashLexer(DelegatingLexer):
+    """
+    Lexer for the Slash programming language.
+    """
+
+    name = 'Slash'
+    aliases = ['slash']
+    filenames = ['*.sl']
+
+    def __init__(self, **options):
+        from pygments.lexers.web import HtmlLexer
+        super(SlashLexer, self).__init__(HtmlLexer, SlashLanguageLexer, **options)
-- 
cgit v1.2.1


From d050666629b06261aed0e0e82648edeefe59e0c6 Mon Sep 17 00:00:00 2001
From: "Frederik ?Freso? S. Olesen" 
Date: Sun, 31 Mar 2019 17:32:01 +0200
Subject: Add lexers for DASM16, Augeas, TOML, and Slash

Lexers copied unmodified from
https://github.com/liluo/pygments-github-lexers
which is available under a 2-clause BSD license (same as pygments),
copyright 2012 to GitHub, Inc.

Fixes #1391 and #1150.
---
 AUTHORS                     |   1 +
 CHANGES                     |   4 +
 pygments/lexers/_mapping.py |   4 +
 pygments/lexers/asm.py      | 106 ++++++++++++++++++++++++-
 pygments/lexers/configs.py  |  79 ++++++++++++++++++-
 pygments/lexers/slash.py    | 184 ++++++++++++++++++++++++++++++++++++++++++++
 6 files changed, 376 insertions(+), 2 deletions(-)
 create mode 100644 pygments/lexers/slash.py

diff --git a/AUTHORS b/AUTHORS
index ed9c547c..e6f90fea 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -221,5 +221,6 @@ Other contributors, listed alphabetically, are:
 * Rob Zimmerman -- Kal lexer
 * Vincent Zurczak -- Roboconf lexer
 * Rostyslav Golda -- FloScript lexer
+* GitHub, Inc -- DASM16, Augeas, TOML, and Slash lexers
 
 Many thanks for all contributions!
diff --git a/CHANGES b/CHANGES
index 9a7426a3..84e68f1e 100644
--- a/CHANGES
+++ b/CHANGES
@@ -18,6 +18,10 @@ Version 2.4.0
   * SGF (PR#780)
   * Slurm (PR#760)
   * VBScript (PR#673)
+  * DASM16
+  * Augeas
+  * TOML
+  * Slash
 
 - Updated lexers:
 
diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py
index f8207e70..2a068d0b 100644
--- a/pygments/lexers/_mapping.py
+++ b/pygments/lexers/_mapping.py
@@ -44,6 +44,7 @@ LEXERS = {
     'ArduinoLexer': ('pygments.lexers.c_like', 'Arduino', ('arduino',), ('*.ino',), ('text/x-arduino',)),
     'AspectJLexer': ('pygments.lexers.jvm', 'AspectJ', ('aspectj',), ('*.aj',), ('text/x-aspectj',)),
     'AsymptoteLexer': ('pygments.lexers.graphics', 'Asymptote', ('asy', 'asymptote'), ('*.asy',), ('text/x-asymptote',)),
+    'AugeasLexer': ('pygments.lexers.configs', 'Augeas', ('augeas',), ('*.aug'), ()),
     'AutoItLexer': ('pygments.lexers.automation', 'AutoIt', ('autoit',), ('*.au3',), ('text/x-autoit',)),
     'AutohotkeyLexer': ('pygments.lexers.automation', 'autohotkey', ('ahk', 'autohotkey'), ('*.ahk', '*.ahkl'), ('text/x-autohotkey',)),
     'AwkLexer': ('pygments.lexers.textedit', 'Awk', ('awk', 'gawk', 'mawk', 'nawk'), ('*.awk',), ('application/x-awk',)),
@@ -121,6 +122,7 @@ LEXERS = {
     'DObjdumpLexer': ('pygments.lexers.asm', 'd-objdump', ('d-objdump',), ('*.d-objdump',), ('text/x-d-objdump',)),
     'DarcsPatchLexer': ('pygments.lexers.diff', 'Darcs Patch', ('dpatch',), ('*.dpatch', '*.darcspatch'), ()),
     'DartLexer': ('pygments.lexers.javascript', 'Dart', ('dart',), ('*.dart',), ('text/x-dart',)),
+    'Dasm16Lexer': ('pygments.lexers.asm', 'DASM16', ('dasm16',), ('*.dasm16', '*.dasm'), ('text/x-dasm16')),
     'DebianControlLexer': ('pygments.lexers.installers', 'Debian Control file', ('control', 'debcontrol'), ('control',), ()),
     'DelphiLexer': ('pygments.lexers.pascal', 'Delphi', ('delphi', 'pas', 'pascal', 'objectpascal'), ('*.pas', '*.dpr'), ('text/x-pascal',)),
     'DgLexer': ('pygments.lexers.python', 'dg', ('dg',), ('*.dg',), ('text/x-dg',)),
@@ -374,6 +376,7 @@ LEXERS = {
     'RubyLexer': ('pygments.lexers.ruby', 'Ruby', ('rb', 'ruby', 'duby'), ('*.rb', '*.rbw', 'Rakefile', '*.rake', '*.gemspec', '*.rbx', '*.duby', 'Gemfile'), ('text/x-ruby', 'application/x-ruby')),
     'RustLexer': ('pygments.lexers.rust', 'Rust', ('rust', 'rs'), ('*.rs', '*.rs.in'), ('text/rust',)),
     'SASLexer': ('pygments.lexers.sas', 'SAS', ('sas',), ('*.SAS', '*.sas'), ('text/x-sas', 'text/sas', 'application/x-sas')),
+    'SlashLexer': ('pygments.lexers.slash', 'Slash', ('slash'), ('*.sl'), ()),
     'SLexer': ('pygments.lexers.r', 'S', ('splus', 's', 'r'), ('*.S', '*.R', '.Rhistory', '.Rprofile', '.Renviron'), ('text/S-plus', 'text/S', 'text/x-r-source', 'text/x-r', 'text/x-R', 'text/x-r-history', 'text/x-r-profile')),
     'SMLLexer': ('pygments.lexers.ml', 'Standard ML', ('sml',), ('*.sml', '*.sig', '*.fun'), ('text/x-standardml', 'application/x-standardml')),
     'SarlLexer': ('pygments.lexers.jvm', 'SARL', ('sarl',), ('*.sarl',), ('text/x-sarl',)),
@@ -420,6 +423,7 @@ LEXERS = {
     'TextLexer': ('pygments.lexers.special', 'Text only', ('text',), ('*.txt',), ('text/plain',)),
     'ThriftLexer': ('pygments.lexers.dsls', 'Thrift', ('thrift',), ('*.thrift',), ('application/x-thrift',)),
     'TodotxtLexer': ('pygments.lexers.textfmts', 'Todotxt', ('todotxt',), ('todo.txt', '*.todotxt'), ('text/x-todo',)),
+    'TOMLLexer': ('pygments.lexers.configs', 'TOML', ('toml',), ('*.toml'), ()),
     'TransactSqlLexer': ('pygments.lexers.sql', 'Transact-SQL', ('tsql', 't-sql'), ('*.sql',), ('text/x-tsql',)),
     'TreetopLexer': ('pygments.lexers.parsers', 'Treetop', ('treetop',), ('*.treetop', '*.tt'), ()),
     'TurtleLexer': ('pygments.lexers.rdf', 'Turtle', ('turtle',), ('*.ttl',), ('text/turtle', 'application/x-turtle')),
diff --git a/pygments/lexers/asm.py b/pygments/lexers/asm.py
index 2f08d510..761b3315 100644
--- a/pygments/lexers/asm.py
+++ b/pygments/lexers/asm.py
@@ -20,7 +20,7 @@ from pygments.token import Text, Name, Number, String, Comment, Punctuation, \
 
 __all__ = ['GasLexer', 'ObjdumpLexer', 'DObjdumpLexer', 'CppObjdumpLexer',
            'CObjdumpLexer', 'HsailLexer', 'LlvmLexer', 'NasmLexer',
-           'NasmObjdumpLexer', 'TasmLexer', 'Ca65Lexer']
+           'NasmObjdumpLexer', 'TasmLexer', 'Ca65Lexer', 'Dasm16Lexer']
 
 
 class GasLexer(RegexLexer):
@@ -650,3 +650,107 @@ class Ca65Lexer(RegexLexer):
         # comments in GAS start with "#"
         if re.match(r'^\s*;', text, re.MULTILINE):
             return 0.9
+
+
+class Dasm16Lexer(RegexLexer):
+    """
+    Simple lexer for DCPU-16 Assembly
+
+    Check http://0x10c.com/doc/dcpu-16.txt
+    """
+    name = 'dasm16'
+    aliases = ['DASM16']
+    filenames = ['*.dasm16', '*.dasm']
+    mimetypes = ['text/x-dasm16']
+
+    INSTRUCTIONS = [
+        'SET',
+        'ADD', 'SUB',
+        'MUL', 'MLI',
+        'DIV', 'DVI',
+        'MOD', 'MDI',
+        'AND', 'BOR', 'XOR',
+        'SHR', 'ASR', 'SHL',
+        'IFB', 'IFC', 'IFE', 'IFN', 'IFG', 'IFA', 'IFL', 'IFU',
+        'ADX', 'SBX',
+        'STI', 'STD',
+        'JSR',
+        'INT', 'IAG', 'IAS', 'RFI', 'IAQ', 'HWN', 'HWQ', 'HWI',
+    ]
+
+    REGISTERS = [
+        'A', 'B', 'C',
+        'X', 'Y', 'Z',
+        'I', 'J',
+        'SP', 'PC', 'EX',
+        'POP', 'PEEK', 'PUSH'
+    ]
+
+    # Regexes yo
+    char = r'[a-zA-Z$._0-9@]'
+    identifier = r'(?:[a-zA-Z$_]' + char + '*|\.' + char + '+)'
+    number = r'[+-]?(?:0[xX][a-zA-Z0-9]+|\d+)'
+    binary_number = r'0b[01_]+'
+    instruction = r'(?i)(' + '|'.join(INSTRUCTIONS) + ')'
+    single_char = r"'\\?" + char + "'"
+    string = r'"(\\"|[^"])*"'
+
+    def guess_identifier(lexer, match):
+        ident = match.group(0)
+        klass = Name.Variable if ident.upper() in lexer.REGISTERS else Name.Label
+        yield match.start(), klass, ident
+
+    tokens = {
+        'root': [
+            include('whitespace'),
+            (':' + identifier, Name.Label),
+            (identifier + ':', Name.Label),
+            (instruction, Name.Function, 'instruction-args'),
+            (r'\.' + identifier, Name.Function, 'data-args'),
+            (r'[\r\n]+', Text)
+        ],
+
+        'numeric' : [
+            (binary_number, Number.Integer),
+            (number, Number.Integer),
+            (single_char, String),
+        ],
+
+        'arg' : [
+            (identifier, guess_identifier),
+            include('numeric')
+        ],
+
+        'deref' : [
+            (r'\+', Punctuation),
+            (r'\]', Punctuation, '#pop'),
+            include('arg'),
+            include('whitespace')
+        ],
+
+        'instruction-line' : [
+            (r'[\r\n]+', Text, '#pop'),
+            (r';.*?$', Comment, '#pop'),
+            include('whitespace')
+        ],
+
+        'instruction-args': [
+            (r',', Punctuation),
+            (r'\[', Punctuation, 'deref'),
+            include('arg'),
+            include('instruction-line')
+        ],
+
+        'data-args' : [
+            (r',', Punctuation),
+            include('numeric'),
+            (string, String),
+            include('instruction-line')
+        ],
+
+        'whitespace': [
+            (r'\n', Text),
+            (r'\s+', Text),
+            (r';.*?\n', Comment)
+        ],
+    }
diff --git a/pygments/lexers/configs.py b/pygments/lexers/configs.py
index 206ec360..7c9bd655 100644
--- a/pygments/lexers/configs.py
+++ b/pygments/lexers/configs.py
@@ -21,7 +21,7 @@ __all__ = ['IniLexer', 'RegeditLexer', 'PropertiesLexer', 'KconfigLexer',
            'Cfengine3Lexer', 'ApacheConfLexer', 'SquidConfLexer',
            'NginxConfLexer', 'LighttpdConfLexer', 'DockerLexer',
            'TerraformLexer', 'TermcapLexer', 'TerminfoLexer',
-           'PkgConfigLexer', 'PacmanConfLexer']
+           'PkgConfigLexer', 'PacmanConfLexer', 'AugeasLexer', 'TOMLLexer']
 
 
 class IniLexer(RegexLexer):
@@ -838,3 +838,80 @@ class PacmanConfLexer(RegexLexer):
             (r'.', Text),
         ],
     }
+
+
+class AugeasLexer(RegexLexer):
+    name = 'Augeas'
+    aliases = ['augeas']
+    filenames = ['*.aug']
+
+    tokens = {
+        'root': [
+            (r'(module)(\s*)([^\s=]+)', bygroups(Keyword.Namespace, Text, Name.Namespace)),
+            (r'(let)(\s*)([^\s=]+)', bygroups(Keyword.Declaration, Text, Name.Variable)),
+            (r'(del|store|value|counter|seq|key|label|autoload|incl|excl|transform|test|get|put)(\s+)', bygroups(Name.Builtin, Text)),
+            (r'(\()([^\:]+)(\:)(unit|string|regexp|lens|tree|filter)(\))', bygroups(Punctuation, Name.Variable, Punctuation, Keyword.Type, Punctuation)),
+            (r'\(\*', Comment.Multiline, 'comment'),
+            (r'[\+=\|\.\*\;\?-]', Operator),
+            (r'[\[\]\(\)\{\}]', Operator),
+            (r'"', String.Double, 'string'),
+            (r'\/', String.Regex, 'regex'),
+            (r'([A-Z]\w*)(\.)(\w+)', bygroups(Name.Namespace, Punctuation, Name.Variable)),
+            (r'.', Name.Variable),
+            (r'\s', Text),
+        ],
+        'string': [
+            (r'\\.', String.Escape),
+            (r'[^"]', String.Double),
+            (r'"', String.Double, '#pop'),
+        ],
+        'regex': [
+            (r'\\.', String.Escape),
+            (r'[^\/]', String.Regex),
+            (r'\/', String.Regex, '#pop'),
+        ],
+        'comment': [
+            (r'[^*\)]', Comment.Multiline),
+            (r'\(\*', Comment.Multiline, '#push'),
+            (r'\*\)', Comment.Multiline, '#pop'),
+            (r'[\*\)]', Comment.Multiline)
+        ],
+    }
+
+
+class TOMLLexer(RegexLexer):
+    """
+    Lexer for TOML, a simple language for config files
+    """
+
+    name = 'TOML'
+    aliases = ['toml']
+    filenames = ['*.toml']
+
+    tokens = {
+        'root': [
+
+            # Basics, comments, strings
+            (r'\s+', Text),
+            (r'#.*?$', Comment.Single),
+            (r'"(\\\\|\\"|[^"])*"', String),
+            (r'(true|false)$', Keyword.Constant),
+            ('[a-zA-Z_][a-zA-Z0-9_\-]*', Name),
+
+            # Datetime
+            (r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z', Number.Integer),
+
+            # Numbers
+            (r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?j?', Number.Float),
+            (r'\d+[eE][+-]?[0-9]+j?', Number.Float),
+            (r'\-?\d+', Number.Integer),
+
+            # Punctuation
+            (r'[]{}:(),;[]', Punctuation),
+            (r'\.', Punctuation),
+
+            # Operators
+            (r'=', Operator)
+
+        ]
+    }
diff --git a/pygments/lexers/slash.py b/pygments/lexers/slash.py
new file mode 100644
index 00000000..f18059f7
--- /dev/null
+++ b/pygments/lexers/slash.py
@@ -0,0 +1,184 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.lexers.slash
+    ~~~~~~~~~~~~~~~~~~~~~
+
+    Lexer for the Slash programming language.
+
+    :copyright: Copyright 2012 by GitHub, Inc
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import ExtendedRegexLexer, bygroups, DelegatingLexer
+from pygments.token import Name, Number, String, Comment, Punctuation, \
+     Other, Keyword, Operator, Whitespace
+
+__all__ = ['SlashLexer']
+
+
+class SlashLanguageLexer(ExtendedRegexLexer):
+    _nkw = r'(?=[^a-zA-Z_0-9])'
+
+    def move_state(new_state):
+        return ("#pop", new_state)
+
+    def right_angle_bracket(lexer, match, ctx):
+        if len(ctx.stack) > 1 and ctx.stack[-2] == "string":
+            ctx.stack.pop()
+        yield match.start(), String.Interpol, "}"
+        ctx.pos = match.end()
+        pass
+
+    tokens = {
+        "root": [
+            (r"<%=",        Comment.Preproc,    move_state("slash")),
+            (r"<%!!",       Comment.Preproc,    move_state("slash")),
+            (r"<%#.*?%>",   Comment.Multiline),
+            (r"<%",         Comment.Preproc,    move_state("slash")),
+            (r".|\n",       Other),
+        ],
+        "string": [
+            (r"\\",         String.Escape,      move_state("string_e")),
+            (r"\"",         String,             move_state("slash")),
+            (r"#\{",        String.Interpol,    "slash"),
+            (r'.|\n',       String),
+        ],
+        "string_e": [
+            (r'n',                  String.Escape,      move_state("string")),
+            (r't',                  String.Escape,      move_state("string")),
+            (r'r',                  String.Escape,      move_state("string")),
+            (r'e',                  String.Escape,      move_state("string")),
+            (r'x[a-fA-F0-9]{2}',    String.Escape,      move_state("string")),
+            (r'.',                  String.Escape,      move_state("string")),
+        ],
+        "regexp": [
+            (r'}[a-z]*',            String.Regex,       move_state("slash")),
+            (r'\\(.|\n)',           String.Regex),
+            (r'{',                  String.Regex,       "regexp_r"),
+            (r'.|\n',               String.Regex),
+        ],
+        "regexp_r": [
+            (r'}[a-z]*',            String.Regex,       "#pop"),
+            (r'\\(.|\n)',           String.Regex),
+            (r'{',                  String.Regex,       "regexp_r"),
+        ],
+        "slash": [
+            (r"%>",                     Comment.Preproc,    move_state("root")),
+            (r"\"",                     String,             move_state("string")),
+            (r"'[a-zA-Z0-9_]+",         String),
+            (r'%r{',                    String.Regex,       move_state("regexp")),
+            (r'/\*.*?\*/',              Comment.Multiline),
+            (r"(#|//).*?\n",            Comment.Single),
+            (r'-?[0-9]+e[+-]?[0-9]+',   Number.Float),
+            (r'-?[0-9]+\.[0-9]+(e[+-]?[0-9]+)?', Number.Float),
+            (r'-?[0-9]+',               Number.Integer),
+            (r'nil'+_nkw,               Name.Builtin),
+            (r'true'+_nkw,              Name.Builtin),
+            (r'false'+_nkw,             Name.Builtin),
+            (r'self'+_nkw,              Name.Builtin),
+            (r'(class)(\s+)([A-Z][a-zA-Z0-9_\']*)',
+                bygroups(Keyword, Whitespace, Name.Class)),
+            (r'class'+_nkw,             Keyword),
+            (r'extends'+_nkw,           Keyword),
+            (r'(def)(\s+)(self)(\s*)(\.)(\s*)([a-z_][a-zA-Z0-9_\']*=?|<<|>>|==|<=>|<=|<|>=|>|\+|-(self)?|~(self)?|\*|/|%|^|&&|&|\||\[\]=?)',
+                bygroups(Keyword, Whitespace, Name.Builtin, Whitespace, Punctuation, Whitespace, Name.Function)),
+            (r'(def)(\s+)([a-z_][a-zA-Z0-9_\']*=?|<<|>>|==|<=>|<=|<|>=|>|\+|-(self)?|~(self)?|\*|/|%|^|&&|&|\||\[\]=?)',
+                bygroups(Keyword, Whitespace, Name.Function)),
+            (r'def'+_nkw,               Keyword),
+            (r'if'+_nkw,                Keyword),
+            (r'elsif'+_nkw,             Keyword),
+            (r'else'+_nkw,              Keyword),
+            (r'unless'+_nkw,            Keyword),
+            (r'for'+_nkw,               Keyword),
+            (r'in'+_nkw,                Keyword),
+            (r'while'+_nkw,             Keyword),
+            (r'until'+_nkw,             Keyword),
+            (r'and'+_nkw,               Keyword),
+            (r'or'+_nkw,                Keyword),
+            (r'not'+_nkw,               Keyword),
+            (r'lambda'+_nkw,            Keyword),
+            (r'try'+_nkw,               Keyword),
+            (r'catch'+_nkw,             Keyword),
+            (r'return'+_nkw,            Keyword),
+            (r'next'+_nkw,              Keyword),
+            (r'last'+_nkw,              Keyword),
+            (r'throw'+_nkw,             Keyword),
+            (r'use'+_nkw,               Keyword),
+            (r'switch'+_nkw,            Keyword),
+            (r'\\',                     Keyword),
+            (r'λ',                      Keyword),
+            (r'__FILE__'+_nkw,          Name.Builtin.Pseudo),
+            (r'__LINE__'+_nkw,          Name.Builtin.Pseudo),
+            (r'[A-Z][a-zA-Z0-9_\']*'+_nkw, Name.Constant),
+            (r'[a-z_][a-zA-Z0-9_\']*'+_nkw, Name),
+            (r'@[a-z_][a-zA-Z0-9_\']*'+_nkw, Name.Variable.Instance),
+            (r'@@[a-z_][a-zA-Z0-9_\']*'+_nkw, Name.Variable.Class),
+            (r'\(',                     Punctuation),
+            (r'\)',                     Punctuation),
+            (r'\[',                     Punctuation),
+            (r'\]',                     Punctuation),
+            (r'\{',                     Punctuation),
+            (r'\}',                     right_angle_bracket),
+            (r';',                      Punctuation),
+            (r',',                      Punctuation),
+            (r'<<=',                    Operator),
+            (r'>>=',                    Operator),
+            (r'<<',                     Operator),
+            (r'>>',                     Operator),
+            (r'==',                     Operator),
+            (r'!=',                     Operator),
+            (r'=>',                     Operator),
+            (r'=',                      Operator),
+            (r'<=>',                    Operator),
+            (r'<=',                     Operator),
+            (r'>=',                     Operator),
+            (r'<',                      Operator),
+            (r'>',                      Operator),
+            (r'\+\+',                   Operator),
+            (r'\+=',                    Operator),
+            (r'-=',                     Operator),
+            (r'\*\*=',                  Operator),
+            (r'\*=',                    Operator),
+            (r'\*\*',                   Operator),
+            (r'\*',                     Operator),
+            (r'/=',                     Operator),
+            (r'\+',                     Operator),
+            (r'-',                      Operator),
+            (r'/',                      Operator),
+            (r'%=',                     Operator),
+            (r'%',                      Operator),
+            (r'^=',                     Operator),
+            (r'&&=',                    Operator),
+            (r'&=',                     Operator),
+            (r'&&',                     Operator),
+            (r'&',                      Operator),
+            (r'\|\|=',                  Operator),
+            (r'\|=',                    Operator),
+            (r'\|\|',                   Operator),
+            (r'\|',                     Operator),
+            (r'!',                      Operator),
+            (r'\.\.\.',                 Operator),
+            (r'\.\.',                   Operator),
+            (r'\.',                     Operator),
+            (r'::',                     Operator),
+            (r':',                      Operator),
+            (r'(\s|\n)+',               Whitespace),
+            (r'[a-z_][a-zA-Z0-9_\']*',  Name.Variable),
+        ],
+    }
+
+
+class SlashLexer(DelegatingLexer):
+    """
+    Lexer for the Slash programming language.
+    """
+
+    name = 'Slash'
+    aliases = ['slash']
+    filenames = ['*.sl']
+
+    def __init__(self, **options):
+        from pygments.lexers.web import HtmlLexer
+        super(SlashLexer, self).__init__(HtmlLexer, SlashLanguageLexer, **options)
-- 
cgit v1.2.1


From b6b5c318fac22504479c13cd0fa55d6ef50429ae Mon Sep 17 00:00:00 2001
From: "Matth?us G. Chajdas" 
Date: Mon, 1 Apr 2019 16:48:13 +0200
Subject: Fix versionadded for Unicon lexers.

---
 pygments/lexers/unicon.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pygments/lexers/unicon.py b/pygments/lexers/unicon.py
index ae825150..7b0cdfe3 100644
--- a/pygments/lexers/unicon.py
+++ b/pygments/lexers/unicon.py
@@ -24,7 +24,7 @@ class UniconLexer(RegexLexer):
     """
     For Unicon source code.
 
-    .. versionadded:: 2.?
+    .. versionadded:: 2.4
     """
 
     name = 'Unicon'
@@ -311,7 +311,7 @@ class UcodeLexer(RegexLexer):
     """
     Lexer for Icon ucode files
 
-    .. versionadded:: 2.?
+    .. versionadded:: 2.4
     """
     name = 'ucode'
     aliases = ['ucode']
-- 
cgit v1.2.1


From e6ad9b8cc716ae1990b3ca6e11d6c1bb24679a36 Mon Sep 17 00:00:00 2001
From: "Matth?us G. Chajdas" 
Date: Mon, 1 Apr 2019 17:18:35 +0200
Subject: Update CHANGES.

---
 CHANGES | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/CHANGES b/CHANGES
index 9a7426a3..15863b27 100644
--- a/CHANGES
+++ b/CHANGES
@@ -17,6 +17,7 @@ Version 2.4.0
   * Hspec (PR#790)
   * SGF (PR#780)
   * Slurm (PR#760)
+  * Unicon (PR#731)
   * VBScript (PR#673)
 
 - Updated lexers:
@@ -35,6 +36,7 @@ Version 2.4.0
 - TypoScript uses ``.typoscript`` now (#1498)
 - Fix catastrophic backtracking in the bash lexer (#1494)
 - Fix incorrect links in the Lisp and R lexer documentation (PR#775)
+- Add support for Markdown reference-style links (PR#753)
 
 Version 2.3.1
 -------------
-- 
cgit v1.2.1


From a5329233321fc0c557448963a747428af413ce8f Mon Sep 17 00:00:00 2001
From: "Matth?us G. Chajdas" 
Date: Mon, 1 Apr 2019 17:34:27 +0200
Subject: Fix nested set warning in Unicon lexers.

---
 pygments/lexers/unicon.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pygments/lexers/unicon.py b/pygments/lexers/unicon.py
index 7b0cdfe3..6301a88b 100644
--- a/pygments/lexers/unicon.py
+++ b/pygments/lexers/unicon.py
@@ -140,7 +140,7 @@ class UniconLexer(RegexLexer):
             (r'[*<>+=/&!?@~\\-]', Operator),
             (r'\^', Operator),
             (r'(\w+)(\s*|[(,])', bygroups(Name, using(this))),
-            (r"([[\]])", Punctuation),
+            (r"([\[\]])", Punctuation),
             (r"(<>|=>|[()|:;,.'`]|[{}]|[%]|[&?])", Punctuation),
             (r'\n+', Text),
         ],
@@ -280,7 +280,7 @@ class IconLexer(RegexLexer):
             (r"'(?:[^\\']|\\.)*'", String.Character),
             (r'[*<>+=/&!?@~\\-]', Operator),
             (r'(\w+)(\s*|[(,])', bygroups(Name, using(this))),
-            (r"([[\]])", Punctuation),
+            (r"([\[\]])", Punctuation),
             (r"(<>|=>|[()|:;,.'`]|[{}]|[%^]|[&?])", Punctuation),
             (r'\n+', Text),
         ],
-- 
cgit v1.2.1


From e566797eb7b56db2dce22c2f116d9a2f0a750827 Mon Sep 17 00:00:00 2001
From: "Frederik ?Freso? S. Olesen" 
Date: Mon, 1 Apr 2019 17:50:02 +0200
Subject: Add `versionadded` to new lexer classes

---
 pygments/lexers/asm.py     | 2 ++
 pygments/lexers/configs.py | 7 +++++++
 pygments/lexers/slash.py   | 2 ++
 3 files changed, 11 insertions(+)

diff --git a/pygments/lexers/asm.py b/pygments/lexers/asm.py
index 761b3315..9ac3fdd9 100644
--- a/pygments/lexers/asm.py
+++ b/pygments/lexers/asm.py
@@ -657,6 +657,8 @@ class Dasm16Lexer(RegexLexer):
     Simple lexer for DCPU-16 Assembly
 
     Check http://0x10c.com/doc/dcpu-16.txt
+
+    .. versionadded:: 2.4
     """
     name = 'dasm16'
     aliases = ['DASM16']
diff --git a/pygments/lexers/configs.py b/pygments/lexers/configs.py
index 7c9bd655..350dcd6f 100644
--- a/pygments/lexers/configs.py
+++ b/pygments/lexers/configs.py
@@ -841,6 +841,11 @@ class PacmanConfLexer(RegexLexer):
 
 
 class AugeasLexer(RegexLexer):
+    """
+    Lexer for Augeas
+
+    .. versionadded:: 2.4
+    """
     name = 'Augeas'
     aliases = ['augeas']
     filenames = ['*.aug']
@@ -882,6 +887,8 @@ class AugeasLexer(RegexLexer):
 class TOMLLexer(RegexLexer):
     """
     Lexer for TOML, a simple language for config files
+
+    .. versionadded:: 2.4
     """
 
     name = 'TOML'
diff --git a/pygments/lexers/slash.py b/pygments/lexers/slash.py
index f18059f7..3ba5e1e6 100644
--- a/pygments/lexers/slash.py
+++ b/pygments/lexers/slash.py
@@ -173,6 +173,8 @@ class SlashLanguageLexer(ExtendedRegexLexer):
 class SlashLexer(DelegatingLexer):
     """
     Lexer for the Slash programming language.
+
+    .. versionadded:: 2.4
     """
 
     name = 'Slash'
-- 
cgit v1.2.1


From 0de93e092cd1e7d14a5d2c360b680f2724d98185 Mon Sep 17 00:00:00 2001
From: nimmajbb 
Date: Mon, 1 Apr 2019 17:15:14 +0100
Subject: some fixes to the kotlin lexer to work with the corda kolin codebase

---
 pygments/lexers/jvm.py |  33 +++++++++++--
 tests/test_kotlin.py   | 131 +++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 161 insertions(+), 3 deletions(-)
 create mode 100644 tests/test_kotlin.py

diff --git a/pygments/lexers/jvm.py b/pygments/lexers/jvm.py
index 5a9a74a9..8de6e9f2 100644
--- a/pygments/lexers/jvm.py
+++ b/pygments/lexers/jvm.py
@@ -1006,7 +1006,7 @@ class KotlinLexer(RegexLexer):
 
     .. versionadded:: 1.5
     """
-
+    
     name = 'Kotlin'
     aliases = ['kotlin']
     filenames = ['*.kt']
@@ -1017,15 +1017,22 @@ class KotlinLexer(RegexLexer):
     kt_name = ('@?[_' + uni.combine('Lu', 'Ll', 'Lt', 'Lm', 'Nl') + ']' +
                '[' + uni.combine('Lu', 'Ll', 'Lt', 'Lm', 'Nl', 'Nd', 'Pc', 'Cf',
                                  'Mn', 'Mc') + ']*')
-    kt_id = '(' + kt_name + '|`' + kt_name + '`)'
+    
+    kt_space_name = ('@?[_' + uni.combine('Lu', 'Ll', 'Lt', 'Lm', 'Nl') + ']' +
+               '[' + uni.combine('Lu', 'Ll', 'Lt', 'Lm', 'Nl', 'Nd', 'Pc', 'Cf',
+                                 'Mn', 'Mc', 'Zs') + ',-]*')
+
+    kt_id = '(' + kt_name + '|`' + kt_space_name + '`)'
 
     tokens = {
         'root': [
             (r'^\s*\[.*?\]', Name.Attribute),
             (r'[^\S\n]+', Text),
+            (r'\s+', Text),
             (r'\\\n', Text),  # line continuation
             (r'//.*?\n', Comment.Single),
             (r'/[*].*?[*]/', Comment.Multiline),
+            (r'""".*?"""', String),
             (r'\n', Text),
             (r'::|!!|\?[:.]', Operator),
             (r'[~!%^&*()+=|\[\]:;,.<>/?-]', Punctuation),
@@ -1035,11 +1042,14 @@ class KotlinLexer(RegexLexer):
             (r"'\\.'|'[^\\]'", String.Char),
             (r"[0-9](\.[0-9]*)?([eE][+-][0-9]+)?[flFL]?|"
              r"0[xX][0-9a-fA-F]+[Ll]?", Number),
-            (r'(class)(\s+)(object)', bygroups(Keyword, Text, Keyword)),
+            (r'(object)(\s+)(:)(\s+)', bygroups(Keyword, Text, Punctuation, Text), 'class'),
+            (r'(companion)(\s+)(object)', bygroups(Keyword, Text, Keyword)),
             (r'(class|interface|object)(\s+)', bygroups(Keyword, Text), 'class'),
             (r'(package|import)(\s+)', bygroups(Keyword, Text), 'package'),
+            (r'(val|var)(\s+)([(])', bygroups(Keyword, Text, Punctuation), 'property_dec'),
             (r'(val|var)(\s+)', bygroups(Keyword, Text), 'property'),
             (r'(fun)(\s+)', bygroups(Keyword, Text), 'function'),
+            (r'(inline fun)(\s+)', bygroups(Keyword, Text), 'function'),
             (r'(abstract|annotation|as|break|by|catch|class|companion|const|'
              r'constructor|continue|crossinline|data|do|dynamic|else|enum|'
              r'external|false|final|finally|for|fun|get|if|import|in|infix|'
@@ -1058,9 +1068,26 @@ class KotlinLexer(RegexLexer):
         'property': [
             (kt_id, Name.Property, '#pop')
         ],
+        'property_dec': [
+            (r'(,)(\s*)', bygroups(Punctuation, Text)),
+            (r'(:)(\s*)', bygroups(Punctuation, Text)),
+            (r'<', Punctuation, 'generic'),
+            (r'([)])', Punctuation, '#pop'),
+            (kt_id, Name.Property)
+        ],
         'function': [
+            (r'<', Punctuation, 'generic'),
+            (r''+kt_id+'([.])'+kt_id, bygroups(Name.Class, Punctuation, Name.Function), '#pop'),
             (kt_id, Name.Function, '#pop')
         ],
+        'generic': [
+            (r'(>)(\s*)', bygroups(Punctuation, Text), '#pop'),
+            (r':',Punctuation),
+            (r'(reified|out|in)\b', Keyword),
+            (r',',Text),
+            (r'\s+',Text),
+            (kt_id,Name)
+        ]
     }
 
 
diff --git a/tests/test_kotlin.py b/tests/test_kotlin.py
new file mode 100644
index 00000000..7c733ad9
--- /dev/null
+++ b/tests/test_kotlin.py
@@ -0,0 +1,131 @@
+# -*- coding: utf-8 -*-
+"""
+    Basic JavaLexer Test
+    ~~~~~~~~~~~~~~~~~~~~
+
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import unittest
+
+from pygments.token import Text, Name, Operator, Keyword, Number, Punctuation, String
+from pygments.lexers import KotlinLexer
+
+class KotlinTest(unittest.TestCase):
+
+    def setUp(self):
+        self.lexer = KotlinLexer()
+        self.maxDiff = None
+    
+    def testCanCopeWithBackTickNamesInFunctions(self):
+        fragment = u'fun `wo bble`'
+        tokens = [
+            (Keyword, u'fun'),
+            (Text, u' '),
+            (Name.Function, u'`wo bble`'),
+            (Text, u'\n')
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+    def testCanCopeWithCommasAndDashesInBackTickNames(self):
+        fragment = u'fun `wo,-bble`'
+        tokens = [
+            (Keyword, u'fun'),
+            (Text, u' '),
+            (Name.Function, u'`wo,-bble`'),
+            (Text, u'\n')
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+    
+    def testCanCopeWithDestructuring(self):
+        fragment = u'val (a, b) = '
+        tokens = [
+            (Keyword, u'val'),
+            (Text, u' '),
+            (Punctuation, u'('),
+            (Name.Property, u'a'),
+            (Punctuation, u','),
+            (Text, u' '),
+            (Name.Property, u'b'),
+            (Punctuation, u')'),
+            (Text, u' '),
+            (Punctuation, u'='),
+            (Text, u' '),
+            (Text, u'\n')
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+    
+    def testCanCopeGenericsInDestructuring(self):
+        fragment = u'val (a: List, b: Set) ='
+        tokens = [
+            (Keyword, u'val'),
+            (Text, u' '),
+            (Punctuation, u'('),
+            (Name.Property, u'a'),
+            (Punctuation, u':'),
+            (Text, u' '),
+            (Name.Property, u'List'),
+            (Punctuation, u'<'),
+            (Name, u'Something'),
+            (Punctuation, u'>'),
+            (Punctuation, u','),
+            (Text, u' '),
+            (Name.Property, u'b'),
+            (Punctuation, u':'),
+            (Text, u' '),
+            (Name.Property, u'Set'),
+            (Punctuation, u'<'),
+            (Name, u'Wobble'),
+            (Punctuation, u'>'),
+            (Punctuation, u')'),
+            (Text, u' '),
+            (Punctuation, u'='),
+            (Text, u'\n')
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+    def testCanCopeWithGenerics(self):
+        fragment = u'inline fun  VaultService.queryBy(): Vault.Page {'
+        tokens = [
+            (Keyword, u'inline fun'),
+            (Text, u' '),
+            (Punctuation, u'<'),
+            (Keyword, u'reified'),
+            (Text, u' '),
+            (Name, u'T'),
+            (Text, u' '),
+            (Punctuation, u':'),
+            (Text, u' '),
+            (Name, u'ContractState'),
+            (Punctuation, u'>'),
+            (Text, u' '),
+            (Name.Class, u'VaultService'),
+            (Punctuation, u'.'),
+            (Name.Function, u'queryBy'),
+            (Punctuation, u'('),
+            (Punctuation, u')'),
+            (Punctuation, u':'),
+            (Text, u' '),
+            (Name, u'Vault'),
+            (Punctuation, u'.'),
+            (Name, u'Page'),
+            (Punctuation, u'<'),
+            (Name, u'T'),
+            (Punctuation, u'>'),
+            (Text, u' '),
+            (Punctuation, u'{'),
+            (Text, u'\n')
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+    def testShouldCopeWithMultilineComments(self):
+        fragment = u'"""\nthis\nis\na\ncomment"""'
+        tokens = [
+            (String, u'"""\nthis\nis\na\ncomment"""'), 
+            (Text, u'\n')
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+if __name__ == '__main__':
+    unittest.main()
-- 
cgit v1.2.1


From 7cd2a4447b61034aa5fcc4fd01c4271968cf6713 Mon Sep 17 00:00:00 2001
From: "Frederik ?Freso? S. Olesen" 
Date: Mon, 1 Apr 2019 18:40:12 +0200
Subject: Add example TOML file

Based on https://github.com/toml-lang/toml/blob/master/README.md
---
 tests/examplefiles/example.toml | 181 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 181 insertions(+)
 create mode 100644 tests/examplefiles/example.toml

diff --git a/tests/examplefiles/example.toml b/tests/examplefiles/example.toml
new file mode 100644
index 00000000..9c60c79f
--- /dev/null
+++ b/tests/examplefiles/example.toml
@@ -0,0 +1,181 @@
+# This is a TOML document comment
+
+title = "TOML example file" # This is an inline comment
+
+[examples]
+# Examples taken from https://github.com/toml-lang/toml/blob/master/README.md
+key = "value"
+bare_key = "value"
+bare-key = "value"
+1234 = "value"
+"127.0.0.1" = "value"
+"character encoding" = "value"
+"ʎǝʞ" = "value"
+'key2' = "value"
+'quoted "value"' = "value"
+name = "Orange"
+physical.color = "orange"
+physical.shape = "round"
+site."google.com" = true
+a.b.c = 1
+a.d = 2
+
+[strings]
+str = "I'm a string. \"You can quote me\". Name\tJos\u00E9\nLocation\tSF."
+str1 = """
+Roses are red
+Violets are blue"""
+str2 = "Roses are red\nViolets are blue"
+str3 = "Roses are red\r\nViolets are blue"
+
+  [strings.equivalents]
+  str1 = "The quick brown fox jumps over the lazy dog."
+  str2 = """
+The quick brown \
+
+
+    fox jumps over \
+      the lazy dog."""
+  str3 = """\
+         The quick brown \
+         fox jumps over \
+         the lazy dog.\
+         """
+
+  [strings.literal]
+  winpath  = 'C:\Users\nodejs\templates'
+  winpath2 = '\\ServerX\admin$\system32\'
+  quoted   = 'Tom "Dubs" Preston-Werner'
+  regex    = '<\i\c*\s*>'
+
+  [strings.multiline]
+  regex2 = '''I [dw]on't need \d{2} apples'''
+  lines  = '''
+The first newline is
+trimmed in raw strings.
+   All other whitespace
+   is preserved.
+'''
+
+[integers]
+int1 = +99
+int2 = 42
+int3 = 0
+int4 = -17
+int5 = 1_000
+int6 = 5_349_221
+int7 = 1_2_3_4_5 # discouraged format
+# hexadecimal with prefix `0x`
+hex1 = 0xDEADBEEF
+hex2 = 0xdeadbeef
+hex3 = 0xdead_beef
+# octal with prefix `0o`
+oct1 = 0o01234567
+oct2 = 0o755 # useful for Unix file permissions
+# binary with prefix `0b`
+bin1 = 0b11010110
+
+[floats]
+# fractional
+flt1 = +1.0
+flt2 = 3.1415
+flt3 = -0.01
+# exponent
+flt4 = 5e+22
+flt5 = 1e6
+flt6 = -2E-2
+# both
+flt7 = 6.626e-34
+# with underscores, for readability
+flt8 = 224_617.445_991_228
+# infinity
+sf1 = inf  # positive infinity
+sf2 = +inf # positive infinity
+sf3 = -inf # negative infinity
+# not a number
+sf4 = nan  # actual sNaN/qNaN encoding is implementation specific
+sf5 = +nan # same as `nan`
+sf6 = -nan # valid, actual encoding is implementation specific
+# plus/minus zero
+sf0_1 = +0.0
+sf0_2 = -0.0
+
+[booleans]
+bool1 = true
+bool2 = false
+
+[datetime.offset]
+odt1 = 1979-05-27T07:32:00Z
+odt2 = 1979-05-27T00:32:00-07:00
+odt3 = 1979-05-27T00:32:00.999999-07:00
+odt4 = 1979-05-27 07:32:00Z
+
+[datetime.local]
+ldt1 = 1979-05-27T07:32:00
+ldt2 = 1979-05-27T00:32:00.999999
+
+[date.local]
+ld1 = 1979-05-27
+
+[time.local]
+lt1 = 07:32:00
+lt2 = 00:32:00.999999
+
+[arrays]
+arr1 = [ 1, 2, 3 ]
+arr2 = [ "red", "yellow", "green" ]
+arr3 = [ [ 1, 2 ], [3, 4, 5] ]
+arr4 = [ "all", 'strings', """are the same""", '''type''']
+arr5 = [ [ 1, 2 ], ["a", "b", "c"] ]
+arr6 = [ 1, 2.0 ] # INVALID
+arr7 = [
+  1, 2, 3
+]
+arr8 = [
+  1,
+  2, # this is ok
+]
+
+["inline tables"]
+name = { first = "Tom", last = "Preston-Werner" }
+point = { x = 1, y = 2 }
+animal = { type.name = "pug" }
+
+["arrays of tables"]
+points = [ { x = 1, y = 2, z = 3 },
+           { x = 7, y = 8, z = 9 },
+           { x = 2, y = 4, z = 8 } ]
+
+  [products]
+
+    [[products]]
+    name = "Hammer"
+    sku = 738594937
+
+    [[products]]
+
+    [[products]]
+    name = "Nail"
+    sku = 284758393
+    color = "gray"
+
+  [fruits]
+
+    [[fruit]]
+      name = "apple"
+
+      [fruit.physical]
+        color = "red"
+        shape = "round"
+
+      [[fruit.variety]]
+        name = "red delicious"
+
+      [[fruit.variety]]
+        name = "granny smith"
+
+    [[fruit]]
+      name = "banana"
+
+      [[fruit.variety]]
+        name = "plantain"
-- 
cgit v1.2.1


From 6385c59af37e013f1b28b5ce1229ee466e136b1a Mon Sep 17 00:00:00 2001
From: "Matth?us G. Chajdas" 
Date: Tue, 2 Apr 2019 14:24:14 +0200
Subject: Fix documentation build using Sphinx 2.0 (fixes #1501.)

---
 CHANGES     | 9 +++++----
 doc/conf.py | 4 ++--
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/CHANGES b/CHANGES
index 15863b27..1447e0ee 100644
--- a/CHANGES
+++ b/CHANGES
@@ -30,13 +30,14 @@ Version 2.4.0
   * Terraform (PR#787)
 
 - Add solarized style (PR#708)
+- Add support for Markdown reference-style links (PR#753)
 - Change ANSI color names (PR#777)
-- Fix rare unicode errors on Python 2.7 (PR#798, #1492)
-- Updated Trove classifiers and ``pip`` requirements (PR#799)
-- TypoScript uses ``.typoscript`` now (#1498)
 - Fix catastrophic backtracking in the bash lexer (#1494)
+- Fix documentation failing to build using Sphinx 2.0 (#1501)
 - Fix incorrect links in the Lisp and R lexer documentation (PR#775)
-- Add support for Markdown reference-style links (PR#753)
+- Fix rare unicode errors on Python 2.7 (PR#798, #1492)
+- TypoScript uses ``.typoscript`` now (#1498)
+- Updated Trove classifiers and ``pip`` requirements (PR#799)
 
 Version 2.3.1
 -------------
diff --git a/doc/conf.py b/doc/conf.py
index 51a91617..00db7d9b 100644
--- a/doc/conf.py
+++ b/doc/conf.py
@@ -125,8 +125,8 @@ html_static_path = ['_static']
 #html_use_smartypants = True
 
 # Custom sidebar templates, maps document names to template names.
-html_sidebars = {'index': 'indexsidebar.html',
-                 'docs/*': 'docssidebar.html'}
+html_sidebars = {'index': ['indexsidebar.html'],
+                 'docs/*': ['docssidebar.html']}
 
 # Additional templates that should be rendered to pages, maps page names to
 # template names.
-- 
cgit v1.2.1


From 72b3cc6c0cb608c302ae4b0d98d49e28a59b5126 Mon Sep 17 00:00:00 2001
From: Mauricio Caceres Bravo 
Date: Tue, 2 Apr 2019 13:32:51 -0400
Subject: Aliased stata-light to stata.

---
 pygments/styles/__init__.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pygments/styles/__init__.py b/pygments/styles/__init__.py
index 0f331e4c..c7050a18 100644
--- a/pygments/styles/__init__.py
+++ b/pygments/styles/__init__.py
@@ -47,6 +47,7 @@ STYLE_MAP = {
     'solarized-dark': 'solarized::SolarizedDarkStyle',
     'solarized-light': 'solarized::SolarizedLightStyle',
     'sas':         'sas::SasStyle',
+    'stata':       'stata_light::StataLightStyle',
     'stata-light': 'stata_light::StataLightStyle',
     'stata-dark':  'stata_dark::StataDarkStyle',
 }
-- 
cgit v1.2.1


From 1ed3f0862974c87af1b22c9402c8803339bfb055 Mon Sep 17 00:00:00 2001
From: "Matth?us G. Chajdas" 
Date: Tue, 2 Apr 2019 20:41:31 +0200
Subject: Fix invalid escape sequence in stata builtins, update CHANGES.

else\sif is invalid, and not needed given else and if are already valid
keywords.
---
 CHANGES                            | 1 +
 pygments/lexers/_stata_builtins.py | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/CHANGES b/CHANGES
index 1447e0ee..747716fd 100644
--- a/CHANGES
+++ b/CHANGES
@@ -27,6 +27,7 @@ Version 2.4.0
   * PHP (#1482)
   * SQL (PR#672)
   * Stan (PR#774)
+  * Stata (PR#800)
   * Terraform (PR#787)
 
 - Add solarized style (PR#708)
diff --git a/pygments/lexers/_stata_builtins.py b/pygments/lexers/_stata_builtins.py
index 13a3dacf..3e5e75b2 100644
--- a/pygments/lexers/_stata_builtins.py
+++ b/pygments/lexers/_stata_builtins.py
@@ -15,7 +15,7 @@ builtins_special = (
 )
 
 builtins_base = (
-    "if", "else", "else\s+if", "in", "foreach", "for", "forv", "forva",
+    "if", "else", "in", "foreach", "for", "forv", "forva",
     "forval", "forvalu", "forvalue", "forvalues", "by", "bys",
     "bysort", "quietly", "qui", "about", "ac",
     "ac_7", "acprplot", "acprplot_7", "adjust", "ado", "adopath",
-- 
cgit v1.2.1


From 075e65ed4e4424c121f7c70e0db9d2ff9e7895f8 Mon Sep 17 00:00:00 2001
From: "Matth?us G. Chajdas" 
Date: Sun, 28 Apr 2019 17:14:55 +0200
Subject: Add TOML example file and improve the lexer a bit.

---
 pygments/lexers/configs.py      |  9 ++++++++-
 tests/examplefiles/example.toml | 15 +++++++++++++++
 2 files changed, 23 insertions(+), 1 deletion(-)
 create mode 100644 tests/examplefiles/example.toml

diff --git a/pygments/lexers/configs.py b/pygments/lexers/configs.py
index 7c9bd655..bf9cb0bb 100644
--- a/pygments/lexers/configs.py
+++ b/pygments/lexers/configs.py
@@ -894,12 +894,19 @@ class TOMLLexer(RegexLexer):
             # Basics, comments, strings
             (r'\s+', Text),
             (r'#.*?$', Comment.Single),
+            # Basic string
             (r'"(\\\\|\\"|[^"])*"', String),
+            # Literal string
+            (r'\'[^\']*\'', String),
             (r'(true|false)$', Keyword.Constant),
             ('[a-zA-Z_][a-zA-Z0-9_\-]*', Name),
 
+            (r'\[.*?\]$', Keyword),
+
             # Datetime
-            (r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z', Number.Integer),
+            # TODO this needs to be expanded, as TOML is rather flexible:
+            # https://github.com/toml-lang/toml#offset-date-time
+            (r'\d{4}-\d{2}-\d{2}(?:T| )\d{2}:\d{2}:\d{2}(?:Z|[-+]\d{2}:\d{2})', Number.Integer),
 
             # Numbers
             (r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?j?', Number.Float),
diff --git a/tests/examplefiles/example.toml b/tests/examplefiles/example.toml
new file mode 100644
index 00000000..40832c22
--- /dev/null
+++ b/tests/examplefiles/example.toml
@@ -0,0 +1,15 @@
+name = "TOML sample file"
+
+[section]
+key = "value"
+literal_string = 'C:\test'
+other_string = '''value'''
+list = [1, 2, 3]
+nested_list = [ [1, 2], [3, 4] ]
+float_variable = 13.37
+
+  [section.nested]
+  boolean_variable = false
+  date_variable = 1969-97-24T16:50:35-00:00
+
+# Comment
\ No newline at end of file
-- 
cgit v1.2.1


From 5256ca6b4b12794a23b848a2dffa28025b8d9d72 Mon Sep 17 00:00:00 2001
From: "Matth?us G. Chajdas" 
Date: Sun, 28 Apr 2019 17:15:44 +0200
Subject: Various fixes and cleanups to the last batch of new languages.

Add to CHANGES, languages, add versionadded, recreate mappings, fix DASM16
alias.
---
 CHANGES                     |  8 ++++----
 doc/languages.rst           |  4 ++++
 pygments/lexers/_mapping.py |  8 ++++----
 pygments/lexers/asm.py      |  6 ++++--
 pygments/lexers/configs.py  | 10 +++++++++-
 pygments/lexers/slash.py    |  5 ++++-
 6 files changed, 29 insertions(+), 12 deletions(-)

diff --git a/CHANGES b/CHANGES
index 6a6a6a17..2b1ebb0b 100644
--- a/CHANGES
+++ b/CHANGES
@@ -12,17 +12,17 @@ Version 2.4.0
 
 - Added lexers:
 
+  * Augeas (PR#807)
   * Charm++ CI (PR#788)
+  * DASM16 (PR#807)
   * FloScript (PR#750)
   * Hspec (PR#790)
   * SGF (PR#780)
+  * Slash (PR#807)
   * Slurm (PR#760)
+  * TOML (PR#807)
   * Unicon (PR#731)
   * VBScript (PR#673)
-  * DASM16
-  * Augeas
-  * TOML
-  * Slash
 
 - Updated lexers:
 
diff --git a/doc/languages.rst b/doc/languages.rst
index 47e3363f..d2508b07 100644
--- a/doc/languages.rst
+++ b/doc/languages.rst
@@ -14,6 +14,7 @@ Programming languages
 * AppleScript
 * Assembly (various)
 * Asymptote
+* `Augeas `_
 * Awk
 * Befunge
 * Boo
@@ -31,6 +32,7 @@ Programming languages
 * `Cython `_
 * `D `_
 * Dart
+* DCPU-16
 * Delphi
 * Dylan
 * `Elm `_
@@ -85,10 +87,12 @@ Programming languages
 * Scheme
 * Scilab
 * `SGF `_
+* `Slash `_
 * `Slurm `_
 * Smalltalk
 * SNOBOL
 * Tcl
+* `TOML `_
 * Vala
 * Verilog
 * VHDL
diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py
index 3c05d147..e8838008 100644
--- a/pygments/lexers/_mapping.py
+++ b/pygments/lexers/_mapping.py
@@ -44,7 +44,7 @@ LEXERS = {
     'ArduinoLexer': ('pygments.lexers.c_like', 'Arduino', ('arduino',), ('*.ino',), ('text/x-arduino',)),
     'AspectJLexer': ('pygments.lexers.jvm', 'AspectJ', ('aspectj',), ('*.aj',), ('text/x-aspectj',)),
     'AsymptoteLexer': ('pygments.lexers.graphics', 'Asymptote', ('asy', 'asymptote'), ('*.asy',), ('text/x-asymptote',)),
-    'AugeasLexer': ('pygments.lexers.configs', 'Augeas', ('augeas',), ('*.aug'), ()),
+    'AugeasLexer': ('pygments.lexers.configs', 'Augeas', ('augeas',), ('*.aug',), ()),
     'AutoItLexer': ('pygments.lexers.automation', 'AutoIt', ('autoit',), ('*.au3',), ('text/x-autoit',)),
     'AutohotkeyLexer': ('pygments.lexers.automation', 'autohotkey', ('ahk', 'autohotkey'), ('*.ahk', '*.ahkl'), ('text/x-autohotkey',)),
     'AwkLexer': ('pygments.lexers.textedit', 'Awk', ('awk', 'gawk', 'mawk', 'nawk'), ('*.awk',), ('application/x-awk',)),
@@ -122,7 +122,7 @@ LEXERS = {
     'DObjdumpLexer': ('pygments.lexers.asm', 'd-objdump', ('d-objdump',), ('*.d-objdump',), ('text/x-d-objdump',)),
     'DarcsPatchLexer': ('pygments.lexers.diff', 'Darcs Patch', ('dpatch',), ('*.dpatch', '*.darcspatch'), ()),
     'DartLexer': ('pygments.lexers.javascript', 'Dart', ('dart',), ('*.dart',), ('text/x-dart',)),
-    'Dasm16Lexer': ('pygments.lexers.asm', 'DASM16', ('dasm16',), ('*.dasm16', '*.dasm'), ('text/x-dasm16')),
+    'Dasm16Lexer': ('pygments.lexers.asm', 'DASM16', ('dasm16',), ('*.dasm16', '*.dasm'), ('text/x-dasm16',)),
     'DebianControlLexer': ('pygments.lexers.installers', 'Debian Control file', ('control', 'debcontrol'), ('control',), ()),
     'DelphiLexer': ('pygments.lexers.pascal', 'Delphi', ('delphi', 'pas', 'pascal', 'objectpascal'), ('*.pas', '*.dpr'), ('text/x-pascal',)),
     'DgLexer': ('pygments.lexers.python', 'dg', ('dg',), ('*.dg',), ('text/x-dg',)),
@@ -377,7 +377,6 @@ LEXERS = {
     'RubyLexer': ('pygments.lexers.ruby', 'Ruby', ('rb', 'ruby', 'duby'), ('*.rb', '*.rbw', 'Rakefile', '*.rake', '*.gemspec', '*.rbx', '*.duby', 'Gemfile'), ('text/x-ruby', 'application/x-ruby')),
     'RustLexer': ('pygments.lexers.rust', 'Rust', ('rust', 'rs'), ('*.rs', '*.rs.in'), ('text/rust',)),
     'SASLexer': ('pygments.lexers.sas', 'SAS', ('sas',), ('*.SAS', '*.sas'), ('text/x-sas', 'text/sas', 'application/x-sas')),
-    'SlashLexer': ('pygments.lexers.slash', 'Slash', ('slash'), ('*.sl'), ()),
     'SLexer': ('pygments.lexers.r', 'S', ('splus', 's', 'r'), ('*.S', '*.R', '.Rhistory', '.Rprofile', '.Renviron'), ('text/S-plus', 'text/S', 'text/x-r-source', 'text/x-r', 'text/x-R', 'text/x-r-history', 'text/x-r-profile')),
     'SMLLexer': ('pygments.lexers.ml', 'Standard ML', ('sml',), ('*.sml', '*.sig', '*.fun'), ('text/x-standardml', 'application/x-standardml')),
     'SarlLexer': ('pygments.lexers.jvm', 'SARL', ('sarl',), ('*.sarl',), ('text/x-sarl',)),
@@ -389,6 +388,7 @@ LEXERS = {
     'ScssLexer': ('pygments.lexers.css', 'SCSS', ('scss',), ('*.scss',), ('text/x-scss',)),
     'ShenLexer': ('pygments.lexers.lisp', 'Shen', ('shen',), ('*.shen',), ('text/x-shen', 'application/x-shen')),
     'SilverLexer': ('pygments.lexers.verification', 'Silver', ('silver',), ('*.sil', '*.vpr'), ()),
+    'SlashLexer': ('pygments.lexers.slash', 'Slash', ('slash',), ('*.sl',), ()),
     'SlimLexer': ('pygments.lexers.webmisc', 'Slim', ('slim',), ('*.slim',), ('text/x-slim',)),
     'SlurmBashLexer': ('pygments.lexers.shell', 'Slurm', ('slurm', 'sbatch'), ('*.sl',), ()),
     'SmaliLexer': ('pygments.lexers.dalvik', 'Smali', ('smali',), ('*.smali',), ('text/smali',)),
@@ -411,6 +411,7 @@ LEXERS = {
     'SwigLexer': ('pygments.lexers.c_like', 'SWIG', ('swig',), ('*.swg', '*.i'), ('text/swig',)),
     'SystemVerilogLexer': ('pygments.lexers.hdl', 'systemverilog', ('systemverilog', 'sv'), ('*.sv', '*.svh'), ('text/x-systemverilog',)),
     'TAPLexer': ('pygments.lexers.testing', 'TAP', ('tap',), ('*.tap',), ()),
+    'TOMLLexer': ('pygments.lexers.configs', 'TOML', ('toml',), ('*.toml',), ()),
     'Tads3Lexer': ('pygments.lexers.int_fiction', 'TADS 3', ('tads3',), ('*.t',), ()),
     'TasmLexer': ('pygments.lexers.asm', 'TASM', ('tasm',), ('*.asm', '*.ASM', '*.tasm'), ('text/x-tasm',)),
     'TclLexer': ('pygments.lexers.tcl', 'Tcl', ('tcl',), ('*.tcl', '*.rvt'), ('text/x-tcl', 'text/x-script.tcl', 'application/x-tcl')),
@@ -424,7 +425,6 @@ LEXERS = {
     'TextLexer': ('pygments.lexers.special', 'Text only', ('text',), ('*.txt',), ('text/plain',)),
     'ThriftLexer': ('pygments.lexers.dsls', 'Thrift', ('thrift',), ('*.thrift',), ('application/x-thrift',)),
     'TodotxtLexer': ('pygments.lexers.textfmts', 'Todotxt', ('todotxt',), ('todo.txt', '*.todotxt'), ('text/x-todo',)),
-    'TOMLLexer': ('pygments.lexers.configs', 'TOML', ('toml',), ('*.toml'), ()),
     'TransactSqlLexer': ('pygments.lexers.sql', 'Transact-SQL', ('tsql', 't-sql'), ('*.sql',), ('text/x-tsql',)),
     'TreetopLexer': ('pygments.lexers.parsers', 'Treetop', ('treetop',), ('*.treetop', '*.tt'), ()),
     'TurtleLexer': ('pygments.lexers.rdf', 'Turtle', ('turtle',), ('*.ttl',), ('text/turtle', 'application/x-turtle')),
diff --git a/pygments/lexers/asm.py b/pygments/lexers/asm.py
index 761b3315..7100868c 100644
--- a/pygments/lexers/asm.py
+++ b/pygments/lexers/asm.py
@@ -657,9 +657,11 @@ class Dasm16Lexer(RegexLexer):
     Simple lexer for DCPU-16 Assembly
 
     Check http://0x10c.com/doc/dcpu-16.txt
+
+    .. versionadded:: 2.4
     """
-    name = 'dasm16'
-    aliases = ['DASM16']
+    name = 'DASM16'
+    aliases = ['dasm16']
     filenames = ['*.dasm16', '*.dasm']
     mimetypes = ['text/x-dasm16']
 
diff --git a/pygments/lexers/configs.py b/pygments/lexers/configs.py
index bf9cb0bb..a3e28dd8 100644
--- a/pygments/lexers/configs.py
+++ b/pygments/lexers/configs.py
@@ -841,6 +841,11 @@ class PacmanConfLexer(RegexLexer):
 
 
 class AugeasLexer(RegexLexer):
+    """
+    Lexer for `Augeas `_.
+
+    .. versionadded:: 2.4
+    """
     name = 'Augeas'
     aliases = ['augeas']
     filenames = ['*.aug']
@@ -881,7 +886,10 @@ class AugeasLexer(RegexLexer):
 
 class TOMLLexer(RegexLexer):
     """
-    Lexer for TOML, a simple language for config files
+    Lexer for `TOML `_, a simple language
+    for config files.
+
+    .. versionadded:: 2.4
     """
 
     name = 'TOML'
diff --git a/pygments/lexers/slash.py b/pygments/lexers/slash.py
index f18059f7..bd73d463 100644
--- a/pygments/lexers/slash.py
+++ b/pygments/lexers/slash.py
@@ -3,7 +3,8 @@
     pygments.lexers.slash
     ~~~~~~~~~~~~~~~~~~~~~
 
-    Lexer for the Slash programming language.
+    Lexer for the `Slash `_ programming
+    language.
 
     :copyright: Copyright 2012 by GitHub, Inc
     :license: BSD, see LICENSE for details.
@@ -173,6 +174,8 @@ class SlashLanguageLexer(ExtendedRegexLexer):
 class SlashLexer(DelegatingLexer):
     """
     Lexer for the Slash programming language.
+
+    .. versionadded:: 2.4
     """
 
     name = 'Slash'
-- 
cgit v1.2.1


From 323d10920c579054c861ee0cfdd9a30d012a4d0a Mon Sep 17 00:00:00 2001
From: "Matth?us G. Chajdas" 
Date: Tue, 30 Apr 2019 09:23:54 +0200
Subject: Improve TOML lexer.

The last found a few shortcomings in the TOML around float parsing. These get
fixed in this commit; additionally, I simplified a bunch of regex.
---
 pygments/lexers/configs.py | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/pygments/lexers/configs.py b/pygments/lexers/configs.py
index 765c67b7..6f2c7c76 100644
--- a/pygments/lexers/configs.py
+++ b/pygments/lexers/configs.py
@@ -855,10 +855,10 @@ class AugeasLexer(RegexLexer):
             (r'(module)(\s*)([^\s=]+)', bygroups(Keyword.Namespace, Text, Name.Namespace)),
             (r'(let)(\s*)([^\s=]+)', bygroups(Keyword.Declaration, Text, Name.Variable)),
             (r'(del|store|value|counter|seq|key|label|autoload|incl|excl|transform|test|get|put)(\s+)', bygroups(Name.Builtin, Text)),
-            (r'(\()([^\:]+)(\:)(unit|string|regexp|lens|tree|filter)(\))', bygroups(Punctuation, Name.Variable, Punctuation, Keyword.Type, Punctuation)),
+            (r'(\()([^:]+)(\:)(unit|string|regexp|lens|tree|filter)(\))', bygroups(Punctuation, Name.Variable, Punctuation, Keyword.Type, Punctuation)),
             (r'\(\*', Comment.Multiline, 'comment'),
-            (r'[\+=\|\.\*\;\?-]', Operator),
-            (r'[\[\]\(\)\{\}]', Operator),
+            (r'[*+\-.;=?|]', Operator),
+            (r'[()\[\]{}]', Operator),
             (r'"', String.Double, 'string'),
             (r'\/', String.Regex, 'regex'),
             (r'([A-Z]\w*)(\.)(\w+)', bygroups(Name.Namespace, Punctuation, Name.Variable)),
@@ -872,14 +872,14 @@ class AugeasLexer(RegexLexer):
         ],
         'regex': [
             (r'\\.', String.Escape),
-            (r'[^\/]', String.Regex),
+            (r'[^/]', String.Regex),
             (r'\/', String.Regex, '#pop'),
         ],
         'comment': [
-            (r'[^*\)]', Comment.Multiline),
+            (r'[^*)]', Comment.Multiline),
             (r'\(\*', Comment.Multiline, '#push'),
             (r'\*\)', Comment.Multiline, '#pop'),
-            (r'[\*\)]', Comment.Multiline)
+            (r'[)*]', Comment.Multiline)
         ],
     }
 
@@ -905,9 +905,10 @@ class TOMLLexer(RegexLexer):
             # Basic string
             (r'"(\\\\|\\"|[^"])*"', String),
             # Literal string
+            (r'\'\'\'(.*)\'\'\'', String),
             (r'\'[^\']*\'', String),
             (r'(true|false)$', Keyword.Constant),
-            ('[a-zA-Z_][a-zA-Z0-9_\-]*', Name),
+            (r'[a-zA-Z_][\w\-]*', Name),
 
             (r'\[.*?\]$', Keyword),
             # Datetime
@@ -918,7 +919,9 @@ class TOMLLexer(RegexLexer):
             # Numbers
             (r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?j?', Number.Float),
             (r'\d+[eE][+-]?[0-9]+j?', Number.Float),
-            (r'\-?\d+', Number.Integer),
+            # Handle +-inf, +-infinity, +-nan
+            (r'[+-]?(?:(inf(?:inity)?)|nan)', Number.Float),
+            (r'[+-]?\d+', Number.Integer),
 
             # Punctuation
             (r'[]{}:(),;[]', Punctuation),
-- 
cgit v1.2.1


From 0b839cf4a0acf37677e7b1a85181a628d9959735 Mon Sep 17 00:00:00 2001
From: "Matth?us G. Chajdas" 
Date: Tue, 30 Apr 2019 17:48:52 +0200
Subject: Simplify a few basic regex.

---
 pygments/lexers/basic.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/pygments/lexers/basic.py b/pygments/lexers/basic.py
index 4d957c2b..1aa3274c 100644
--- a/pygments/lexers/basic.py
+++ b/pygments/lexers/basic.py
@@ -550,7 +550,7 @@ class VBScriptLexer(RegexLexer):
             (r'[a-z_][a-z0-9_]*', Name),
             (r'\b_\n', Operator),
             (words(r'(),.:'), Punctuation),
-            ('.+(\n)?', Error)
+            (r'.+(\n)?', Error)
         ],
         'dim_more': [
             (r'(\s*)(,)(\s*)([a-z_][a-z0-9]*)', bygroups(Whitespace, Punctuation, Whitespace, Name.Variable)),
@@ -570,7 +570,7 @@ class BBCBasicLexer(RegexLexer):
     BBC Basic was supplied on the BBC Micro, and later Acorn RISC OS.
     It is also used by BBC Basic For Windows.
 
-    .. versionadded:: 2.4????
+    .. versionadded:: 2.4
     """
     base_keywords = ['OTHERWISE', 'AND', 'DIV', 'EOR', 'MOD', 'OR', 'ERROR',
                      'LINE', 'OFF', 'STEP', 'SPC', 'TAB', 'ELSE', 'THEN',
@@ -620,9 +620,9 @@ class BBCBasicLexer(RegexLexer):
             (r':', Comment.Preproc),
 
             # Some special cases to make functions come out nicer
-            (r'(DEF)(\s*)(FN|PROC)([A-Za-z_@][A-Za-z0-9_@]*)',
+            (r'(DEF)(\s*)(FN|PROC)([A-Za-z_@][\w@]*)',
              bygroups(Keyword.Declaration, Whitespace, Keyword.Declaration, Name.Function)),
-            (r'(FN|PROC)([A-Za-z_@][A-Za-z0-9_@]*)',
+            (r'(FN|PROC)([A-Za-z_@][\w@]*)',
              bygroups(Keyword, Name.Function)),
 
             (r'(GOTO|GOSUB|THEN|RESTORE)(\s*)(\d+)',
@@ -644,7 +644,7 @@ class BBCBasicLexer(RegexLexer):
             (r'[+-]?[0-9]+E[+-]?[0-9]+', Number.Float),
             (r'[+-]?\d+', Number.Integer),
 
-            (r'([A-Za-z_@][A-Za-z0-9_@]*[%$]?)', Name.Variable),
+            (r'([A-Za-z_@][\w@]*[%$]?)', Name.Variable),
             (r'([+\-]=|[$!|?+\-*/%^=><();]|>=|<=|<>|<<|>>|>>>|,)', Operator),
         ],
         'string': [
-- 
cgit v1.2.1


From 601e6e86e45f31e23a7b29e1265d68aeba5e1366 Mon Sep 17 00:00:00 2001
From: "Matth?us G. Chajdas" 
Date: Tue, 30 Apr 2019 17:50:49 +0200
Subject: Update CHANGES, language list.

---
 CHANGES                 |  2 ++
 doc/languages.rst       |  2 ++
 pygments/lexers/pony.py | 10 ++++++----
 3 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/CHANGES b/CHANGES
index 2b1ebb0b..62d2f113 100644
--- a/CHANGES
+++ b/CHANGES
@@ -13,10 +13,12 @@ Version 2.4.0
 - Added lexers:
 
   * Augeas (PR#807)
+  * BBC Basic (PR#806)
   * Charm++ CI (PR#788)
   * DASM16 (PR#807)
   * FloScript (PR#750)
   * Hspec (PR#790)
+  * Pony (PR#627)
   * SGF (PR#780)
   * Slash (PR#807)
   * Slurm (PR#760)
diff --git a/doc/languages.rst b/doc/languages.rst
index d2508b07..d96b1ba8 100644
--- a/doc/languages.rst
+++ b/doc/languages.rst
@@ -16,6 +16,7 @@ Programming languages
 * Asymptote
 * `Augeas `_
 * Awk
+* BBC Basic
 * Befunge
 * Boo
 * BrainFuck
@@ -72,6 +73,7 @@ Programming languages
 * OCaml
 * PHP
 * `Perl 5 `_ and `Perl 6 `_
+* `Pony `_
 * PovRay
 * PostScript
 * PowerShell
diff --git a/pygments/lexers/pony.py b/pygments/lexers/pony.py
index 486e404b..13239047 100644
--- a/pygments/lexers/pony.py
+++ b/pygments/lexers/pony.py
@@ -2,7 +2,7 @@
 """
     pygments.lexers.pony
     ~~~~~~~~~~~~~~~~~~~~
- 
+
     Lexers for Pony and related languages.
 
     :copyright: Copyright 2006-2016 by the Pygments team, see AUTHORS.
@@ -19,14 +19,16 @@ __all__ = ['PonyLexer']
 class PonyLexer(RegexLexer):
     """
     For Pony source code.
+
+    .. versionadded:: 2.4
     """
-    
+
     name = 'Pony'
     aliases = ['pony']
     filenames = ['*.pony']
-    
+
     _caps = r'(iso|trn|ref|val|box|tag)'
-    
+
     tokens = {
         'root': [
             (r'\n', Text),
-- 
cgit v1.2.1