diff options
author | Gaurav Jain <gaurav@gauravjain.org> | 2014-05-17 10:06:29 -0400 |
---|---|---|
committer | Gaurav Jain <gaurav@gauravjain.org> | 2014-05-17 10:06:29 -0400 |
commit | b4328f0a0cd095ce34eb4614cfd3267d695ddf8f (patch) | |
tree | 6af6ad8b23f89c47746c280f7e2bf8cbefcd4ce6 | |
parent | 16cc059837eb83f96f35caf207c1e0778673f63c (diff) | |
parent | 778edd1d5ff974b1d4084af964151d9f87f05efd (diff) | |
download | pygments-b4328f0a0cd095ce34eb4614cfd3267d695ddf8f.tar.gz |
Merged birkenfeld/pygments-main into default
42 files changed, 3649 insertions, 1180 deletions
@@ -28,7 +28,7 @@ Other contributors, listed alphabetically, are: * Pierre Bourdon -- bugfixes * Hiram Chirino -- Scaml and Jade lexers * Ian Cooper -- VGL lexer -* David Corbett -- Inform lexers +* David Corbett -- Inform and Jasmin lexers * Leaf Corcoran -- MoonScript lexer * Christopher Creutzig -- MuPAD lexer * Daniël W. Crompton - Pike lexer @@ -115,12 +115,14 @@ Other contributors, listed alphabetically, are: * Jesper Noehr -- HTML formatter "anchorlinenos" * Mike Nolta -- Julia lexer * Jonas Obrist -- BBCode lexer +* Edward O'Callaghan -- Cryptol lexer * David Oliva -- Rebol lexer * Pat Pannuto -- nesC lexer * Jon Parise -- Protocol buffers lexer * Ronny Pfannschmidt -- BBCode lexer * Benjamin Peterson -- Test suite refactoring * Dominik Picheta -- Nimrod lexer +* Andrew Pinkham -- RTF Formatter Refactoring * Clément Prévost -- UrbiScript lexer * raichoo -- Idris lexer * Kashif Rasul -- CUDA lexer diff --git a/doc/languages.rst b/doc/languages.rst index 426a576b..0f98c583 100644 --- a/doc/languages.rst +++ b/doc/languages.rst @@ -25,6 +25,7 @@ Programming languages * ColdFusion * Common Lisp * Coq +* Cryptol (incl. Literate Cryptol) * `Cython <http://cython.org>`_ * `D <http://digitalmars.com/d>`_ * Dart @@ -65,7 +66,8 @@ Programming languages * PowerShell * Prolog * `Python <http://www.python.org>`_ 2.x and 3.x (incl. console sessions and tracebacks) -* Rebol +* `REBOL <http://www.rebol.com>`_ +* `Red <http://www.red-lang.org>`_ * Redcode * `Ruby <http://www.ruby-lang.org>`_ (incl. irb sessions) * Rust diff --git a/pygments/formatters/html.py b/pygments/formatters/html.py index 3bc60e8a..970e595b 100644 --- a/pygments/formatters/html.py +++ b/pygments/formatters/html.py @@ -523,7 +523,8 @@ class HtmlFormatter(Formatter): self.cssfile) except AttributeError: print('Note: Cannot determine output file name, ' \ - 'using current directory as base for the CSS file name', file=sys.stderr) + 'using current directory as base for the CSS file name', + file=sys.stderr) cssfilename = self.cssfile # write CSS file only if noclobber_cssfile isn't given as an option. try: diff --git a/pygments/formatters/latex.py b/pygments/formatters/latex.py index 413cca63..352684b0 100644 --- a/pygments/formatters/latex.py +++ b/pygments/formatters/latex.py @@ -375,9 +375,9 @@ class LatexFormatter(Formatter): if len(sep1) > 0: b,sep2,text = text.partition(self.right) if len(sep2) > 0: - value = value + escape_tex(a, self.commandprefix) + b + value += escape_tex(a, self.commandprefix) + b else: - value = value + escape_tex(a + sep1 + b, self.commandprefix) + value += escape_tex(a + sep1 + b, self.commandprefix) else: value = value + escape_tex(a, self.commandprefix) else: diff --git a/pygments/formatters/other.py b/pygments/formatters/other.py index b6e4bc58..c8269b19 100644 --- a/pygments/formatters/other.py +++ b/pygments/formatters/other.py @@ -118,11 +118,11 @@ class RawTokenFormatter(Formatter): TESTCASE_BEFORE = u'''\ def testNeedsName(self): fragment = %r - expected = [ + tokens = [ ''' TESTCASE_AFTER = u'''\ ] - self.assertEqual(expected, list(self.lexer.get_tokens(fragment))) + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) ''' diff --git a/pygments/formatters/rtf.py b/pygments/formatters/rtf.py index 4b03f8a7..cf65a927 100644 --- a/pygments/formatters/rtf.py +++ b/pygments/formatters/rtf.py @@ -10,7 +10,7 @@ """ from pygments.formatter import Formatter -from pygments.util import get_int_opt +from pygments.util import get_int_opt, _surrogatepair __all__ = ['RtfFormatter'] @@ -22,6 +22,10 @@ class RtfFormatter(Formatter): documents with color information and other useful stuff. Perfect for Copy and Paste into Microsoft® Word® documents. + Please note that ``encoding`` and ``outencoding`` options are ignored. + The RTF format is ASCII natively, but handles unicode characters correctly + thanks to escape sequences. + .. versionadded:: 0.6 Additional options accepted: @@ -74,28 +78,27 @@ class RtfFormatter(Formatter): # escape text text = self._escape(text) - if self.encoding in ('utf-8', 'utf-16', 'utf-32'): - encoding = 'iso-8859-15' - else: - encoding = self.encoding or 'iso-8859-15' buf = [] for c in text: - if ord(c) > 128: - ansic = c.encode(encoding, 'ignore') - if ansic and ord(ansic) > 128: - ansic = '\\\'%x' % ord(ansic) - else: - ansic = '?' - buf.append(r'\ud{\u%d%s}' % (ord(c), ansic)) - else: + cn = ord(c) + if cn < (2**7): + # ASCII character buf.append(str(c)) + elif (2**7) <= cn < (2**16): + # single unicode escape sequence + buf.append(r'{\u%d}' % cn) + elif (2**16) <= cn: + # RTF limits unicode to 16 bits. + # Force surrogate pairs + h,l = _surrogatepair(cn) + buf.append(r'{\u%d}{\u%d}' % (h,l)) return ''.join(buf).replace('\n', '\\par\n') def format_unencoded(self, tokensource, outfile): # rtf 1.8 header - outfile.write(r'{\rtf1\ansi\deff0' + outfile.write(r'{\rtf1\ansi\uc0\deff0' r'{\fonttbl{\f0\fmodern\fprq1\fcharset0%s;}}' r'{\colortbl;' % (self.fontface and ' ' + self._escape(self.fontface) or @@ -114,7 +117,7 @@ class RtfFormatter(Formatter): int(color[4:6], 16) )) offset += 1 - outfile.write(r'}\f0') + outfile.write(r'}\f0 ') if self.fontsize: outfile.write(r'\fs%d' % (self.fontsize)) diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py index c77ff1dc..462c6501 100644 --- a/pygments/lexers/_mapping.py +++ b/pygments/lexers/_mapping.py @@ -81,6 +81,7 @@ LEXERS = { 'CppLexer': ('pygments.lexers.compiled', 'C++', ('cpp', 'c++'), ('*.cpp', '*.hpp', '*.c++', '*.h++', '*.cc', '*.hh', '*.cxx', '*.hxx', '*.C', '*.H', '*.cp', '*.CPP'), ('text/x-c++hdr', 'text/x-c++src')), 'CppObjdumpLexer': ('pygments.lexers.asm', 'cpp-objdump', ('cpp-objdump', 'c++-objdumb', 'cxx-objdump'), ('*.cpp-objdump', '*.c++-objdump', '*.cxx-objdump'), ('text/x-cpp-objdump',)), 'CrocLexer': ('pygments.lexers.agile', 'Croc', ('croc',), ('*.croc',), ('text/x-crocsrc',)), + 'CryptolLexer': ('pygments.lexers.functional', 'Cryptol', ('cryptol', 'cry'), ('*.cry',), ('text/x-cryptol',)), 'CssDjangoLexer': ('pygments.lexers.templates', 'CSS+Django/Jinja', ('css+django', 'css+jinja'), (), ('text/css+django', 'text/css+jinja')), 'CssErbLexer': ('pygments.lexers.templates', 'CSS+Ruby', ('css+erb', 'css+ruby'), (), ('text/css+ruby',)), 'CssGenshiLexer': ('pygments.lexers.templates', 'CSS+Genshi Text', ('css+genshitext', 'css+genshi'), (), ('text/css+genshi',)), @@ -165,6 +166,7 @@ LEXERS = { 'IrcLogsLexer': ('pygments.lexers.text', 'IRC logs', ('irc',), ('*.weechatlog',), ('text/x-irclog',)), 'JadeLexer': ('pygments.lexers.web', 'Jade', ('jade',), ('*.jade',), ('text/x-jade',)), 'JagsLexer': ('pygments.lexers.math', 'JAGS', ('jags',), ('*.jag', '*.bug'), ()), + 'JasminLexer': ('pygments.lexers.jvm', 'Jasmin', ('jasmin', 'jasminxt'), ('*.j',), ()), 'JavaLexer': ('pygments.lexers.jvm', 'Java', ('java',), ('*.java',), ('text/x-java',)), 'JavascriptDjangoLexer': ('pygments.lexers.templates', 'JavaScript+Django/Jinja', ('js+django', 'javascript+django', 'js+jinja', 'javascript+jinja'), (), ('application/x-javascript+django', 'application/x-javascript+jinja', 'text/x-javascript+django', 'text/x-javascript+jinja', 'text/javascript+django', 'text/javascript+jinja')), 'JavascriptErbLexer': ('pygments.lexers.templates', 'JavaScript+Ruby', ('js+erb', 'javascript+erb', 'js+ruby', 'javascript+ruby'), (), ('application/x-javascript+ruby', 'text/x-javascript+ruby', 'text/javascript+ruby')), @@ -189,6 +191,7 @@ LEXERS = { 'LighttpdConfLexer': ('pygments.lexers.text', 'Lighttpd configuration file', ('lighty', 'lighttpd'), (), ('text/x-lighttpd-conf',)), 'LimboLexer': ('pygments.lexers.inferno', 'Limbo', ('limbo',), ('*.b',), ('text/limbo',)), 'LiterateAgdaLexer': ('pygments.lexers.functional', 'Literate Agda', ('lagda', 'literate-agda'), ('*.lagda',), ('text/x-literate-agda',)), + 'LiterateCryptolLexer': ('pygments.lexers.functional', 'Literate Cryptol', ('lcry', 'literate-cryptol', 'lcryptol'), ('*.lcry',), ('text/x-literate-cryptol',)), 'LiterateHaskellLexer': ('pygments.lexers.functional', 'Literate Haskell', ('lhs', 'literate-haskell', 'lhaskell'), ('*.lhs',), ('text/x-literate-haskell',)), 'LiterateIdrisLexer': ('pygments.lexers.functional', 'Literate Idris', ('lidr', 'literate-idris', 'lidris'), ('*.lidr',), ('text/x-literate-idris',)), 'LiveScriptLexer': ('pygments.lexers.web', 'LiveScript', ('live-script', 'livescript'), ('*.ls',), ('text/livescript',)), @@ -272,7 +275,7 @@ LEXERS = { 'QmlLexer': ('pygments.lexers.web', 'QML', ('qml',), ('*.qml',), ('application/x-qml',)), 'RConsoleLexer': ('pygments.lexers.math', 'RConsole', ('rconsole', 'rout'), ('*.Rout',), ()), 'RPMSpecLexer': ('pygments.lexers.other', 'RPMSpec', ('spec',), ('*.spec',), ('text/x-rpm-spec',)), - 'RacketLexer': ('pygments.lexers.functional', 'Racket', ('racket', 'rkt'), ('*.rkt', '*.rktl'), ('text/x-racket', 'application/x-racket')), + 'RacketLexer': ('pygments.lexers.functional', 'Racket', ('racket', 'rkt'), ('*.rkt', '*.rktd', '*.rktl'), ('text/x-racket', 'application/x-racket')), 'RagelCLexer': ('pygments.lexers.parsers', 'Ragel in C Host', ('ragel-c',), ('*.rl',), ()), 'RagelCppLexer': ('pygments.lexers.parsers', 'Ragel in CPP Host', ('ragel-cpp',), ('*.rl',), ()), 'RagelDLexer': ('pygments.lexers.parsers', 'Ragel in D Host', ('ragel-d',), ('*.rl',), ()), @@ -283,9 +286,9 @@ LEXERS = { 'RagelRubyLexer': ('pygments.lexers.parsers', 'Ragel in Ruby Host', ('ragel-ruby', 'ragel-rb'), ('*.rl',), ()), 'RawTokenLexer': ('pygments.lexers.special', 'Raw token data', ('raw',), (), ('application/x-pygments-tokens',)), 'RdLexer': ('pygments.lexers.math', 'Rd', ('rd',), ('*.Rd',), ('text/x-r-doc',)), - 'RebolLexer': ('pygments.lexers.other', 'REBOL', ('rebol',), ('*.r', '*.r3'), ('text/x-rebol',)), + 'RebolLexer': ('pygments.lexers.other', 'REBOL', ('rebol',), ('*.r', '*.r3', '*.reb'), ('text/x-rebol',)), + 'RedLexer': ('pygments.lexers.other', 'Red', ('red', 'red/system'), ('*.red', '*.reds'), ('text/x-red', 'text/x-red-system')), 'RedcodeLexer': ('pygments.lexers.other', 'Redcode', ('redcode',), ('*.cw',), ()), - 'RedLexer': ('pygments.lexers.other', 'Red', ('red','red/system'), ('*.red', '*.reds'), ('text/x-red','text/x-red-system',)), 'RegeditLexer': ('pygments.lexers.text', 'reg', ('registry',), ('*.reg',), ('text/x-windows-registry',)), 'RexxLexer': ('pygments.lexers.other', 'Rexx', ('rexx', 'arexx'), ('*.rexx', '*.rex', '*.rx', '*.arexx'), ('text/x-rexx',)), 'RhtmlLexer': ('pygments.lexers.templates', 'RHTML', ('rhtml', 'html+erb', 'html+ruby'), ('*.rhtml',), ('text/html+ruby',)), diff --git a/pygments/lexers/_stan_builtins.py b/pygments/lexers/_stan_builtins.py index fa45738d..583b44a8 100644 --- a/pygments/lexers/_stan_builtins.py +++ b/pygments/lexers/_stan_builtins.py @@ -4,7 +4,7 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ This file contains the names of functions for Stan used by - `pygments.lexers.math.StanLexer`. + ``pygments.lexers.math.StanLexer``. These builtins are from Stan language v2.2.0. :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. @@ -91,9 +91,11 @@ FUNCTIONS = [ 'Phi', 'diag_post_multiply', 'diag_pre_multiply', 'diagonal', + 'digamma', 'dims', 'dirichlet_log', 'dirichlet_rng', + 'distance', 'dot_product', 'dot_self', 'double_exponential_ccdf_log', @@ -170,7 +172,6 @@ FUNCTIONS = [ 'Phi', 'lgamma', 'lkj_corr_log', 'lkj_corr_rng', - 'lkj_cov_log', 'lmgamma', 'log', 'log10', @@ -285,6 +286,7 @@ FUNCTIONS = [ 'Phi', 'sqrt', 'sqrt2', 'square', + 'squared_distance', 'step', 'student_t_ccdf_log', 'student_t_cdf', @@ -303,6 +305,7 @@ FUNCTIONS = [ 'Phi', 'trace', 'trace_gen_quad_form', 'trace_quad_form', + 'trigamma', 'trunc', 'uniform_ccdf_log', 'uniform_cdf', @@ -324,7 +327,6 @@ DISTRIBUTIONS = [ 'bernoulli', 'beta', 'beta_binomial', 'binomial', - 'binomial_coefficient', 'binomial_logit', 'categorical', 'categorical_logit', @@ -342,7 +344,6 @@ DISTRIBUTIONS = [ 'bernoulli', 'inv_gamma', 'inv_wishart', 'lkj_corr', - 'lkj_cov', 'logistic', 'lognormal', 'multi_normal', @@ -350,7 +351,6 @@ DISTRIBUTIONS = [ 'bernoulli', 'multi_normal_prec', 'multi_student_t', 'multinomial', - 'multiply', 'neg_binomial', 'normal', 'ordered_logistic', @@ -451,3 +451,4 @@ RESERVED = [ 'alignas', 'wchar_t', 'xor', 'xor_eq'] + diff --git a/pygments/lexers/agile.py b/pygments/lexers/agile.py index a3e60f59..cd105126 100644 --- a/pygments/lexers/agile.py +++ b/pygments/lexers/agile.py @@ -119,20 +119,20 @@ class PythonLexer(RegexLexer): ('`.*?`', String.Backtick), ], 'name': [ - (r'@[a-zA-Z0-9_.]+', Name.Decorator), - ('[a-zA-Z_][a-zA-Z0-9_]*', Name), + (r'@[\w.]+', Name.Decorator), + ('[a-zA-Z_]\w*', Name), ], 'funcname': [ - ('[a-zA-Z_][a-zA-Z0-9_]*', Name.Function, '#pop') + ('[a-zA-Z_]\w*', Name.Function, '#pop') ], 'classname': [ - ('[a-zA-Z_][a-zA-Z0-9_]*', Name.Class, '#pop') + ('[a-zA-Z_]\w*', Name.Class, '#pop') ], 'import': [ (r'(?:[ \t]|\\\n)+', Text), (r'as\b', Keyword.Namespace), (r',', Operator), - (r'[a-zA-Z_][a-zA-Z0-9_.]*', Name.Namespace), + (r'[a-zA-Z_][\w.]*', Name.Namespace), (r'', Text, '#pop') # all else: go back ], 'fromimport': [ @@ -142,7 +142,7 @@ class PythonLexer(RegexLexer): # never be a module name (r'None\b', Name.Builtin.Pseudo, '#pop'), # sadly, in "raise x from y" y will be highlighted as namespace too - (r'[a-zA-Z_.][a-zA-Z0-9_.]*', Name.Namespace), + (r'[a-zA-Z_.][\w.]*', Name.Namespace), # anything else here also means "raise x from y" and is therefore # not an error (r'', Text, '#pop'), @@ -152,7 +152,7 @@ class PythonLexer(RegexLexer): r'U[a-fA-F0-9]{8}|x[a-fA-F0-9]{2}|[0-7]{1,3})', String.Escape) ], 'strings': [ - (r'%(\([a-zA-Z0-9_]+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?' + (r'%(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?' '[hlL]?[diouxXeEfFgGcrs%]', String.Interpol), (r'[^\\\'"%\n]+', String), # quotes, percents and backslashes must be parsed one at a time @@ -255,7 +255,7 @@ class Python3Lexer(RegexLexer): ] tokens['backtick'] = [] tokens['name'] = [ - (r'@[a-zA-Z0-9_]+', Name.Decorator), + (r'@\w+', Name.Decorator), (uni_name, Name), ] tokens['funcname'] = [ @@ -406,7 +406,7 @@ class PythonTracebackLexer(RegexLexer): bygroups(Text, Comment, Text)), # for doctests... (r'^([^:]+)(: )(.+)(\n)', bygroups(Generic.Error, Text, Name, Text), '#pop'), - (r'^([a-zA-Z_][a-zA-Z0-9_]*)(:?\n)', + (r'^([a-zA-Z_]\w*)(:?\n)', bygroups(Generic.Error, Text), '#pop') ], } @@ -445,7 +445,7 @@ class Python3TracebackLexer(RegexLexer): bygroups(Text, Comment, Text)), # for doctests... (r'^([^:]+)(: )(.+)(\n)', bygroups(Generic.Error, Text, Name, Text), '#pop'), - (r'^([a-zA-Z_][a-zA-Z0-9_]*)(:?\n)', + (r'^([a-zA-Z_]\w*)(:?\n)', bygroups(Generic.Error, Text), '#pop') ], } @@ -535,7 +535,7 @@ class RubyLexer(ExtendedRegexLexer): (r":'(\\\\|\\'|[^'])*'", String.Symbol), (r"'(\\\\|\\'|[^'])*'", String.Single), (r':"', String.Symbol, 'simple-sym'), - (r'([a-zA-Z_][a-zA-Z0-9_]*)(:)(?!:)', + (r'([a-zA-Z_]\w*)(:)(?!:)', bygroups(String.Symbol, Punctuation)), # Since Ruby 1.9 (r'"', String.Double, 'simple-string'), (r'(?<!\.)`', String.Backtick, 'simple-backtick'), @@ -621,8 +621,8 @@ class RubyLexer(ExtendedRegexLexer): r'rescue|raise|retry|return|super|then|undef|unless|until|when|' r'while|yield)\b', Keyword), # start of function, class and module names - (r'(module)(\s+)([a-zA-Z_][a-zA-Z0-9_]*' - r'(?:::[a-zA-Z_][a-zA-Z0-9_]*)*)', + (r'(module)(\s+)([a-zA-Z_]\w*' + r'(?:::[a-zA-Z_]\w*)*)', bygroups(Keyword, Text, Name.Namespace)), (r'(def)(\s+)', bygroups(Keyword, Text), 'funcname'), (r'def(?=[*%&^`~+-/\[<>=])', Keyword, 'funcname'), @@ -713,9 +713,9 @@ class RubyLexer(ExtendedRegexLexer): (r'([\d]+(?:_\d+)*)(\s*)([/?])?', bygroups(Number.Integer, Text, Operator)), # Names - (r'@@[a-zA-Z_][a-zA-Z0-9_]*', Name.Variable.Class), - (r'@[a-zA-Z_][a-zA-Z0-9_]*', Name.Variable.Instance), - (r'\$[a-zA-Z0-9_]+', Name.Variable.Global), + (r'@@[a-zA-Z_]\w*', Name.Variable.Class), + (r'@[a-zA-Z_]\w*', Name.Variable.Instance), + (r'\$\w+', Name.Variable.Global), (r'\$[!@&`\'+~=/\\,;.<>_*$?:"]', Name.Variable.Global), (r'\$-[0adFiIlpvw]', Name.Variable.Global), (r'::', Operator), @@ -725,7 +725,7 @@ class RubyLexer(ExtendedRegexLexer): r'(\\([\\abefnrstv#"\']|x[a-fA-F0-9]{1,2}|[0-7]{1,3})|\S)' r'(?!\w)', String.Char), - (r'[A-Z][a-zA-Z0-9_]+', Name.Constant), + (r'[A-Z]\w+', Name.Constant), # this is needed because ruby attributes can look # like keywords (class) or like this: ` ?!? (r'(\.|::)([a-zA-Z_]\w*[\!\?]?|[*%&^`~+-/\[<>=])', @@ -739,7 +739,7 @@ class RubyLexer(ExtendedRegexLexer): ], 'funcname': [ (r'\(', Punctuation, 'defexpr'), - (r'(?:([a-zA-Z_][a-zA-Z0-9_]*)(\.))?' + (r'(?:([a-zA-Z_]\w*)(\.))?' r'([a-zA-Z_]\w*[\!\?]?|\*\*?|[-+]@?|' r'[/%&|^`~]|\[\]=?|<<|>>|<=?>|>=?|===?)', bygroups(Name.Class, Operator, Name.Function), '#pop'), @@ -762,8 +762,8 @@ class RubyLexer(ExtendedRegexLexer): ], 'string-intp': [ (r'#{', String.Interpol, 'in-intp'), - (r'#@@?[a-zA-Z_][a-zA-Z0-9_]*', String.Interpol), - (r'#\$[a-zA-Z_][a-zA-Z0-9_]*', String.Interpol) + (r'#@@?[a-zA-Z_]\w*', String.Interpol), + (r'#\$[a-zA-Z_]\w*', String.Interpol) ], 'string-intp-escaped': [ include('string-intp'), @@ -814,7 +814,7 @@ class RubyConsoleLexer(Lexer): aliases = ['rbcon', 'irb'] mimetypes = ['text/x-ruby-shellsession'] - _prompt_re = re.compile('irb\([a-zA-Z_][a-zA-Z0-9_]*\):\d{3}:\d+[>*"\'] ' + _prompt_re = re.compile('irb\([a-zA-Z_]\w*\):\d{3}:\d+[>*"\'] ' '|>> |\?> ') def get_tokens_unprocessed(self, text): @@ -875,7 +875,7 @@ class PerlLexer(RegexLexer): (r'(case|continue|do|else|elsif|for|foreach|if|last|my|' r'next|our|redo|reset|then|unless|until|while|use|' r'print|new|BEGIN|CHECK|INIT|END|return)\b', Keyword), - (r'(format)(\s+)([a-zA-Z0-9_]+)(\s*)(=)(\s*\n)', + (r'(format)(\s+)(\w+)(\s*)(=)(\s*\n)', bygroups(Keyword, Text, Name, Text, Punctuation, Text), 'format'), (r'(eq|lt|gt|le|ge|ne|not|and|or|cmp)\b', Operator.Word), # common delimiters @@ -928,7 +928,7 @@ class PerlLexer(RegexLexer): r'utime|values|vec|wait|waitpid|wantarray|warn|write' r')\b', Name.Builtin), (r'((__(DATA|DIE|WARN)__)|(STD(IN|OUT|ERR)))\b', Name.Builtin.Pseudo), - (r'<<([\'"]?)([a-zA-Z_][a-zA-Z0-9_]*)\1;?\n.*?\n\2\n', String), + (r'<<([\'"]?)([a-zA-Z_]\w*)\1;?\n.*?\n\2\n', String), (r'__END__', Comment.Preproc, 'end-part'), (r'\$\^[ADEFHILMOPSTWX]', Name.Variable.Global), (r"\$[\\\"\[\]'&`+*.,;=%~?@$!<>(^|/-](?!\w)", Name.Variable.Global), @@ -966,14 +966,14 @@ class PerlLexer(RegexLexer): (r'\s+', Text), (r'\{', Punctuation, '#pop'), # hash syntax? (r'\)|,', Punctuation, '#pop'), # argument specifier - (r'[a-zA-Z0-9_]+::', Name.Namespace), - (r'[a-zA-Z0-9_:]+', Name.Variable, '#pop'), + (r'\w+::', Name.Namespace), + (r'[\w:]+', Name.Variable, '#pop'), ], 'name': [ - (r'[a-zA-Z0-9_]+::', Name.Namespace), - (r'[a-zA-Z0-9_:]+', Name, '#pop'), - (r'[A-Z_]+(?=[^a-zA-Z0-9_])', Name.Constant, '#pop'), - (r'(?=[^a-zA-Z0-9_])', Text, '#pop'), + (r'\w+::', Name.Namespace), + (r'[\w:]+', Name, '#pop'), + (r'[A-Z_]+(?=\W)', Name.Constant, '#pop'), + (r'(?=\W)', Text, '#pop'), ], 'modulename': [ (r'[a-zA-Z_]\w*', Name.Namespace, '#pop') @@ -1085,7 +1085,7 @@ class LuaLexer(RegexLexer): (r'(function)\b', Keyword, 'funcname'), - (r'[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)?', Name), + (r'[A-Za-z_]\w*(\.[A-Za-z_]\w*)?', Name), ("'", String.Single, combined('stringescape', 'sqs')), ('"', String.Double, combined('stringescape', 'dqs')) @@ -1093,7 +1093,7 @@ class LuaLexer(RegexLexer): 'funcname': [ (r'\s+', Text), - ('(?:([A-Za-z_][A-Za-z0-9_]*)(\.))?([A-Za-z_][A-Za-z0-9_]*)', + ('(?:([A-Za-z_]\w*)(\.))?([A-Za-z_]\w*)', bygroups(Name.Class, Punctuation, Name.Function), '#pop'), # inline function ('\(', Punctuation, '#pop'), @@ -1176,20 +1176,20 @@ class MoonScriptLexer(LuaLexer): (r'[^\S\n]+', Text), (r'(?s)\[(=*)\[.*?\]\1\]', String), (r'(->|=>)', Name.Function), - (r':[a-zA-Z_][a-zA-Z0-9_]*', Name.Variable), + (r':[a-zA-Z_]\w*', Name.Variable), (r'(==|!=|~=|<=|>=|\.\.\.|\.\.|[=+\-*/%^<>#!.\\:])', Operator), (r'[;,]', Punctuation), (r'[\[\]\{\}\(\)]', Keyword.Type), - (r'[a-zA-Z_][a-zA-Z0-9_]*:', Name.Variable), + (r'[a-zA-Z_]\w*:', Name.Variable), (r"(class|extends|if|then|super|do|with|import|export|" r"while|elseif|return|for|in|from|when|using|else|" r"and|or|not|switch|break)\b", Keyword), (r'(true|false|nil)\b', Keyword.Constant), (r'(and|or|not)\b', Operator.Word), (r'(self)\b', Name.Builtin.Pseudo), - (r'@@?([a-zA-Z_][a-zA-Z0-9_]*)?', Name.Variable.Class), + (r'@@?([a-zA-Z_]\w*)?', Name.Variable.Class), (r'[A-Z]\w*', Name.Class), # proper name - (r'[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)?', Name), + (r'[A-Za-z_]\w*(\.[A-Za-z_]\w*)?', Name), ("'", String.Single, combined('stringescape', 'sqs')), ('"', String.Double, combined('stringescape', 'dqs')) ], @@ -1320,7 +1320,7 @@ class IoLexer(RegexLexer): # names (r'(Object|list|List|Map|args|Sequence|Coroutine|File)\b', Name.Builtin), - ('[a-zA-Z_][a-zA-Z0-9_]*', Name), + ('[a-zA-Z_]\w*', Name), # numbers (r'(\d+\.?\d*|\d*\.\d+)([eE][+-]?[0-9]+)?', Number.Float), (r'\d+', Number.Integer) @@ -1857,14 +1857,14 @@ class FancyLexer(RegexLexer): r'FalseClass|Tuple|Symbol|Stack|Set|FancySpec|Method|Package|' r'Range)\b', Name.Builtin), # functions - (r'[a-zA-Z]([a-zA-Z0-9_]|[-+?!=*/^><%])*:', Name.Function), + (r'[a-zA-Z](\w|[-+?!=*/^><%])*:', Name.Function), # operators, must be below functions (r'[-+*/~,<>=&!?%^\[\]\.$]+', Operator), - ('[A-Z][a-zA-Z0-9_]*', Name.Constant), - ('@[a-zA-Z_][a-zA-Z0-9_]*', Name.Variable.Instance), - ('@@[a-zA-Z_][a-zA-Z0-9_]*', Name.Variable.Class), + ('[A-Z]\w*', Name.Constant), + ('@[a-zA-Z_]\w*', Name.Variable.Instance), + ('@@[a-zA-Z_]\w*', Name.Variable.Class), ('@@?', Operator), - ('[a-zA-Z_][a-zA-Z0-9_]*', Name), + ('[a-zA-Z_]\w*', Name), # numbers - / checks are necessary to avoid mismarking regexes, # see comment in RubyLexer (r'(0[oO]?[0-7]+(?:_[0-7]+)*)(\s*)([/?])?', @@ -1949,7 +1949,7 @@ class DgLexer(RegexLexer): r'U[a-fA-F0-9]{8}|x[a-fA-F0-9]{2}|[0-7]{1,3})', String.Escape) ], 'string': [ - (r'%(\([a-zA-Z0-9_]+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?' + (r'%(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?' '[hlL]?[diouxXeEfFgGcrs%]', String.Interpol), (r'[^\\\'"%\n]+', String), # quotes, percents and backslashes must be parsed one at a time @@ -1987,7 +1987,7 @@ class Perl6Lexer(ExtendedRegexLexer): mimetypes = ['text/x-perl6', 'application/x-perl6'] flags = re.MULTILINE | re.DOTALL | re.UNICODE - PERL6_IDENTIFIER_RANGE = "['a-zA-Z0-9_:-]" # if you alter this, search for a copy made of it below + PERL6_IDENTIFIER_RANGE = "['\w:-]" PERL6_KEYWORDS = ( 'BEGIN', 'CATCH', 'CHECK', 'CONTROL', 'END', 'ENTER', 'FIRST', 'INIT', @@ -2078,62 +2078,80 @@ class Perl6Lexer(ExtendedRegexLexer): # Perl 6 has a *lot* of possible bracketing characters # this list was lifted from STD.pm6 (https://github.com/perl6/std) PERL6_BRACKETS = { - u'\u0028' : u'\u0029', u'\u003c' : u'\u003e', u'\u005b' : u'\u005d', u'\u007b' : u'\u007d', - u'\u00ab' : u'\u00bb', u'\u0f3a' : u'\u0f3b', u'\u0f3c' : u'\u0f3d', u'\u169b' : u'\u169c', - u'\u2018' : u'\u2019', u'\u201a' : u'\u2019', u'\u201b' : u'\u2019', u'\u201c' : u'\u201d', - u'\u201e' : u'\u201d', u'\u201f' : u'\u201d', u'\u2039' : u'\u203a', u'\u2045' : u'\u2046', - u'\u207d' : u'\u207e', u'\u208d' : u'\u208e', u'\u2208' : u'\u220b', u'\u2209' : u'\u220c', - u'\u220a' : u'\u220d', u'\u2215' : u'\u29f5', u'\u223c' : u'\u223d', u'\u2243' : u'\u22cd', - u'\u2252' : u'\u2253', u'\u2254' : u'\u2255', u'\u2264' : u'\u2265', u'\u2266' : u'\u2267', - u'\u2268' : u'\u2269', u'\u226a' : u'\u226b', u'\u226e' : u'\u226f', u'\u2270' : u'\u2271', - u'\u2272' : u'\u2273', u'\u2274' : u'\u2275', u'\u2276' : u'\u2277', u'\u2278' : u'\u2279', - u'\u227a' : u'\u227b', u'\u227c' : u'\u227d', u'\u227e' : u'\u227f', u'\u2280' : u'\u2281', - u'\u2282' : u'\u2283', u'\u2284' : u'\u2285', u'\u2286' : u'\u2287', u'\u2288' : u'\u2289', - u'\u228a' : u'\u228b', u'\u228f' : u'\u2290', u'\u2291' : u'\u2292', u'\u2298' : u'\u29b8', - u'\u22a2' : u'\u22a3', u'\u22a6' : u'\u2ade', u'\u22a8' : u'\u2ae4', u'\u22a9' : u'\u2ae3', - u'\u22ab' : u'\u2ae5', u'\u22b0' : u'\u22b1', u'\u22b2' : u'\u22b3', u'\u22b4' : u'\u22b5', - u'\u22b6' : u'\u22b7', u'\u22c9' : u'\u22ca', u'\u22cb' : u'\u22cc', u'\u22d0' : u'\u22d1', - u'\u22d6' : u'\u22d7', u'\u22d8' : u'\u22d9', u'\u22da' : u'\u22db', u'\u22dc' : u'\u22dd', - u'\u22de' : u'\u22df', u'\u22e0' : u'\u22e1', u'\u22e2' : u'\u22e3', u'\u22e4' : u'\u22e5', - u'\u22e6' : u'\u22e7', u'\u22e8' : u'\u22e9', u'\u22ea' : u'\u22eb', u'\u22ec' : u'\u22ed', - u'\u22f0' : u'\u22f1', u'\u22f2' : u'\u22fa', u'\u22f3' : u'\u22fb', u'\u22f4' : u'\u22fc', - u'\u22f6' : u'\u22fd', u'\u22f7' : u'\u22fe', u'\u2308' : u'\u2309', u'\u230a' : u'\u230b', - u'\u2329' : u'\u232a', u'\u23b4' : u'\u23b5', u'\u2768' : u'\u2769', u'\u276a' : u'\u276b', - u'\u276c' : u'\u276d', u'\u276e' : u'\u276f', u'\u2770' : u'\u2771', u'\u2772' : u'\u2773', - u'\u2774' : u'\u2775', u'\u27c3' : u'\u27c4', u'\u27c5' : u'\u27c6', u'\u27d5' : u'\u27d6', - u'\u27dd' : u'\u27de', u'\u27e2' : u'\u27e3', u'\u27e4' : u'\u27e5', u'\u27e6' : u'\u27e7', - u'\u27e8' : u'\u27e9', u'\u27ea' : u'\u27eb', u'\u2983' : u'\u2984', u'\u2985' : u'\u2986', - u'\u2987' : u'\u2988', u'\u2989' : u'\u298a', u'\u298b' : u'\u298c', u'\u298d' : u'\u298e', - u'\u298f' : u'\u2990', u'\u2991' : u'\u2992', u'\u2993' : u'\u2994', u'\u2995' : u'\u2996', - u'\u2997' : u'\u2998', u'\u29c0' : u'\u29c1', u'\u29c4' : u'\u29c5', u'\u29cf' : u'\u29d0', - u'\u29d1' : u'\u29d2', u'\u29d4' : u'\u29d5', u'\u29d8' : u'\u29d9', u'\u29da' : u'\u29db', - u'\u29f8' : u'\u29f9', u'\u29fc' : u'\u29fd', u'\u2a2b' : u'\u2a2c', u'\u2a2d' : u'\u2a2e', - u'\u2a34' : u'\u2a35', u'\u2a3c' : u'\u2a3d', u'\u2a64' : u'\u2a65', u'\u2a79' : u'\u2a7a', - u'\u2a7d' : u'\u2a7e', u'\u2a7f' : u'\u2a80', u'\u2a81' : u'\u2a82', u'\u2a83' : u'\u2a84', - u'\u2a8b' : u'\u2a8c', u'\u2a91' : u'\u2a92', u'\u2a93' : u'\u2a94', u'\u2a95' : u'\u2a96', - u'\u2a97' : u'\u2a98', u'\u2a99' : u'\u2a9a', u'\u2a9b' : u'\u2a9c', u'\u2aa1' : u'\u2aa2', - u'\u2aa6' : u'\u2aa7', u'\u2aa8' : u'\u2aa9', u'\u2aaa' : u'\u2aab', u'\u2aac' : u'\u2aad', - u'\u2aaf' : u'\u2ab0', u'\u2ab3' : u'\u2ab4', u'\u2abb' : u'\u2abc', u'\u2abd' : u'\u2abe', - u'\u2abf' : u'\u2ac0', u'\u2ac1' : u'\u2ac2', u'\u2ac3' : u'\u2ac4', u'\u2ac5' : u'\u2ac6', - u'\u2acd' : u'\u2ace', u'\u2acf' : u'\u2ad0', u'\u2ad1' : u'\u2ad2', u'\u2ad3' : u'\u2ad4', - u'\u2ad5' : u'\u2ad6', u'\u2aec' : u'\u2aed', u'\u2af7' : u'\u2af8', u'\u2af9' : u'\u2afa', - u'\u2e02' : u'\u2e03', u'\u2e04' : u'\u2e05', u'\u2e09' : u'\u2e0a', u'\u2e0c' : u'\u2e0d', - u'\u2e1c' : u'\u2e1d', u'\u2e20' : u'\u2e21', u'\u3008' : u'\u3009', u'\u300a' : u'\u300b', - u'\u300c' : u'\u300d', u'\u300e' : u'\u300f', u'\u3010' : u'\u3011', u'\u3014' : u'\u3015', - u'\u3016' : u'\u3017', u'\u3018' : u'\u3019', u'\u301a' : u'\u301b', u'\u301d' : u'\u301e', - u'\ufd3e' : u'\ufd3f', u'\ufe17' : u'\ufe18', u'\ufe35' : u'\ufe36', u'\ufe37' : u'\ufe38', - u'\ufe39' : u'\ufe3a', u'\ufe3b' : u'\ufe3c', u'\ufe3d' : u'\ufe3e', u'\ufe3f' : u'\ufe40', - u'\ufe41' : u'\ufe42', u'\ufe43' : u'\ufe44', u'\ufe47' : u'\ufe48', u'\ufe59' : u'\ufe5a', - u'\ufe5b' : u'\ufe5c', u'\ufe5d' : u'\ufe5e', u'\uff08' : u'\uff09', u'\uff1c' : u'\uff1e', - u'\uff3b' : u'\uff3d', u'\uff5b' : u'\uff5d', u'\uff5f' : u'\uff60', u'\uff62' : u'\uff63', + u'\u0028' : u'\u0029', u'\u003c' : u'\u003e', u'\u005b' : u'\u005d', + u'\u007b' : u'\u007d', u'\u00ab' : u'\u00bb', u'\u0f3a' : u'\u0f3b', + u'\u0f3c' : u'\u0f3d', u'\u169b' : u'\u169c', u'\u2018' : u'\u2019', + u'\u201a' : u'\u2019', u'\u201b' : u'\u2019', u'\u201c' : u'\u201d', + u'\u201e' : u'\u201d', u'\u201f' : u'\u201d', u'\u2039' : u'\u203a', + u'\u2045' : u'\u2046', u'\u207d' : u'\u207e', u'\u208d' : u'\u208e', + u'\u2208' : u'\u220b', u'\u2209' : u'\u220c', u'\u220a' : u'\u220d', + u'\u2215' : u'\u29f5', u'\u223c' : u'\u223d', u'\u2243' : u'\u22cd', + u'\u2252' : u'\u2253', u'\u2254' : u'\u2255', u'\u2264' : u'\u2265', + u'\u2266' : u'\u2267', u'\u2268' : u'\u2269', u'\u226a' : u'\u226b', + u'\u226e' : u'\u226f', u'\u2270' : u'\u2271', u'\u2272' : u'\u2273', + u'\u2274' : u'\u2275', u'\u2276' : u'\u2277', u'\u2278' : u'\u2279', + u'\u227a' : u'\u227b', u'\u227c' : u'\u227d', u'\u227e' : u'\u227f', + u'\u2280' : u'\u2281', u'\u2282' : u'\u2283', u'\u2284' : u'\u2285', + u'\u2286' : u'\u2287', u'\u2288' : u'\u2289', u'\u228a' : u'\u228b', + u'\u228f' : u'\u2290', u'\u2291' : u'\u2292', u'\u2298' : u'\u29b8', + u'\u22a2' : u'\u22a3', u'\u22a6' : u'\u2ade', u'\u22a8' : u'\u2ae4', + u'\u22a9' : u'\u2ae3', u'\u22ab' : u'\u2ae5', u'\u22b0' : u'\u22b1', + u'\u22b2' : u'\u22b3', u'\u22b4' : u'\u22b5', u'\u22b6' : u'\u22b7', + u'\u22c9' : u'\u22ca', u'\u22cb' : u'\u22cc', u'\u22d0' : u'\u22d1', + u'\u22d6' : u'\u22d7', u'\u22d8' : u'\u22d9', u'\u22da' : u'\u22db', + u'\u22dc' : u'\u22dd', u'\u22de' : u'\u22df', u'\u22e0' : u'\u22e1', + u'\u22e2' : u'\u22e3', u'\u22e4' : u'\u22e5', u'\u22e6' : u'\u22e7', + u'\u22e8' : u'\u22e9', u'\u22ea' : u'\u22eb', u'\u22ec' : u'\u22ed', + u'\u22f0' : u'\u22f1', u'\u22f2' : u'\u22fa', u'\u22f3' : u'\u22fb', + u'\u22f4' : u'\u22fc', u'\u22f6' : u'\u22fd', u'\u22f7' : u'\u22fe', + u'\u2308' : u'\u2309', u'\u230a' : u'\u230b', u'\u2329' : u'\u232a', + u'\u23b4' : u'\u23b5', u'\u2768' : u'\u2769', u'\u276a' : u'\u276b', + u'\u276c' : u'\u276d', u'\u276e' : u'\u276f', u'\u2770' : u'\u2771', + u'\u2772' : u'\u2773', u'\u2774' : u'\u2775', u'\u27c3' : u'\u27c4', + u'\u27c5' : u'\u27c6', u'\u27d5' : u'\u27d6', u'\u27dd' : u'\u27de', + u'\u27e2' : u'\u27e3', u'\u27e4' : u'\u27e5', u'\u27e6' : u'\u27e7', + u'\u27e8' : u'\u27e9', u'\u27ea' : u'\u27eb', u'\u2983' : u'\u2984', + u'\u2985' : u'\u2986', u'\u2987' : u'\u2988', u'\u2989' : u'\u298a', + u'\u298b' : u'\u298c', u'\u298d' : u'\u298e', u'\u298f' : u'\u2990', + u'\u2991' : u'\u2992', u'\u2993' : u'\u2994', u'\u2995' : u'\u2996', + u'\u2997' : u'\u2998', u'\u29c0' : u'\u29c1', u'\u29c4' : u'\u29c5', + u'\u29cf' : u'\u29d0', u'\u29d1' : u'\u29d2', u'\u29d4' : u'\u29d5', + u'\u29d8' : u'\u29d9', u'\u29da' : u'\u29db', u'\u29f8' : u'\u29f9', + u'\u29fc' : u'\u29fd', u'\u2a2b' : u'\u2a2c', u'\u2a2d' : u'\u2a2e', + u'\u2a34' : u'\u2a35', u'\u2a3c' : u'\u2a3d', u'\u2a64' : u'\u2a65', + u'\u2a79' : u'\u2a7a', u'\u2a7d' : u'\u2a7e', u'\u2a7f' : u'\u2a80', + u'\u2a81' : u'\u2a82', u'\u2a83' : u'\u2a84', u'\u2a8b' : u'\u2a8c', + u'\u2a91' : u'\u2a92', u'\u2a93' : u'\u2a94', u'\u2a95' : u'\u2a96', + u'\u2a97' : u'\u2a98', u'\u2a99' : u'\u2a9a', u'\u2a9b' : u'\u2a9c', + u'\u2aa1' : u'\u2aa2', u'\u2aa6' : u'\u2aa7', u'\u2aa8' : u'\u2aa9', + u'\u2aaa' : u'\u2aab', u'\u2aac' : u'\u2aad', u'\u2aaf' : u'\u2ab0', + u'\u2ab3' : u'\u2ab4', u'\u2abb' : u'\u2abc', u'\u2abd' : u'\u2abe', + u'\u2abf' : u'\u2ac0', u'\u2ac1' : u'\u2ac2', u'\u2ac3' : u'\u2ac4', + u'\u2ac5' : u'\u2ac6', u'\u2acd' : u'\u2ace', u'\u2acf' : u'\u2ad0', + u'\u2ad1' : u'\u2ad2', u'\u2ad3' : u'\u2ad4', u'\u2ad5' : u'\u2ad6', + u'\u2aec' : u'\u2aed', u'\u2af7' : u'\u2af8', u'\u2af9' : u'\u2afa', + u'\u2e02' : u'\u2e03', u'\u2e04' : u'\u2e05', u'\u2e09' : u'\u2e0a', + u'\u2e0c' : u'\u2e0d', u'\u2e1c' : u'\u2e1d', u'\u2e20' : u'\u2e21', + u'\u3008' : u'\u3009', u'\u300a' : u'\u300b', u'\u300c' : u'\u300d', + u'\u300e' : u'\u300f', u'\u3010' : u'\u3011', u'\u3014' : u'\u3015', + u'\u3016' : u'\u3017', u'\u3018' : u'\u3019', u'\u301a' : u'\u301b', + u'\u301d' : u'\u301e', u'\ufd3e' : u'\ufd3f', u'\ufe17' : u'\ufe18', + u'\ufe35' : u'\ufe36', u'\ufe37' : u'\ufe38', u'\ufe39' : u'\ufe3a', + u'\ufe3b' : u'\ufe3c', u'\ufe3d' : u'\ufe3e', u'\ufe3f' : u'\ufe40', + u'\ufe41' : u'\ufe42', u'\ufe43' : u'\ufe44', u'\ufe47' : u'\ufe48', + u'\ufe59' : u'\ufe5a', u'\ufe5b' : u'\ufe5c', u'\ufe5d' : u'\ufe5e', + u'\uff08' : u'\uff09', u'\uff1c' : u'\uff1e', u'\uff3b' : u'\uff3d', + u'\uff5b' : u'\uff5d', u'\uff5f' : u'\uff60', u'\uff62' : u'\uff63', } def _build_word_match(words, boundary_regex_fragment = None, prefix = '', suffix = ''): if boundary_regex_fragment is None: - return r'\b(' + prefix + r'|'.join([ re.escape(x) for x in words]) + suffix + r')\b' + return r'\b(' + prefix + r'|'.join([ re.escape(x) for x in words]) + \ + suffix + r')\b' else: - return r'(?<!' + boundary_regex_fragment + ')' + prefix + '(' + \ - r'|'.join([ re.escape(x) for x in words]) + r')' + suffix + '(?!' + boundary_regex_fragment + ')' + return r'(?<!' + boundary_regex_fragment + r')' + prefix + r'(' + \ + r'|'.join([ re.escape(x) for x in words]) + r')' + suffix + r'(?!' + \ + boundary_regex_fragment + r')' def brackets_callback(token_class): def callback(lexer, match, context): @@ -2223,10 +2241,10 @@ class Perl6Lexer(ExtendedRegexLexer): context.pos = match.end() context.stack.append('root') - # If you're modifying these rules, be careful if you need to process '{' or '}' characters. - # We have special logic for processing these characters (due to the fact that you can nest - # Perl 6 code in regex blocks), so if you need to process one of them, make sure you also - # process the corresponding one! + # If you're modifying these rules, be careful if you need to process '{' or '}' + # characters. We have special logic for processing these characters (due to the fact + # that you can nest Perl 6 code in regex blocks), so if you need to process one of + # them, make sure you also process the corresponding one! tokens = { 'common' : [ (r'#[`|=](?P<delimiter>(?P<first_char>[' + ''.join(PERL6_BRACKETS) + r'])(?P=first_char)*)', brackets_callback(Comment.Multiline)), @@ -2234,7 +2252,8 @@ class Perl6Lexer(ExtendedRegexLexer): (r'^(\s*)=begin\s+(\w+)\b.*?^\1=end\s+\2', Comment.Multiline), (r'^(\s*)=for.*?\n\s*?\n', Comment.Multiline), (r'^=.*?\n\s*?\n', Comment.Multiline), - (r'(regex|token|rule)(\s*' + PERL6_IDENTIFIER_RANGE + '+:sym)', bygroups(Keyword, Name), 'token-sym-brackets'), + (r'(regex|token|rule)(\s*' + PERL6_IDENTIFIER_RANGE + '+:sym)', + bygroups(Keyword, Name), 'token-sym-brackets'), (r'(regex|token|rule)(?!' + PERL6_IDENTIFIER_RANGE + ')(\s*' + PERL6_IDENTIFIER_RANGE + '+)?', bygroups(Keyword, Name), 'pre-token'), # deal with a special case in the Perl 6 grammar (role q { ... }) (r'(role)(\s+)(q)(\s*)', bygroups(Keyword, Text, Name, Text)), @@ -2242,24 +2261,28 @@ class Perl6Lexer(ExtendedRegexLexer): (_build_word_match(PERL6_BUILTIN_CLASSES, PERL6_IDENTIFIER_RANGE, suffix = '(?::[UD])?'), Name.Builtin), (_build_word_match(PERL6_BUILTINS, PERL6_IDENTIFIER_RANGE), Name.Builtin), # copied from PerlLexer - (r'[$@%&][.^:?=!~]?' + PERL6_IDENTIFIER_RANGE + u'+(?:<<.*?>>|<.*?>|«.*?»)*', Name.Variable), + (r'[$@%&][.^:?=!~]?' + PERL6_IDENTIFIER_RANGE + u'+(?:<<.*?>>|<.*?>|«.*?»)*', + Name.Variable), (r'\$[!/](?:<<.*?>>|<.*?>|«.*?»)*', Name.Variable.Global), (r'::\?\w+', Name.Variable.Global), - (r'[$@%&]\*' + PERL6_IDENTIFIER_RANGE + u'+(?:<<.*?>>|<.*?>|«.*?»)*', Name.Variable.Global), + (r'[$@%&]\*' + PERL6_IDENTIFIER_RANGE + u'+(?:<<.*?>>|<.*?>|«.*?»)*', + Name.Variable.Global), (r'\$(?:<.*?>)+', Name.Variable), (r'(?:q|qq|Q)[a-zA-Z]?\s*(?P<adverbs>:[\w\s:]+)?\s*(?P<delimiter>(?P<first_char>[^0-9a-zA-Z:\s])(?P=first_char)*)', brackets_callback(String)), # copied from PerlLexer (r'0_?[0-7]+(_[0-7]+)*', Number.Oct), (r'0x[0-9A-Fa-f]+(_[0-9A-Fa-f]+)*', Number.Hex), (r'0b[01]+(_[01]+)*', Number.Bin), - (r'(?i)(\d*(_\d*)*\.\d+(_\d*)*|\d+(_\d*)*\.\d+(_\d*)*)(e[+-]?\d+)?', Number.Float), + (r'(?i)(\d*(_\d*)*\.\d+(_\d*)*|\d+(_\d*)*\.\d+(_\d*)*)(e[+-]?\d+)?', + Number.Float), (r'(?i)\d+(_\d*)*e[+-]?\d+(_\d*)*', Number.Float), (r'\d+(_\d+)*', Number.Integer), (r'(?<=~~)\s*/(?:\\\\|\\/|.)*?/', String.Regex), (r'(?<=[=(,])\s*/(?:\\\\|\\/|.)*?/', String.Regex), (r'm\w+(?=\()', Name), (r'(?:m|ms|rx)\s*(?P<adverbs>:[\w\s:]+)?\s*(?P<delimiter>(?P<first_char>[^0-9a-zA-Z_:\s])(?P=first_char)*)', brackets_callback(String.Regex)), - (r'(?:s|ss|tr)\s*(?::[\w\s:]+)?\s*/(?:\\\\|\\/|.)*?/(?:\\\\|\\/|.)*?/', String.Regex), + (r'(?:s|ss|tr)\s*(?::[\w\s:]+)?\s*/(?:\\\\|\\/|.)*?/(?:\\\\|\\/|.)*?/', + String.Regex), (r'<[^\s=].*?\S>', String), (_build_word_match(PERL6_OPERATORS), Operator), (r'[0-9a-zA-Z_]' + PERL6_IDENTIFIER_RANGE + '*', Name), @@ -2322,8 +2345,8 @@ class Perl6Lexer(ExtendedRegexLexer): rating = False # check for my/our/has declarations - # copied PERL6_IDENTIFIER_RANGE from above; not happy about that - if re.search("(?:my|our|has)\s+(?:['a-zA-Z0-9_:-]+\s+)?[$@%&(]", text): + if re.search("(?:my|our|has)\s+(?:" + Perl6Lexer.PERL6_IDENTIFIER_RANGE + \ + "+\s+)?[$@%&(]", text): rating = 0.8 saw_perl_decl = True @@ -2511,7 +2534,7 @@ class ChaiscriptLexer(RegexLexer): (r'(true|false)\b', Keyword.Constant), (r'(eval|throw)\b', Name.Builtin), (r'`\S+`', Name.Builtin), - (r'[$a-zA-Z_][a-zA-Z0-9_]*', Name.Other), + (r'[$a-zA-Z_]\w*', Name.Other), (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float), (r'0x[0-9a-fA-F]+', Number.Hex), (r'[0-9]+', Number.Integer), diff --git a/pygments/lexers/asm.py b/pygments/lexers/asm.py index fc361e2f..35092f52 100644 --- a/pygments/lexers/asm.py +++ b/pygments/lexers/asm.py @@ -318,8 +318,8 @@ class NasmLexer(RegexLexer): filenames = ['*.asm', '*.ASM'] mimetypes = ['text/x-nasm'] - identifier = r'[a-zA-Z$._?][a-zA-Z0-9$._?#@~]*' - hexn = r'(?:0[xX][0-9a-fA-F]+|$0[0-9a-fA-F]*|[0-9]+[0-9a-fA-F]*h)' + identifier = r'[a-z$._?][\w$.?#@~]*' + hexn = r'(?:0[xX][0-9a-f]+|$0[0-9a-f]*|[0-9]+[0-9a-f]*h)' octn = r'[0-7]+q' binn = r'[01]+b' decn = r'[0-9]+' @@ -417,7 +417,7 @@ class Ca65Lexer(RegexLexer): r'|cl[cvdi]|se[cdi]|jmp|jsr|bne|beq|bpl|bmi|bvc|bvs|bcc|bcs' r'|p[lh][ap]|rt[is]|brk|nop|ta[xy]|t[xy]a|txs|tsx|and|ora|eor' r'|bit)\b', Keyword), - (r'\.[a-z0-9_]+', Keyword.Pseudo), + (r'\.\w+', Keyword.Pseudo), (r'[-+~*/^&|!<>=]', Operator), (r'"[^"\n]*.', String), (r"'[^'\n]*.", String.Char), diff --git a/pygments/lexers/compiled.py b/pygments/lexers/compiled.py index d70cf38f..c2c24cd2 100644 --- a/pygments/lexers/compiled.py +++ b/pygments/lexers/compiled.py @@ -87,23 +87,23 @@ class CFamilyLexer(RegexLexer): r'declspec|finally|int64|try|leave|wchar_t|w64|unaligned|' r'raise|noop|identifier|forceinline|assume)\b', Keyword.Reserved), (r'(true|false|NULL)\b', Name.Builtin), - (r'([a-zA-Z_][a-zA-Z0-9_]*)(\s*)(:)(?!:)', bygroups(Name.Label, Text, Punctuation)), - ('[a-zA-Z_][a-zA-Z0-9_]*', Name), + (r'([a-zA-Z_]\w*)(\s*)(:)(?!:)', bygroups(Name.Label, Text, Punctuation)), + ('[a-zA-Z_]\w*', Name), ], 'root': [ include('whitespace'), # functions - (r'((?:[a-zA-Z0-9_*\s])+?(?:\s|[*]))' # return arguments - r'([a-zA-Z_][a-zA-Z0-9_]*)' # method name - r'(\s*\([^;]*?\))' # signature + (r'((?:[\w*\s])+?(?:\s|[*]))' # return arguments + r'([a-zA-Z_]\w*)' # method name + r'(\s*\([^;]*?\))' # signature r'(' + _ws + r')?({)', bygroups(using(this), Name.Function, using(this), using(this), Punctuation), 'function'), # function declarations - (r'((?:[a-zA-Z0-9_*\s])+?(?:\s|[*]))' # return arguments - r'([a-zA-Z_][a-zA-Z0-9_]*)' # method name - r'(\s*\([^;]*?\))' # signature + (r'((?:[\w*\s])+?(?:\s|[*]))' # return arguments + r'([a-zA-Z_]\w*)' # method name + r'(\s*\([^;]*?\))' # signature r'(' + _ws + r')?(;)', bygroups(using(this), Name.Function, using(this), using(this), Punctuation)), @@ -223,7 +223,7 @@ class CppLexer(CFamilyLexer): (r'(__offload|__blockingoffload|__outer)\b', Keyword.Pseudo), ], 'classname': [ - (r'[a-zA-Z_][a-zA-Z0-9_]*', Name.Class, '#pop'), + (r'[a-zA-Z_]\w*', Name.Class, '#pop'), # template specification (r'\s*(?=>)', Text, '#pop'), ], @@ -268,7 +268,7 @@ class PikeLexer(CppLexer): inherit, ], 'classname': [ - (r'[a-zA-Z_][a-zA-Z0-9_]*', Name.Class, '#pop'), + (r'[a-zA-Z_]\w*', Name.Class, '#pop'), # template specification (r'\s*(?=>)', Text, '#pop'), ], @@ -289,9 +289,12 @@ class SwigLexer(CppLexer): tokens = { 'statements': [ - (r'(%[a-z_][a-z0-9_]*)', Name.Function), # SWIG directives - ('\$\**\&?[a-zA-Z0-9_]+', Name), # Special variables - (r'##*[a-zA-Z_][a-zA-Z0-9_]*', Comment.Preproc), # Stringification / additional preprocessor directives + # SWIG directives + (r'(%[a-z_][a-z0-9_]*)', Name.Function), + # Special variables + ('\$\**\&?\w+', Name), + # Stringification / additional preprocessor directives + (r'##*[a-zA-Z_]\w*', Comment.Preproc), inherit, ], } @@ -360,7 +363,7 @@ class ECLexer(CLexer): inherit, ], 'classname': [ - (r'[a-zA-Z_][a-zA-Z0-9_]*', Name.Class, '#pop'), + (r'[a-zA-Z_]\w*', Name.Class, '#pop'), # template specification (r'\s*(?=>)', Text, '#pop'), ], @@ -1155,7 +1158,7 @@ class DylanLexer(RegexLexer): 'type-error-value', 'type-for-copy', 'type-union', 'union', 'values', 'vector', 'zero?']) - valid_name = '\\\\?[a-zA-Z0-9' + re.escape('!&*<>|^$%@_-+~?/=') + ']+' + valid_name = '\\\\?[a-z0-9' + re.escape('!&*<>|^$%@_-+~?/=') + ']+' def get_tokens_unprocessed(self, text): for index, token, value in RegexLexer.get_tokens_unprocessed(self, text): @@ -1184,7 +1187,7 @@ class DylanLexer(RegexLexer): (r'//.*?\n', Comment.Single), # lid header - (r'([A-Za-z0-9-]+)(:)([ \t]*)(.*(?:\n[ \t].+)*)', + (r'([a-z0-9-]+)(:)([ \t]*)(.*(?:\n[ \t].+)*)', bygroups(Name.Attribute, Operator, Text, String)), ('', Text, 'code') # no header match, switch to code @@ -1201,7 +1204,7 @@ class DylanLexer(RegexLexer): # strings and characters (r'"', String, 'string'), - (r"'(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,2}|[^\\\'\n])'", String.Char), + (r"'(\\.|\\[0-7]{1,3}|\\x[a-f0-9]{1,2}|[^\\\'\n])'", String.Char), # binary integer (r'#[bB][01]+', Number), @@ -1216,7 +1219,7 @@ class DylanLexer(RegexLexer): (r'[-+]?\d+', Number.Integer), # hex integer - (r'#[xX][0-9a-fA-F]+', Number.Hex), + (r'#[xX][0-9a-f]+', Number.Hex), # Macro parameters (r'(\?' + valid_name + ')(:)' @@ -1240,7 +1243,7 @@ class DylanLexer(RegexLexer): (r'#"', String.Symbol, 'keyword'), # #rest, #key, #all-keys, etc. - (r'#[a-zA-Z0-9-]+', Keyword), + (r'#[a-z0-9-]+', Keyword), # required-init-keyword: style keywords. (valid_name + ':', Keyword), @@ -1269,7 +1272,7 @@ class DylanLexer(RegexLexer): ], 'string': [ (r'"', String, '#pop'), - (r'\\([\\abfnrtv"\']|x[a-fA-F0-9]{2,4}|[0-7]{1,3})', String.Escape), + (r'\\([\\abfnrtv"\']|x[a-f0-9]{2,4}|[0-7]{1,3})', String.Escape), (r'[^\\"\n]+', String), # all other characters (r'\\\n', String), # line continuation (r'\\', String), # stray backslash @@ -1371,9 +1374,9 @@ def objective(baselexer): # Matches [ <ws>? identifier <ws> ( identifier <ws>? ] | identifier? : ) # (note the identifier is *optional* when there is a ':'!) - _oc_message = re.compile(r'\[\s*[a-zA-Z_][a-zA-Z0-9_]*\s+' - r'(?:[a-zA-Z_][a-zA-Z0-9_]*\s*\]|' - r'(?:[a-zA-Z_][a-zA-Z0-9_]*)?:)') + _oc_message = re.compile(r'\[\s*[a-zA-Z_]\w*\s+' + r'(?:[a-zA-Z_]\w*\s*\]|' + r'(?:[a-zA-Z_]\w*)?:)') class GeneratedObjectiveCVariant(baselexer): """ @@ -1394,11 +1397,11 @@ def objective(baselexer): (r'@\[', Literal, 'literal_array'), (r'@\{', Literal, 'literal_dictionary'), (r'(@selector|@private|@protected|@public|@encode|' - r'@synchronized|@try|@throw|@catch|@finally|@end|@property|' + r'@synchronized|@try|@throw|@catch|@finally|@end|@property|@synthesize|' r'__bridge|__bridge_transfer|__autoreleasing|__block|__weak|__strong|' r'weak|strong|copy|retain|assign|unsafe_unretained|atomic|nonatomic|' r'readonly|readwrite|setter|getter|typeof|in|out|inout|release|class|' - r'@synthesize|@dynamic|@optional|@required|@autoreleasepool)\b', Keyword), + r'@dynamic|@optional|@required|@autoreleasepool)\b', Keyword), (r'(id|instancetype|Class|IMP|SEL|BOOL|IBOutlet|IBAction|unichar)\b', Keyword.Type), (r'@(true|false|YES|NO)\n', Name.Builtin), @@ -1417,24 +1420,26 @@ def objective(baselexer): ], 'oc_classname' : [ # interface definition that inherits - ('([a-zA-Z$_][a-zA-Z0-9$_]*)(\s*:\s*)([a-zA-Z$_][a-zA-Z0-9$_]*)?(\s*)({)', - bygroups(Name.Class, Text, Name.Class, Text, Punctuation), ('#pop', 'oc_ivars')), - ('([a-zA-Z$_][a-zA-Z0-9$_]*)(\s*:\s*)([a-zA-Z$_][a-zA-Z0-9$_]*)?', + ('([a-zA-Z$_][\w$]*)(\s*:\s*)([a-zA-Z$_][\w$]*)?(\s*)({)', + bygroups(Name.Class, Text, Name.Class, Text, Punctuation), + ('#pop', 'oc_ivars')), + ('([a-zA-Z$_][\w$]*)(\s*:\s*)([a-zA-Z$_][\w$]*)?', bygroups(Name.Class, Text, Name.Class), '#pop'), # interface definition for a category - ('([a-zA-Z$_][a-zA-Z0-9$_]*)(\s*)(\([a-zA-Z$_][a-zA-Z0-9$_]*\))(\s*)({)', - bygroups(Name.Class, Text, Name.Label, Text, Punctuation), ('#pop', 'oc_ivars')), - ('([a-zA-Z$_][a-zA-Z0-9$_]*)(\s*)(\([a-zA-Z$_][a-zA-Z0-9$_]*\))', + ('([a-zA-Z$_][\w$]*)(\s*)(\([a-zA-Z$_][\w$]*\))(\s*)({)', + bygroups(Name.Class, Text, Name.Label, Text, Punctuation), + ('#pop', 'oc_ivars')), + ('([a-zA-Z$_][\w$]*)(\s*)(\([a-zA-Z$_][\w$]*\))', bygroups(Name.Class, Text, Name.Label), '#pop'), # simple interface / implementation - ('([a-zA-Z$_][a-zA-Z0-9$_]*)(\s*)({)', + ('([a-zA-Z$_][\w$]*)(\s*)({)', bygroups(Name.Class, Text, Punctuation), ('#pop', 'oc_ivars')), - ('([a-zA-Z$_][a-zA-Z0-9$_]*)', Name.Class, '#pop') + ('([a-zA-Z$_][\w$]*)', Name.Class, '#pop') ], 'oc_forward_classname' : [ - ('([a-zA-Z$_][a-zA-Z0-9$_]*)(\s*,\s*)', + ('([a-zA-Z$_][\w$]*)(\s*,\s*)', bygroups(Name.Class, Text), 'oc_forward_classname'), - ('([a-zA-Z$_][a-zA-Z0-9$_]*)(\s*;?)', + ('([a-zA-Z$_][\w$]*)(\s*;?)', bygroups(Name.Class, Text), '#pop') ], 'oc_ivars' : [ @@ -1448,7 +1453,7 @@ def objective(baselexer): # methods (r'^([-+])(\s*)' # method marker r'(\(.*?\))?(\s*)' # return type - r'([a-zA-Z$_][a-zA-Z0-9$_]*:?)', # begin of method name + r'([a-zA-Z$_][\w$]*:?)', # begin of method name bygroups(Punctuation, Text, using(this), Text, Name.Function), 'method'), @@ -1460,9 +1465,9 @@ def objective(baselexer): # discussion in Issue 789 (r',', Punctuation), (r'\.\.\.', Punctuation), - (r'(\(.*?\))(\s*)([a-zA-Z$_][a-zA-Z0-9$_]*)', + (r'(\(.*?\))(\s*)([a-zA-Z$_][\w$]*)', bygroups(using(this), Text, Name.Variable)), - (r'[a-zA-Z$_][a-zA-Z0-9$_]*:', Name.Function), + (r'[a-zA-Z$_][\w$]*:', Name.Function), (';', Punctuation, '#pop'), ('{', Punctuation, 'function'), ('', Text, '#pop'), @@ -1568,7 +1573,7 @@ class FortranLexer(RegexLexer): (r'!.*\n', Comment), include('strings'), include('core'), - (r'[a-z][a-z0-9_]*', Name.Variable), + (r'[a-z]\w*', Name.Variable), include('nums'), (r'[\s]+', Text), ], @@ -1652,9 +1657,9 @@ class FortranLexer(RegexLexer): ], 'nums': [ - (r'\d+(?![.Ee])(_[a-z][a-z0-9_]+)?', Number.Integer), - (r'[+-]?\d*\.\d+([eE][-+]?\d+)?(_[a-z][a-z0-9_]+)?', Number.Float), - (r'[+-]?\d+\.\d*([eE][-+]?\d+)?(_[a-z][a-z0-9_]+)?', Number.Float), + (r'\d+(?![.e])(_[a-z]\w+)?', Number.Integer), + (r'[+-]?\d*\.\d+(e[-+]?\d+)?(_[a-z]\w+)?', Number.Float), + (r'[+-]?\d+\.\d*(e[-+]?\d+)?(_[a-z]\w+)?', Number.Float), ], } @@ -1743,20 +1748,20 @@ class PrologLexer(RegexLexer): (r'_', Keyword), # The don't-care variable (r'([a-z]+)(:)', bygroups(Name.Namespace, Punctuation)), (u'([a-z\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]' - u'[a-zA-Z0-9_$\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]*)' + u'[\w$\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]*)' u'(\\s*)(:-|-->)', bygroups(Name.Function, Text, Operator)), # function defn (u'([a-z\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]' - u'[a-zA-Z0-9_$\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]*)' + u'[\w$\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]*)' u'(\\s*)(\\()', bygroups(Name.Function, Text, Punctuation)), (u'[a-z\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]' - u'[a-zA-Z0-9_$\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]*', + u'[\w$\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]*', String.Atom), # atom, characters # This one includes ! (u'[#&*+\\-./:<=>?@\\\\^~\u00a1-\u00bf\u2010-\u303f]+', String.Atom), # atom, graphics - (r'[A-Z_][A-Za-z0-9_]*', Name.Variable), + (r'[A-Z_]\w*', Name.Variable), (u'\\s+|[\u2000-\u200f\ufff0-\ufffe\uffef]', Text), ], 'nested-comment': [ @@ -1862,38 +1867,38 @@ class CythonLexer(RegexLexer): ('`.*?`', String.Backtick), ], 'name': [ - (r'@[a-zA-Z0-9_]+', Name.Decorator), - ('[a-zA-Z_][a-zA-Z0-9_]*', Name), + (r'@\w+', Name.Decorator), + ('[a-zA-Z_]\w*', Name), ], 'funcname': [ - ('[a-zA-Z_][a-zA-Z0-9_]*', Name.Function, '#pop') + ('[a-zA-Z_]\w*', Name.Function, '#pop') ], 'cdef': [ (r'(public|readonly|extern|api|inline)\b', Keyword.Reserved), (r'(struct|enum|union|class)\b', Keyword), - (r'([a-zA-Z_][a-zA-Z0-9_]*)(\s*)(?=[(:#=]|$)', + (r'([a-zA-Z_]\w*)(\s*)(?=[(:#=]|$)', bygroups(Name.Function, Text), '#pop'), - (r'([a-zA-Z_][a-zA-Z0-9_]*)(\s*)(,)', + (r'([a-zA-Z_]\w*)(\s*)(,)', bygroups(Name.Function, Text, Punctuation)), (r'from\b', Keyword, '#pop'), (r'as\b', Keyword), (r':', Punctuation, '#pop'), (r'(?=["\'])', Text, '#pop'), - (r'[a-zA-Z_][a-zA-Z0-9_]*', Keyword.Type), + (r'[a-zA-Z_]\w*', Keyword.Type), (r'.', Text), ], 'classname': [ - ('[a-zA-Z_][a-zA-Z0-9_]*', Name.Class, '#pop') + ('[a-zA-Z_]\w*', Name.Class, '#pop') ], 'import': [ (r'(\s+)(as)(\s+)', bygroups(Text, Keyword, Text)), - (r'[a-zA-Z_][a-zA-Z0-9_.]*', Name.Namespace), + (r'[a-zA-Z_][\w.]*', Name.Namespace), (r'(\s*)(,)(\s*)', bygroups(Text, Operator, Text)), (r'', Text, '#pop') # all else: go back ], 'fromimport': [ (r'(\s+)(c?import)\b', bygroups(Text, Keyword), '#pop'), - (r'[a-zA-Z_.][a-zA-Z0-9_.]*', Name.Namespace), + (r'[a-zA-Z_.][\w.]*', Name.Namespace), # ``cdef foo from "header"``, or ``for foo from 0 < i < 10`` (r'', Text, '#pop'), ], @@ -1985,14 +1990,14 @@ class ValaLexer(RegexLexer): 'namespace'), (r'(class|errordomain|interface|struct)(\s+)', bygroups(Keyword.Declaration, Text), 'class'), - (r'(\.)([a-zA-Z_][a-zA-Z0-9_]*)', + (r'(\.)([a-zA-Z_]\w*)', bygroups(Operator, Name.Attribute)), # void is an actual keyword, others are in glib-2.0.vapi (r'(void|bool|char|double|float|int|int8|int16|int32|int64|long|' r'short|size_t|ssize_t|string|time_t|uchar|uint|uint8|uint16|' r'uint32|uint64|ulong|unichar|ushort)\b', Keyword.Type), (r'(true|false|null)\b', Name.Builtin), - ('[a-zA-Z_][a-zA-Z0-9_]*', Name), + ('[a-zA-Z_]\w*', Name), ], 'root': [ include('whitespace'), @@ -2018,10 +2023,10 @@ class ValaLexer(RegexLexer): (r'.*?\n', Comment), ], 'class': [ - (r'[a-zA-Z_][a-zA-Z0-9_]*', Name.Class, '#pop') + (r'[a-zA-Z_]\w*', Name.Class, '#pop') ], 'namespace': [ - (r'[a-zA-Z_][a-zA-Z0-9_.]*', Name.Namespace, '#pop') + (r'[a-zA-Z_][\w.]*', Name.Namespace, '#pop') ], } @@ -2045,9 +2050,9 @@ class OocLexer(RegexLexer): r'while|do|switch|case|as|in|version|return|true|false|null)\b', Keyword), (r'include\b', Keyword, 'include'), - (r'(cover)([ \t]+)(from)([ \t]+)([a-zA-Z0-9_]+[*@]?)', + (r'(cover)([ \t]+)(from)([ \t]+)(\w+[*@]?)', bygroups(Keyword, Text, Keyword, Text, Name.Class)), - (r'(func)((?:[ \t]|\\\n)+)(~[a-z_][a-zA-Z0-9_]*)', + (r'(func)((?:[ \t]|\\\n)+)(~[a-z_]\w*)', bygroups(Keyword, Text, Name.Function)), (r'\bfunc\b', Keyword), # Note: %= and ^= not listed on http://ooc-lang.org/syntax @@ -2058,11 +2063,11 @@ class OocLexer(RegexLexer): (r'(\.)([ \t]*)([a-z]\w*)', bygroups(Operator, Text, Name.Function)), (r'[A-Z][A-Z0-9_]+', Name.Constant), - (r'[A-Z][a-zA-Z0-9_]*([@*]|\[[ \t]*\])?', Name.Class), + (r'[A-Z]\w*([@*]|\[[ \t]*\])?', Name.Class), - (r'([a-z][a-zA-Z0-9_]*(?:~[a-z][a-zA-Z0-9_]*)?)((?:[ \t]|\\\n)*)(?=\()', + (r'([a-z]\w*(?:~[a-z]\w*)?)((?:[ \t]|\\\n)*)(?=\()', bygroups(Name.Function, Text)), - (r'[a-z][a-zA-Z0-9_]*', Name.Variable), + (r'[a-z]\w*', Name.Variable), # : introduces types (r'[:(){}\[\];,]', Punctuation), @@ -2444,7 +2449,7 @@ class AdaLexer(RegexLexer): (r'task|protected', Keyword.Declaration), (r'(subtype)(\s+)', bygroups(Keyword.Declaration, Text)), (r'(end)(\s+)', bygroups(Keyword.Reserved, Text), 'end'), - (r'(pragma)(\s+)([a-zA-Z0-9_]+)', bygroups(Keyword.Reserved, Text, + (r'(pragma)(\s+)(\w+)', bygroups(Keyword.Reserved, Text, Comment.Preproc)), (r'(true|false|null)\b', Keyword.Constant), (r'(Address|Byte|Boolean|Character|Controlled|Count|Cursor|' @@ -2487,7 +2492,7 @@ class AdaLexer(RegexLexer): (r'[0-9_]+', Number.Integer), ], 'attribute' : [ - (r"(')([a-zA-Z0-9_]+)", bygroups(Punctuation, Name.Attribute)), + (r"(')(\w+)", bygroups(Punctuation, Name.Attribute)), ], 'subprogram' : [ (r'\(', Punctuation, ('#pop', 'formal_part')), @@ -2498,7 +2503,7 @@ class AdaLexer(RegexLexer): ], 'end' : [ ('(if|case|record|loop|select)', Keyword.Reserved), - ('"[^"]+"|[a-zA-Z0-9_.]+', Name.Function), + ('"[^"]+"|[\w.]+', Name.Function), ('\s+', Text), (';', Punctuation, '#pop'), ], @@ -2538,7 +2543,7 @@ class AdaLexer(RegexLexer): ('is', Keyword.Reserved, '#pop'), (';', Punctuation, '#pop'), ('\(', Punctuation, 'package_instantiation'), - ('([a-zA-Z0-9_.]+)', Name.Class), + ('([\w.]+)', Name.Class), include('root'), ], 'package_instantiation': [ @@ -2581,7 +2586,7 @@ class Modula2Lexer(RegexLexer): (r'\s+', Text), # whitespace ], 'identifiers': [ - (r'([a-zA-Z_\$][a-zA-Z0-9_\$]*)', Name), + (r'([a-zA-Z_\$][\w\$]*)', Name), ], 'numliterals': [ (r'[01]+B', Number.Binary), # binary number (ObjM2) @@ -2759,9 +2764,9 @@ class BlitzMaxLexer(RegexLexer): bmax_vopwords = r'\b(Shl|Shr|Sar|Mod)\b' bmax_sktypes = r'@{1,2}|[!#$%]' bmax_lktypes = r'\b(Int|Byte|Short|Float|Double|Long)\b' - bmax_name = r'[a-z_][a-z0-9_]*' + bmax_name = r'[a-z_]\w*' bmax_var = (r'(%s)(?:(?:([ \t]*)(%s)|([ \t]*:[ \t]*\b(?:Shl|Shr|Sar|Mod)\b)' - r'|([ \t]*)([:])([ \t]*)(?:%s|(%s)))(?:([ \t]*)(Ptr))?)') % \ + r'|([ \t]*)(:)([ \t]*)(?:%s|(%s)))(?:([ \t]*)(Ptr))?)') % \ (bmax_name, bmax_sktypes, bmax_lktypes, bmax_name) bmax_func = bmax_var + r'?((?:[ \t]|\.\.\n)*)([(])' @@ -2854,7 +2859,7 @@ class BlitzBasicLexer(RegexLexer): r'Abs|Sgn|Handle|Int|Float|Str|' r'First|Last|Before|After)\b') bb_sktypes = r'@{1,2}|[#$%]' - bb_name = r'[a-z][a-z0-9_]*' + bb_name = r'[a-z]\w*' bb_var = (r'(%s)(?:([ \t]*)(%s)|([ \t]*)([.])([ \t]*)(?:(%s)))?') % \ (bb_name, bb_sktypes, bb_name) @@ -2999,7 +3004,7 @@ class NimrodLexer(RegexLexer): # Numbers (r'[0-9][0-9_]*(?=([eE.]|\'[fF](32|64)))', Number.Float, ('float-suffix', 'float-number')), - (r'0[xX][a-fA-F0-9][a-fA-F0-9_]*', Number.Hex, 'int-suffix'), + (r'0[xX][a-f0-9][a-f0-9_]*', Number.Hex, 'int-suffix'), (r'0[bB][01][01_]*', Number, 'int-suffix'), (r'0o[0-7][0-7_]*', Number.Oct, 'int-suffix'), (r'[0-9][0-9_]*', Number.Integer, 'int-suffix'), @@ -3008,7 +3013,7 @@ class NimrodLexer(RegexLexer): (r'.+$', Error), ], 'chars': [ - (r'\\([\\abcefnrtvl"\']|x[a-fA-F0-9]{2}|[0-9]{1,3})', String.Escape), + (r'\\([\\abcefnrtvl"\']|x[a-f0-9]{2}|[0-9]{1,3})', String.Escape), (r"'", String.Char, '#pop'), (r".", String.Char) ], @@ -3022,7 +3027,7 @@ class NimrodLexer(RegexLexer): # newlines are an error (use "nl" state) ], 'dqs': [ - (r'\\([\\abcefnrtvl"\']|\n|x[a-fA-F0-9]{2}|[0-9]{1,3})', + (r'\\([\\abcefnrtvl"\']|\n|x[a-f0-9]{2}|[0-9]{1,3})', String.Escape), (r'"', String, '#pop'), include('strings') @@ -3078,7 +3083,7 @@ class FantomLexer(RegexLexer): dict ( pod = r'[\"\w\.]+', eos = r'\n|;', - id = r'[a-zA-Z_][a-zA-Z0-9_]*', + id = r'[a-zA-Z_]\w*', # all chars which can be part of type definition. Starts with # either letter, or [ (maps), or | (funcs) type = r'(?:\[|[a-zA-Z_]|\|)[:\w\[\]\|\->\?]*?', @@ -3363,7 +3368,7 @@ class RustLexer(RegexLexer): r"""|\\u[0-9a-fA-F]{4}|\\U[0-9a-fA-F]{8}|.)'""", String.Char), # Lifetime - (r"""'[a-zA-Z_][a-zA-Z0-9_]*""", Name.Label), + (r"""'[a-zA-Z_]\w*""", Name.Label), # Binary Literal (r'0b[01_]+', Number, 'number_lit'), # Octal Literal @@ -3491,10 +3496,10 @@ class MonkeyLexer(RegexLexer): filenames = ['*.monkey'] mimetypes = ['text/x-monkey'] - name_variable = r'[a-z_][a-zA-Z0-9_]*' - name_function = r'[A-Z][a-zA-Z0-9_]*' + name_variable = r'[a-z_]\w*' + name_function = r'[A-Z]\w*' name_constant = r'[A-Z_][A-Z0-9_]*' - name_class = r'[A-Z][a-zA-Z0-9_]*' + name_class = r'[A-Z]\w*' name_module = r'[a-z0-9_]*' keyword_type = r'(?:Int|Float|String|Bool|Object|Array|Void)' @@ -3836,12 +3841,12 @@ class LogosLexer(ObjectiveCppLexer): tokens = { 'statements': [ (r'(%orig|%log)\b', Keyword), - (r'(%c)\b(\()(\s*)([a-zA-Z$_][a-zA-Z0-9$_]*)(\s*)(\))', + (r'(%c)\b(\()(\s*)([a-zA-Z$_][\w$]*)(\s*)(\))', bygroups(Keyword, Punctuation, Text, Name.Class, Text, Punctuation)), (r'(%init)\b(\()', bygroups(Keyword, Punctuation), 'logos_init_directive'), (r'(%init)(?=\s*;)', bygroups(Keyword)), - (r'(%hook|%group)(\s+)([a-zA-Z$_][a-zA-Z0-9$_]+)', + (r'(%hook|%group)(\s+)([a-zA-Z$_][\w$]+)', bygroups(Keyword, Text, Name.Class), '#pop'), (r'(%subclass)(\s+)', bygroups(Keyword, Text), ('#pop', 'logos_classname')), @@ -3850,20 +3855,20 @@ class LogosLexer(ObjectiveCppLexer): 'logos_init_directive' : [ ('\s+', Text), (',', Punctuation, ('logos_init_directive', '#pop')), - ('([a-zA-Z$_][a-zA-Z0-9$_]*)(\s*)(=)(\s*)([^);]*)', + ('([a-zA-Z$_][\w$]*)(\s*)(=)(\s*)([^);]*)', bygroups(Name.Class, Text, Punctuation, Text, Text)), - ('([a-zA-Z$_][a-zA-Z0-9$_]*)', Name.Class), + ('([a-zA-Z$_][\w$]*)', Name.Class), ('\)', Punctuation, '#pop'), ], 'logos_classname' : [ - ('([a-zA-Z$_][a-zA-Z0-9$_]*)(\s*:\s*)([a-zA-Z$_][a-zA-Z0-9$_]*)?', + ('([a-zA-Z$_][\w$]*)(\s*:\s*)([a-zA-Z$_][\w$]*)?', bygroups(Name.Class, Text, Name.Class), '#pop'), - ('([a-zA-Z$_][a-zA-Z0-9$_]*)', Name.Class, '#pop') + ('([a-zA-Z$_][\w$]*)', Name.Class, '#pop') ], 'root': [ (r'(%subclass)(\s+)', bygroups(Keyword, Text), 'logos_classname'), - (r'(%hook|%group)(\s+)([a-zA-Z$_][a-zA-Z0-9$_]+)', + (r'(%hook|%group)(\s+)([a-zA-Z$_][\w$]+)', bygroups(Keyword, Text, Name.Class)), (r'(%config)(\s*\(\s*)(\w+)(\s*=\s*)(.*?)(\s*\)\s*)', bygroups(Keyword, Text, Name.Variable, Text, String, Text)), @@ -3950,13 +3955,13 @@ class ChapelLexer(RegexLexer): (r'[:;,.?()\[\]{}]', Punctuation), # identifiers - (r'[a-zA-Z_][a-zA-Z0-9_$]*', Name.Other), + (r'[a-zA-Z_][\w$]*', Name.Other), ], 'classname': [ - (r'[a-zA-Z_][a-zA-Z0-9_$]*', Name.Class, '#pop'), + (r'[a-zA-Z_][\w$]*', Name.Class, '#pop'), ], 'procname': [ - (r'[a-zA-Z_][a-zA-Z0-9_$]*', Name.Function, '#pop'), + (r'[a-zA-Z_][\w$]*', Name.Function, '#pop'), ], } @@ -3992,7 +3997,7 @@ class EiffelLexer(RegexLexer): (r"'([^'%]|%'|%%)'", String.Char), (r"(//|\\\\|>=|<=|:=|/=|~|/~|[\\\?!#%&@|+/\-=\>\*$<|^\[\]])", Operator), (r"([{}():;,.])", Punctuation), - (r'([a-z][a-zA-Z0-9_]*)|([A-Z][A-Z0-9_]*[a-z][a-zA-Z0-9_]*)', Name), + (r'([a-z]\w*)|([A-Z][A-Z0-9_]*[a-z]\w*)', Name), (r'([A-Z][A-Z0-9_]*)', Name.Class), (r'\n+', Text), ], diff --git a/pygments/lexers/dalvik.py b/pygments/lexers/dalvik.py index 901b7c5a..ea09cecb 100644 --- a/pygments/lexers/dalvik.py +++ b/pygments/lexers/dalvik.py @@ -9,6 +9,8 @@ :license: BSD, see LICENSE for details. """ +import re + from pygments.lexer import RegexLexer, include, bygroups from pygments.token import Keyword, Text, Comment, Name, String, Number, \ Punctuation @@ -73,22 +75,22 @@ class SmaliLexer(RegexLexer): (r'[0-9]+L?', Number.Integer), ], 'field': [ - (r'(\$?\b)([A-Za-z0-9_$]*)(:)', + (r'(\$?\b)([\w$]*)(:)', bygroups(Punctuation, Name.Variable, Punctuation)), ], 'method': [ (r'<(?:cl)?init>', Name.Function), # constructor - (r'(\$?\b)([A-Za-z0-9_$]*)(\()', + (r'(\$?\b)([\w$]*)(\()', bygroups(Punctuation, Name.Function, Punctuation)), ], 'label': [ - (r':[A-Za-z0-9_]+', Name.Label), + (r':\w+', Name.Label), ], 'class': [ # class names in the form Lcom/namespace/ClassName; # I only want to color the ClassName part, so the namespace part is # treated as 'Text' - (r'(L)((?:[A-Za-z0-9_$]+/)*)([A-Za-z0-9_$]+)(;)', + (r'(L)((?:[\w$]+/)*)([\w$]+)(;)', bygroups(Keyword.Type, Text, Name.Class, Text)), ], 'punctuation': [ @@ -102,3 +104,22 @@ class SmaliLexer(RegexLexer): (r'#.*?\n', Comment), ], } + + def analyse_text(text): + score = 0 + if re.search(r'^\s*\.class\s', text, re.MULTILINE): + score += 0.5 + if re.search(r'\b((check-cast|instance-of|throw-verification-error' + r')\b|(-to|add|[ais]get|[ais]put|and|cmpl|const|div|' + r'if|invoke|move|mul|neg|not|or|rem|return|rsub|shl|' + r'shr|sub|ushr)[-/])|{|}', text, re.MULTILINE): + score += 0.3 + if re.search(r'(\.(catchall|epilogue|restart local|prologue)|' + r'\b(array-data|class-change-error|declared-synchronized|' + r'(field|inline|vtable)@0x[0-9a-fA-F]|generic-error|' + r'illegal-class-access|illegal-field-access|' + r'illegal-method-access|instantiation-error|no-error|' + r'no-such-class|no-such-field|no-such-method|' + r'packed-switch|sparse-switch))\b', text, re.MULTILINE): + score += 0.6 + return score diff --git a/pygments/lexers/dotnet.py b/pygments/lexers/dotnet.py index a281e6ab..5a07cd0d 100644 --- a/pygments/lexers/dotnet.py +++ b/pygments/lexers/dotnet.py @@ -58,7 +58,7 @@ class CSharpLexer(RegexLexer): # see http://www.ecma-international.org/publications/files/ECMA-ST/Ecma-334.pdf levels = { - 'none': '@?[_a-zA-Z][a-zA-Z0-9_]*', + 'none': '@?[_a-zA-Z]\w*', 'basic': ('@?[_' + uni.Lu + uni.Ll + uni.Lt + uni.Lm + uni.Nl + ']' + '[' + uni.Lu + uni.Ll + uni.Lt + uni.Lm + uni.Nl + uni.Nd + uni.Pc + uni.Cf + uni.Mn + uni.Mc + ']*'), @@ -170,7 +170,7 @@ class NemerleLexer(RegexLexer): # http://www.ecma-international.org/publications/files/ECMA-ST/Ecma-334.pdf levels = dict( - none = '@?[_a-zA-Z][a-zA-Z0-9_]*', + none = '@?[_a-zA-Z]\w*', basic = ('@?[_' + uni.Lu + uni.Ll + uni.Lt + uni.Lm + uni.Nl + ']' + '[' + uni.Lu + uni.Ll + uni.Lt + uni.Lm + uni.Nl + uni.Nd + uni.Pc + uni.Cf + uni.Mn + uni.Mc + ']*'), @@ -336,7 +336,7 @@ class BooLexer(RegexLexer): (r'"""(\\\\|\\"|.*?)"""', String.Double), (r'"(\\\\|\\"|[^"]*?)"', String.Double), (r"'(\\\\|\\'|[^']*?)'", String.Single), - (r'[a-zA-Z_][a-zA-Z0-9_]*', Name), + (r'[a-zA-Z_]\w*', Name), (r'(\d+\.\d*|\d*\.\d+)([fF][+-]?[0-9]+)?', Number.Float), (r'[0-9][0-9\.]*(ms?|d|h|s)', Number), (r'0\d+', Number.Oct), @@ -351,13 +351,13 @@ class BooLexer(RegexLexer): ('[*/]', Comment.Multiline) ], 'funcname': [ - ('[a-zA-Z_][a-zA-Z0-9_]*', Name.Function, '#pop') + ('[a-zA-Z_]\w*', Name.Function, '#pop') ], 'classname': [ - ('[a-zA-Z_][a-zA-Z0-9_]*', Name.Class, '#pop') + ('[a-zA-Z_]\w*', Name.Class, '#pop') ], 'namespace': [ - ('[a-zA-Z_][a-zA-Z0-9_.]*', Name.Namespace, '#pop') + ('[a-zA-Z_][\w.]*', Name.Namespace, '#pop') ] } @@ -425,7 +425,7 @@ class VbNetLexer(RegexLexer): r'<=|>=|<>|[-&*/\\^+=<>]', Operator), ('"', String, 'string'), - ('[a-zA-Z_][a-zA-Z0-9_]*[%&@!#$]?', Name), + ('[a-z_]\w*[%&@!#$]?', Name), ('#.*?#', Literal.Date), (r'(\d+\.\d*|\d*\.\d+)([fF][+-]?[0-9]+)?', Number.Float), (r'\d+([SILDFR]|US|UI|UL)?', Number.Integer), @@ -439,17 +439,17 @@ class VbNetLexer(RegexLexer): (r'[^"]+', String), ], 'dim': [ - (r'[a-z_][a-z0-9_]*', Name.Variable, '#pop'), + (r'[a-z_]\w*', Name.Variable, '#pop'), (r'', Text, '#pop'), # any other syntax ], 'funcname': [ - (r'[a-z_][a-z0-9_]*', Name.Function, '#pop'), + (r'[a-z_]\w*', Name.Function, '#pop'), ], 'classname': [ - (r'[a-z_][a-z0-9_]*', Name.Class, '#pop'), + (r'[a-z_]\w*', Name.Class, '#pop'), ], 'namespace': [ - (r'[a-z_][a-z0-9_.]*', Name.Namespace, '#pop'), + (r'[a-z_][\w.]*', Name.Namespace, '#pop'), ], 'end': [ (r'\s+', Text), @@ -460,7 +460,8 @@ class VbNetLexer(RegexLexer): } def analyse_text(text): - if re.search(r'^\s*#If', text, re.I) or re.search(r'^\s*(Module|Namespace)', re.I): + if (re.search(r'^[ \t]*#If', text, re.I) + or re.search(r'^[ \t]*(Module|Namespace)', re.I)): return 0.5 @@ -594,9 +595,9 @@ class FSharpLexer(RegexLexer): 'root': [ (r'\s+', Text), (r'\(\)|\[\]', Name.Builtin.Pseudo), - (r'\b(?<!\.)([A-Z][A-Za-z0-9_\']*)(?=\s*\.)', + (r'\b(?<!\.)([A-Z][\w\']*)(?=\s*\.)', Name.Namespace, 'dotted'), - (r'\b([A-Z][A-Za-z0-9_\']*)', Name), + (r'\b([A-Z][\w\']*)', Name), (r'///.*?\n', String.Doc), (r'//.*?\n', Comment.Single), (r'\(\*(?!\))', Comment, 'comment'), @@ -605,13 +606,13 @@ class FSharpLexer(RegexLexer): (r'"""', String, 'tqs'), (r'"', String, 'string'), - (r'\b(open|module)(\s+)([a-zA-Z0-9_.]+)', + (r'\b(open|module)(\s+)([\w.]+)', bygroups(Keyword, Text, Name.Namespace)), - (r'\b(let!?)(\s+)([a-zA-Z0-9_]+)', + (r'\b(let!?)(\s+)(\w+)', bygroups(Keyword, Text, Name.Variable)), - (r'\b(type)(\s+)([a-zA-Z0-9_]+)', + (r'\b(type)(\s+)(\w+)', bygroups(Keyword, Text, Name.Class)), - (r'\b(member|override)(\s+)([a-zA-Z0-9_]+)(\.)([a-zA-Z0-9_]+)', + (r'\b(member|override)(\s+)(\w+)(\.)(\w+)', bygroups(Keyword, Text, Name, Punctuation, Name.Function)), (r'\b(%s)\b' % '|'.join(keywords), Keyword), (r'(%s)' % '|'.join(keyopts), Operator), @@ -640,9 +641,9 @@ class FSharpLexer(RegexLexer): 'dotted': [ (r'\s+', Text), (r'\.', Punctuation), - (r'[A-Z][A-Za-z0-9_\']*(?=\s*\.)', Name.Namespace), - (r'[A-Z][A-Za-z0-9_\']*', Name, '#pop'), - (r'[a-z_][A-Za-z0-9_\']*', Name, '#pop'), + (r'[A-Z][\w\']*(?=\s*\.)', Name.Namespace), + (r'[A-Z][\w\']*', Name, '#pop'), + (r'[a-z_][\w\']*', Name, '#pop'), # e.g. dictionary index access (r'', Text, '#pop'), ], diff --git a/pygments/lexers/functional.py b/pygments/lexers/functional.py index af9918a0..0c978254 100644 --- a/pygments/lexers/functional.py +++ b/pygments/lexers/functional.py @@ -14,12 +14,15 @@ import re from pygments.lexer import Lexer, RegexLexer, bygroups, include, do_insertions from pygments.token import Text, Comment, Operator, Keyword, Name, \ String, Number, Punctuation, Literal, Generic, Error +from pygments import unistring as uni -__all__ = ['RacketLexer', 'SchemeLexer', 'CommonLispLexer', 'HaskellLexer', - 'AgdaLexer', 'LiterateHaskellLexer', 'LiterateAgdaLexer', - 'SMLLexer', 'OcamlLexer', 'ErlangLexer', 'ErlangShellLexer', - 'OpaLexer', 'CoqLexer', 'NewLispLexer', 'NixLexer', 'ElixirLexer', - 'ElixirConsoleLexer', 'KokaLexer', 'IdrisLexer', 'LiterateIdrisLexer'] +__all__ = ['RacketLexer', 'SchemeLexer', 'CommonLispLexer', 'CryptolLexer', + 'HaskellLexer', 'AgdaLexer', 'LiterateCryptolLexer', + 'LiterateHaskellLexer', 'LiterateAgdaLexer', 'SMLLexer', + 'OcamlLexer', 'ErlangLexer', 'ErlangShellLexer', 'OpaLexer', + 'CoqLexer', 'NewLispLexer', 'NixLexer', 'ElixirLexer', + 'ElixirConsoleLexer', 'KokaLexer', 'IdrisLexer', + 'LiterateIdrisLexer'] line_re = re.compile('.*?\n') @@ -27,126 +30,219 @@ line_re = re.compile('.*?\n') class RacketLexer(RegexLexer): """ - Lexer for `Racket <http://racket-lang.org/>`_ source code (formerly known as - PLT Scheme). + Lexer for `Racket <http://racket-lang.org/>`_ source code (formerly + known as PLT Scheme). .. versionadded:: 1.6 """ name = 'Racket' aliases = ['racket', 'rkt'] - filenames = ['*.rkt', '*.rktl'] + filenames = ['*.rkt', '*.rktd', '*.rktl'] mimetypes = ['text/x-racket', 'application/x-racket'] - # From namespace-mapped-symbols - keywords = [ - '#%app', '#%datum', '#%expression', '#%module-begin', + # Generated by example.rkt + _keywords = [ + '#%app', '#%datum', '#%declare', '#%expression', '#%module-begin', '#%plain-app', '#%plain-lambda', '#%plain-module-begin', - '#%provide', '#%require', '#%stratified-body', '#%top', - '#%top-interaction', '#%variable-reference', '...', 'and', 'begin', - 'begin-for-syntax', 'begin0', 'case', 'case-lambda', 'cond', - 'datum->syntax-object', 'define', 'define-for-syntax', - 'define-struct', 'define-syntax', 'define-syntax-rule', - 'define-syntaxes', 'define-values', 'define-values-for-syntax', - 'delay', 'do', 'expand-path', 'fluid-let', 'hash-table-copy', - 'hash-table-count', 'hash-table-for-each', 'hash-table-get', - 'hash-table-iterate-first', 'hash-table-iterate-key', - 'hash-table-iterate-next', 'hash-table-iterate-value', - 'hash-table-map', 'hash-table-put!', 'hash-table-remove!', - 'hash-table?', 'if', 'lambda', 'let', 'let*', 'let*-values', - 'let-struct', 'let-syntax', 'let-syntaxes', 'let-values', 'let/cc', - 'let/ec', 'letrec', 'letrec-syntax', 'letrec-syntaxes', - 'letrec-syntaxes+values', 'letrec-values', 'list-immutable', - 'make-hash-table', 'make-immutable-hash-table', 'make-namespace', - 'module', 'module-identifier=?', 'module-label-identifier=?', - 'module-template-identifier=?', 'module-transformer-identifier=?', - 'namespace-transformer-require', 'or', 'parameterize', - 'parameterize*', 'parameterize-break', 'provide', - 'provide-for-label', 'provide-for-syntax', 'quasiquote', + '#%printing-module-begin', '#%provide', '#%require', + '#%stratified-body', '#%top', '#%top-interaction', + '#%variable-reference', '->', '->*', '->*m', '->d', '->dm', '->i', + '->m', '...', ':do-in', '==', '=>', '_', 'absent', 'abstract', + 'all-defined-out', 'all-from-out', 'and', 'any', 'augment', 'augment*', + 'augment-final', 'augment-final*', 'augride', 'augride*', 'begin', + 'begin-for-syntax', 'begin0', 'case', 'case->', 'case->m', + 'case-lambda', 'class', 'class*', 'class-field-accessor', + 'class-field-mutator', 'class/c', 'class/derived', 'combine-in', + 'combine-out', 'command-line', 'compound-unit', 'compound-unit/infer', + 'cond', 'contract', 'contract-out', 'contract-struct', 'contracted', + 'define', 'define-compound-unit', 'define-compound-unit/infer', + 'define-contract-struct', 'define-custom-hash-types', + 'define-custom-set-types', 'define-for-syntax', + 'define-local-member-name', 'define-logger', 'define-match-expander', + 'define-member-name', 'define-module-boundary-contract', + 'define-namespace-anchor', 'define-opt/c', 'define-sequence-syntax', + 'define-serializable-class', 'define-serializable-class*', + 'define-signature', 'define-signature-form', 'define-struct', + 'define-struct/contract', 'define-struct/derived', 'define-syntax', + 'define-syntax-rule', 'define-syntaxes', 'define-unit', + 'define-unit-binding', 'define-unit-from-context', + 'define-unit/contract', 'define-unit/new-import-export', + 'define-unit/s', 'define-values', 'define-values-for-export', + 'define-values-for-syntax', 'define-values/invoke-unit', + 'define-values/invoke-unit/infer', 'define/augment', + 'define/augment-final', 'define/augride', 'define/contract', + 'define/final-prop', 'define/match', 'define/overment', + 'define/override', 'define/override-final', 'define/private', + 'define/public', 'define/public-final', 'define/pubment', + 'define/subexpression-pos-prop', 'delay', 'delay/idle', 'delay/name', + 'delay/strict', 'delay/sync', 'delay/thread', 'do', 'else', 'except', + 'except-in', 'except-out', 'export', 'extends', 'failure-cont', + 'false', 'false/c', 'field', 'field-bound?', 'file', + 'flat-murec-contract', 'flat-rec-contract', 'for', 'for*', 'for*/and', + 'for*/first', 'for*/fold', 'for*/fold/derived', 'for*/hash', + 'for*/hasheq', 'for*/hasheqv', 'for*/last', 'for*/list', 'for*/lists', + 'for*/mutable-set', 'for*/mutable-seteq', 'for*/mutable-seteqv', + 'for*/or', 'for*/product', 'for*/set', 'for*/seteq', 'for*/seteqv', + 'for*/sum', 'for*/vector', 'for*/weak-set', 'for*/weak-seteq', + 'for*/weak-seteqv', 'for-label', 'for-meta', 'for-syntax', + 'for-template', 'for/and', 'for/first', 'for/fold', 'for/fold/derived', + 'for/hash', 'for/hasheq', 'for/hasheqv', 'for/last', 'for/list', + 'for/lists', 'for/mutable-set', 'for/mutable-seteq', + 'for/mutable-seteqv', 'for/or', 'for/product', 'for/set', 'for/seteq', + 'for/seteqv', 'for/sum', 'for/vector', 'for/weak-set', + 'for/weak-seteq', 'for/weak-seteqv', 'gen:custom-write', 'gen:dict', + 'gen:equal+hash', 'gen:set', 'gen:stream', 'generic', 'get-field', + 'if', 'implies', 'import', 'include', 'include-at/relative-to', + 'include-at/relative-to/reader', 'include/reader', 'inherit', + 'inherit-field', 'inherit/inner', 'inherit/super', 'init', + 'init-depend', 'init-field', 'init-rest', 'inner', 'inspect', + 'instantiate', 'interface', 'interface*', 'invoke-unit', + 'invoke-unit/infer', 'lambda', 'lazy', 'let', 'let*', 'let*-values', + 'let-syntax', 'let-syntaxes', 'let-values', 'let/cc', 'let/ec', + 'letrec', 'letrec-syntax', 'letrec-syntaxes', 'letrec-syntaxes+values', + 'letrec-values', 'lib', 'link', 'local', 'local-require', 'log-debug', + 'log-error', 'log-fatal', 'log-info', 'log-warning', 'match', 'match*', + 'match*/derived', 'match-define', 'match-define-values', + 'match-lambda', 'match-lambda*', 'match-lambda**', 'match-let', + 'match-let*', 'match-let*-values', 'match-let-values', 'match-letrec', + 'match/derived', 'match/values', 'member-name-key', 'method-contract?', + 'mixin', 'module', 'module*', 'module+', 'nand', 'new', 'nor', + 'object-contract', 'object/c', 'only', 'only-in', 'only-meta-in', + 'open', 'opt/c', 'or', 'overment', 'overment*', 'override', + 'override*', 'override-final', 'override-final*', 'parameterize', + 'parameterize*', 'parameterize-break', 'parametric->/c', 'place', + 'place*', 'planet', 'prefix', 'prefix-in', 'prefix-out', 'private', + 'private*', 'prompt-tag/c', 'protect-out', 'provide', + 'provide-signature-elements', 'provide/contract', 'public', 'public*', + 'public-final', 'public-final*', 'pubment', 'pubment*', 'quasiquote', 'quasisyntax', 'quasisyntax/loc', 'quote', 'quote-syntax', - 'quote-syntax/prune', 'require', 'require-for-label', - 'require-for-syntax', 'require-for-template', 'set!', - 'set!-values', 'syntax', 'syntax-case', 'syntax-case*', - 'syntax-id-rules', 'syntax-object->datum', 'syntax-rules', - 'syntax/loc', 'time', 'transcript-off', 'transcript-on', 'unless', - 'unquote', 'unquote-splicing', 'unsyntax', 'unsyntax-splicing', - 'when', 'with-continuation-mark', 'with-handlers', - 'with-handlers*', 'with-syntax', u'λ' + 'quote-syntax/prune', 'recontract-out', 'recursive-contract', + 'relative-in', 'rename', 'rename-in', 'rename-inner', 'rename-out', + 'rename-super', 'require', 'send', 'send*', 'send+', 'send-generic', + 'send/apply', 'send/keyword-apply', 'set!', 'set!-values', + 'set-field!', 'shared', 'stream', 'stream-cons', 'struct', 'struct*', + 'struct-copy', 'struct-field-index', 'struct-out', 'struct/c', + 'struct/ctc', 'struct/dc', 'submod', 'super', 'super-instantiate', + 'super-make-object', 'super-new', 'syntax', 'syntax-case', + 'syntax-case*', 'syntax-id-rules', 'syntax-rules', 'syntax/loc', 'tag', + 'this', 'this%', 'thunk', 'thunk*', 'time', 'unconstrained-domain->', + 'unit', 'unit-from-context', 'unit/c', 'unit/new-import-export', + 'unit/s', 'unless', 'unquote', 'unquote-splicing', 'unsyntax', + 'unsyntax-splicing', 'values/drop', 'when', 'with-continuation-mark', + 'with-contract', 'with-handlers', 'with-handlers*', 'with-method', + 'with-syntax', u'λ' ] - # From namespace-mapped-symbols - builtins = [ - '*', '+', '-', '/', '<', '<=', '=', '>', '>=', - 'abort-current-continuation', 'abs', 'absolute-path?', 'acos', - 'add1', 'alarm-evt', 'always-evt', 'andmap', 'angle', 'append', - 'apply', 'arithmetic-shift', 'arity-at-least', - 'arity-at-least-value', 'arity-at-least?', 'asin', 'assoc', 'assq', - 'assv', 'atan', 'banner', 'bitwise-and', 'bitwise-bit-field', - 'bitwise-bit-set?', 'bitwise-ior', 'bitwise-not', 'bitwise-xor', - 'boolean?', 'bound-identifier=?', 'box', 'box-immutable', 'box?', - 'break-enabled', 'break-thread', 'build-path', - 'build-path/convention-type', 'byte-pregexp', 'byte-pregexp?', + # Generated by example.rkt + _builtins = [ + '*', '+', '-', '/', '<', '</c', '<=', '<=/c', '=', '=/c', '>', '>/c', + '>=', '>=/c', 'abort-current-continuation', 'abs', 'absolute-path?', + 'acos', 'add-between', 'add1', 'alarm-evt', 'always-evt', 'and/c', + 'andmap', 'angle', 'any/c', 'append', 'append*', 'append-map', 'apply', + 'argmax', 'argmin', 'arithmetic-shift', 'arity-at-least', + 'arity-at-least-value', 'arity-at-least?', 'arity-checking-wrapper', + 'arity-includes?', 'arity=?', 'asin', 'assf', 'assoc', 'assq', 'assv', + 'atan', 'bad-number-of-results', 'banner', 'base->-doms/c', + 'base->-rngs/c', 'base->?', 'between/c', 'bitwise-and', + 'bitwise-bit-field', 'bitwise-bit-set?', 'bitwise-ior', 'bitwise-not', + 'bitwise-xor', 'blame-add-car-context', 'blame-add-cdr-context', + 'blame-add-context', 'blame-add-missing-party', + 'blame-add-nth-arg-context', 'blame-add-or-context', + 'blame-add-range-context', 'blame-add-unknown-context', + 'blame-context', 'blame-contract', 'blame-fmt->-string', + 'blame-negative', 'blame-original?', 'blame-positive', + 'blame-replace-negative', 'blame-source', 'blame-swap', + 'blame-swapped?', 'blame-update', 'blame-value', 'blame?', 'boolean=?', + 'boolean?', 'bound-identifier=?', 'box', 'box-cas!', 'box-immutable', + 'box-immutable/c', 'box/c', 'box?', 'break-enabled', 'break-thread', + 'build-chaperone-contract-property', 'build-compound-type-name', + 'build-contract-property', 'build-flat-contract-property', + 'build-list', 'build-path', 'build-path/convention-type', + 'build-string', 'build-vector', 'byte-pregexp', 'byte-pregexp?', 'byte-ready?', 'byte-regexp', 'byte-regexp?', 'byte?', 'bytes', 'bytes->immutable-bytes', 'bytes->list', 'bytes->path', - 'bytes->path-element', 'bytes->string/latin-1', - 'bytes->string/locale', 'bytes->string/utf-8', 'bytes-append', + 'bytes->path-element', 'bytes->string/latin-1', 'bytes->string/locale', + 'bytes->string/utf-8', 'bytes-append', 'bytes-append*', 'bytes-close-converter', 'bytes-convert', 'bytes-convert-end', - 'bytes-converter?', 'bytes-copy', 'bytes-copy!', 'bytes-fill!', - 'bytes-length', 'bytes-open-converter', 'bytes-ref', 'bytes-set!', - 'bytes-utf-8-index', 'bytes-utf-8-length', 'bytes-utf-8-ref', - 'bytes<?', 'bytes=?', 'bytes>?', 'bytes?', 'caaaar', 'caaadr', - 'caaar', 'caadar', 'caaddr', 'caadr', 'caar', 'cadaar', 'cadadr', - 'cadar', 'caddar', 'cadddr', 'caddr', 'cadr', - 'call-in-nested-thread', 'call-with-break-parameterization', - 'call-with-composable-continuation', - 'call-with-continuation-barrier', 'call-with-continuation-prompt', - 'call-with-current-continuation', 'call-with-escape-continuation', - 'call-with-exception-handler', - 'call-with-immediate-continuation-mark', 'call-with-input-file', - 'call-with-output-file', 'call-with-parameterization', - 'call-with-semaphore', 'call-with-semaphore/enable-break', - 'call-with-values', 'call/cc', 'call/ec', 'car', 'cdaaar', - 'cdaadr', 'cdaar', 'cdadar', 'cdaddr', 'cdadr', 'cdar', 'cddaar', - 'cddadr', 'cddar', 'cdddar', 'cddddr', 'cdddr', 'cddr', 'cdr', - 'ceiling', 'channel-get', 'channel-put', 'channel-put-evt', - 'channel-try-get', 'channel?', 'chaperone-box', 'chaperone-evt', - 'chaperone-hash', 'chaperone-of?', 'chaperone-procedure', + 'bytes-converter?', 'bytes-copy', 'bytes-copy!', + 'bytes-environment-variable-name?', 'bytes-fill!', 'bytes-join', + 'bytes-length', 'bytes-no-nuls?', 'bytes-open-converter', 'bytes-ref', + 'bytes-set!', 'bytes-utf-8-index', 'bytes-utf-8-length', + 'bytes-utf-8-ref', 'bytes<?', 'bytes=?', 'bytes>?', 'bytes?', 'caaaar', + 'caaadr', 'caaar', 'caadar', 'caaddr', 'caadr', 'caar', 'cadaar', + 'cadadr', 'cadar', 'caddar', 'cadddr', 'caddr', 'cadr', + 'call-in-nested-thread', 'call-with-atomic-output-file', + 'call-with-break-parameterization', + 'call-with-composable-continuation', 'call-with-continuation-barrier', + 'call-with-continuation-prompt', 'call-with-current-continuation', + 'call-with-default-reading-parameterization', + 'call-with-escape-continuation', 'call-with-exception-handler', + 'call-with-file-lock/timeout', 'call-with-immediate-continuation-mark', + 'call-with-input-bytes', 'call-with-input-file', + 'call-with-input-file*', 'call-with-input-string', + 'call-with-output-bytes', 'call-with-output-file', + 'call-with-output-file*', 'call-with-output-string', + 'call-with-parameterization', 'call-with-semaphore', + 'call-with-semaphore/enable-break', 'call-with-values', 'call/cc', + 'call/ec', 'car', 'cdaaar', 'cdaadr', 'cdaar', 'cdadar', 'cdaddr', + 'cdadr', 'cdar', 'cddaar', 'cddadr', 'cddar', 'cdddar', 'cddddr', + 'cdddr', 'cddr', 'cdr', 'ceiling', 'channel-get', 'channel-put', + 'channel-put-evt', 'channel-put-evt?', 'channel-try-get', 'channel/c', + 'channel?', 'chaperone-box', 'chaperone-channel', + 'chaperone-continuation-mark-key', 'chaperone-contract-property?', + 'chaperone-contract?', 'chaperone-evt', 'chaperone-hash', + 'chaperone-of?', 'chaperone-procedure', 'chaperone-prompt-tag', 'chaperone-struct', 'chaperone-struct-type', 'chaperone-vector', 'chaperone?', 'char->integer', 'char-alphabetic?', 'char-blank?', 'char-ci<=?', 'char-ci<?', 'char-ci=?', 'char-ci>=?', 'char-ci>?', 'char-downcase', 'char-foldcase', 'char-general-category', 'char-graphic?', 'char-iso-control?', 'char-lower-case?', - 'char-numeric?', 'char-punctuation?', 'char-ready?', - 'char-symbolic?', 'char-title-case?', 'char-titlecase', - 'char-upcase', 'char-upper-case?', 'char-utf-8-length', - 'char-whitespace?', 'char<=?', 'char<?', 'char=?', 'char>=?', - 'char>?', 'char?', 'check-duplicate-identifier', - 'checked-procedure-check-and-extract', 'choice-evt', + 'char-numeric?', 'char-punctuation?', 'char-ready?', 'char-symbolic?', + 'char-title-case?', 'char-titlecase', 'char-upcase', + 'char-upper-case?', 'char-utf-8-length', 'char-whitespace?', 'char<=?', + 'char<?', 'char=?', 'char>=?', 'char>?', 'char?', + 'check-duplicate-identifier', 'checked-procedure-check-and-extract', + 'choice-evt', 'class->interface', 'class-info', 'class?', 'cleanse-path', 'close-input-port', 'close-output-port', - 'collect-garbage', 'collection-file-path', 'collection-path', - 'compile', 'compile-allow-set!-undefined', - 'compile-context-preservation-enabled', + 'coerce-chaperone-contract', 'coerce-chaperone-contracts', + 'coerce-contract', 'coerce-contract/f', 'coerce-contracts', + 'coerce-flat-contract', 'coerce-flat-contracts', 'collect-garbage', + 'collection-file-path', 'collection-path', 'compile', + 'compile-allow-set!-undefined', 'compile-context-preservation-enabled', 'compile-enforce-module-constants', 'compile-syntax', 'compiled-expression?', 'compiled-module-expression?', - 'complete-path?', 'complex?', 'cons', - 'continuation-mark-set->context', 'continuation-mark-set->list', - 'continuation-mark-set->list*', 'continuation-mark-set-first', - 'continuation-mark-set?', 'continuation-marks', - 'continuation-prompt-available?', 'continuation-prompt-tag?', - 'continuation?', 'copy-file', 'cos', - 'current-break-parameterization', 'current-code-inspector', - 'current-command-line-arguments', 'current-compile', - 'current-continuation-marks', 'current-custodian', - 'current-directory', 'current-drive', 'current-error-port', + 'complete-path?', 'complex?', 'compose', 'compose1', 'conjugate', + 'cons', 'cons/c', 'cons?', 'const', 'continuation-mark-key/c', + 'continuation-mark-key?', 'continuation-mark-set->context', + 'continuation-mark-set->list', 'continuation-mark-set->list*', + 'continuation-mark-set-first', 'continuation-mark-set?', + 'continuation-marks', 'continuation-prompt-available?', + 'continuation-prompt-tag?', 'continuation?', + 'contract-continuation-mark-key', 'contract-first-order', + 'contract-first-order-passes?', 'contract-name', 'contract-proc', + 'contract-projection', 'contract-property?', + 'contract-random-generate', 'contract-stronger?', + 'contract-struct-exercise', 'contract-struct-generate', + 'contract-val-first-projection', 'contract?', 'convert-stream', + 'copy-directory/files', 'copy-file', 'copy-port', 'cos', 'cosh', + 'count', 'current-blame-format', 'current-break-parameterization', + 'current-code-inspector', 'current-command-line-arguments', + 'current-compile', 'current-compiled-file-roots', + 'current-continuation-marks', 'current-contract-region', + 'current-custodian', 'current-directory', 'current-directory-for-user', + 'current-drive', 'current-environment-variables', 'current-error-port', 'current-eval', 'current-evt-pseudo-random-generator', - 'current-gc-milliseconds', 'current-get-interaction-input-port', - 'current-inexact-milliseconds', 'current-input-port', - 'current-inspector', 'current-library-collection-paths', + 'current-future', 'current-gc-milliseconds', + 'current-get-interaction-input-port', 'current-inexact-milliseconds', + 'current-input-port', 'current-inspector', + 'current-library-collection-links', 'current-library-collection-paths', 'current-load', 'current-load-extension', 'current-load-relative-directory', 'current-load/use-compiled', - 'current-locale', 'current-memory-use', 'current-milliseconds', - 'current-module-declare-name', 'current-module-declare-source', - 'current-module-name-resolver', 'current-namespace', + 'current-locale', 'current-logger', 'current-memory-use', + 'current-milliseconds', 'current-module-declare-name', + 'current-module-declare-source', 'current-module-name-resolver', + 'current-module-path-for-load', 'current-namespace', 'current-output-port', 'current-parameterization', 'current-preserved-thread-cell-values', 'current-print', 'current-process-milliseconds', 'current-prompt-read', @@ -155,133 +251,229 @@ class RacketLexer(RegexLexer): 'current-security-guard', 'current-subprocess-custodian-mode', 'current-thread', 'current-thread-group', 'current-thread-initial-stack-size', - 'current-write-relative-directory', 'custodian-box-value', - 'custodian-box?', 'custodian-limit-memory', + 'current-write-relative-directory', 'curry', 'curryr', + 'custodian-box-value', 'custodian-box?', 'custodian-limit-memory', 'custodian-managed-list', 'custodian-memory-accounting-available?', 'custodian-require-memory', 'custodian-shutdown-all', 'custodian?', 'custom-print-quotable-accessor', 'custom-print-quotable?', - 'custom-write-accessor', 'custom-write?', 'date', 'date*', - 'date*-nanosecond', 'date*-time-zone-name', 'date*?', 'date-day', - 'date-dst?', 'date-hour', 'date-minute', 'date-month', - 'date-second', 'date-time-zone-offset', 'date-week-day', - 'date-year', 'date-year-day', 'date?', 'datum-intern-literal', - 'default-continuation-prompt-tag', 'delete-directory', - 'delete-file', 'denominator', 'directory-exists?', - 'directory-list', 'display', 'displayln', 'dump-memory-stats', - 'dynamic-require', 'dynamic-require-for-syntax', 'dynamic-wind', - 'eof', 'eof-object?', 'ephemeron-value', 'ephemeron?', 'eprintf', - 'eq-hash-code', 'eq?', 'equal-hash-code', - 'equal-secondary-hash-code', 'equal?', 'equal?/recur', - 'eqv-hash-code', 'eqv?', 'error', 'error-display-handler', - 'error-escape-handler', 'error-print-context-length', - 'error-print-source-location', 'error-print-width', - 'error-value->string-handler', 'eval', 'eval-jit-enabled', - 'eval-syntax', 'even?', 'evt?', 'exact->inexact', 'exact-integer?', - 'exact-nonnegative-integer?', 'exact-positive-integer?', 'exact?', - 'executable-yield-handler', 'exit', 'exit-handler', 'exn', - 'exn-continuation-marks', 'exn-message', 'exn:break', - 'exn:break-continuation', 'exn:break?', 'exn:fail', - 'exn:fail:contract', 'exn:fail:contract:arity', - 'exn:fail:contract:arity?', 'exn:fail:contract:continuation', - 'exn:fail:contract:continuation?', - 'exn:fail:contract:divide-by-zero', + 'custom-write-accessor', 'custom-write-property-proc', 'custom-write?', + 'date', 'date*', 'date*-nanosecond', 'date*-time-zone-name', 'date*?', + 'date-day', 'date-dst?', 'date-hour', 'date-minute', 'date-month', + 'date-second', 'date-time-zone-offset', 'date-week-day', 'date-year', + 'date-year-day', 'date?', 'datum->syntax', 'datum-intern-literal', + 'default-continuation-prompt-tag', 'degrees->radians', + 'delete-directory', 'delete-directory/files', 'delete-file', + 'denominator', 'dict->list', 'dict-can-functional-set?', + 'dict-can-remove-keys?', 'dict-clear', 'dict-clear!', 'dict-copy', + 'dict-count', 'dict-empty?', 'dict-for-each', 'dict-has-key?', + 'dict-implements/c', 'dict-implements?', 'dict-iter-contract', + 'dict-iterate-first', 'dict-iterate-key', 'dict-iterate-next', + 'dict-iterate-value', 'dict-key-contract', 'dict-keys', 'dict-map', + 'dict-mutable?', 'dict-ref', 'dict-ref!', 'dict-remove', + 'dict-remove!', 'dict-set', 'dict-set!', 'dict-set*', 'dict-set*!', + 'dict-update', 'dict-update!', 'dict-value-contract', 'dict-values', + 'dict?', 'directory-exists?', 'directory-list', 'display', + 'display-lines', 'display-lines-to-file', 'display-to-file', + 'displayln', 'double-flonum?', 'drop', 'drop-right', 'dropf', + 'dropf-right', 'dump-memory-stats', 'dup-input-port', + 'dup-output-port', 'dynamic-get-field', 'dynamic-place', + 'dynamic-place*', 'dynamic-require', 'dynamic-require-for-syntax', + 'dynamic-send', 'dynamic-set-field!', 'dynamic-wind', 'eighth', + 'empty', 'empty-sequence', 'empty-stream', 'empty?', + 'environment-variables-copy', 'environment-variables-names', + 'environment-variables-ref', 'environment-variables-set!', + 'environment-variables?', 'eof', 'eof-evt', 'eof-object?', + 'ephemeron-value', 'ephemeron?', 'eprintf', 'eq-contract-val', + 'eq-contract?', 'eq-hash-code', 'eq?', 'equal-contract-val', + 'equal-contract?', 'equal-hash-code', 'equal-secondary-hash-code', + 'equal<%>', 'equal?', 'equal?/recur', 'eqv-hash-code', 'eqv?', 'error', + 'error-display-handler', 'error-escape-handler', + 'error-print-context-length', 'error-print-source-location', + 'error-print-width', 'error-value->string-handler', 'eval', + 'eval-jit-enabled', 'eval-syntax', 'even?', 'evt/c', 'evt?', + 'exact->inexact', 'exact-ceiling', 'exact-floor', 'exact-integer?', + 'exact-nonnegative-integer?', 'exact-positive-integer?', 'exact-round', + 'exact-truncate', 'exact?', 'executable-yield-handler', 'exit', + 'exit-handler', 'exn', 'exn-continuation-marks', 'exn-message', + 'exn:break', 'exn:break-continuation', 'exn:break:hang-up', + 'exn:break:hang-up?', 'exn:break:terminate', 'exn:break:terminate?', + 'exn:break?', 'exn:fail', 'exn:fail:contract', + 'exn:fail:contract:arity', 'exn:fail:contract:arity?', + 'exn:fail:contract:blame', 'exn:fail:contract:blame-object', + 'exn:fail:contract:blame?', 'exn:fail:contract:continuation', + 'exn:fail:contract:continuation?', 'exn:fail:contract:divide-by-zero', 'exn:fail:contract:divide-by-zero?', 'exn:fail:contract:non-fixnum-result', - 'exn:fail:contract:non-fixnum-result?', - 'exn:fail:contract:variable', 'exn:fail:contract:variable-id', - 'exn:fail:contract:variable?', 'exn:fail:contract?', - 'exn:fail:filesystem', 'exn:fail:filesystem:exists', - 'exn:fail:filesystem:exists?', 'exn:fail:filesystem:version', + 'exn:fail:contract:non-fixnum-result?', 'exn:fail:contract:variable', + 'exn:fail:contract:variable-id', 'exn:fail:contract:variable?', + 'exn:fail:contract?', 'exn:fail:filesystem', + 'exn:fail:filesystem:errno', 'exn:fail:filesystem:errno-errno', + 'exn:fail:filesystem:errno?', 'exn:fail:filesystem:exists', + 'exn:fail:filesystem:exists?', 'exn:fail:filesystem:missing-module', + 'exn:fail:filesystem:missing-module-path', + 'exn:fail:filesystem:missing-module?', 'exn:fail:filesystem:version', 'exn:fail:filesystem:version?', 'exn:fail:filesystem?', - 'exn:fail:network', 'exn:fail:network?', 'exn:fail:out-of-memory', - 'exn:fail:out-of-memory?', 'exn:fail:read', + 'exn:fail:network', 'exn:fail:network:errno', + 'exn:fail:network:errno-errno', 'exn:fail:network:errno?', + 'exn:fail:network?', 'exn:fail:object', 'exn:fail:object?', + 'exn:fail:out-of-memory', 'exn:fail:out-of-memory?', 'exn:fail:read', 'exn:fail:read-srclocs', 'exn:fail:read:eof', 'exn:fail:read:eof?', - 'exn:fail:read:non-char', 'exn:fail:read:non-char?', - 'exn:fail:read?', 'exn:fail:syntax', 'exn:fail:syntax-exprs', - 'exn:fail:syntax:unbound', 'exn:fail:syntax:unbound?', - 'exn:fail:syntax?', 'exn:fail:unsupported', + 'exn:fail:read:non-char', 'exn:fail:read:non-char?', 'exn:fail:read?', + 'exn:fail:syntax', 'exn:fail:syntax-exprs', + 'exn:fail:syntax:missing-module', + 'exn:fail:syntax:missing-module-path', + 'exn:fail:syntax:missing-module?', 'exn:fail:syntax:unbound', + 'exn:fail:syntax:unbound?', 'exn:fail:syntax?', 'exn:fail:unsupported', 'exn:fail:unsupported?', 'exn:fail:user', 'exn:fail:user?', - 'exn:fail?', 'exn:srclocs-accessor', 'exn:srclocs?', 'exn?', 'exp', - 'expand', 'expand-once', 'expand-syntax', 'expand-syntax-once', - 'expand-syntax-to-top-form', 'expand-to-top-form', - 'expand-user-path', 'expt', 'file-exists?', - 'file-or-directory-identity', 'file-or-directory-modify-seconds', - 'file-or-directory-permissions', 'file-position', 'file-size', - 'file-stream-buffer-mode', 'file-stream-port?', - 'filesystem-root-list', 'find-executable-path', - 'find-library-collection-paths', 'find-system-path', 'fixnum?', + 'exn:fail?', 'exn:misc:match?', 'exn:missing-module-accessor', + 'exn:missing-module?', 'exn:srclocs-accessor', 'exn:srclocs?', 'exn?', + 'exp', 'expand', 'expand-once', 'expand-syntax', 'expand-syntax-once', + 'expand-syntax-to-top-form', 'expand-to-top-form', 'expand-user-path', + 'explode-path', 'expt', 'externalizable<%>', 'false?', 'field-names', + 'fifth', 'file->bytes', 'file->bytes-lines', 'file->lines', + 'file->list', 'file->string', 'file->value', 'file-exists?', + 'file-name-from-path', 'file-or-directory-identity', + 'file-or-directory-modify-seconds', 'file-or-directory-permissions', + 'file-position', 'file-position*', 'file-size', + 'file-stream-buffer-mode', 'file-stream-port?', 'file-truncate', + 'filename-extension', 'filesystem-change-evt', + 'filesystem-change-evt-cancel', 'filesystem-change-evt?', + 'filesystem-root-list', 'filter', 'filter-map', 'filter-not', + 'filter-read-input-port', 'find-executable-path', 'find-files', + 'find-library-collection-links', 'find-library-collection-paths', + 'find-relative-path', 'find-system-path', 'findf', 'first', 'fixnum?', + 'flat-contract', 'flat-contract-predicate', 'flat-contract-property?', + 'flat-contract?', 'flat-named-contract', 'flatten', 'floating-point-bytes->real', 'flonum?', 'floor', 'flush-output', - 'for-each', 'force', 'format', 'fprintf', 'free-identifier=?', - 'gcd', 'generate-temporaries', 'gensym', 'get-output-bytes', - 'get-output-string', 'getenv', 'global-port-print-handler', - 'guard-evt', 'handle-evt', 'handle-evt?', 'hash', 'hash-equal?', - 'hash-eqv?', 'hash-has-key?', 'hash-placeholder?', 'hash-ref!', - 'hasheq', 'hasheqv', 'identifier-binding', - 'identifier-label-binding', 'identifier-prune-lexical-context', + 'fold-files', 'foldl', 'foldr', 'for-each', 'force', 'format', + 'fourth', 'fprintf', 'free-identifier=?', 'free-label-identifier=?', + 'free-template-identifier=?', 'free-transformer-identifier=?', + 'fsemaphore-count', 'fsemaphore-post', 'fsemaphore-try-wait?', + 'fsemaphore-wait', 'fsemaphore?', 'future', 'future?', + 'futures-enabled?', 'gcd', 'generate-member-key', + 'generate-temporaries', 'generic-set?', 'generic?', 'gensym', + 'get-output-bytes', 'get-output-string', 'get-preference', + 'get/build-val-first-projection', 'getenv', + 'global-port-print-handler', 'group-execute-bit', 'group-read-bit', + 'group-write-bit', 'guard-evt', 'handle-evt', 'handle-evt?', + 'has-contract?', 'hash', 'hash->list', 'hash-clear', 'hash-clear!', + 'hash-copy', 'hash-copy-clear', 'hash-count', 'hash-empty?', + 'hash-eq?', 'hash-equal?', 'hash-eqv?', 'hash-for-each', + 'hash-has-key?', 'hash-iterate-first', 'hash-iterate-key', + 'hash-iterate-next', 'hash-iterate-value', 'hash-keys', 'hash-map', + 'hash-placeholder?', 'hash-ref', 'hash-ref!', 'hash-remove', + 'hash-remove!', 'hash-set', 'hash-set!', 'hash-set*', 'hash-set*!', + 'hash-update', 'hash-update!', 'hash-values', 'hash-weak?', 'hash/c', + 'hash?', 'hasheq', 'hasheqv', 'identifier-binding', + 'identifier-binding-symbol', 'identifier-label-binding', + 'identifier-prune-lexical-context', 'identifier-prune-to-source-module', 'identifier-remove-from-definition-context', 'identifier-template-binding', 'identifier-transformer-binding', - 'identifier?', 'imag-part', 'immutable?', 'impersonate-box', - 'impersonate-hash', 'impersonate-procedure', 'impersonate-struct', - 'impersonate-vector', 'impersonator-of?', - 'impersonator-prop:application-mark', - 'impersonator-property-accessor-procedure?', - 'impersonator-property?', 'impersonator?', 'inexact->exact', - 'inexact-real?', 'inexact?', 'input-port?', 'inspector?', - 'integer->char', 'integer->integer-bytes', - 'integer-bytes->integer', 'integer-length', 'integer-sqrt', - 'integer-sqrt/remainder', 'integer?', + 'identifier?', 'identity', 'imag-part', 'immutable?', + 'impersonate-box', 'impersonate-channel', + 'impersonate-continuation-mark-key', 'impersonate-hash', + 'impersonate-procedure', 'impersonate-prompt-tag', + 'impersonate-struct', 'impersonate-vector', 'impersonator-contract?', + 'impersonator-ephemeron', 'impersonator-of?', + 'impersonator-prop:application-mark', 'impersonator-prop:contracted', + 'impersonator-property-accessor-procedure?', 'impersonator-property?', + 'impersonator?', 'implementation?', 'implementation?/c', 'in-bytes', + 'in-bytes-lines', 'in-cycle', 'in-dict', 'in-dict-keys', + 'in-dict-pairs', 'in-dict-values', 'in-directory', 'in-hash', + 'in-hash-keys', 'in-hash-pairs', 'in-hash-values', 'in-indexed', + 'in-input-port-bytes', 'in-input-port-chars', 'in-lines', 'in-list', + 'in-mlist', 'in-naturals', 'in-parallel', 'in-permutations', 'in-port', + 'in-producer', 'in-range', 'in-sequences', 'in-set', 'in-stream', + 'in-string', 'in-value', 'in-values*-sequence', 'in-values-sequence', + 'in-vector', 'inexact->exact', 'inexact-real?', 'inexact?', + 'infinite?', 'input-port-append', 'input-port?', 'inspector?', + 'instanceof/c', 'integer->char', 'integer->integer-bytes', + 'integer-bytes->integer', 'integer-in', 'integer-length', + 'integer-sqrt', 'integer-sqrt/remainder', 'integer?', + 'interface->method-names', 'interface-extension?', 'interface?', 'internal-definition-context-seal', 'internal-definition-context?', - 'keyword->string', 'keyword<?', 'keyword?', 'kill-thread', 'lcm', - 'length', 'liberal-define-context?', 'link-exists?', 'list', - 'list*', 'list->bytes', 'list->string', 'list->vector', 'list-ref', - 'list-tail', 'list?', 'load', 'load-extension', - 'load-on-demand-enabled', 'load-relative', - 'load-relative-extension', 'load/cd', 'load/use-compiled', - 'local-expand', 'local-expand/capture-lifts', - 'local-transformer-expand', - 'local-transformer-expand/capture-lifts', 'locale-string-encoding', - 'log', 'magnitude', 'make-arity-at-least', 'make-bytes', - 'make-channel', 'make-continuation-prompt-tag', 'make-custodian', - 'make-custodian-box', 'make-date', 'make-date*', - 'make-derived-parameter', 'make-directory', 'make-ephemeron', - 'make-exn', 'make-exn:break', 'make-exn:fail', - 'make-exn:fail:contract', 'make-exn:fail:contract:arity', + 'is-a?', 'is-a?/c', 'keyword->string', 'keyword-apply', 'keyword<?', + 'keyword?', 'keywords-match', 'kill-thread', 'last', 'last-pair', + 'lcm', 'length', 'liberal-define-context?', 'link-exists?', 'list', + 'list*', 'list->bytes', 'list->mutable-set', 'list->mutable-seteq', + 'list->mutable-seteqv', 'list->set', 'list->seteq', 'list->seteqv', + 'list->string', 'list->vector', 'list->weak-set', 'list->weak-seteq', + 'list->weak-seteqv', 'list-ref', 'list-tail', 'list/c', 'list?', + 'listof', 'load', 'load-extension', 'load-on-demand-enabled', + 'load-relative', 'load-relative-extension', 'load/cd', + 'load/use-compiled', 'local-expand', 'local-expand/capture-lifts', + 'local-transformer-expand', 'local-transformer-expand/capture-lifts', + 'locale-string-encoding', 'log', 'log-level?', 'log-max-level', + 'log-message', 'log-receiver?', 'logger-name', 'logger?', 'magnitude', + 'make-arity-at-least', 'make-base-empty-namespace', + 'make-base-namespace', 'make-bytes', 'make-channel', + 'make-chaperone-contract', 'make-continuation-mark-key', + 'make-continuation-prompt-tag', 'make-contract', 'make-custodian', + 'make-custodian-box', 'make-custom-hash', 'make-custom-hash-types', + 'make-custom-set', 'make-custom-set-types', 'make-date', 'make-date*', + 'make-derived-parameter', 'make-directory', 'make-directory*', + 'make-do-sequence', 'make-empty-namespace', + 'make-environment-variables', 'make-ephemeron', 'make-exn', + 'make-exn:break', 'make-exn:break:hang-up', 'make-exn:break:terminate', + 'make-exn:fail', 'make-exn:fail:contract', + 'make-exn:fail:contract:arity', 'make-exn:fail:contract:blame', 'make-exn:fail:contract:continuation', 'make-exn:fail:contract:divide-by-zero', 'make-exn:fail:contract:non-fixnum-result', 'make-exn:fail:contract:variable', 'make-exn:fail:filesystem', - 'make-exn:fail:filesystem:exists', + 'make-exn:fail:filesystem:errno', 'make-exn:fail:filesystem:exists', + 'make-exn:fail:filesystem:missing-module', 'make-exn:fail:filesystem:version', 'make-exn:fail:network', + 'make-exn:fail:network:errno', 'make-exn:fail:object', 'make-exn:fail:out-of-memory', 'make-exn:fail:read', 'make-exn:fail:read:eof', 'make-exn:fail:read:non-char', - 'make-exn:fail:syntax', 'make-exn:fail:syntax:unbound', - 'make-exn:fail:unsupported', 'make-exn:fail:user', - 'make-file-or-directory-link', 'make-hash-placeholder', - 'make-hasheq-placeholder', 'make-hasheqv', - 'make-hasheqv-placeholder', 'make-immutable-hasheqv', - 'make-impersonator-property', 'make-input-port', 'make-inspector', - 'make-known-char-range-list', 'make-output-port', 'make-parameter', - 'make-pipe', 'make-placeholder', 'make-polar', - 'make-prefab-struct', 'make-pseudo-random-generator', + 'make-exn:fail:syntax', 'make-exn:fail:syntax:missing-module', + 'make-exn:fail:syntax:unbound', 'make-exn:fail:unsupported', + 'make-exn:fail:user', 'make-file-or-directory-link', + 'make-flat-contract', 'make-fsemaphore', 'make-generic', + 'make-handle-get-preference-locked', 'make-hash', + 'make-hash-placeholder', 'make-hasheq', 'make-hasheq-placeholder', + 'make-hasheqv', 'make-hasheqv-placeholder', + 'make-immutable-custom-hash', 'make-immutable-hash', + 'make-immutable-hasheq', 'make-immutable-hasheqv', + 'make-impersonator-property', 'make-input-port', + 'make-input-port/read-to-peek', 'make-inspector', + 'make-keyword-procedure', 'make-known-char-range-list', + 'make-limited-input-port', 'make-list', 'make-lock-file-name', + 'make-log-receiver', 'make-logger', 'make-mixin-contract', + 'make-mutable-custom-set', 'make-none/c', 'make-object', + 'make-output-port', 'make-parameter', 'make-phantom-bytes', + 'make-pipe', 'make-pipe-with-specials', 'make-placeholder', + 'make-polar', 'make-prefab-struct', 'make-primitive-class', + 'make-proj-contract', 'make-pseudo-random-generator', 'make-reader-graph', 'make-readtable', 'make-rectangular', 'make-rename-transformer', 'make-resolved-module-path', 'make-security-guard', 'make-semaphore', 'make-set!-transformer', - 'make-shared-bytes', 'make-sibling-inspector', - 'make-special-comment', 'make-srcloc', 'make-string', - 'make-struct-field-accessor', 'make-struct-field-mutator', - 'make-struct-type', 'make-struct-type-property', - 'make-syntax-delta-introducer', 'make-syntax-introducer', - 'make-thread-cell', 'make-thread-group', 'make-vector', - 'make-weak-box', 'make-weak-hasheqv', 'make-will-executor', 'map', - 'max', 'mcar', 'mcdr', 'mcons', 'member', 'memq', 'memv', 'min', - 'module->exports', 'module->imports', 'module->language-info', - 'module->namespace', 'module-compiled-exports', + 'make-shared-bytes', 'make-sibling-inspector', 'make-special-comment', + 'make-srcloc', 'make-string', 'make-struct-field-accessor', + 'make-struct-field-mutator', 'make-struct-type', + 'make-struct-type-property', 'make-syntax-delta-introducer', + 'make-syntax-introducer', 'make-temporary-file', + 'make-tentative-pretty-print-output-port', 'make-thread-cell', + 'make-thread-group', 'make-vector', 'make-weak-box', + 'make-weak-custom-hash', 'make-weak-custom-set', 'make-weak-hash', + 'make-weak-hasheq', 'make-weak-hasheqv', 'make-will-executor', 'map', + 'match-equality-test', 'matches-arity-exactly?', 'max', 'mcar', 'mcdr', + 'mcons', 'member', 'member-name-key-hash-code', 'member-name-key=?', + 'member-name-key?', 'memf', 'memq', 'memv', 'merge-input', + 'method-in-interface?', 'min', 'mixin-contract', 'module->exports', + 'module->imports', 'module->language-info', 'module->namespace', + 'module-compiled-cross-phase-persistent?', 'module-compiled-exports', 'module-compiled-imports', 'module-compiled-language-info', - 'module-compiled-name', 'module-path-index-join', + 'module-compiled-name', 'module-compiled-submodules', + 'module-declared?', 'module-path-index-join', 'module-path-index-resolve', 'module-path-index-split', - 'module-path-index?', 'module-path?', 'module-predefined?', - 'module-provide-protected?', 'modulo', 'mpair?', 'nack-guard-evt', + 'module-path-index-submodule', 'module-path-index?', 'module-path?', + 'module-predefined?', 'module-provide-protected?', 'modulo', 'mpair?', + 'mutable-set', 'mutable-seteq', 'mutable-seteqv', 'n->th', + 'nack-guard-evt', 'namespace-anchor->empty-namespace', + 'namespace-anchor->namespace', 'namespace-anchor?', 'namespace-attach-module', 'namespace-attach-module-declaration', 'namespace-base-phase', 'namespace-mapped-symbols', 'namespace-module-identifier', 'namespace-module-registry', @@ -289,136 +481,212 @@ class RacketLexer(RegexLexer): 'namespace-require/copy', 'namespace-require/expansion-time', 'namespace-set-variable-value!', 'namespace-symbol->identifier', 'namespace-syntax-introduce', 'namespace-undefine-variable!', - 'namespace-unprotect-module', 'namespace-variable-value', - 'namespace?', 'negative?', 'never-evt', 'newline', - 'normal-case-path', 'not', 'null', 'null?', 'number->string', - 'number?', 'numerator', 'object-name', 'odd?', 'open-input-bytes', + 'namespace-unprotect-module', 'namespace-variable-value', 'namespace?', + 'nan?', 'natural-number/c', 'negate', 'negative?', 'never-evt', + u'new-∀/c', u'new-∃/c', 'newline', 'ninth', 'non-empty-listof', + 'none/c', 'normal-case-path', 'normalize-arity', 'normalize-path', + 'normalized-arity?', 'not', 'not/c', 'null', 'null?', 'number->string', + 'number?', 'numerator', 'object%', 'object->vector', 'object-info', + 'object-interface', 'object-method-arity-includes?', 'object-name', + 'object=?', 'object?', 'odd?', 'one-of/c', 'open-input-bytes', 'open-input-file', 'open-input-output-file', 'open-input-string', - 'open-output-bytes', 'open-output-file', 'open-output-string', - 'ormap', 'output-port?', 'pair?', 'parameter-procedure=?', - 'parameter?', 'parameterization?', 'path->bytes', - 'path->complete-path', 'path->directory-path', 'path->string', - 'path-add-suffix', 'path-convention-type', 'path-element->bytes', - 'path-element->string', 'path-for-some-system?', - 'path-list-string->path-list', 'path-replace-suffix', - 'path-string?', 'path?', 'peek-byte', 'peek-byte-or-special', - 'peek-bytes', 'peek-bytes!', 'peek-bytes-avail!', - 'peek-bytes-avail!*', 'peek-bytes-avail!/enable-break', - 'peek-char', 'peek-char-or-special', 'peek-string', 'peek-string!', - 'pipe-content-length', 'placeholder-get', 'placeholder-set!', - 'placeholder?', 'poll-guard-evt', 'port-closed-evt', - 'port-closed?', 'port-commit-peeked', 'port-count-lines!', - 'port-count-lines-enabled', 'port-display-handler', - 'port-file-identity', 'port-file-unlock', 'port-next-location', - 'port-print-handler', 'port-progress-evt', - 'port-provides-progress-evts?', 'port-read-handler', - 'port-try-file-lock?', 'port-write-handler', 'port-writes-atomic?', - 'port-writes-special?', 'port?', 'positive?', - 'prefab-key->struct-type', 'prefab-struct-key', 'pregexp', - 'pregexp?', 'primitive-closure?', 'primitive-result-arity', - 'primitive?', 'print', 'print-as-expression', + 'open-output-bytes', 'open-output-file', 'open-output-nowhere', + 'open-output-string', 'or/c', 'order-of-magnitude', 'ormap', + 'other-execute-bit', 'other-read-bit', 'other-write-bit', + 'output-port?', 'pair?', 'parameter-procedure=?', 'parameter/c', + 'parameter?', 'parameterization?', 'parse-command-line', 'partition', + 'path->bytes', 'path->complete-path', 'path->directory-path', + 'path->string', 'path-add-suffix', 'path-convention-type', + 'path-element->bytes', 'path-element->string', 'path-element?', + 'path-for-some-system?', 'path-list-string->path-list', 'path-only', + 'path-replace-suffix', 'path-string?', 'path<?', 'path?', + 'pathlist-closure', 'peek-byte', 'peek-byte-or-special', 'peek-bytes', + 'peek-bytes!', 'peek-bytes!-evt', 'peek-bytes-avail!', + 'peek-bytes-avail!*', 'peek-bytes-avail!-evt', + 'peek-bytes-avail!/enable-break', 'peek-bytes-evt', 'peek-char', + 'peek-char-or-special', 'peek-string', 'peek-string!', + 'peek-string!-evt', 'peek-string-evt', 'peeking-input-port', + 'permutations', 'phantom-bytes?', 'pi', 'pi.f', 'pipe-content-length', + 'place-break', 'place-channel', 'place-channel-get', + 'place-channel-put', 'place-channel-put/get', 'place-channel?', + 'place-dead-evt', 'place-enabled?', 'place-kill', 'place-location?', + 'place-message-allowed?', 'place-sleep', 'place-wait', 'place?', + 'placeholder-get', 'placeholder-set!', 'placeholder?', + 'poll-guard-evt', 'port->bytes', 'port->bytes-lines', 'port->lines', + 'port->list', 'port->string', 'port-closed-evt', 'port-closed?', + 'port-commit-peeked', 'port-count-lines!', 'port-count-lines-enabled', + 'port-counts-lines?', 'port-display-handler', 'port-file-identity', + 'port-file-unlock', 'port-next-location', 'port-print-handler', + 'port-progress-evt', 'port-provides-progress-evts?', + 'port-read-handler', 'port-try-file-lock?', 'port-write-handler', + 'port-writes-atomic?', 'port-writes-special?', 'port?', 'positive?', + 'predicate/c', 'prefab-key->struct-type', 'prefab-key?', + 'prefab-struct-key', 'preferences-lock-file-mode', 'pregexp', + 'pregexp?', 'pretty-display', 'pretty-format', 'pretty-print', + 'pretty-print-.-symbol-without-bars', + 'pretty-print-abbreviate-read-macros', 'pretty-print-columns', + 'pretty-print-current-style-table', 'pretty-print-depth', + 'pretty-print-exact-as-decimal', 'pretty-print-extend-style-table', + 'pretty-print-handler', 'pretty-print-newline', + 'pretty-print-post-print-hook', 'pretty-print-pre-print-hook', + 'pretty-print-print-hook', 'pretty-print-print-line', + 'pretty-print-remap-stylable', 'pretty-print-show-inexactness', + 'pretty-print-size-hook', 'pretty-print-style-table?', + 'pretty-printing', 'pretty-write', 'primitive-closure?', + 'primitive-result-arity', 'primitive?', 'print', 'print-as-expression', 'print-boolean-long-form', 'print-box', 'print-graph', 'print-hash-table', 'print-mpair-curly-braces', 'print-pair-curly-braces', 'print-reader-abbreviations', 'print-struct', 'print-syntax-width', 'print-unreadable', - 'print-vector-length', 'printf', 'procedure->method', - 'procedure-arity', 'procedure-arity-includes?', 'procedure-arity?', + 'print-vector-length', 'printable/c', 'printable<%>', 'printf', + 'procedure->method', 'procedure-arity', 'procedure-arity-includes/c', + 'procedure-arity-includes?', 'procedure-arity?', 'procedure-closure-contents-eq?', 'procedure-extract-target', - 'procedure-reduce-arity', 'procedure-rename', - 'procedure-struct-type?', 'procedure?', 'promise?', - 'prop:arity-string', 'prop:checked-procedure', - 'prop:custom-print-quotable', 'prop:custom-write', - 'prop:equal+hash', 'prop:evt', 'prop:exn:srclocs', + 'procedure-keywords', 'procedure-reduce-arity', + 'procedure-reduce-keyword-arity', 'procedure-rename', + 'procedure-struct-type?', 'procedure?', 'process', 'process*', + 'process*/ports', 'process/ports', 'processor-count', 'progress-evt?', + 'promise-forced?', 'promise-running?', 'promise/c', 'promise?', + 'prop:arity-string', 'prop:chaperone-contract', + 'prop:checked-procedure', 'prop:contract', 'prop:contracted', + 'prop:custom-print-quotable', 'prop:custom-write', 'prop:dict', + 'prop:dict/contract', 'prop:equal+hash', 'prop:evt', + 'prop:exn:missing-module', 'prop:exn:srclocs', 'prop:flat-contract', 'prop:impersonator-of', 'prop:input-port', - 'prop:liberal-define-context', 'prop:output-port', - 'prop:procedure', 'prop:rename-transformer', - 'prop:set!-transformer', 'pseudo-random-generator->vector', + 'prop:liberal-define-context', 'prop:opt-chaperone-contract', + 'prop:opt-chaperone-contract-get-test', 'prop:opt-chaperone-contract?', + 'prop:output-port', 'prop:place-location', 'prop:procedure', + 'prop:rename-transformer', 'prop:sequence', 'prop:set!-transformer', + 'prop:stream', 'proper-subset?', 'pseudo-random-generator->vector', 'pseudo-random-generator-vector?', 'pseudo-random-generator?', - 'putenv', 'quotient', 'quotient/remainder', 'raise', - 'raise-arity-error', 'raise-mismatch-error', 'raise-syntax-error', - 'raise-type-error', 'raise-user-error', 'random', 'random-seed', - 'rational?', 'rationalize', 'read', 'read-accept-bar-quote', - 'read-accept-box', 'read-accept-compiled', 'read-accept-dot', - 'read-accept-graph', 'read-accept-infix-dot', 'read-accept-lang', - 'read-accept-quasiquote', 'read-accept-reader', 'read-byte', - 'read-byte-or-special', 'read-bytes', 'read-bytes!', - 'read-bytes-avail!', 'read-bytes-avail!*', - 'read-bytes-avail!/enable-break', 'read-bytes-line', - 'read-case-sensitive', 'read-char', 'read-char-or-special', - 'read-curly-brace-as-paren', 'read-decimal-as-inexact', - 'read-eval-print-loop', 'read-language', 'read-line', - 'read-on-demand-source', 'read-square-bracket-as-paren', - 'read-string', 'read-string!', 'read-syntax', + 'put-preferences', 'putenv', 'quotient', 'quotient/remainder', + 'radians->degrees', 'raise', 'raise-argument-error', + 'raise-arguments-error', 'raise-arity-error', 'raise-blame-error', + 'raise-contract-error', 'raise-mismatch-error', + 'raise-not-cons-blame-error', 'raise-range-error', + 'raise-result-error', 'raise-syntax-error', 'raise-type-error', + 'raise-user-error', 'random', 'random-seed', 'range', 'rational?', + 'rationalize', 'read', 'read-accept-bar-quote', 'read-accept-box', + 'read-accept-compiled', 'read-accept-dot', 'read-accept-graph', + 'read-accept-infix-dot', 'read-accept-lang', 'read-accept-quasiquote', + 'read-accept-reader', 'read-byte', 'read-byte-or-special', + 'read-bytes', 'read-bytes!', 'read-bytes!-evt', 'read-bytes-avail!', + 'read-bytes-avail!*', 'read-bytes-avail!-evt', + 'read-bytes-avail!/enable-break', 'read-bytes-evt', 'read-bytes-line', + 'read-bytes-line-evt', 'read-case-sensitive', 'read-char', + 'read-char-or-special', 'read-curly-brace-as-paren', + 'read-decimal-as-inexact', 'read-eval-print-loop', 'read-language', + 'read-line', 'read-line-evt', 'read-on-demand-source', + 'read-square-bracket-as-paren', 'read-string', 'read-string!', + 'read-string!-evt', 'read-string-evt', 'read-syntax', 'read-syntax/recursive', 'read/recursive', 'readtable-mapping', - 'readtable?', 'real->double-flonum', 'real->floating-point-bytes', - 'real->single-flonum', 'real-part', 'real?', 'regexp', - 'regexp-match', 'regexp-match-peek', 'regexp-match-peek-immediate', - 'regexp-match-peek-positions', + 'readtable?', 'real->decimal-string', 'real->double-flonum', + 'real->floating-point-bytes', 'real->single-flonum', 'real-in', + 'real-part', 'real?', 'reencode-input-port', 'reencode-output-port', + 'regexp', 'regexp-match', 'regexp-match*', 'regexp-match-evt', + 'regexp-match-exact?', 'regexp-match-peek', + 'regexp-match-peek-immediate', 'regexp-match-peek-positions', + 'regexp-match-peek-positions*', 'regexp-match-peek-positions-immediate', 'regexp-match-peek-positions-immediate/end', 'regexp-match-peek-positions/end', 'regexp-match-positions', - 'regexp-match-positions/end', 'regexp-match/end', 'regexp-match?', - 'regexp-max-lookbehind', 'regexp-replace', 'regexp-replace*', - 'regexp?', 'relative-path?', 'remainder', + 'regexp-match-positions*', 'regexp-match-positions/end', + 'regexp-match/end', 'regexp-match?', 'regexp-max-lookbehind', + 'regexp-quote', 'regexp-replace', 'regexp-replace*', + 'regexp-replace-quote', 'regexp-replaces', 'regexp-split', + 'regexp-try-match', 'regexp?', 'relative-path?', 'relocate-input-port', + 'relocate-output-port', 'remainder', 'remove', 'remove*', + 'remove-duplicates', 'remq', 'remq*', 'remv', 'remv*', 'rename-file-or-directory', 'rename-transformer-target', - 'rename-transformer?', 'resolve-path', 'resolved-module-path-name', - 'resolved-module-path?', 'reverse', 'round', 'seconds->date', - 'security-guard?', 'semaphore-peek-evt', 'semaphore-post', - 'semaphore-try-wait?', 'semaphore-wait', - 'semaphore-wait/enable-break', 'semaphore?', - 'set!-transformer-procedure', 'set!-transformer?', 'set-box!', - 'set-mcar!', 'set-mcdr!', 'set-port-next-location!', - 'shared-bytes', 'shell-execute', 'simplify-path', 'sin', - 'single-flonum?', 'sleep', 'special-comment-value', - 'special-comment?', 'split-path', 'sqrt', 'srcloc', - 'srcloc-column', 'srcloc-line', 'srcloc-position', 'srcloc-source', - 'srcloc-span', 'srcloc?', 'string', 'string->bytes/latin-1', - 'string->bytes/locale', 'string->bytes/utf-8', + 'rename-transformer?', 'reroot-path', 'resolve-path', + 'resolved-module-path-name', 'resolved-module-path?', 'rest', + 'reverse', 'round', 'second', 'seconds->date', 'security-guard?', + 'semaphore-peek-evt', 'semaphore-peek-evt?', 'semaphore-post', + 'semaphore-try-wait?', 'semaphore-wait', 'semaphore-wait/enable-break', + 'semaphore?', 'sequence->list', 'sequence->stream', + 'sequence-add-between', 'sequence-andmap', 'sequence-append', + 'sequence-count', 'sequence-filter', 'sequence-fold', + 'sequence-for-each', 'sequence-generate', 'sequence-generate*', + 'sequence-length', 'sequence-map', 'sequence-ormap', 'sequence-ref', + 'sequence-tail', 'sequence?', 'set', 'set!-transformer-procedure', + 'set!-transformer?', 'set->list', 'set->stream', 'set-add', 'set-add!', + 'set-box!', 'set-clear', 'set-clear!', 'set-copy', 'set-copy-clear', + 'set-count', 'set-empty?', 'set-eq?', 'set-equal?', 'set-eqv?', + 'set-first', 'set-for-each', 'set-implements/c', 'set-implements?', + 'set-intersect', 'set-intersect!', 'set-map', 'set-mcar!', 'set-mcdr!', + 'set-member?', 'set-mutable?', 'set-phantom-bytes!', + 'set-port-next-location!', 'set-remove', 'set-remove!', 'set-rest', + 'set-subtract', 'set-subtract!', 'set-symmetric-difference', + 'set-symmetric-difference!', 'set-union', 'set-union!', 'set-weak?', + 'set/c', 'set=?', 'set?', 'seteq', 'seteqv', 'seventh', 'sgn', + 'shared-bytes', 'shell-execute', 'shrink-path-wrt', 'shuffle', + 'simple-form-path', 'simplify-path', 'sin', 'single-flonum?', 'sinh', + 'sixth', 'skip-projection-wrapper?', 'sleep', + 'some-system-path->string', 'sort', 'special-comment-value', + 'special-comment?', 'special-filter-input-port', 'split-at', + 'split-at-right', 'split-path', 'splitf-at', 'splitf-at-right', 'sqr', + 'sqrt', 'srcloc', 'srcloc->string', 'srcloc-column', 'srcloc-line', + 'srcloc-position', 'srcloc-source', 'srcloc-span', 'srcloc?', + 'stop-after', 'stop-before', 'stream->list', 'stream-add-between', + 'stream-andmap', 'stream-append', 'stream-count', 'stream-empty?', + 'stream-filter', 'stream-first', 'stream-fold', 'stream-for-each', + 'stream-length', 'stream-map', 'stream-ormap', 'stream-ref', + 'stream-rest', 'stream-tail', 'stream?', 'string', + 'string->bytes/latin-1', 'string->bytes/locale', 'string->bytes/utf-8', 'string->immutable-string', 'string->keyword', 'string->list', 'string->number', 'string->path', 'string->path-element', - 'string->symbol', 'string->uninterned-symbol', - 'string->unreadable-symbol', 'string-append', 'string-ci<=?', - 'string-ci<?', 'string-ci=?', 'string-ci>=?', 'string-ci>?', - 'string-copy', 'string-copy!', 'string-downcase', 'string-fill!', - 'string-foldcase', 'string-length', 'string-locale-ci<?', - 'string-locale-ci=?', 'string-locale-ci>?', - 'string-locale-downcase', 'string-locale-upcase', + 'string->some-system-path', 'string->symbol', + 'string->uninterned-symbol', 'string->unreadable-symbol', + 'string-append', 'string-append*', 'string-ci<=?', 'string-ci<?', + 'string-ci=?', 'string-ci>=?', 'string-ci>?', 'string-copy', + 'string-copy!', 'string-downcase', 'string-environment-variable-name?', + 'string-fill!', 'string-foldcase', 'string-join', 'string-len/c', + 'string-length', 'string-locale-ci<?', 'string-locale-ci=?', + 'string-locale-ci>?', 'string-locale-downcase', 'string-locale-upcase', 'string-locale<?', 'string-locale=?', 'string-locale>?', - 'string-normalize-nfc', 'string-normalize-nfd', - 'string-normalize-nfkc', 'string-normalize-nfkd', 'string-ref', - 'string-set!', 'string-titlecase', 'string-upcase', - 'string-utf-8-length', 'string<=?', 'string<?', 'string=?', - 'string>=?', 'string>?', 'string?', 'struct->vector', + 'string-no-nuls?', 'string-normalize-nfc', 'string-normalize-nfd', + 'string-normalize-nfkc', 'string-normalize-nfkd', + 'string-normalize-spaces', 'string-ref', 'string-replace', + 'string-set!', 'string-split', 'string-titlecase', 'string-trim', + 'string-upcase', 'string-utf-8-length', 'string<=?', 'string<?', + 'string=?', 'string>=?', 'string>?', 'string?', 'struct->vector', 'struct-accessor-procedure?', 'struct-constructor-procedure?', 'struct-info', 'struct-mutator-procedure?', 'struct-predicate-procedure?', 'struct-type-info', 'struct-type-make-constructor', 'struct-type-make-predicate', - 'struct-type-property-accessor-procedure?', + 'struct-type-property-accessor-procedure?', 'struct-type-property/c', 'struct-type-property?', 'struct-type?', 'struct:arity-at-least', 'struct:date', 'struct:date*', 'struct:exn', 'struct:exn:break', + 'struct:exn:break:hang-up', 'struct:exn:break:terminate', 'struct:exn:fail', 'struct:exn:fail:contract', - 'struct:exn:fail:contract:arity', + 'struct:exn:fail:contract:arity', 'struct:exn:fail:contract:blame', 'struct:exn:fail:contract:continuation', 'struct:exn:fail:contract:divide-by-zero', 'struct:exn:fail:contract:non-fixnum-result', 'struct:exn:fail:contract:variable', 'struct:exn:fail:filesystem', + 'struct:exn:fail:filesystem:errno', 'struct:exn:fail:filesystem:exists', + 'struct:exn:fail:filesystem:missing-module', 'struct:exn:fail:filesystem:version', 'struct:exn:fail:network', + 'struct:exn:fail:network:errno', 'struct:exn:fail:object', 'struct:exn:fail:out-of-memory', 'struct:exn:fail:read', 'struct:exn:fail:read:eof', 'struct:exn:fail:read:non-char', - 'struct:exn:fail:syntax', 'struct:exn:fail:syntax:unbound', - 'struct:exn:fail:unsupported', 'struct:exn:fail:user', - 'struct:srcloc', 'struct?', 'sub1', 'subbytes', 'subprocess', - 'subprocess-group-enabled', 'subprocess-kill', 'subprocess-pid', - 'subprocess-status', 'subprocess-wait', 'subprocess?', 'substring', - 'symbol->string', 'symbol-interned?', 'symbol-unreadable?', - 'symbol?', 'sync', 'sync/enable-break', 'sync/timeout', - 'sync/timeout/enable-break', 'syntax->list', 'syntax-arm', - 'syntax-column', 'syntax-disarm', 'syntax-e', 'syntax-line', - 'syntax-local-bind-syntaxes', 'syntax-local-certifier', - 'syntax-local-context', 'syntax-local-expand-expression', - 'syntax-local-get-shadower', 'syntax-local-introduce', - 'syntax-local-lift-context', 'syntax-local-lift-expression', + 'struct:exn:fail:syntax', 'struct:exn:fail:syntax:missing-module', + 'struct:exn:fail:syntax:unbound', 'struct:exn:fail:unsupported', + 'struct:exn:fail:user', 'struct:srcloc', + 'struct:wrapped-extra-arg-arrow', 'struct?', 'sub1', 'subbytes', + 'subclass?', 'subclass?/c', 'subprocess', 'subprocess-group-enabled', + 'subprocess-kill', 'subprocess-pid', 'subprocess-status', + 'subprocess-wait', 'subprocess?', 'subset?', 'substring', + 'symbol->string', 'symbol-interned?', 'symbol-unreadable?', 'symbol<?', + 'symbol=?', 'symbol?', 'symbols', 'sync', 'sync/enable-break', + 'sync/timeout', 'sync/timeout/enable-break', 'syntax->datum', + 'syntax->list', 'syntax-arm', 'syntax-column', 'syntax-disarm', + 'syntax-e', 'syntax-line', 'syntax-local-bind-syntaxes', + 'syntax-local-certifier', 'syntax-local-context', + 'syntax-local-expand-expression', 'syntax-local-get-shadower', + 'syntax-local-introduce', 'syntax-local-lift-context', + 'syntax-local-lift-expression', 'syntax-local-lift-module-end-declaration', 'syntax-local-lift-provide', 'syntax-local-lift-require', 'syntax-local-lift-values-expression', @@ -427,163 +695,229 @@ class RacketLexer(RegexLexer): 'syntax-local-module-defined-identifiers', 'syntax-local-module-exports', 'syntax-local-module-required-identifiers', 'syntax-local-name', - 'syntax-local-phase-level', + 'syntax-local-phase-level', 'syntax-local-submodules', 'syntax-local-transforming-module-provides?', 'syntax-local-value', - 'syntax-local-value/immediate', 'syntax-original?', - 'syntax-position', 'syntax-property', - 'syntax-property-symbol-keys', 'syntax-protect', 'syntax-rearm', - 'syntax-recertify', 'syntax-shift-phase-level', 'syntax-source', - 'syntax-source-module', 'syntax-span', 'syntax-taint', + 'syntax-local-value/immediate', 'syntax-original?', 'syntax-position', + 'syntax-property', 'syntax-property-symbol-keys', 'syntax-protect', + 'syntax-rearm', 'syntax-recertify', 'syntax-shift-phase-level', + 'syntax-source', 'syntax-source-module', 'syntax-span', 'syntax-taint', 'syntax-tainted?', 'syntax-track-origin', 'syntax-transforming-module-expression?', 'syntax-transforming?', - 'syntax?', 'system-big-endian?', 'system-idle-evt', - 'system-language+country', 'system-library-subpath', - 'system-path-convention-type', 'system-type', 'tan', - 'tcp-abandon-port', 'tcp-accept', 'tcp-accept-evt', - 'tcp-accept-ready?', 'tcp-accept/enable-break', 'tcp-addresses', - 'tcp-close', 'tcp-connect', 'tcp-connect/enable-break', - 'tcp-listen', 'tcp-listener?', 'tcp-port?', 'terminal-port?', - 'thread', 'thread-cell-ref', 'thread-cell-set!', 'thread-cell?', - 'thread-dead-evt', 'thread-dead?', 'thread-group?', - 'thread-resume', 'thread-resume-evt', 'thread-rewind-receive', - 'thread-running?', 'thread-suspend', 'thread-suspend-evt', + 'syntax/c', 'syntax?', 'system', 'system*', 'system*/exit-code', + 'system-big-endian?', 'system-idle-evt', 'system-language+country', + 'system-library-subpath', 'system-path-convention-type', 'system-type', + 'system/exit-code', 'tail-marks-match?', 'take', 'take-right', 'takef', + 'takef-right', 'tan', 'tanh', 'tcp-abandon-port', 'tcp-accept', + 'tcp-accept-evt', 'tcp-accept-ready?', 'tcp-accept/enable-break', + 'tcp-addresses', 'tcp-close', 'tcp-connect', + 'tcp-connect/enable-break', 'tcp-listen', 'tcp-listener?', 'tcp-port?', + 'tentative-pretty-print-port-cancel', + 'tentative-pretty-print-port-transfer', 'tenth', 'terminal-port?', + 'the-unsupplied-arg', 'third', 'thread', 'thread-cell-ref', + 'thread-cell-set!', 'thread-cell-values?', 'thread-cell?', + 'thread-dead-evt', 'thread-dead?', 'thread-group?', 'thread-receive', + 'thread-receive-evt', 'thread-resume', 'thread-resume-evt', + 'thread-rewind-receive', 'thread-running?', 'thread-send', + 'thread-suspend', 'thread-suspend-evt', 'thread-try-receive', 'thread-wait', 'thread/suspend-to-kill', 'thread?', 'time-apply', - 'truncate', 'udp-addresses', 'udp-bind!', 'udp-bound?', - 'udp-close', 'udp-connect!', 'udp-connected?', 'udp-open-socket', - 'udp-receive!', 'udp-receive!*', 'udp-receive!-evt', - 'udp-receive!/enable-break', 'udp-receive-ready-evt', 'udp-send', - 'udp-send*', 'udp-send-evt', 'udp-send-ready-evt', 'udp-send-to', - 'udp-send-to*', 'udp-send-to-evt', 'udp-send-to/enable-break', - 'udp-send/enable-break', 'udp?', 'unbox', - 'uncaught-exception-handler', 'use-collection-link-paths', + 'touch', 'transplant-input-port', 'transplant-output-port', 'true', + 'truncate', 'udp-addresses', 'udp-bind!', 'udp-bound?', 'udp-close', + 'udp-connect!', 'udp-connected?', 'udp-multicast-interface', + 'udp-multicast-join-group!', 'udp-multicast-leave-group!', + 'udp-multicast-loopback?', 'udp-multicast-set-interface!', + 'udp-multicast-set-loopback!', 'udp-multicast-set-ttl!', + 'udp-multicast-ttl', 'udp-open-socket', 'udp-receive!', + 'udp-receive!*', 'udp-receive!-evt', 'udp-receive!/enable-break', + 'udp-receive-ready-evt', 'udp-send', 'udp-send*', 'udp-send-evt', + 'udp-send-ready-evt', 'udp-send-to', 'udp-send-to*', 'udp-send-to-evt', + 'udp-send-to/enable-break', 'udp-send/enable-break', 'udp?', 'unbox', + 'uncaught-exception-handler', 'unit?', 'unspecified-dom', + 'unsupplied-arg?', 'use-collection-link-paths', 'use-compiled-file-paths', 'use-user-specific-search-paths', - 'values', 'variable-reference->empty-namespace', + 'user-execute-bit', 'user-read-bit', 'user-write-bit', + 'value-contract', 'values', 'variable-reference->empty-namespace', 'variable-reference->module-base-phase', 'variable-reference->module-declaration-inspector', - 'variable-reference->module-source', - 'variable-reference->namespace', 'variable-reference->phase', + 'variable-reference->module-path-index', + 'variable-reference->module-source', 'variable-reference->namespace', + 'variable-reference->phase', 'variable-reference->resolved-module-path', 'variable-reference-constant?', 'variable-reference?', 'vector', 'vector->immutable-vector', 'vector->list', - 'vector->pseudo-random-generator', - 'vector->pseudo-random-generator!', 'vector->values', - 'vector-fill!', 'vector-immutable', 'vector-length', 'vector-ref', - 'vector-set!', 'vector-set-performance-stats!', 'vector?', - 'version', 'void', 'void?', 'weak-box-value', 'weak-box?', - 'will-execute', 'will-executor?', 'will-register', - 'will-try-execute', 'with-input-from-file', 'with-output-to-file', - 'wrap-evt', 'write', 'write-byte', 'write-bytes', + 'vector->pseudo-random-generator', 'vector->pseudo-random-generator!', + 'vector->values', 'vector-append', 'vector-argmax', 'vector-argmin', + 'vector-copy', 'vector-copy!', 'vector-count', 'vector-drop', + 'vector-drop-right', 'vector-fill!', 'vector-filter', + 'vector-filter-not', 'vector-immutable', 'vector-immutable/c', + 'vector-immutableof', 'vector-length', 'vector-map', 'vector-map!', + 'vector-member', 'vector-memq', 'vector-memv', 'vector-ref', + 'vector-set!', 'vector-set*!', 'vector-set-performance-stats!', + 'vector-split-at', 'vector-split-at-right', 'vector-take', + 'vector-take-right', 'vector/c', 'vector?', 'vectorof', 'version', + 'void', 'void?', 'weak-box-value', 'weak-box?', 'weak-set', + 'weak-seteq', 'weak-seteqv', 'will-execute', 'will-executor?', + 'will-register', 'will-try-execute', 'with-input-from-bytes', + 'with-input-from-file', 'with-input-from-string', + 'with-output-to-bytes', 'with-output-to-file', 'with-output-to-string', + 'would-be-future', 'wrap-evt', 'wrapped-extra-arg-arrow', + 'wrapped-extra-arg-arrow-extra-neg-party-argument', + 'wrapped-extra-arg-arrow-real-func', 'wrapped-extra-arg-arrow?', + 'writable<%>', 'write', 'write-byte', 'write-bytes', 'write-bytes-avail', 'write-bytes-avail*', 'write-bytes-avail-evt', 'write-bytes-avail/enable-break', 'write-char', 'write-special', - 'write-special-avail*', 'write-special-evt', 'write-string', 'zero?' + 'write-special-avail*', 'write-special-evt', 'write-string', + 'write-to-file', 'xor', 'zero?', '~.a', '~.s', '~.v', '~a', '~e', '~r', + '~s', '~v' ] - # From SchemeLexer - valid_name = r'[a-zA-Z0-9!$%&*+,/:<=>?@^_~|-]+' + _opening_parenthesis = r'[([{]' + _closing_parenthesis = r'[)\]}]' + _delimiters = r'()[\]{}",\'`;\s' + _symbol = r'(?u)(?:\|[^|]*\||\\[\w\W]|[^|\\%s]+)+' % _delimiters + _number_prefix = r'(?:#e)?(?:#b|(?:#d)?)(?:#e)?' + _exponent = r'(?:[defls][-+]?\d+)' + _inexact_simple_no_hashes = r'(?:\d+(?:/\d+|\.\d*)?|\.\d+)' + _inexact_simple = (r'(?:%s|(?:\d+#+(?:\.#*|/\d+#*)?|\.\d+#+|' + r'\d+(?:\.\d*#+|/\d+#+)))' % _inexact_simple_no_hashes) + _inexact_normal_no_hashes = r'(?:%s%s?)' % (_inexact_simple_no_hashes, + _exponent) + _inexact_normal = r'(?:%s%s?)' % (_inexact_simple, _exponent) + _inexact_special = r'(?:(?:inf|nan)\.[0f])' + _inexact_real = r'(?:[-+]?%s|[-+]%s)' % (_inexact_normal, + _inexact_special) + _inexact_unsigned = r'(?:%s|%s)' % (_inexact_normal, _inexact_special) tokens = { - 'root' : [ - (r';.*$', Comment.Single), - (r'#\|[^|]+\|#', Comment.Multiline), - - # whitespaces - usually not relevant - (r'\s+', Text), - - ## numbers: Keep in mind Racket reader hash prefixes, - ## which can denote the base or the type. These don't map - ## neatly onto pygments token types; some judgment calls - ## here. Note that none of these regexps attempt to - ## exclude identifiers that start with a number, such as a - ## variable named "100-Continue". - - # #b - (r'#b[-+]?[01]+\.[01]+', Number.Float), - (r'#b[01]+e[-+]?[01]+', Number.Float), - (r'#b[-+]?[01]/[01]+', Number), - (r'#b[-+]?[01]+', Number.Integer), - (r'#b\S*', Error), - - # #d OR no hash prefix - (r'(#d)?[-+]?\d+\.\d+', Number.Float), - (r'(#d)?\d+e[-+]?\d+', Number.Float), - (r'(#d)?[-+]?\d+/\d+', Number), - (r'(#d)?[-+]?\d+', Number.Integer), - (r'#d\S*', Error), - - # #e - (r'#e[-+]?\d+\.\d+', Number.Float), - (r'#e\d+e[-+]?\d+', Number.Float), - (r'#e[-+]?\d+/\d+', Number), - (r'#e[-+]?\d+', Number), - (r'#e\S*', Error), - - # #i is always inexact-real, i.e. float - (r'#i[-+]?\d+\.\d+', Number.Float), - (r'#i\d+e[-+]?\d+', Number.Float), - (r'#i[-+]?\d+/\d+', Number.Float), - (r'#i[-+]?\d+', Number.Float), - (r'#i\S*', Error), + 'root': [ + (_closing_parenthesis, Error), + (r'(?!\Z)', Text, 'unquoted-datum') + ], + 'datum': [ + (r'(?s)#;|#![ /]([^\\\n]|\\.)*', Comment), + (u';[^\\n\\r\x85\u2028\u2029]*', Comment.Single), + (r'#\|', Comment.Multiline, 'block-comment'), + + # Whitespaces + (r'(?u)\s+', Text), + + # Numbers: Keep in mind Racket reader hash prefixes, which + # can denote the base or the type. These don't map neatly + # onto Pygments token types; some judgment calls here. + + # #b or #d or no prefix + (r'(?i)%s[-+]?\d+(?=[%s])' % (_number_prefix, _delimiters), + Number.Integer, '#pop'), + (r'(?i)%s[-+]?(\d+(\.\d*)?|\.\d+)([deflst][-+]?\d+)?(?=[%s])' % + (_number_prefix, _delimiters), Number.Float, '#pop'), + (r'(?i)%s[-+]?(%s([-+]%s?i)?|[-+]%s?i)(?=[%s])' % + (_number_prefix, _inexact_normal_no_hashes, + _inexact_normal_no_hashes, _inexact_normal_no_hashes, + _delimiters), Number, '#pop'), + + # Inexact without explicit #i + (r'(?i)(#[bd])?(%s([-+]%s?i)?|[-+]%s?i|%s@%s)(?=[%s])' % + (_inexact_real, _inexact_unsigned, _inexact_unsigned, + _inexact_real, _inexact_real, _delimiters), Number.Float, + '#pop'), + + # The remaining extflonums + (r'(?i)(([-+]?%st[-+]?\d+)|[-+](inf|nan)\.t)(?=[%s])' % + (_inexact_simple, _delimiters), Number.Float, '#pop'), # #o - (r'#o[-+]?[0-7]+\.[0-7]+', Number.Oct), - (r'#o[0-7]+e[-+]?[0-7]+', Number.Oct), - (r'#o[-+]?[0-7]+/[0-7]+', Number.Oct), - (r'#o[-+]?[0-7]+', Number.Oct), - (r'#o\S*', Error), + (r'(?i)(#[ei])?#o%s' % _symbol, Number.Oct, '#pop'), # #x - (r'#x[-+]?[0-9a-fA-F]+\.[0-9a-fA-F]+', Number.Hex), - # the exponent variation (e.g. #x1e1) is N/A - (r'#x[-+]?[0-9a-fA-F]+/[0-9a-fA-F]+', Number.Hex), - (r'#x[-+]?[0-9a-fA-F]+', Number.Hex), - (r'#x\S*', Error), - - - # strings, symbols and characters - (r'"(\\\\|\\"|[^"])*"', String), - (r"'" + valid_name, String.Symbol), - (r"#\\([()/'\"._!§$%& ?=+-]{1}|[a-zA-Z0-9]+)", String.Char), - (r'#rx".+"', String.Regex), - (r'#px".+"', String.Regex), - - # constants - (r'(#t|#f)', Name.Constant), - - # keyword argument names (e.g. #:keyword) - (r'#:\S+', Keyword.Declaration), - - # #lang - (r'#lang \S+', Keyword.Namespace), - - # special operators - (r"('|#|`|,@|,|\.)", Operator), - - # highlight the keywords - ('(%s)' % '|'.join([ - re.escape(entry) + ' ' for entry in keywords]), - Keyword - ), - - # first variable in a quoted string like - # '(this is syntactic sugar) - (r"(?<='\()" + valid_name, Name.Variable), - (r"(?<=#\()" + valid_name, Name.Variable), - - # highlight the builtins - ("(?<=\()(%s)" % '|'.join([ - re.escape(entry) + ' ' for entry in builtins]), - Name.Builtin - ), - - # the remaining functions; handle both ( and [ - (r'(?<=(\(|\[|\{))' + valid_name, Name.Function), - - # find the remaining variables - (valid_name, Name.Variable), - - # the famous parentheses! - (r'(\(|\)|\[|\]|\{|\})', Punctuation), + (r'(?i)(#[ei])?#x%s' % _symbol, Number.Hex, '#pop'), + + # #i is always inexact, i.e. float + (r'(?i)(#[bd])?#i%s' % _symbol, Number.Float, '#pop'), + + # Strings and characters + (r'#?"', String.Double, ('#pop', 'string')), + (r'#<<(.+)\n(^(?!\1$).*$\n)*^\1$', String.Heredoc, '#pop'), + (r'#\\(u[\da-fA-F]{1,4}|U[\da-fA-F]{1,8})', String.Char, '#pop'), + (r'(?is)#\\([0-7]{3}|[a-z]+|.)', String.Char, '#pop'), + (r'(?s)#[pr]x#?"(\\?.)*?"', String.Regex, '#pop'), + + # Constants + (r'#(true|false|[tTfF])', Name.Constant, '#pop'), + + # Keyword argument names (e.g. #:keyword) + (r'#:%s' % _symbol, Keyword.Declaration, '#pop'), + + # Reader extensions + (r'(#lang |#!)(\S+)', + bygroups(Keyword.Namespace, Name.Namespace)), + (r'#reader', Keyword.Namespace, 'quoted-datum'), + + # Other syntax + (r"(?i)\.(?=[%s])|#c[is]|#['`]|#,@?" % _delimiters, Operator), + (r"'|#[s&]|#hash(eqv?)?|#\d*(?=%s)" % _opening_parenthesis, + Operator, ('#pop', 'quoted-datum')) + ], + 'datum*': [ + (r'`|,@?', Operator), + (_symbol, String.Symbol, '#pop'), + (r'[|\\]', Error), + (r'', Text, '#pop') + ], + 'list': [ + (_closing_parenthesis, Punctuation, '#pop') + ], + 'unquoted-datum': [ + include('datum'), + (r'quote(?=[%s])' % _delimiters, Keyword, + ('#pop', 'quoted-datum')), + (r'`', Operator, ('#pop', 'quasiquoted-datum')), + (r'quasiquote(?=[%s])' % _delimiters, Keyword, + ('#pop', 'quasiquoted-datum')), + (_opening_parenthesis, Punctuation, ('#pop', 'unquoted-list')), + (r'(?u)(%s)(?=[%s])' % ('|'.join( + [re.escape(entry) for entry in _keywords]), _delimiters), + Keyword, '#pop'), + (r'(?u)(%s)(?=[%s])' % ('|'.join( + [re.escape(entry) for entry in _builtins]), _delimiters), + Name.Builtin, '#pop'), + (_symbol, Name, '#pop'), + include('datum*') + ], + 'unquoted-list': [ + include('list'), + (r'(?!\Z)', Text, 'unquoted-datum') + ], + 'quasiquoted-datum': [ + include('datum'), + (r',@?', Operator, ('#pop', 'unquoted-datum')), + (r'unquote(-splicing)?(?=[%s])' % _delimiters, Keyword, + ('#pop', 'unquoted-datum')), + (_opening_parenthesis, Punctuation, ('#pop', 'quasiquoted-list')), + include('datum*') + ], + 'quasiquoted-list': [ + include('list'), + (r'(?!\Z)', Text, 'quasiquoted-datum') + ], + 'quoted-datum': [ + include('datum'), + (_opening_parenthesis, Punctuation, ('#pop', 'quoted-list')), + include('datum*') + ], + 'quoted-list': [ + include('list'), + (r'(?!\Z)', Text, 'quoted-datum') + ], + 'block-comment': [ + (r'#\|', Comment.Multiline, '#push'), + (r'\|#', Comment.Multiline, '#pop'), + (r'[^#|]+|.', Comment.Multiline) ], + 'string': [ + (r'"', String.Double, '#pop'), + (r'(?s)\\([0-7]{1,3}|x[\da-fA-F]{1,2}|u[\da-fA-F]{1,4}|' + r'U[\da-fA-F]{1,8}|.)', String.Escape), + (r'[^\\"]+', String.Double) + ] } @@ -732,7 +1066,7 @@ class CommonLispLexer(RegexLexer): ### couple of useful regexes # characters that are not macro-characters and can be used to begin a symbol - nonmacro = r'\\.|[a-zA-Z0-9!$%&*+-/<=>?@\[\]^_{}~]' + nonmacro = r'\\.|[\w!$%&*+-/<=>?@\[\]^{}~]' constituent = nonmacro + '|[#.:]' terminated = r'(?=[ "()\'\n,;`])' # whitespace or terminating macro characters @@ -850,10 +1184,10 @@ class CommonLispLexer(RegexLexer): (r'#[oO][+-]?[0-7]+(/[0-7]+)?', Number.Oct), # hex rational - (r'#[xX][+-]?[0-9a-fA-F]+(/[0-9a-fA-F]+)?', Number.Hex), + (r'#[xX][+-]?[0-9a-f]+(/[0-9a-f]+)?', Number.Hex), # radix rational - (r'#\d+[rR][+-]?[0-9a-zA-Z]+(/[0-9a-zA-Z]+)?', Number), + (r'#\d+[rR][+-]?[0-9a-z]+(/[0-9a-z]+)?', Number), # complex (r'(#[cC])(\()', bygroups(Number, Punctuation), 'body'), @@ -894,6 +1228,142 @@ class CommonLispLexer(RegexLexer): } +class CryptolLexer(RegexLexer): + """ + FIXME: A Cryptol2 lexer based on the lexemes defined in the Haskell 98 Report. + + .. versionadded:: 2.0 + """ + name = 'Cryptol' + aliases = ['cryptol', 'cry'] + filenames = ['*.cry'] + mimetypes = ['text/x-cryptol'] + + reserved = ['Arith','Bit','Cmp','False','Inf','True','else', + 'export','extern','fin','if','import','inf','lg2', + 'max','min','module','newtype','pragma','property', + 'then','type','where','width'] + ascii = ['NUL','SOH','[SE]TX','EOT','ENQ','ACK', + 'BEL','BS','HT','LF','VT','FF','CR','S[OI]','DLE', + 'DC[1-4]','NAK','SYN','ETB','CAN', + 'EM','SUB','ESC','[FGRU]S','SP','DEL'] + + tokens = { + 'root': [ + # Whitespace: + (r'\s+', Text), + #(r'--\s*|.*$', Comment.Doc), + (r'//.*$', Comment.Single), + (r'/\*', Comment.Multiline, 'comment'), + # Lexemes: + # Identifiers + (r'\bimport\b', Keyword.Reserved, 'import'), + (r'\bmodule\b', Keyword.Reserved, 'module'), + (r'\berror\b', Name.Exception), + (r'\b(%s)(?!\')\b' % '|'.join(reserved), Keyword.Reserved), + (r'^[_a-z][\w\']*', Name.Function), + (r"'?[_a-z][\w']*", Name), + (r"('')?[A-Z][\w\']*", Keyword.Type), + # Operators + (r'\\(?![:!#$%&*+.\\/<=>?@^|~-]+)', Name.Function), # lambda operator + (r'(<-|::|->|=>|=)(?![:!#$%&*+.\\/<=>?@^|~-]+)', Operator.Word), # specials + (r':[:!#$%&*+.\\/<=>?@^|~-]*', Keyword.Type), # Constructor operators + (r'[:!#$%&*+.\\/<=>?@^|~-]+', Operator), # Other operators + # Numbers + (r'\d+[eE][+-]?\d+', Number.Float), + (r'\d+\.\d+([eE][+-]?\d+)?', Number.Float), + (r'0[oO][0-7]+', Number.Oct), + (r'0[xX][\da-fA-F]+', Number.Hex), + (r'\d+', Number.Integer), + # Character/String Literals + (r"'", String.Char, 'character'), + (r'"', String, 'string'), + # Special + (r'\[\]', Keyword.Type), + (r'\(\)', Name.Builtin), + (r'[][(),;`{}]', Punctuation), + ], + 'import': [ + # Import statements + (r'\s+', Text), + (r'"', String, 'string'), + # after "funclist" state + (r'\)', Punctuation, '#pop'), + (r'qualified\b', Keyword), + # import X as Y + (r'([A-Z][a-zA-Z0-9_.]*)(\s+)(as)(\s+)([A-Z][a-zA-Z0-9_.]*)', + bygroups(Name.Namespace, Text, Keyword, Text, Name), '#pop'), + # import X hiding (functions) + (r'([A-Z][a-zA-Z0-9_.]*)(\s+)(hiding)(\s+)(\()', + bygroups(Name.Namespace, Text, Keyword, Text, Punctuation), 'funclist'), + # import X (functions) + (r'([A-Z][a-zA-Z0-9_.]*)(\s+)(\()', + bygroups(Name.Namespace, Text, Punctuation), 'funclist'), + # import X + (r'[a-zA-Z0-9_.]+', Name.Namespace, '#pop'), + ], + 'module': [ + (r'\s+', Text), + (r'([A-Z][a-zA-Z0-9_.]*)(\s+)(\()', + bygroups(Name.Namespace, Text, Punctuation), 'funclist'), + (r'[A-Z][a-zA-Z0-9_.]*', Name.Namespace, '#pop'), + ], + 'funclist': [ + (r'\s+', Text), + (r'[A-Z][a-zA-Z0-9_]*', Keyword.Type), + (r'(_[\w\']+|[a-z][\w\']*)', Name.Function), + (r'--(?![!#$%&*+./<=>?@\^|_~:\\]).*?$', Comment.Single), + (r'{-', Comment.Multiline, 'comment'), + (r',', Punctuation), + (r'[:!#$%&*+.\\/<=>?@^|~-]+', Operator), + # (HACK, but it makes sense to push two instances, believe me) + (r'\(', Punctuation, ('funclist', 'funclist')), + (r'\)', Punctuation, '#pop:2'), + ], + 'comment': [ + # Multiline Comments + (r'[^/\*]+', Comment.Multiline), + (r'/\*', Comment.Multiline, '#push'), + (r'\*/', Comment.Multiline, '#pop'), + (r'[\*/]', Comment.Multiline), + ], + 'character': [ + # Allows multi-chars, incorrectly. + (r"[^\\']'", String.Char, '#pop'), + (r"\\", String.Escape, 'escape'), + ("'", String.Char, '#pop'), + ], + 'string': [ + (r'[^\\"]+', String), + (r"\\", String.Escape, 'escape'), + ('"', String, '#pop'), + ], + 'escape': [ + (r'[abfnrtv"\'&\\]', String.Escape, '#pop'), + (r'\^[][A-Z@\^_]', String.Escape, '#pop'), + ('|'.join(ascii), String.Escape, '#pop'), + (r'o[0-7]+', String.Escape, '#pop'), + (r'x[\da-fA-F]+', String.Escape, '#pop'), + (r'\d+', String.Escape, '#pop'), + (r'\s+\\', String.Escape, '#pop'), + ], + } + + EXTRA_KEYWORDS = ['join', 'split', 'reverse', 'transpose', 'width', + 'length', 'tail', '<<', '>>', '<<<', '>>>', 'const', + 'reg', 'par', 'seq', 'ASSERT', 'undefined', 'error', + 'trace'] + + def get_tokens_unprocessed(self, text): + stack = ['root'] + for index, token, value in \ + RegexLexer.get_tokens_unprocessed(self, text, stack): + if token is Name and value in self.EXTRA_KEYWORDS: + yield index, Name.Builtin, value + else: + yield index, token, value + + class HaskellLexer(RegexLexer): """ A Haskell lexer based on the lexemes defined in the Haskell 98 Report. @@ -905,6 +1375,8 @@ class HaskellLexer(RegexLexer): filenames = ['*.hs'] mimetypes = ['text/x-haskell'] + flags = re.MULTILINE | re.UNICODE + reserved = ['case','class','data','default','deriving','do','else', 'if','in','infix[lr]?','instance', 'let','newtype','of','then','type','where','_'] @@ -926,9 +1398,9 @@ class HaskellLexer(RegexLexer): (r'\bmodule\b', Keyword.Reserved, 'module'), (r'\berror\b', Name.Exception), (r'\b(%s)(?!\')\b' % '|'.join(reserved), Keyword.Reserved), - (r'^[_a-z][\w\']*', Name.Function), - (r"'?[_a-z][\w']*", Name), - (r"('')?[A-Z][\w\']*", Keyword.Type), + (r'^[_' + uni.Ll + r'][\w\']*', Name.Function), + (r"'?[_" + uni.Ll + r"'][\w']*", Name), + (r"('')?[" + uni.Lu + r"][\w\']*", Keyword.Type), # Operators (r'\\(?![:!#$%&*+.\\/<=>?@^|~-]+)', Name.Function), # lambda operator (r'(<-|::|->|=>|=)(?![:!#$%&*+.\\/<=>?@^|~-]+)', Operator.Word), # specials @@ -956,27 +1428,27 @@ class HaskellLexer(RegexLexer): (r'\)', Punctuation, '#pop'), (r'qualified\b', Keyword), # import X as Y - (r'([A-Z][a-zA-Z0-9_.]*)(\s+)(as)(\s+)([A-Z][a-zA-Z0-9_.]*)', + (r'([' + uni.Lu + r'][\w.]*)(\s+)(as)(\s+)([' + uni.Lu + r'][\w.]*)', bygroups(Name.Namespace, Text, Keyword, Text, Name), '#pop'), # import X hiding (functions) - (r'([A-Z][a-zA-Z0-9_.]*)(\s+)(hiding)(\s+)(\()', + (r'([' + uni.Lu + r'][\w.]*)(\s+)(hiding)(\s+)(\()', bygroups(Name.Namespace, Text, Keyword, Text, Punctuation), 'funclist'), # import X (functions) - (r'([A-Z][a-zA-Z0-9_.]*)(\s+)(\()', + (r'([' + uni.Lu + r'][\w.]*)(\s+)(\()', bygroups(Name.Namespace, Text, Punctuation), 'funclist'), # import X - (r'[a-zA-Z0-9_.]+', Name.Namespace, '#pop'), + (r'[\w.]+', Name.Namespace, '#pop'), ], 'module': [ (r'\s+', Text), - (r'([A-Z][a-zA-Z0-9_.]*)(\s+)(\()', + (r'([' + uni.Lu + r'][\w.]*)(\s+)(\()', bygroups(Name.Namespace, Text, Punctuation), 'funclist'), - (r'[A-Z][a-zA-Z0-9_.]*', Name.Namespace, '#pop'), + (r'[' + uni.Lu + r'][\w.]*', Name.Namespace, '#pop'), ], 'funclist': [ (r'\s+', Text), - (r'[A-Z][a-zA-Z0-9_]*', Keyword.Type), - (r'(_[\w\']+|[a-z][\w\']*)', Name.Function), + (r'[' + uni.Lu + r']\w*', Keyword.Type), + (r'(_[\w\']+|[' + uni.Ll + r'][\w\']*)', Name.Function), (r'--(?![!#$%&*+./<=>?@\^|_~:\\]).*?$', Comment.Single), (r'{-', Comment.Multiline, 'comment'), (r',', Punctuation), @@ -1007,7 +1479,7 @@ class HaskellLexer(RegexLexer): ], 'escape': [ (r'[abfnrtv"\'&\\]', String.Escape, '#pop'), - (r'\^[][A-Z@\^_]', String.Escape, '#pop'), + (r'\^[][' + uni.Lu + r'@\^_]', String.Escape, '#pop'), ('|'.join(ascii), String.Escape, '#pop'), (r'o[0-7]+', String.Escape, '#pop'), (r'x[\da-fA-F]+', String.Escape, '#pop'), @@ -1061,7 +1533,7 @@ class IdrisLexer(RegexLexer): (r'\b(%s)(?!\')\b' % '|'.join(reserved), Keyword.Reserved), (r'(import|module)(\s+)', bygroups(Keyword.Reserved, Text), 'module'), (r"('')?[A-Z][\w\']*", Keyword.Type), - (r'[a-z][A-Za-z0-9_\']*', Text), + (r'[a-z][\w\']*', Text), # Special Symbols (r'(<-|::|->|=>|=)', Operator.Word), # specials (r'([\(\)\{\}\[\]:!#$%&*+.\\/<=>?@^|~-]+)', Operator.Word), # specials @@ -1078,13 +1550,13 @@ class IdrisLexer(RegexLexer): ], 'module': [ (r'\s+', Text), - (r'([A-Z][a-zA-Z0-9_.]*)(\s+)(\()', + (r'([A-Z][\w.]*)(\s+)(\()', bygroups(Name.Namespace, Text, Punctuation), 'funclist'), - (r'[A-Z][a-zA-Z0-9_.]*', Name.Namespace, '#pop'), + (r'[A-Z][\w.]*', Name.Namespace, '#pop'), ], 'funclist': [ (r'\s+', Text), - (r'[A-Z][a-zA-Z0-9_]*', Keyword.Type), + (r'[A-Z]\w*', Keyword.Type), (r'(_[\w\']+|[a-z][\w\']*)', Name.Function), (r'--.*$', Comment.Single), (r'{-', Comment.Multiline, 'comment'), @@ -1163,7 +1635,7 @@ class AgdaLexer(RegexLexer): (r'\b(Set|Prop)\b', Keyword.Type), # Special Symbols (r'(\(|\)|\{|\})', Operator), - (u'(\\.{1,3}|\\||[\u039B]|[\u2200]|[\u2192]|:|=|->)', Operator.Word), + (u'(\\.{1,3}|\\||\u039B|\u2200|\u2192|:|=|->)', Operator.Word), # Numbers (r'\d+[eE][+-]?\d+', Number.Float), (r'\d+\.\d+([eE][+-]?\d+)?', Number.Float), @@ -1184,7 +1656,7 @@ class AgdaLexer(RegexLexer): ], 'module': [ (r'{-', Comment.Multiline, 'comment'), - (r'[a-zA-Z][a-zA-Z0-9_.]*', Name, '#pop'), + (r'[a-zA-Z][\w.]*', Name, '#pop'), (r'[^a-zA-Z]*', Text) ], 'comment': HaskellLexer.tokens['comment'], @@ -1259,6 +1731,29 @@ class LiterateLexer(Lexer): yield item +class LiterateCryptolLexer(LiterateLexer): + """ + For Literate Cryptol (Bird-style or LaTeX) source. + + Additional options accepted: + + `litstyle` + If given, must be ``"bird"`` or ``"latex"``. If not given, the style + is autodetected: if the first non-whitespace character in the source + is a backslash or percent character, LaTeX is assumed, else Bird. + + .. versionadded:: 0.9 + """ + name = 'Literate Cryptol' + aliases = ['lcry', 'literate-cryptol', 'lcryptol'] + filenames = ['*.lcry'] + mimetypes = ['text/x-literate-cryptol'] + + def __init__(self, **options): + crylexer = CryptolLexer(**options) + LiterateLexer.__init__(self, crylexer, **options) + + class LiterateHaskellLexer(LiterateLexer): """ For Literate Haskell (Bird-style or LaTeX) source. @@ -1360,7 +1855,7 @@ class SMLLexer(RegexLexer): nonid_reserved = [ '(', ')', '[', ']', '{', '}', ',', ';', '...', '_' ] - alphanumid_re = r"[a-zA-Z][a-zA-Z0-9_']*" + alphanumid_re = r"[a-zA-Z][\w']*" symbolicid_re = r"[!%&$#+\-/:<=>?@\\~`^|*]+" # A character constant is a sequence of the form #s, where s is a string @@ -1450,7 +1945,7 @@ class SMLLexer(RegexLexer): (r'\b(type|eqtype)\b(?!\')', Keyword.Reserved, 'tname'), # Regular identifiers, long and otherwise - (r'\'[0-9a-zA-Z_\']*', Name.Decorator), + (r'\'[\w\']*', Name.Decorator), (r'(%s)(\.)' % alphanumid_re, long_id_callback, "dotted"), (r'(%s)' % alphanumid_re, id_callback), (r'(%s)' % symbolicid_re, id_callback), @@ -1697,9 +2192,9 @@ class OcamlLexer(RegexLexer): 'root': [ (r'\s+', Text), (r'false|true|\(\)|\[\]', Name.Builtin.Pseudo), - (r'\b([A-Z][A-Za-z0-9_\']*)(?=\s*\.)', + (r'\b([A-Z][\w\']*)(?=\s*\.)', Name.Namespace, 'dotted'), - (r'\b([A-Z][A-Za-z0-9_\']*)', Name.Class), + (r'\b([A-Z][\w\']*)', Name.Class), (r'\(\*(?![)])', Comment, 'comment'), (r'\b(%s)\b' % '|'.join(keywords), Keyword), (r'(%s)' % '|'.join(keyopts[::-1]), Operator), @@ -1739,9 +2234,9 @@ class OcamlLexer(RegexLexer): 'dotted': [ (r'\s+', Text), (r'\.', Punctuation), - (r'[A-Z][A-Za-z0-9_\']*(?=\s*\.)', Name.Namespace), - (r'[A-Z][A-Za-z0-9_\']*', Name.Class, '#pop'), - (r'[a-z_][A-Za-z0-9_\']*', Name, '#pop'), + (r'[A-Z][\w\']*(?=\s*\.)', Name.Namespace), + (r'[A-Z][\w\']*', Name.Class, '#pop'), + (r'[a-z_][\w\']*', Name, '#pop'), ], } @@ -1801,9 +2296,9 @@ class ErlangLexer(RegexLexer): 'div', 'not', 'or', 'orelse', 'rem', 'xor' ] - atom_re = r"(?:[a-z][a-zA-Z0-9_]*|'[^\n']*[^\\]')" + atom_re = r"(?:[a-z]\w*|'[^\n']*[^\\]')" - variable_re = r'(?:[A-Z_][a-zA-Z0-9_]*)' + variable_re = r'(?:[A-Z_]\w*)' escape_re = r'(?:\\(?:[bdefnrstv\'"\\/]|[0-7][0-7]?[0-7]?|\^[a-zA-Z]))' @@ -2309,9 +2804,9 @@ class CoqLexer(RegexLexer): (r'\b(%s)\b' % '|'.join(keywords4), Keyword), (r'\b(%s)\b' % '|'.join(keywords5), Keyword.Pseudo), (r'\b(%s)\b' % '|'.join(keywords6), Keyword.Reserved), - (r'\b([A-Z][A-Za-z0-9_\']*)(?=\s*\.)', + (r'\b([A-Z][\w\']*)(?=\s*\.)', Name.Namespace, 'dotted'), - (r'\b([A-Z][A-Za-z0-9_\']*)', Name.Class), + (r'\b([A-Z][\w\']*)', Name.Class), (r'(%s)' % '|'.join(keyopts[::-1]), Operator), (r'(%s|%s)?%s' % (infix_syms, prefix_syms, operators), Operator), (r'\b(%s)\b' % '|'.join(word_operators), Operator.Word), @@ -2348,8 +2843,8 @@ class CoqLexer(RegexLexer): 'dotted': [ (r'\s+', Text), (r'\.', Punctuation), - (r'[A-Z][A-Za-z0-9_\']*(?=\s*\.)', Name.Namespace), - (r'[A-Z][A-Za-z0-9_\']*', Name.Class, '#pop'), + (r'[A-Z][\w\']*(?=\s*\.)', Name.Namespace), + (r'[A-Z][\w\']*', Name.Class, '#pop'), (r'[a-z][a-z0-9_\']*', Name, '#pop'), (r'', Text, '#pop') ], @@ -2437,7 +2932,7 @@ class NewLispLexer(RegexLexer): ] # valid names - valid_name = r'([a-zA-Z0-9!$%&*+.,/<=>?@^_~|-])+|(\[.*?\])+' + valid_name = r'([\w!$%&*+.,/<=>?@^~|-])+|(\[.*?\])+' tokens = { 'root': [ @@ -2552,15 +3047,15 @@ class NixLexer(RegexLexer): (r"''", String.Single, 'singlequote'), # paths - (r'[a-zA-Z0-9._+-]*(\/[a-zA-Z0-9._+-]+)+', Literal), - (r'\<[a-zA-Z0-9._+-]+(\/[a-zA-Z0-9._+-]+)*\>', Literal), + (r'[\w.+-]*(\/[\w.+-]+)+', Literal), + (r'\<[\w.+-]+(\/[\w.+-]+)*\>', Literal), # urls - (r'[a-zA-Z][a-zA-Z0-9\+\-\.]*\:[a-zA-Z0-9%/?:@&=+$,\\_.!~*\'-]+', Literal), + (r'[a-zA-Z][a-zA-Z0-9\+\-\.]*\:[\w%/?:@&=+$,\\.!~*\'-]+', Literal), # names of variables - (r'[a-zA-Z0-9-_]+\s*=', String.Symbol), - (r'[a-zA-Z_][a-zA-Z0-9_\'-]*', Text), + (r'[\w-]+\s*=', String.Symbol), + (r'[a-zA-Z_][\w\'-]*', Text), ], 'comment': [ @@ -2679,7 +3174,7 @@ class ElixirLexer(RegexLexer): (r':"', String.Symbol, 'interpoling_symbol'), (r'\b(nil|true|false)\b(?![?!])|\b[A-Z]\w*\b', Name.Constant), (r'\b(__(FILE|LINE|MODULE|MAIN|FUNCTION)__)\b(?![?!])', Name.Builtin.Pseudo), - (r'[a-zA-Z_!][\w_]*[!\?]?', Name), + (r'[a-zA-Z_!]\w*[!\?]?', Name), (r'[(){};,/\|:\\\[\]]', Punctuation), (r'@[a-zA-Z_]\w*|&\d', Name.Variable), (r'\b(0[xX][0-9A-Fa-f]+|\d(_?\d)*(\.(?![^\d\s])' diff --git a/pygments/lexers/graph.py b/pygments/lexers/graph.py index fccba5a4..6aa446c7 100644 --- a/pygments/lexers/graph.py +++ b/pygments/lexers/graph.py @@ -73,7 +73,7 @@ class CypherLexer(RegexLexer): (r'\s+', Whitespace), ], 'barewords': [ - (r'[a-z][a-zA-Z0-9_]*', Name), + (r'[a-z]\w*', Name), (r'\d+', Number), ], } diff --git a/pygments/lexers/hdl.py b/pygments/lexers/hdl.py index 1ebe4e5c..6240cfe0 100644 --- a/pygments/lexers/hdl.py +++ b/pygments/lexers/hdl.py @@ -54,7 +54,7 @@ class VerilogLexer(RegexLexer): (r'\*/', Error), (r'[~!%^&*+=|?:<>/-]', Operator), (r'[()\[\],.;\']', Punctuation), - (r'`[a-zA-Z_][a-zA-Z0-9_]*', Name.Constant), + (r'`[a-zA-Z_]\w*', Name.Constant), (r'^(\s*)(package)(\s+)', bygroups(Text, Keyword.Namespace, Text)), (r'^(\s*)(import)(\s+)', bygroups(Text, Keyword.Namespace, Text), @@ -96,8 +96,8 @@ class VerilogLexer(RegexLexer): r'bit|logic|reg|' r'supply0|supply1|tri|triand|trior|tri0|tri1|trireg|uwire|wire|wand|wor' r'shortreal|real|realtime)\b', Keyword.Type), - ('[a-zA-Z_][a-zA-Z0-9_]*:(?!:)', Name.Label), - ('[a-zA-Z_][a-zA-Z0-9_]*', Name), + ('[a-zA-Z_]\w*:(?!:)', Name.Label), + ('[a-zA-Z_]\w*', Name), ], 'string': [ (r'"', String, '#pop'), @@ -115,7 +115,7 @@ class VerilogLexer(RegexLexer): (r'\n', Comment.Preproc, '#pop'), ], 'import': [ - (r'[a-zA-Z0-9_:]+\*?', Name.Namespace, '#pop') + (r'[\w:]+\*?', Name.Namespace, '#pop') ] } @@ -169,7 +169,7 @@ class SystemVerilogLexer(RegexLexer): (r'\*/', Error), (r'[~!%^&*+=|?:<>/-]', Operator), (r'[()\[\],.;\']', Punctuation), - (r'`[a-zA-Z_][a-zA-Z0-9_]*', Name.Constant), + (r'`[a-zA-Z_]\w*', Name.Constant), (r'(accept_on|alias|always|always_comb|always_ff|always_latch|' r'and|assert|assign|assume|automatic|before|begin|bind|bins|' @@ -230,11 +230,11 @@ class SystemVerilogLexer(RegexLexer): r'bit|logic|reg|' r'supply0|supply1|tri|triand|trior|tri0|tri1|trireg|uwire|wire|wand|wor' r'shortreal|real|realtime)\b', Keyword.Type), - ('[a-zA-Z_][a-zA-Z0-9_]*:(?!:)', Name.Label), - ('[a-zA-Z_][a-zA-Z0-9_]*', Name), + ('[a-zA-Z_]\w*:(?!:)', Name.Label), + ('[a-zA-Z_]\w*', Name), ], 'classname': [ - (r'[a-zA-Z_][a-zA-Z0-9_]*', Name.Class, '#pop'), + (r'[a-zA-Z_]\w*', Name.Class, '#pop'), ], 'string': [ (r'"', String, '#pop'), @@ -252,7 +252,7 @@ class SystemVerilogLexer(RegexLexer): (r'\n', Comment.Preproc, '#pop'), ], 'import': [ - (r'[a-zA-Z0-9_:]+\*?', Name.Namespace, '#pop') + (r'[\w:]+\*?', Name.Namespace, '#pop') ] } @@ -290,19 +290,19 @@ class VhdlLexer(RegexLexer): (r'--(?![!#$%&*+./<=>?@\^|_~]).*?$', Comment.Single), (r"'(U|X|0|1|Z|W|L|H|-)'", String.Char), (r'[~!%^&*+=|?:<>/-]', Operator), - (r"'[a-zA-Z_][a-zA-Z0-9_]*", Name.Attribute), + (r"'[a-z_]\w*", Name.Attribute), (r'[()\[\],.;\']', Punctuation), (r'"[^\n\\]*"', String), - (r'(library)(\s+)([a-zA-Z_][a-zA-Z0-9_]*)', + (r'(library)(\s+)([a-z_]\w*)', bygroups(Keyword, Text, Name.Namespace)), (r'(use)(\s+)(entity)', bygroups(Keyword, Text, Keyword)), - (r'(use)(\s+)([a-zA-Z_][\.a-zA-Z0-9_]*)', + (r'(use)(\s+)([a-z_][\.\w]*)', bygroups(Keyword, Text, Name.Namespace)), - (r'(entity|component)(\s+)([a-zA-Z_][a-zA-Z0-9_]*)', + (r'(entity|component)(\s+)([a-z_]\w*)', bygroups(Keyword, Text, Name.Class)), - (r'(architecture|configuration)(\s+)([a-zA-Z_][a-zA-Z0-9_]*)(\s+)' - r'(of)(\s+)([a-zA-Z_][a-zA-Z0-9_]*)(\s+)(is)', + (r'(architecture|configuration)(\s+)([a-z_]\w*)(\s+)' + r'(of)(\s+)([a-z_]\w*)(\s+)(is)', bygroups(Keyword, Text, Name.Class, Text, Keyword, Text, Name.Class, Text, Keyword)), @@ -312,11 +312,11 @@ class VhdlLexer(RegexLexer): include('keywords'), include('numbers'), - (r'[a-zA-Z_][a-zA-Z0-9_]*', Name), + (r'[a-z_]\w*', Name), ], 'endblock': [ include('keywords'), - (r'[a-zA-Z_][a-zA-Z0-9_]*', Name.Class), + (r'[a-z_]\w*', Name.Class), (r'(\s+)', Text), (r';', Punctuation, '#pop'), ], @@ -345,11 +345,11 @@ class VhdlLexer(RegexLexer): r'while|with|xnor|xor)\b', Keyword), ], 'numbers': [ - (r'\d{1,2}#[0-9a-fA-F_]+#?', Number.Integer), + (r'\d{1,2}#[0-9a-f_]+#?', Number.Integer), (r'[0-1_]+(\.[0-1_])', Number.Integer), (r'\d+', Number.Integer), (r'(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+', Number.Float), - (r'H"[0-9a-fA-F_]+"', Number.Oct), + (r'H"[0-9a-f_]+"', Number.Oct), (r'O"[0-7_]+"', Number.Oct), (r'B"[0-1_]+"', Number.Oct), ], diff --git a/pygments/lexers/inferno.py b/pygments/lexers/inferno.py index 56f02b98..6aecafdb 100644 --- a/pygments/lexers/inferno.py +++ b/pygments/lexers/inferno.py @@ -35,7 +35,7 @@ class LimboLexer(RegexLexer): tokens = { 'whitespace': [ - (r'^(\s*)([a-zA-Z_][a-zA-Z0-9_]*:(\s*)\n)', + (r'^(\s*)([a-zA-Z_]\w*:(\s*)\n)', bygroups(Text, Name.Label)), (r'\n', Text), (r'\s+', Text), @@ -64,7 +64,7 @@ class LimboLexer(RegexLexer): (r'(byte|int|big|real|string|array|chan|list|adt' r'|fn|ref|of|module|self|type)\b', Keyword.Type), (r'(con|iota|nil)\b', Keyword.Constant), - ('[a-zA-Z_][a-zA-Z0-9_]*', Name), + ('[a-zA-Z_]\w*', Name), ], 'statement' : [ include('whitespace'), diff --git a/pygments/lexers/jvm.py b/pygments/lexers/jvm.py index 4b91e723..4633c66f 100644 --- a/pygments/lexers/jvm.py +++ b/pygments/lexers/jvm.py @@ -21,7 +21,7 @@ from pygments import unistring as uni __all__ = ['JavaLexer', 'ScalaLexer', 'GosuLexer', 'GosuTemplateLexer', 'GroovyLexer', 'IokeLexer', 'ClojureLexer', 'ClojureScriptLexer', 'KotlinLexer', 'XtendLexer', 'AspectJLexer', 'CeylonLexer', - 'PigLexer', 'GoloLexer'] + 'PigLexer', 'GoloLexer', 'JasminLexer'] class JavaLexer(RegexLexer): @@ -34,14 +34,19 @@ class JavaLexer(RegexLexer): filenames = ['*.java'] mimetypes = ['text/x-java'] - flags = re.MULTILINE | re.DOTALL + flags = re.MULTILINE | re.DOTALL | re.UNICODE tokens = { 'root': [ (r'[^\S\n]+', Text), (r'//.*?\n', Comment.Single), (r'/\*.*?\*/', Comment.Multiline), - (r'@[a-zA-Z_][a-zA-Z0-9_\.]*', Name.Decorator), + # method names + (r'((?:(?:[^\W\d]|\$)[\w\.\[\]\$<>]*\s+)+?)' # return arguments + r'((?:[^\W\d]|\$)[\w\$]*)' # method name + r'(\s*)(\()', # signature start + bygroups(using(this), Name.Function, Text, Operator)), + (r'@[^\W\d][\w\.]*', Name.Decorator), (r'(assert|break|case|catch|continue|default|do|else|finally|for|' r'if|goto|instanceof|new|return|switch|this|throw|try|while)\b', Keyword), @@ -50,20 +55,15 @@ class JavaLexer(RegexLexer): r'transient|volatile)\b', Keyword.Declaration), (r'(boolean|byte|char|double|float|int|long|short|void)\b', Keyword.Type), - # method names - (r'^(\s*(?:[a-zA-Z_][a-zA-Z0-9_\.\[\]<>]*\s+)+?)' # return arguments - r'([a-zA-Z_][a-zA-Z0-9_]*)' # method name - r'(\s*)(\()', # signature start - bygroups(using(this), Name.Function, Text, Operator)), (r'(package)(\s+)', bygroups(Keyword.Namespace, Text)), (r'(true|false|null)\b', Keyword.Constant), (r'(class|interface)(\s+)', bygroups(Keyword.Declaration, Text), 'class'), (r'(import)(\s+)', bygroups(Keyword.Namespace, Text), 'import'), (r'"(\\\\|\\"|[^"])*"', String), (r"'\\.'|'[^\\]'|'\\u[0-9a-fA-F]{4}'", String.Char), - (r'(\.)([a-zA-Z_][a-zA-Z0-9_]*)', bygroups(Operator, Name.Attribute)), - (r'[a-zA-Z_][a-zA-Z0-9_]*:', Name.Label), - (r'[a-zA-Z_\$][a-zA-Z0-9_]*', Name), + (r'(\.)((?:[^\W\d]|\$)[\w\$]*)', bygroups(Operator, Name.Attribute)), + (r'([^\W\d]|\$)[\w\$]*:', Name.Label), + (r'([^\W\d]|\$)[\w\$]*', Name), (r'[~\^\*!%&\[\]\(\)\{\}<>\|+=:;,./?-]', Operator), (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float), (r'0x[0-9a-fA-F]+', Number.Hex), @@ -71,10 +71,10 @@ class JavaLexer(RegexLexer): (r'\n', Text) ], 'class': [ - (r'[a-zA-Z_][a-zA-Z0-9_]*', Name.Class, '#pop') + (r'([^\W\d]|\$)[\w\$]*', Name.Class, '#pop') ], 'import': [ - (r'[a-zA-Z0-9_.]+\*?', Name.Namespace, '#pop') + (r'[\w.]+\*?', Name.Namespace, '#pop') ], } @@ -340,14 +340,14 @@ class GosuLexer(RegexLexer): tokens = { 'root': [ # method names - (r'^(\s*(?:[a-zA-Z_][a-zA-Z0-9_\.\[\]]*\s+)+?)' # modifiers etc. - r'([a-zA-Z_][a-zA-Z0-9_]*)' # method name - r'(\s*)(\()', # signature start + (r'^(\s*(?:[a-zA-Z_][\w\.\[\]]*\s+)+?)' # modifiers etc. + r'([a-zA-Z_]\w*)' # method name + r'(\s*)(\()', # signature start bygroups(using(this), Name.Function, Text, Operator)), (r'[^\S\n]+', Text), (r'//.*?\n', Comment.Single), (r'/\*.*?\*/', Comment.Multiline), - (r'@[a-zA-Z_][a-zA-Z0-9_\.]*', Name.Decorator), + (r'@[a-zA-Z_][\w\.]*', Name.Decorator), (r'(in|as|typeof|statictypeof|typeis|typeas|if|else|foreach|for|' r'index|while|do|continue|break|return|try|catch|finally|this|' r'throw|new|switch|case|default|eval|super|outer|classpath|' @@ -360,16 +360,16 @@ class GosuLexer(RegexLexer): Keyword.Type), (r'(package)(\s+)', bygroups(Keyword.Namespace, Text)), (r'(true|false|null|NaN|Infinity)\b', Keyword.Constant), - (r'(class|interface|enhancement|enum)(\s+)([a-zA-Z_][a-zA-Z0-9_]*)', + (r'(class|interface|enhancement|enum)(\s+)([a-zA-Z_]\w*)', bygroups(Keyword.Declaration, Text, Name.Class)), - (r'(uses)(\s+)([a-zA-Z0-9_.]+\*?)', + (r'(uses)(\s+)([\w.]+\*?)', bygroups(Keyword.Namespace, Text, Name.Namespace)), (r'"', String, 'string'), - (r'(\??[\.#])([a-zA-Z_][a-zA-Z0-9_]*)', + (r'(\??[\.#])([a-zA-Z_]\w*)', bygroups(Operator, Name.Attribute)), - (r'(:)([a-zA-Z_][a-zA-Z0-9_]*)', + (r'(:)([a-zA-Z_]\w*)', bygroups(Operator, Name.Attribute)), - (r'[a-zA-Z_\$][a-zA-Z0-9_]*', Name), + (r'[a-zA-Z_\$]\w*', Name), (r'and|or|not|[\\~\^\*!%&\[\]\(\)\{\}<>\|+=:;,./?-]', Operator), (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float), (r'[0-9]+', Number.Integer), @@ -438,14 +438,14 @@ class GroovyLexer(RegexLexer): tokens = { 'root': [ # method names - (r'^(\s*(?:[a-zA-Z_][a-zA-Z0-9_\.\[\]]*\s+)+?)' # return arguments - r'([a-zA-Z_][a-zA-Z0-9_]*)' # method name - r'(\s*)(\()', # signature start + (r'^(\s*(?:[a-zA-Z_][\w\.\[\]]*\s+)+?)' # return arguments + r'([a-zA-Z_]\w*)' # method name + r'(\s*)(\()', # signature start bygroups(using(this), Name.Function, Text, Operator)), (r'[^\S\n]+', Text), (r'//.*?\n', Comment.Single), (r'/\*.*?\*/', Comment.Multiline), - (r'@[a-zA-Z_][a-zA-Z0-9_\.]*', Name.Decorator), + (r'@[a-zA-Z_][\w\.]*', Name.Decorator), (r'(assert|break|case|catch|continue|default|do|else|finally|for|' r'if|goto|instanceof|new|return|switch|this|throw|try|while|in|as)\b', Keyword), @@ -464,9 +464,9 @@ class GroovyLexer(RegexLexer): (r'\$/((?!/\$).)*/\$', String), (r'/(\\\\|\\"|[^/])*/', String), (r"'\\.'|'[^\\]'|'\\u[0-9a-fA-F]{4}'", String.Char), - (r'(\.)([a-zA-Z_][a-zA-Z0-9_]*)', bygroups(Operator, Name.Attribute)), - (r'[a-zA-Z_][a-zA-Z0-9_]*:', Name.Label), - (r'[a-zA-Z_\$][a-zA-Z0-9_]*', Name), + (r'(\.)([a-zA-Z_]\w*)', bygroups(Operator, Name.Attribute)), + (r'[a-zA-Z_]\w*:', Name.Label), + (r'[a-zA-Z_\$]\w*', Name), (r'[~\^\*!%&\[\]\(\)\{\}<>\|+=:;,./?-]', Operator), (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float), (r'0x[0-9a-fA-F]+', Number.Hex), @@ -474,10 +474,10 @@ class GroovyLexer(RegexLexer): (r'\n', Text) ], 'class': [ - (r'[a-zA-Z_][a-zA-Z0-9_]*', Name.Class, '#pop') + (r'[a-zA-Z_]\w*', Name.Class, '#pop') ], 'import': [ - (r'[a-zA-Z0-9_.]+\*?', Name.Namespace, '#pop') + (r'[\w.]+\*?', Name.Namespace, '#pop') ], } @@ -550,8 +550,8 @@ class IokeLexer(RegexLexer): (r'#r\[', String.Regex, 'squareRegexp'), #Symbols - (r':[a-zA-Z0-9_!:?]+', String.Symbol), - (r'[a-zA-Z0-9_!:?]+:(?![a-zA-Z0-9_!?])', String.Other), + (r':[\w!:?]+', String.Symbol), + (r'[\w!:?]+:(?![\w!?])', String.Other), (r':"(\\\\|\\"|[^"])*"', String.Symbol), #Documentation @@ -564,10 +564,10 @@ class IokeLexer(RegexLexer): (r'#\[', String, 'squareText'), #Mimic - (r'[a-zA-Z0-9_][a-zA-Z0-9!?_:]+(?=\s*=.*mimic\s)', Name.Entity), + (r'\w[a-zA-Z0-9!?_:]+(?=\s*=.*mimic\s)', Name.Entity), #Assignment - (r'[a-zA-Z_][a-zA-Z0-9_!:?]*(?=[\s]*[+*/-]?=[^=].*($|\.))', + (r'[a-zA-Z_][\w!:?]*(?=[\s]*[+*/-]?=[^=].*($|\.))', Name.Variable), # keywords @@ -658,17 +658,17 @@ class IokeLexer(RegexLexer): r'\+>|!>|<>|&>|%>|#>|\@>|\/>|\*>|\?>|\|>|\^>|~>|\$>|<\->|\->|' r'<<|>>|\*\*|\?\||\?&|\|\||>|<|\*|\/|%|\+|\-|&|\^|\||=|\$|!|~|' u'\\?|#|\u2260|\u2218|\u2208|\u2209)', Operator), - (r'(and|nand|or|xor|nor|return|import)(?![a-zA-Z0-9_!?])', + (r'(and|nand|or|xor|nor|return|import)(?![\w!?])', Operator), # Punctuation (r'(\`\`|\`|\'\'|\'|\.|\,|@@|@|\[|\]|\(|\)|{|})', Punctuation), #kinds - (r'[A-Z][a-zA-Z0-9_!:?]*', Name.Class), + (r'[A-Z][\w!:?]*', Name.Class), #default cellnames - (r'[a-z_][a-zA-Z0-9_!:?]*', Name) + (r'[a-z_][\w!:?]*', Name) ] } @@ -840,14 +840,14 @@ class TeaLangLexer(RegexLexer): tokens = { 'root': [ # method names - (r'^(\s*(?:[a-zA-Z_][a-zA-Z0-9_\.\[\]]*\s+)+?)' # return arguments - r'([a-zA-Z_][a-zA-Z0-9_]*)' # method name - r'(\s*)(\()', # signature start + (r'^(\s*(?:[a-zA-Z_][\w\.\[\]]*\s+)+?)' # return arguments + r'([a-zA-Z_]\w*)' # method name + r'(\s*)(\()', # signature start bygroups(using(this), Name.Function, Text, Operator)), (r'[^\S\n]+', Text), (r'//.*?\n', Comment.Single), (r'/\*.*?\*/', Comment.Multiline), - (r'@[a-zA-Z_][a-zA-Z0-9_\.]*', Name.Decorator), + (r'@[a-zA-Z_][\w\.]*', Name.Decorator), (r'(and|break|else|foreach|if|in|not|or|reverse)\b', Keyword), (r'(as|call|define)\b', Keyword.Declaration), @@ -856,9 +856,9 @@ class TeaLangLexer(RegexLexer): (r'(import)(\s+)', bygroups(Keyword.Namespace, Text), 'import'), (r'"(\\\\|\\"|[^"])*"', String), (r'\'(\\\\|\\\'|[^\'])*\'', String), - (r'(\.)([a-zA-Z_][a-zA-Z0-9_]*)', bygroups(Operator, Name.Attribute)), - (r'[a-zA-Z_][a-zA-Z0-9_]*:', Name.Label), - (r'[a-zA-Z_\$][a-zA-Z0-9_]*', Name), + (r'(\.)([a-zA-Z_]\w*)', bygroups(Operator, Name.Attribute)), + (r'[a-zA-Z_]\w*:', Name.Label), + (r'[a-zA-Z_\$]\w*', Name), (r'(isa|[.]{3}|[.]{2}|[=#!<>+-/%&;,.\*\\\(\)\[\]\{\}])', Operator), (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float), (r'0x[0-9a-fA-F]+', Number.Hex), @@ -866,10 +866,10 @@ class TeaLangLexer(RegexLexer): (r'\n', Text) ], 'template': [ - (r'[a-zA-Z_][a-zA-Z0-9_]*', Name.Class, '#pop') + (r'[a-zA-Z_]\w*', Name.Class, '#pop') ], 'import': [ - (r'[a-zA-Z0-9_.]+\*?', Name.Namespace, '#pop') + (r'[\w.]+\*?', Name.Namespace, '#pop') ], } @@ -894,9 +894,9 @@ class CeylonLexer(RegexLexer): tokens = { 'root': [ # method names - (r'^(\s*(?:[a-zA-Z_][a-zA-Z0-9_\.\[\]]*\s+)+?)' # return arguments - r'([a-zA-Z_][a-zA-Z0-9_]*)' # method name - r'(\s*)(\()', # signature start + (r'^(\s*(?:[a-zA-Z_][\w\.\[\]]*\s+)+?)' # return arguments + r'([a-zA-Z_]\w*)' # method name + r'(\s*)(\()', # signature start bygroups(using(this), Name.Function, Text, Operator)), (r'[^\S\n]+', Text), (r'//.*?\n', Comment.Single), @@ -919,10 +919,10 @@ class CeylonLexer(RegexLexer): (r'"(\\\\|\\"|[^"])*"', String), (r"'\\.'|'[^\\]'|'\\\{#[0-9a-fA-F]{4}\}'", String.Char), (r'".*``.*``.*"', String.Interpol), - (r'(\.)([a-z_][a-zA-Z0-9_]*)', + (r'(\.)([a-z_]\w*)', bygroups(Operator, Name.Attribute)), - (r'[a-zA-Z_][a-zA-Z0-9_]*:', Name.Label), - (r'[a-zA-Z_][a-zA-Z0-9_]*', Name), + (r'[a-zA-Z_]\w*:', Name.Label), + (r'[a-zA-Z_]\w*', Name), (r'[~\^\*!%&\[\]\(\)\{\}<>\|+=:;,./?-]', Operator), (r'\d{1,3}(_\d{3})+\.\d{1,3}(_\d{3})+[kMGTPmunpf]?', Number.Float), (r'\d{1,3}(_\d{3})+\.[0-9]+([eE][+-]?[0-9]+)?[kMGTPmunpf]?', @@ -939,10 +939,10 @@ class CeylonLexer(RegexLexer): (r'\n', Text) ], 'class': [ - (r'[A-Za-z_][a-zA-Z0-9_]*', Name.Class, '#pop') + (r'[A-Za-z_]\w*', Name.Class, '#pop') ], 'import': [ - (r'[a-z][a-zA-Z0-9_.]*', + (r'[a-z][\w.]*', Name.Namespace, '#pop') ], 'comment': [ @@ -1034,14 +1034,14 @@ class XtendLexer(RegexLexer): tokens = { 'root': [ # method names - (r'^(\s*(?:[a-zA-Z_][a-zA-Z0-9_\.\[\]]*\s+)+?)' # return arguments - r'([a-zA-Z_$][a-zA-Z0-9_$]*)' # method name - r'(\s*)(\()', # signature start + (r'^(\s*(?:[a-zA-Z_][\w\.\[\]]*\s+)+?)' # return arguments + r'([a-zA-Z_$][\w$]*)' # method name + r'(\s*)(\()', # signature start bygroups(using(this), Name.Function, Text, Operator)), (r'[^\S\n]+', Text), (r'//.*?\n', Comment.Single), (r'/\*.*?\*/', Comment.Multiline), - (r'@[a-zA-Z_][a-zA-Z0-9_\.]*', Name.Decorator), + (r'@[a-zA-Z_][\w\.]*', Name.Decorator), (r'(assert|break|case|catch|continue|default|do|else|finally|for|' r'if|goto|instanceof|new|return|switch|this|throw|try|while|IF|' r'ELSE|ELSEIF|ENDIF|FOR|ENDFOR|SEPARATOR|BEFORE|AFTER)\b', @@ -1060,8 +1060,8 @@ class XtendLexer(RegexLexer): (u'(\u00BB)', String, 'template'), (r'"(\\\\|\\"|[^"])*"', String), (r"'(\\\\|\\'|[^'])*'", String), - (r'[a-zA-Z_][a-zA-Z0-9_]*:', Name.Label), - (r'[a-zA-Z_\$][a-zA-Z0-9_]*', Name), + (r'[a-zA-Z_]\w*:', Name.Label), + (r'[a-zA-Z_\$]\w*', Name), (r'[~\^\*!%&\[\]\(\)\{\}<>\|+=:;,./?-]', Operator), (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float), (r'0x[0-9a-fA-F]+', Number.Hex), @@ -1069,10 +1069,10 @@ class XtendLexer(RegexLexer): (r'\n', Text) ], 'class': [ - (r'[a-zA-Z_][a-zA-Z0-9_]*', Name.Class, '#pop') + (r'[a-zA-Z_]\w*', Name.Class, '#pop') ], 'import': [ - (r'[a-zA-Z0-9_.]+\*?', Name.Namespace, '#pop') + (r'[\w.]+\*?', Name.Namespace, '#pop') ], 'template': [ (r"'''", String, '#pop'), @@ -1112,7 +1112,7 @@ class PigLexer(RegexLexer): (r'0x[0-9a-f]+', Number.Hex), (r'[0-9]+L?', Number.Integer), (r'\n', Text), - (r'([a-z_][a-z0-9_]*)(\s*)(\()', + (r'([a-z_]\w*)(\s*)(\()', bygroups(Name.Function, Text, Punctuation)), (r'[()#:]', Text), (r'[^(:#\'\")\s]+', Text), @@ -1175,8 +1175,8 @@ class GoloLexer(RegexLexer): (r'(module|import)(\s+)', bygroups(Keyword.Namespace, Text), 'modname'), - (r'\b([a-zA-Z_][a-z$A-Z0-9._]*)(::)', bygroups(Name.Namespace, Punctuation)), - (r'\b([a-zA-Z_][a-z$A-Z0-9_]*(?:\.[a-zA-Z_][a-z$A-Z0-9_]*)+)\b', Name.Namespace), + (r'\b([a-zA-Z_][\w$.]*)(::)', bygroups(Name.Namespace, Punctuation)), + (r'\b([a-zA-Z_][\w$]*(?:\.[a-zA-Z_][\w$]*)+)\b', Name.Namespace), (r'(let|var)(\s+)', bygroups(Keyword.Declaration, Text), @@ -1201,7 +1201,7 @@ class GoloLexer(RegexLexer): bygroups(Name.Builtin, Punctuation)), (r'(print|println|readln|raise|fun' r'|asInterfaceInstance)\b', Name.Builtin), - (r'(`?[a-zA-Z_][a-z$A-Z0-9_]*)(\()', + (r'(`?[a-zA-Z_][\w$]*)(\()', bygroups(Name.Function, Punctuation)), (r'-?[\d_]*\.[\d_]*([eE][+-]?\d[\d_]*)?F?', Number.Float), @@ -1210,7 +1210,7 @@ class GoloLexer(RegexLexer): (r'-?\d[\d_]*L', Number.Integer.Long), (r'-?\d[\d_]*', Number.Integer), - ('`?[a-zA-Z_][a-z$A-Z0-9_]*', Name), + ('`?[a-zA-Z_][\w$]*', Name), (r'"""', String, combined('stringescape', 'triplestring')), (r'"', String, combined('stringescape', 'doublestring')), @@ -1220,16 +1220,16 @@ class GoloLexer(RegexLexer): ], 'funcname': [ - (r'`?[a-zA-Z_][a-z$A-Z0-9_]*', Name.Function, '#pop'), + (r'`?[a-zA-Z_][\w$]*', Name.Function, '#pop'), ], 'modname': [ - (r'[a-zA-Z_][a-z$A-Z0-9._]*\*?', Name.Namespace, '#pop') + (r'[a-zA-Z_][\w$.]*\*?', Name.Namespace, '#pop') ], 'structname': [ - (r'`?[a-zA-Z0-9_.]+\*?', Name.Class, '#pop') + (r'`?[\w.]+\*?', Name.Class, '#pop') ], 'varname': [ - (r'`?[a-zA-Z_][a-z$A-Z0-9_]*', Name.Variable, '#pop'), + (r'`?[a-zA-Z_][\w$]*', Name.Variable, '#pop'), ], 'string': [ (r'[^\\\'"\n]+', String), @@ -1258,3 +1258,262 @@ class GoloLexer(RegexLexer): (r'(==|<=|<|>=|>|!=)', Operator), ], } + + +class JasminLexer(RegexLexer): + """ + For `Jasmin <http://jasmin.sourceforge.net/>`_ assembly code. + + .. versionadded:: 2.0 + """ + + name = 'Jasmin' + aliases = ['jasmin', 'jasminxt'] + filenames = ['*.j'] + + _whitespace = r' \n\t\r' + _ws = r'(?:[%s]+)' % _whitespace + _separator = r'%s:=' % _whitespace + _break = r'(?=[%s]|$)' % _separator + _name = r'[^%s]+' % _separator + _unqualified_name = r'(?:[^%s.;\[/]+)' % _separator + + tokens = { + 'default': [ + (r'\n', Text, '#pop'), + (r"'", String.Single, ('#pop', 'quote')), + (r'"', String.Double, 'string'), + (r'=', Punctuation), + (r':', Punctuation, 'label'), + (_ws, Text), + (r';.*', Comment.Single), + (r'(\$[-+])?0x-?[\da-fA-F]+%s' % _break, Number.Hex), + (r'(\$[-+]|\+)?-?\d+%s' % _break, Number.Integer), + (r'-?(\d+\.\d*|\.\d+)([eE][-+]?\d+)?[fFdD]?' + r'[\x00-\x08\x0b\x0c\x0e-\x1f]*%s' % _break, Number.Float), + (r'\$%s' % _name, Name.Variable), + + # Directives + (r'\.annotation%s' % _break, Keyword.Reserved, 'annotation'), + (r'(\.attribute|\.bytecode|\.debug|\.deprecated|\.enclosing|' + r'\.interface|\.line|\.signature|\.source|\.stack|\.var|abstract|' + r'annotation|bridge|class|default|enum|field|final|fpstrict|' + r'interface|native|private|protected|public|signature|static|' + r'synchronized|synthetic|transient|varargs|volatile)%s' % _break, + Keyword.Reserved), + (r'\.catch%s' % _break, Keyword.Reserved, 'caught-exception'), + (r'(\.class|\.implements|\.inner|\.super|inner|invisible|' + r'invisibleparam|outer|visible|visibleparam)%s' % _break, + Keyword.Reserved, 'class/convert-dots'), + (r'\.field%s' % _break, Keyword.Reserved, + ('descriptor/convert-dots', 'field')), + (r'(\.end|\.limit|use)%s' % _break, Keyword.Reserved, + 'no-verification'), + (r'\.method%s' % _break, Keyword.Reserved, 'method'), + (r'\.set%s' % _break, Keyword.Reserved, 'var'), + (r'\.throws%s' % _break, Keyword.Reserved, 'exception'), + (r'(from|offset|to|using)%s' % _break, Keyword.Reserved, 'label'), + (r'is%s' % _break, Keyword.Reserved, + ('descriptor/convert-dots', 'var')), + (r'(locals|stack)%s' % _break, Keyword.Reserved, 'verification'), + (r'method%s' % _break, Keyword.Reserved, 'enclosing-method'), + + # Instructions + (r'(aaload|aastore|aconst_null|aload|aload_0|aload_1|aload_2|' + r'aload_3|aload_w|areturn|arraylength|astore|astore_0|astore_1|' + r'astore_2|astore_3|astore_w|athrow|baload|bastore|bipush|' + r'breakpoint|caload|castore|d2f|d2i|d2l|dadd|daload|dastore|' + r'dcmpg|dcmpl|dconst_0|dconst_1|ddiv|dload|dload_0|dload_1|' + r'dload_2|dload_3|dload_w|dmul|dneg|drem|dreturn|dstore|dstore_0|' + r'dstore_1|dstore_2|dstore_3|dstore_w|dsub|dup|dup2|dup2_x1|' + r'dup2_x2|dup_x1|dup_x2|f2d|f2i|f2l|fadd|faload|fastore|fcmpg|' + r'fcmpl|fconst_0|fconst_1|fconst_2|fdiv|fload|fload_0|fload_1|' + r'fload_2|fload_3|fload_w|fmul|fneg|frem|freturn|fstore|fstore_0|' + r'fstore_1|fstore_2|fstore_3|fstore_w|fsub|i2b|i2c|i2d|i2f|i2l|' + r'i2s|iadd|iaload|iand|iastore|iconst_0|iconst_1|iconst_2|' + r'iconst_3|iconst_4|iconst_5|iconst_m1|idiv|iinc|iinc_w|iload|' + r'iload_0|iload_1|iload_2|iload_3|iload_w|imul|ineg|int2byte|' + r'int2char|int2short|ior|irem|ireturn|ishl|ishr|istore|istore_0|' + r'istore_1|istore_2|istore_3|istore_w|isub|iushr|ixor|l2d|l2f|' + r'l2i|ladd|laload|land|lastore|lcmp|lconst_0|lconst_1|ldc2_w|' + r'ldiv|lload|lload_0|lload_1|lload_2|lload_3|lload_w|lmul|lneg|' + r'lookupswitch|lor|lrem|lreturn|lshl|lshr|lstore|lstore_0|' + r'lstore_1|lstore_2|lstore_3|lstore_w|lsub|lushr|lxor|' + r'monitorenter|monitorexit|nop|pop|pop2|ret|ret_w|return|saload|' + r'sastore|sipush|swap)%s' % _break, Keyword.Reserved), + (r'(anewarray|checkcast|instanceof|ldc|ldc_w|new)%s' % _break, + Keyword.Reserved, 'class/no-dots'), + (r'(invokedynamic|invokeinterface|invokenonvirtual|invokespecial|' + r'invokestatic|invokevirtual)%s' % _break, Keyword.Reserved, + 'invocation'), + (r'(getfield|putfield)%s' % _break, Keyword.Reserved, + ('descriptor/no-dots', 'field')), + (r'(getstatic|putstatic)%s' % _break, Keyword.Reserved, + ('descriptor/no-dots', 'static')), + (r'(goto|goto_w|if_acmpeq|if_acmpne|if_icmpeq|if_icmpge|if_icmpgt|' + r'if_icmple|if_icmplt|if_icmpne|ifeq|ifge|ifgt|ifle|iflt|ifne|' + r'ifnonnull|ifnull|jsr|jsr_w)%s' % _break, Keyword.Reserved, + 'label'), + (r'(multianewarray|newarray)%s' % _break, Keyword.Reserved, + 'descriptor/convert-dots'), + (r'tableswitch%s' % _break, Keyword.Reserved, 'table') + ], + 'quote': [ + (r"'", String.Single, '#pop'), + (r'\\u[\da-fA-F]{4}', String.Escape), + (r"[^'\\]+", String.Single) + ], + 'string': [ + (r'"', String.Double, '#pop'), + (r'\\([nrtfb"\'\\]|u[\da-fA-F]{4}|[0-3]?[0-7]{1,2})', + String.Escape), + (r'[^"\\]+', String.Double) + ], + 'root': [ + (r'\n+', Text), + (r"'", String.Single, 'quote'), + include('default'), + (r'(%s)([ \t\r]*)(:)' % _name, + bygroups(Name.Label, Text, Punctuation)), + (_name, String.Other) + ], + 'annotation': [ + (r'\n', Text, ('#pop', 'annotation-body')), + (r'default%s' % _break, Keyword.Reserved, + ('#pop', 'annotation-default')), + include('default') + ], + 'annotation-body': [ + (r'\n+', Text), + (r'\.end%s' % _break, Keyword.Reserved, '#pop'), + include('default'), + (_name, String.Other, ('annotation-items', 'descriptor/no-dots')) + ], + 'annotation-default': [ + (r'\n+', Text), + (r'\.end%s' % _break, Keyword.Reserved, '#pop'), + include('default'), + (r'', Text, ('annotation-items', 'descriptor/no-dots')) + ], + 'annotation-items': [ + (r"'", String.Single, 'quote'), + include('default'), + (_name, String.Other) + ], + 'caught-exception': [ + (r'all%s' % _break, Keyword, '#pop'), + include('exception') + ], + 'class/convert-dots': [ + include('default'), + (r'(L)((?:%s[/.])*)(%s)(;)' % (_unqualified_name, _name), + bygroups(Keyword.Type, Name.Namespace, Name.Class, Punctuation), + '#pop'), + (r'((?:%s[/.])*)(%s)' % (_unqualified_name, _name), + bygroups(Name.Namespace, Name.Class), '#pop') + ], + 'class/no-dots': [ + include('default'), + (r'\[+', Punctuation, ('#pop', 'descriptor/no-dots')), + (r'(L)((?:%s/)*)(%s)(;)' % (_unqualified_name, _name), + bygroups(Keyword.Type, Name.Namespace, Name.Class, Punctuation), + '#pop'), + (r'((?:%s/)*)(%s)' % (_unqualified_name, _name), + bygroups(Name.Namespace, Name.Class), '#pop') + ], + 'descriptor/convert-dots': [ + include('default'), + (r'\[+', Punctuation), + (r'(L)((?:%s[/.])*)(%s?)(;)' % (_unqualified_name, _name), + bygroups(Keyword.Type, Name.Namespace, Name.Class, Punctuation), + '#pop'), + (r'[^%s\[)L]*' % _separator, Keyword.Type, '#pop') + ], + 'descriptor/no-dots': [ + include('default'), + (r'\[+', Punctuation), + (r'(L)((?:%s/)*)(%s)(;)' % (_unqualified_name, _name), + bygroups(Keyword.Type, Name.Namespace, Name.Class, Punctuation), + '#pop'), + (r'[^%s\[)L]*' % _separator, Keyword.Type, '#pop') + ], + 'descriptors/convert-dots': [ + (r'\)', Punctuation, '#pop'), + (r'', Text, 'descriptor/convert-dots') + ], + 'enclosing-method': [ + (_ws, Text), + (r'(?=[^%s]*\()' % _separator, Text, ('#pop', 'invocation')), + (r'', Text, ('#pop', 'class/convert-dots')) + ], + 'exception': [ + include('default'), + (r'((?:%s[/.])*)(%s)' % (_unqualified_name, _name), + bygroups(Name.Namespace, Name.Exception), '#pop') + ], + 'field': [ + (r'static%s' % _break, Keyword.Reserved, ('#pop', 'static')), + include('default'), + (r'((?:%s[/.](?=[^%s]*[/.]))*)(%s[/.])?(%s)' % + (_unqualified_name, _separator, _unqualified_name, _name), + bygroups(Name.Namespace, Name.Class, Name.Variable.Instance), + '#pop') + ], + 'invocation': [ + include('default'), + (r'((?:%s[/.](?=[^%s(]*[/.]))*)(%s[/.])?(%s)(\()' % + (_unqualified_name, _separator, _unqualified_name, _name), + bygroups(Name.Namespace, Name.Class, Name.Function, Punctuation), + ('#pop', 'descriptor/convert-dots', 'descriptors/convert-dots', + 'descriptor/convert-dots')) + ], + 'label': [ + include('default'), + (_name, Name.Label, '#pop') + ], + 'method': [ + include('default'), + (r'(%s)(\()' % _name, bygroups(Name.Function, Punctuation), + ('#pop', 'descriptor/convert-dots', 'descriptors/convert-dots', + 'descriptor/convert-dots')) + ], + 'no-verification': [ + (r'(locals|method|stack)%s' % _break, Keyword.Reserved, '#pop'), + include('default') + ], + 'static': [ + include('default'), + (r'((?:%s[/.](?=[^%s]*[/.]))*)(%s[/.])?(%s)' % + (_unqualified_name, _separator, _unqualified_name, _name), + bygroups(Name.Namespace, Name.Class, Name.Variable.Class), '#pop') + ], + 'table': [ + (r'\n+', Text), + (r'default%s' % _break, Keyword.Reserved, '#pop'), + include('default'), + (_name, Name.Label) + ], + 'var': [ + include('default'), + (_name, Name.Variable, '#pop') + ], + 'verification': [ + include('default'), + (r'(Double|Float|Integer|Long|Null|Top|UninitializedThis)%s' % + _break, Keyword, '#pop'), + (r'Object%s' % _break, Keyword, ('#pop', 'class/no-dots')), + (r'Uninitialized%s' % _break, Keyword, ('#pop', 'label')) + ] + } + + def analyse_text(text): + score = 0 + if re.search(r'^\s*\.class\s', text, re.MULTILINE): + score += 0.5 + if re.search(r'^\s*[a-z]+_[a-z]+\b', text, re.MULTILINE): + score += 0.3 + if re.search(r'^\s*\.(attribute|bytecode|debug|deprecated|enclosing|' + r'inner|interface|limit|set|signature|stack)\b', text, + re.MULTILINE): + score += 0.6 + return score diff --git a/pygments/lexers/math.py b/pygments/lexers/math.py index cfbfc432..de766bd2 100644 --- a/pygments/lexers/math.py +++ b/pygments/lexers/math.py @@ -97,8 +97,8 @@ class JuliaLexer(RegexLexer): (r'[E]?"', String, combined('stringescape', 'string')), # names - (r'@[a-zA-Z0-9_.]+', Name.Decorator), - (r'[a-zA-Z_][a-zA-Z0-9_]*', Name), + (r'@[\w.]+', Name.Decorator), + (r'[a-zA-Z_]\w*', Name), # numbers (r'(\d+(_\d+)+\.\d*|\d*\.\d+(_\d+)+)([eEf][+-]?[0-9]+)?', Number.Float), @@ -116,13 +116,13 @@ class JuliaLexer(RegexLexer): ], 'funcname': [ - ('[a-zA-Z_][a-zA-Z0-9_]*', Name.Function, '#pop'), + ('[a-zA-Z_]\w*', Name.Function, '#pop'), ('\([^\s\w{]{1,2}\)', Operator, '#pop'), ('[^\s\w{]{1,2}', Operator, '#pop'), ], 'typename': [ - ('[a-zA-Z_][a-zA-Z0-9_]*', Name.Class, '#pop') + ('[a-zA-Z_]\w*', Name.Class, '#pop') ], 'stringescape': [ @@ -133,7 +133,7 @@ class JuliaLexer(RegexLexer): 'string': [ (r'"', String, '#pop'), (r'\\\\|\\"|\\\n', String.Escape), # included here for raw strings - (r'\$(\([a-zA-Z0-9_]+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?', + (r'\$(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?', String.Interpol), (r'[^\\"$]+', String), # quotes, dollar signs, and backslashes must be parsed one at a time @@ -357,7 +357,7 @@ class MatlabLexer(RegexLexer): (r'\d+', Number.Integer), (r'(?<![\w\)\].])\'', String, 'string'), - ('[a-zA-Z_][a-zA-Z0-9_]*', Name), + ('[a-zA-Z_]\w*', Name), (r'.', Text), ], 'string': [ @@ -816,7 +816,7 @@ class OctaveLexer(RegexLexer): (r'(?<=[\w\)\].])\'+', Operator), (r'(?<![\w\)\].])\'', String, 'string'), - ('[a-zA-Z_][a-zA-Z0-9_]*', Name), + ('[a-zA-Z_]\w*', Name), (r'.', Text), ], 'string': [ @@ -881,7 +881,7 @@ class ScilabLexer(RegexLexer): (r'\d+[eEf][+-]?[0-9]+', Number.Float), (r'\d+', Number.Integer), - ('[a-zA-Z_][a-zA-Z0-9_]*', Name), + ('[a-zA-Z_]\w*', Name), (r'.', Text), ], 'string': [ @@ -1461,7 +1461,7 @@ class BugsLexer(RegexLexer): % r'|'.join(_FUNCTIONS + _DISTRIBUTIONS), Name.Builtin), # Regular variable names - (r'[A-Za-z][A-Za-z0-9_.]*', Name), + (r'[A-Za-z][\w.]*', Name), # Number Literals (r'[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?', Number), # Punctuation @@ -1521,7 +1521,7 @@ class JagsLexer(RegexLexer): ], 'names' : [ # Regular variable names - (r'[a-zA-Z][a-zA-Z0-9_.]*\b', Name), + (r'[a-zA-Z][\w.]*\b', Name), ], 'comments' : [ # do not use stateful comments @@ -1572,9 +1572,8 @@ class JagsLexer(RegexLexer): class StanLexer(RegexLexer): """Pygments Lexer for Stan models. - The Stan modeling language is specified in the *Stan 2.0.1 - Modeling Language Manual* `pdf - <https://github.com/stan-dev/stan/releases/download/v2.0.1/stan-reference-2.0.1.pdf>`__ + The Stan modeling language is specified in the *Stan Modeling Language User's Guide and Reference Manual, v2.2.0*, + `pdf <https://github.com/stan-dev/stan/releases/download/v2.2.0/stan-reference-2.2.0.pdf>`__. .. versionadded:: 1.6 """ @@ -1619,10 +1618,10 @@ class StanLexer(RegexLexer): + _stan_builtins.DISTRIBUTIONS), Name.Builtin), # Special names ending in __, like lp__ - (r'[A-Za-z][A-Za-z0-9_]*__\b', Name.Builtin.Pseudo), + (r'[A-Za-z]\w*__\b', Name.Builtin.Pseudo), (r'(%s)\b' % r'|'.join(_stan_builtins.RESERVED), Keyword.Reserved), # Regular variable names - (r'[A-Za-z][A-Za-z0-9_]*\b', Name), + (r'[A-Za-z]\w*\b', Name), # Real Literals (r'-?[0-9]+(\.[0-9]+)?[eE]-?[0-9]+', Number.Float), (r'-?[0-9]*\.[0-9]*', Number.Float), @@ -1656,6 +1655,8 @@ class IDLLexer(RegexLexer): filenames = ['*.pro'] mimetypes = ['text/idl'] + flags = re.IGNORECASE | re.MULTILINE + _RESERVED = ['and', 'begin', 'break', 'case', 'common', 'compile_opt', 'continue', 'do', 'else', 'end', 'endcase', 'elseelse', 'endfor', 'endforeach', 'endif', 'endrep', 'endswitch', @@ -1677,7 +1678,7 @@ class IDLLexer(RegexLexer): 'broyden', 'butterworth', 'bytarr', 'byte', 'byteorder', 'bytscl', 'caldat', 'calendar', 'call_external', 'call_function', 'call_method', 'call_procedure', 'canny', - 'catch', 'cd', 'cdf_[0-9a-za-z_]*', 'ceil', 'chebyshev', + 'catch', 'cd', 'cdf_\w*', 'ceil', 'chebyshev', 'check_math', 'chisqr_cvf', 'chisqr_pdf', 'choldc', 'cholsol', 'cindgen', 'cir_3pnt', 'close', 'cluster', 'cluster_tree', 'clust_wts', @@ -1711,7 +1712,7 @@ class IDLLexer(RegexLexer): 'dlm_load', 'dlm_register', 'doc_library', 'double', 'draw_roi', 'edge_dog', 'efont', 'eigenql', 'eigenvec', 'ellipse', 'elmhes', 'emboss', 'empty', 'enable_sysrtn', - 'eof', 'eos_[0-9a-za-z_]*', 'erase', 'erf', 'erfc', 'erfcx', + 'eof', 'eos_\w*', 'erase', 'erf', 'erfc', 'erfcx', 'erode', 'errorplot', 'errplot', 'estimator_filter', 'execute', 'exit', 'exp', 'expand', 'expand_path', 'expint', 'extrac', 'extract_slice', 'factorial', 'fft', 'filepath', @@ -1728,11 +1729,11 @@ class IDLLexer(RegexLexer): 'gauss_cvf', 'gauss_pdf', 'gauss_smooth', 'getenv', 'getwindows', 'get_drive_list', 'get_dxf_objects', 'get_kbrd', 'get_login_info', 'get_lun', 'get_screen_size', - 'greg2jul', 'grib_[0-9a-za-z_]*', 'grid3', 'griddata', + 'greg2jul', 'grib_\w*', 'grid3', 'griddata', 'grid_input', 'grid_tps', 'gs_iter', - 'h5[adfgirst]_[0-9a-za-z_]*', 'h5_browser', 'h5_close', + 'h5[adfgirst]_\w*', 'h5_browser', 'h5_close', 'h5_create', 'h5_get_libversion', 'h5_open', 'h5_parse', - 'hanning', 'hash', 'hdf_[0-9a-za-z_]*', 'heap_free', + 'hanning', 'hash', 'hdf_\w*', 'heap_free', 'heap_gc', 'heap_nosave', 'heap_refcount', 'heap_save', 'help', 'hilbert', 'histogram', 'hist_2d', 'hist_equal', 'hls', 'hough', 'hqr', 'hsv', 'h_eq_ct', 'h_eq_int', @@ -1780,7 +1781,7 @@ class IDLLexer(RegexLexer): 'modifyct', 'moment', 'morph_close', 'morph_distance', 'morph_gradient', 'morph_hitormiss', 'morph_open', 'morph_thin', 'morph_tophat', 'multi', 'm_correlate', - 'ncdf_[0-9a-za-z_]*', 'newton', 'noise_hurl', 'noise_pick', + 'ncdf_\w*', 'newton', 'noise_hurl', 'noise_pick', 'noise_scatter', 'noise_slur', 'norm', 'n_elements', 'n_params', 'n_tags', 'objarr', 'obj_class', 'obj_destroy', 'obj_hasmethod', 'obj_isa', 'obj_new', 'obj_valid', @@ -2182,7 +2183,7 @@ class IgorLexer(RegexLexer): # Compiler directives. (r'^#(include|pragma|define|ifdef|ifndef|endif)', Name.Decorator), - (r'[^a-zA-Z"/]+$', Text), + (r'[^a-z"/]+$', Text), (r'.', Text), ], } diff --git a/pygments/lexers/other.py b/pygments/lexers/other.py index 454f0e42..d4d1d34b 100644 --- a/pygments/lexers/other.py +++ b/pygments/lexers/other.py @@ -95,7 +95,7 @@ class LSLLexer(RegexLexer): (lsl_invalid_unimplemented, Error), (lsl_reserved_godmode, Keyword.Reserved), (lsl_reserved_log, Keyword.Reserved), - (r'\b([a-zA-Z_][a-zA-Z0-9_]*)\b', Name.Variable), + (r'\b([a-zA-Z_]\w*)\b', Name.Variable), (r'(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d*', Number.Float), (r'(\d+\.\d*|\.\d+)', Number.Float), (r'0[xX][0-9a-fA-F]+', Number.Hex), @@ -156,15 +156,15 @@ class ECLLexer(RegexLexer): include('hash'), (r'"', String, 'string'), (r'\'', String, 'string'), - (r'(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+[LlUu]*', Number.Float), + (r'(\d+\.\d*|\.\d+|\d+)e[+-]?\d+[lu]*', Number.Float), (r'(\d+\.\d*|\.\d+|\d+[fF])[fF]?', Number.Float), - (r'0x[0-9a-fA-F]+[LlUu]*', Number.Hex), - (r'0[0-7]+[LlUu]*', Number.Oct), + (r'0x[0-9a-f]+[lu]*', Number.Hex), + (r'0[0-7]+[lu]*', Number.Oct), (r'\d+[LlUu]*', Number.Integer), (r'\*/', Error), (r'[~!%^&*+=|?:<>/-]+', Operator), (r'[{}()\[\],.;]', Punctuation), - (r'[a-zA-Z_][a-zA-Z0-9_]*', Name), + (r'[a-z_]\w*', Name), ], 'hash': [ (r'^#.*$', Comment.Preproc), @@ -514,7 +514,7 @@ class LogtalkLexer(RegexLexer): (r'0x[0-9a-fA-F]+', Number), (r'\d+\.?\d*((e|E)(\+|-)?\d+)?', Number), # Variables - (r'([A-Z_][a-zA-Z0-9_]*)', Name.Variable), + (r'([A-Z_]\w*)', Name.Variable), # Event handlers (r'(after|before)(?=[(])', Keyword), # Execution-context methods @@ -630,7 +630,7 @@ class LogtalkLexer(RegexLexer): # Ponctuation (r'[()\[\],.|]', Text), # Atoms - (r"[a-z][a-zA-Z0-9_]*", Text), + (r"[a-z]\w*", Text), (r"'", String, 'quoted_atom'), ], @@ -661,8 +661,8 @@ class LogtalkLexer(RegexLexer): (r'op(?=[(])', Keyword, 'root'), (r'(c(alls|oinductive)|reexport|use(s|_module))(?=[(])', Keyword, 'root'), - (r'[a-z][a-zA-Z0-9_]*(?=[(])', Text, 'root'), - (r'[a-z][a-zA-Z0-9_]*[.]', Text, 'root'), + (r'[a-z]\w*(?=[(])', Text, 'root'), + (r'[a-z]\w*[.]', Text, 'root'), ], 'entityrelations': [ @@ -675,9 +675,9 @@ class LogtalkLexer(RegexLexer): (r'0x[0-9a-fA-F]+', Number), (r'\d+\.?\d*((e|E)(\+|-)?\d+)?', Number), # Variables - (r'([A-Z_][a-zA-Z0-9_]*)', Name.Variable), + (r'([A-Z_]\w*)', Name.Variable), # Atoms - (r"[a-z][a-zA-Z0-9_]*", Text), + (r"[a-z]\w*", Text), (r"'", String, 'quoted_atom'), # Strings (r'"(\\\\|\\"|[^"])*"', String), @@ -747,11 +747,11 @@ class GnuplotLexer(RegexLexer): (_shortened_many('pwd$', 're$read', 'res$et', 'scr$eendump', 'she$ll', 'test$'), Keyword, 'noargs'), - ('([a-zA-Z_][a-zA-Z0-9_]*)(\s*)(=)', + ('([a-zA-Z_]\w*)(\s*)(=)', bygroups(Name.Variable, Text, Operator), 'genericargs'), - ('([a-zA-Z_][a-zA-Z0-9_]*)(\s*\(.*?\)\s*)(=)', + ('([a-zA-Z_]\w*)(\s*\(.*?\)\s*)(=)', bygroups(Name.Function, Text, Operator), 'genericargs'), - (r'@[a-zA-Z_][a-zA-Z0-9_]*', Name.Constant), # macros + (r'@[a-zA-Z_]\w*', Name.Constant), # macros (r';', Keyword), ], 'comment': [ @@ -797,10 +797,10 @@ class GnuplotLexer(RegexLexer): ('[,.~!%^&*+=|?:<>/-]', Operator), ('[{}()\[\]]', Punctuation), (r'(eq|ne)\b', Operator.Word), - (r'([a-zA-Z_][a-zA-Z0-9_]*)(\s*)(\()', + (r'([a-zA-Z_]\w*)(\s*)(\()', bygroups(Name.Function, Text, Punctuation)), - (r'[a-zA-Z_][a-zA-Z0-9_]*', Name), - (r'@[a-zA-Z_][a-zA-Z0-9_]*', Name.Constant), # macros + (r'[a-zA-Z_]\w*', Name), + (r'@[a-zA-Z_]\w*', Name.Constant), # macros (r'\\\n', Text), ], 'optionarg': [ @@ -1306,9 +1306,9 @@ class ModelicaLexer(RegexLexer): (r'\d+[Ll]?', Number.Integer), (r'[~!%^&*+=|?:<>/-]', Operator), (r'(true|false|NULL|Real|Integer|Boolean)\b', Name.Builtin), - (r'([a-zA-Z_][\w]*|[\'][^\']+[\'])' + (r'([a-z_][\w]*|\'[^\']+\')' r'([\[\d,:\]]*)' - r'(\.([a-zA-Z_][\w]*|[\'][^\']+[\']))+' + r'(\.([a-z_][\w]*|\'[^\']+\'))+' r'([\[\d,:\]]*)', Name.Class), (r'(\'[\w\+\-\*\/\^]+\'|\w+)', Name), (r'[()\[\]{},.;]', Punctuation), @@ -1316,16 +1316,16 @@ class ModelicaLexer(RegexLexer): ], 'root': [ include('whitespace'), - include('keywords'), include('classes'), include('functions'), + include('keywords'), include('operators'), (r'("<html>|<html>)', Name.Tag, 'html-content'), include('statements'), ], 'keywords': [ (r'(algorithm|annotation|break|connect|constant|constrainedby|' - r'discrete|each|else|elseif|elsewhen|encapsulated|enumeration|' + r'discrete|each|end|else|elseif|elsewhen|encapsulated|enumeration|' r'equation|exit|expandable|extends|' r'external|false|final|flow|for|if|import|impure|in|initial\sequation|' r'inner|input|loop|nondiscrete|outer|output|parameter|partial|' @@ -1350,7 +1350,7 @@ class ModelicaLexer(RegexLexer): 'classes': [ (r'(operator)?(\s+)?(block|class|connector|end|function|model|' r'operator|package|record|type)(\s+)' - r'((?!if|for|when|while)[A-Za-z_]\w*|[\'][^\']+[\'])([;]?)', + r'((?!if|for|when|while)[a-z_]\w*|\'[^\']+\')([;]?)', bygroups(Keyword, Text, Keyword, Text, Name.Class, Text)) ], 'quoted_ident': [ @@ -1359,7 +1359,7 @@ class ModelicaLexer(RegexLexer): ], 'string': [ (r'"', String, '#pop'), - (r'\\([\\abfnrtv"\']|x[a-fA-F0-9]{2,4}|[0-7]{1,3})', + (r'\\([\\abfnrtv"\']|x[a-f0-9]{2,4}|[0-7]{1,3})', String.Escape), (r'[^\\"\n]+', String), # all other characters (r'\\\n', String), # line continuation @@ -1380,14 +1380,14 @@ class RebolLexer(RegexLexer): """ name = 'REBOL' aliases = ['rebol'] - filenames = ['*.r', '*.r3'] + filenames = ['*.r', '*.r3', '*.reb'] mimetypes = ['text/x-rebol'] flags = re.IGNORECASE | re.MULTILINE re.IGNORECASE - escape_re = r'(?:\^\([0-9a-fA-F]{1,4}\)*)' + escape_re = r'(?:\^\([0-9a-f]{1,4}\)*)' def word_callback(lexer, match): word = match.group() @@ -1483,9 +1483,9 @@ class RebolLexer(RegexLexer): 'script': [ (r'\s+', Text), (r'#"', String.Char, 'char'), - (r'#{[0-9a-fA-F]*}', Number.Hex), + (r'#{[0-9a-f]*}', Number.Hex), (r'2#{', Number.Hex, 'bin2'), - (r'64#{[0-9a-zA-Z+/=\s]*}', Number.Hex), + (r'64#{[0-9a-z+/=\s]*}', Number.Hex), (r'"', String, 'string'), (r'{', String, 'string2'), (r';#+.*\n', Comment.Special), @@ -1493,9 +1493,9 @@ class RebolLexer(RegexLexer): (r';.*\n', Comment), (r'%"', Name.Decorator, 'stringFile'), (r'%[^(\^{^")\s\[\]]+', Name.Decorator), - (r'[+-]?([a-zA-Z]{1,3})?\$\d+(\.\d+)?', Number.Float), # money + (r'[+-]?([a-z]{1,3})?\$\d+(\.\d+)?', Number.Float), # money (r'[+-]?\d+\:\d+(\:\d+)?(\.\d+)?', String.Other), # time - (r'\d+[\-\/][0-9a-zA-Z]+[\-\/]\d+(\/\d+\:\d+((\:\d+)?' + (r'\d+[\-\/][0-9a-z]+[\-\/]\d+(\/\d+\:\d+((\:\d+)?' r'([\.\d+]?([+-]?\d+:\d+)?)?)?)?', String.Other), # date (r'\d+(\.\d+)+\.\d+', Keyword.Constant), # tuple (r'\d+[xX]\d+', Keyword.Constant), # pair @@ -1503,13 +1503,16 @@ class RebolLexer(RegexLexer): (r'[+-]?\d+(\'\d+)?[\.,]\d*', Number.Float), (r'[+-]?\d+(\'\d+)?', Number), (r'[\[\]\(\)]', Generic.Strong), - (r'[a-zA-Z]+[^(\^{"\s:)]*://[^(\^{"\s)]*', Name.Decorator), # url + (r'[a-z]+[^(\^{"\s:)]*://[^(\^{"\s)]*', Name.Decorator), # url (r'mailto:[^(\^{"@\s)]+@[^(\^{"@\s)]+', Name.Decorator), # url (r'[^(\^{"@\s)]+@[^(\^{"@\s)]+', Name.Decorator), # email - (r'comment\s', Comment, 'comment'), + (r'comment\s"', Comment, 'commentString1'), + (r'comment\s{', Comment, 'commentString2'), + (r'comment\s\[', Comment, 'commentBlock'), + (r'comment\s[^(\s{\"\[]+', Comment), (r'/[^(\^{^")\s/[\]]*', Name.Attribute), (r'([^(\^{^")\s/[\]]+)(?=[:({"\s/\[\]])', word_callback), - (r'<[a-zA-Z0-9:._-]*>', Name.Tag), + (r'<[\w:.-]*>', Name.Tag), (r'<[^(<>\s")]+', Name.Tag, 'tag'), (r'([^(\^{^")\s]+)', Text), ], @@ -1561,12 +1564,6 @@ class RebolLexer(RegexLexer): (r'([0-1]\s*){8}', Number.Hex), (r'}', Number.Hex, '#pop'), ], - 'comment': [ - (r'"', Comment, 'commentString1'), - (r'{', Comment, 'commentString2'), - (r'\[', Comment, 'commentBlock'), - (r'[^(\s{\"\[]+', Comment, '#pop'), - ], 'commentString1': [ (r'[^(\^")]+', Comment), (escape_re, Comment), @@ -1585,7 +1582,9 @@ class RebolLexer(RegexLexer): 'commentBlock': [ (r'\[', Comment, '#push'), (r'\]', Comment, '#pop'), - (r'[^(\[\])]+', Comment), + (r'"', Comment, "commentString1"), + (r'{', Comment, "commentString2"), + (r'[^(\[\]\"{)]+', Comment), ], } def analyse_text(text): @@ -1620,7 +1619,7 @@ class ABAPLexer(RegexLexer): (r'\".*?\n', Comment.Single), ], 'variable-names': [ - (r'<[\S_]+>', Name.Variable), + (r'<\S+>', Name.Variable), (r'\w[\w~]*(?:(\[\])|->\*)?', Name.Variable), ], 'root': [ @@ -1804,15 +1803,15 @@ class NewspeakLexer(RegexLexer): 'root' : [ (r'\b(Newsqueak2)\b',Keyword.Declaration), (r"'[^']*'",String), - (r'\b(class)(\s+)([a-zA-Z0-9_]+)(\s*)', + (r'\b(class)(\s+)(\w+)(\s*)', bygroups(Keyword.Declaration,Text,Name.Class,Text)), (r'\b(mixin|self|super|private|public|protected|nil|true|false)\b', Keyword), - (r'([a-zA-Z0-9_]+\:)(\s*)([a-zA-Z_]\w+)', + (r'(\w+\:)(\s*)([a-zA-Z_]\w+)', bygroups(Name.Function,Text,Name.Variable)), - (r'([a-zA-Z0-9_]+)(\s*)(=)', + (r'(\w+)(\s*)(=)', bygroups(Name.Attribute,Text,Operator)), - (r'<[a-zA-Z0-9_]+>', Comment.Special), + (r'<\w+>', Comment.Special), include('expressionstat'), include('whitespace') ], @@ -1891,7 +1890,9 @@ class GherkinLexer(RegexLexer): (r"[^\|]", Name.Variable), ], 'scenario_sections_on_stack': [ - (feature_element_keywords, bygroups(Name.Function, Keyword, Keyword, Name.Function), "feature_elements_on_stack"), + (feature_element_keywords, + bygroups(Name.Function, Keyword, Keyword, Name.Function), + "feature_elements_on_stack"), ], 'narrative': [ include('scenario_sections_on_stack'), @@ -2021,23 +2022,23 @@ class AsymptoteLexer(RegexLexer): r'bounds|coord|frame|guide|horner|int|linefit|marginT|pair|pen|' r'picture|position|real|revolution|slice|splitface|ticksgridT|' r'tickvalues|tree|triple|vertex|void)\b', Keyword.Type), - ('[a-zA-Z_][a-zA-Z0-9_]*:(?!:)', Name.Label), - ('[a-zA-Z_][a-zA-Z0-9_]*', Name), + ('[a-zA-Z_]\w*:(?!:)', Name.Label), + ('[a-zA-Z_]\w*', Name), ], 'root': [ include('whitespace'), # functions - (r'((?:[a-zA-Z0-9_*\s])+?(?:\s|\*))' # return arguments - r'([a-zA-Z_][a-zA-Z0-9_]*)' # method name - r'(\s*\([^;]*?\))' # signature + (r'((?:[\w*\s])+?(?:\s|\*))' # return arguments + r'([a-zA-Z_]\w*)' # method name + r'(\s*\([^;]*?\))' # signature r'(' + _ws + r')({)', bygroups(using(this), Name.Function, using(this), using(this), Punctuation), 'function'), # function declarations - (r'((?:[a-zA-Z0-9_*\s])+?(?:\s|\*))' # return arguments - r'([a-zA-Z_][a-zA-Z0-9_]*)' # method name - r'(\s*\([^;]*?\))' # signature + (r'((?:[\w*\s])+?(?:\s|\*))' # return arguments + r'([a-zA-Z_]\w*)' # method name + r'(\s*\([^;]*?\))' # signature r'(' + _ws + r')(;)', bygroups(using(this), Name.Function, using(this), using(this), Punctuation)), @@ -2190,7 +2191,7 @@ class AutohotkeyLexer(RegexLexer): (r'^;.*?$', Comment.Singleline), (r'[]{}(),;[]', Punctuation), (r'(in|is|and|or|not)\b', Operator.Word), - (r'\%[a-zA-Z_#@$][a-zA-Z0-9_#@$]*\%', Name.Variable), + (r'\%[a-zA-Z_#@$][\w#@$]*\%', Name.Variable), (r'!=|==|:=|\.=|<<|>>|[-~+/*%=<>&^|?:!.]', Operator), include('commands'), include('labels'), @@ -2198,7 +2199,7 @@ class AutohotkeyLexer(RegexLexer): include('builtInVariables'), (r'"', String, combined('stringescape', 'dqs')), include('numbers'), - (r'[a-zA-Z_#@$][a-zA-Z0-9_#@$]*', Name), + (r'[a-zA-Z_#@$][\w#@$]*', Name), (r'\\|\'', Text), (r'\`([\,\%\`abfnrtv\-\+;])', String.Escape), include('garbage'), @@ -2392,7 +2393,7 @@ class MaqlLexer(RegexLexer): r'SYNCHRONIZE|TYPE|DEFAULT|ORDER|ASC|DESC|HYPERLINK|' r'INCLUDE|TEMPLATE|MODIFY)\b', Keyword), # FUNCNAME - (r'[a-zA-Z]\w*\b', Name.Function), + (r'[a-z]\w*\b', Name.Function), # Comments (r'#.*', Comment.Single), # Punctuation @@ -2427,7 +2428,7 @@ class GoodDataCLLexer(RegexLexer): # Comments (r'#.*', Comment.Single), # Function call - (r'[a-zA-Z]\w*', Name.Function), + (r'[a-z]\w*', Name.Function), # Argument list (r'\(', Token.Punctuation, 'args-list'), # Punctuation @@ -2438,7 +2439,7 @@ class GoodDataCLLexer(RegexLexer): 'args-list': [ (r'\)', Token.Punctuation, '#pop'), (r',', Token.Punctuation), - (r'[a-zA-Z]\w*', Name.Variable), + (r'[a-z]\w*', Name.Variable), (r'=', Operator), (r'"', Literal.String, 'string-literal'), (r'[0-9]+(?:\.[0-9]+)?(?:[eE][+-]?[0-9]{1,3})?', Literal.Number), @@ -2490,18 +2491,18 @@ class ProtoBufLexer(RegexLexer): (r'0[0-7]+[LlUu]*', Number.Oct), (r'\d+[LlUu]*', Number.Integer), (r'[+-=]', Operator), - (r'([a-zA-Z_][a-zA-Z0-9_\.]*)([ \t]*)(=)', + (r'([a-zA-Z_][\w\.]*)([ \t]*)(=)', bygroups(Name.Attribute, Text, Operator)), - ('[a-zA-Z_][a-zA-Z0-9_\.]*', Name), + ('[a-zA-Z_][\w\.]*', Name), ], 'package': [ - (r'[a-zA-Z_][a-zA-Z0-9_]*', Name.Namespace, '#pop') + (r'[a-zA-Z_]\w*', Name.Namespace, '#pop') ], 'message': [ - (r'[a-zA-Z_][a-zA-Z0-9_]*', Name.Class, '#pop') + (r'[a-zA-Z_]\w*', Name.Class, '#pop') ], 'type': [ - (r'[a-zA-Z_][a-zA-Z0-9_]*', Name, '#pop') + (r'[a-zA-Z_]\w*', Name, '#pop') ], } @@ -2524,12 +2525,12 @@ class HybrisLexer(RegexLexer): 'root': [ # method names (r'^(\s*(?:function|method|operator\s+)+?)' - r'([a-zA-Z_][a-zA-Z0-9_]*)' + r'([a-zA-Z_]\w*)' r'(\s*)(\()', bygroups(Keyword, Name.Function, Text, Operator)), (r'[^\S\n]+', Text), (r'//.*?\n', Comment.Single), (r'/\*.*?\*/', Comment.Multiline), - (r'@[a-zA-Z_][a-zA-Z0-9_\.]*', Name.Decorator), + (r'@[a-zA-Z_][\w\.]*', Name.Decorator), (r'(break|case|catch|next|default|do|else|finally|for|foreach|of|' r'unless|if|new|return|switch|me|throw|try|while)\b', Keyword), (r'(extends|private|protected|public|static|throws|function|method|' @@ -2565,10 +2566,10 @@ class HybrisLexer(RegexLexer): r'Exception)\b', Keyword.Type), (r'"(\\\\|\\"|[^"])*"', String), (r"'\\.'|'[^\\]'|'\\u[0-9a-f]{4}'", String.Char), - (r'(\.)([a-zA-Z_][a-zA-Z0-9_]*)', + (r'(\.)([a-zA-Z_]\w*)', bygroups(Operator, Name.Attribute)), - (r'[a-zA-Z_][a-zA-Z0-9_]*:', Name.Label), - (r'[a-zA-Z_\$][a-zA-Z0-9_]*', Name), + (r'[a-zA-Z_]\w*:', Name.Label), + (r'[a-zA-Z_\$]\w*', Name), (r'[~\^\*!%&\[\]\(\)\{\}<>\|+=:;,./?\-@]+', Operator), (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float), (r'0x[0-9a-f]+', Number.Hex), @@ -2576,10 +2577,10 @@ class HybrisLexer(RegexLexer): (r'\n', Text), ], 'class': [ - (r'[a-zA-Z_][a-zA-Z0-9_]*', Name.Class, '#pop') + (r'[a-zA-Z_]\w*', Name.Class, '#pop') ], 'import': [ - (r'[a-zA-Z0-9_.]+\*?', Name.Namespace, '#pop') + (r'[\w.]+\*?', Name.Namespace, '#pop') ], } @@ -2628,7 +2629,7 @@ class AwkLexer(RegexLexer): (r'(ARGC|ARGIND|ARGV|CONVFMT|ENVIRON|ERRNO|FIELDWIDTHS|FILENAME|FNR|FS|' r'IGNORECASE|NF|NR|OFMT|OFS|ORFS|RLENGTH|RS|RSTART|RT|' r'SUBSEP)\b', Name.Builtin), - (r'[$a-zA-Z_][a-zA-Z0-9_]*', Name.Other), + (r'[$a-zA-Z_]\w*', Name.Other), (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float), (r'0x[0-9a-fA-F]+', Number.Hex), (r'[0-9]+', Number.Integer), @@ -2845,7 +2846,7 @@ class UrbiscriptLexer(ExtendedRegexLexer): Operator.Word), (r'[{}\[\]()]+', Punctuation), (r'(?:;|\||,|&|\?|!)+', Punctuation), - (r'[$a-zA-Z_][a-zA-Z0-9_]*', Name.Other), + (r'[$a-zA-Z_]\w*', Name.Other), (r'0x[0-9a-fA-F]+', Number.Hex), # Float, Integer, Angle and Duration (r'(?:[0-9]+(?:(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?)?' @@ -3051,7 +3052,7 @@ class MscgenLexer(RegexLexer): aliases = ['mscgen', 'msc'] filenames = ['*.msc'] - _var = r'([a-zA-Z0-9_]+|"(?:\\"|[^"])*")' + _var = r'(\w+|"(?:\\"|[^"])*")' tokens = { 'root': [ @@ -3201,9 +3202,9 @@ class VGLLexer(RegexLexer): (r'(true|false|null|empty|error|locked)', Keyword.Constant), (r'[~\^\*\#!%&\[\]\(\)<>\|+=:;,./?-]', Operator), (r'"[^"]*"', String), - (r'(\.)([a-z_\$][a-z0-9_\$]*)', bygroups(Operator, Name.Attribute)), + (r'(\.)([a-z_\$][\w\$]*)', bygroups(Operator, Name.Attribute)), (r'[0-9][0-9]*(\.[0-9]+(e[+\-]?[0-9]+)?)?', Number), - (r'[a-z_\$][a-z0-9_\$]*', Name), + (r'[a-z_\$][\w\$]*', Name), (r'[\r\n]+', Text), (r'\s+', Text) ] @@ -3253,7 +3254,7 @@ class SourcePawnLexer(RegexLexer): r'public|return|sizeof|static|decl|struct|switch)\b', Keyword), (r'(bool|Float)\b', Keyword.Type), (r'(true|false)\b', Keyword.Constant), - ('[a-zA-Z_][a-zA-Z0-9_]*', Name), + ('[a-zA-Z_]\w*', Name), ], 'string': [ (r'"', String, '#pop'), @@ -3353,7 +3354,7 @@ class PuppetLexer(RegexLexer): ], 'names': [ - ('[a-zA-Z_][a-zA-Z0-9_]*', Name.Attribute), + ('[a-zA-Z_]\w*', Name.Attribute), (r'(\$\S+)(\[)(\S+)(\])', bygroups(Name.Variable, Punctuation, String, Punctuation)), (r'\$\S+', Name.Variable), @@ -3590,9 +3591,9 @@ class RPMSpecLexer(RegexLexer): 'interpol': [ (r'%\{?__[a-z_]+\}?', Name.Function), (r'%\{?_([a-z_]+dir|[a-z_]+path|prefix)\}?', Keyword.Pseudo), - (r'%\{\?[A-Za-z0-9_]+\}', Name.Variable), + (r'%\{\?\w+\}', Name.Variable), (r'\$\{?RPM_[A-Z0-9_]+\}?', Name.Variable.Global), - (r'%\{[a-zA-Z][a-zA-Z0-9_]+\}', Keyword.Constant), + (r'%\{[a-zA-Z]\w+\}', Keyword.Constant), ] } @@ -3723,7 +3724,7 @@ class AutoItLexer(RegexLexer): (r'(#comments-start|#cs).*?(#comments-end|#ce)', Comment.Multiline), (r'[\[\]{}(),;]', Punctuation), (r'(and|or|not)\b', Operator.Word), - (r'[\$|@][a-zA-Z_][a-zA-Z0-9_]*', Name.Variable), + (r'[\$|@][a-zA-Z_]\w*', Name.Variable), (r'!=|==|:=|\.=|<<|>>|[-~+/*%=<>&^|?:!.]', Operator), include('commands'), include('labels'), @@ -3731,7 +3732,7 @@ class AutoItLexer(RegexLexer): include('builtInMarcros'), (r'"', String, combined('stringescape', 'dqs')), include('numbers'), - (r'[a-zA-Z_#@$][a-zA-Z0-9_#@$]*', Name), + (r'[a-zA-Z_#@$][\w#@$]*', Name), (r'\\|\'', Text), (r'\`([\,\%\`abfnrtv\-\+;])', String.Escape), (r'_\n', Text), # Line continuation @@ -3799,15 +3800,15 @@ class RexxLexer(RegexLexer): (r'"', String, 'string_double'), (r"'", String, 'string_single'), (r'[0-9]+(\.[0-9]+)?(e[+-]?[0-9])?', Number), - (r'([a-z_][a-z0-9_]*)(\s*)(:)(\s*)(procedure)\b', + (r'([a-z_]\w*)(\s*)(:)(\s*)(procedure)\b', bygroups(Name.Function, Whitespace, Operator, Whitespace, Keyword.Declaration)), - (r'([a-z_][a-z0-9_]*)(\s*)(:)', + (r'([a-z_]\w*)(\s*)(:)', bygroups(Name.Label, Whitespace, Operator)), include('function'), include('keyword'), include('operator'), - (r'[a-z_][a-z0-9_]*', Text), + (r'[a-z_]\w*', Text), ], 'function': [ (r'(abbrev|abs|address|arg|b2x|bitand|bitor|bitxor|c2d|c2x|' @@ -3855,7 +3856,7 @@ class RexxLexer(RegexLexer): _ADDRESS_PATTERN = _c(r'^\s*address\s+') _DO_WHILE_PATTERN = _c(r'^\s*do\s+while\b') _IF_THEN_DO_PATTERN = _c(r'^\s*if\b.+\bthen\s+do\s*$') - _PROCEDURE_PATTERN = _c(r'^\s*([a-z_][a-z0-9_]*)(\s*)(:)(\s*)(procedure)\b') + _PROCEDURE_PATTERN = _c(r'^\s*([a-z_]\w*)(\s*)(:)(\s*)(procedure)\b') _ELSE_DO_PATTERN = _c(r'\belse\s+do\s*$') _PARSE_ARG_PATTERN = _c(r'^\s*parse\s+(upper\s+)?(arg|value)\b') PATTERNS_AND_WEIGHTS = ( @@ -4000,12 +4001,12 @@ class AmbientTalkLexer(RegexLexer): (r'"(\\\\|\\"|[^"])*"', String), (r'\|', Punctuation, 'arglist'), (r'<:|[\^\*!%&<>+=,./?-]|:=', Operator), - (r"`[a-zA-Z_][a-zA-Z0-9_]*", String.Symbol), - (r"[a-zA-Z_][a-zA-Z0-9_]*:", Name.Function), + (r"`[a-zA-Z_]\w*", String.Symbol), + (r"[a-zA-Z_]\w*:", Name.Function), (r"[\{\}()\[\];`]", Punctuation), (r'(self|super)\b', Name.Variable.Instance), - (r"[a-zA-Z_][a-zA-Z0-9_]*", Name.Variable), - (r"@[a-zA-Z_][a-zA-Z0-9_]*", Name.Class), + (r"[a-zA-Z_]\w*", Name.Variable), + (r"@[a-zA-Z_]\w*", Name.Class), (r"@\[", Name.Class, 'annotations'), include('numbers'), ], @@ -4014,9 +4015,9 @@ class AmbientTalkLexer(RegexLexer): (r'\d+', Number.Integer) ], 'namespace': [ - (r'[a-zA-Z_][a-zA-Z0-9_]*\.', Name.Namespace), - (r'[a-zA-Z_][a-zA-Z0-9_]*:', Name.Function , '#pop'), - (r'[a-zA-Z_][a-zA-Z0-9_]*(?!\.)', Name.Function , '#pop') + (r'[a-zA-Z_]\w*\.', Name.Namespace), + (r'[a-zA-Z_]\w*:', Name.Function , '#pop'), + (r'[a-zA-Z_]\w*(?!\.)', Name.Function , '#pop') ], 'annotations' : [ (r"(.*?)\]", Name.Class, '#pop') @@ -4024,7 +4025,7 @@ class AmbientTalkLexer(RegexLexer): 'arglist' : [ (r'\|', Punctuation, '#pop'), (r'\s*(,)\s*', Punctuation), - (r'[a-zA-Z_][a-zA-Z0-9_]*', Name.Variable), + (r'[a-zA-Z_]\w*', Name.Variable), ], } @@ -4071,7 +4072,7 @@ class PawnLexer(RegexLexer): r'public|return|sizeof|tagof|state|goto)\b', Keyword), (r'(bool|Float)\b', Keyword.Type), (r'(true|false)\b', Keyword.Constant), - ('[a-zA-Z_][a-zA-Z0-9_]*', Name), + ('[a-zA-Z_]\w*', Name), ], 'string': [ (r'"', String, '#pop'), @@ -4230,7 +4231,7 @@ class PanLexer(RegexLexer): 'curly': [ (r'}', Keyword, '#pop'), (r':-', Keyword), - (r'[a-zA-Z0-9_]+', Name.Variable), + (r'\w+', Name.Variable), (r'[^}:"\'`$]+', Punctuation), (r':', Punctuation), include('root'), @@ -4254,7 +4255,7 @@ class RedLexer(RegexLexer): flags = re.IGNORECASE | re.MULTILINE - escape_re = r'(?:\^\([0-9a-fA-F]{1,4}\)*)' + escape_re = r'(?:\^\([0-9a-f]{1,4}\)*)' def word_callback(lexer, match): word = match.group() @@ -4274,22 +4275,27 @@ class RedLexer(RegexLexer): r'power|remainder|round|subtract|even\?|odd\?|and~|complement|or~|xor~|' r'append|at|back|change|clear|copy|find|head|head\?|index\?|insert|' r'length\?|next|pick|poke|remove|reverse|select|sort|skip|swap|tail|tail\?|' - r'take|trim|create|close|delete|modify|open|open\?|query|read|rename|update|write)$', word): + r'take|trim|create|close|delete|modify|open|open\?|query|read|rename|' + r'update|write)$', word): yield match.start(), Name.Function, word elif re.match( - r'(yes|on|no|off|true|false|tab|cr|lf|newline|escape|slash|sp|space|null|none|crlf|dot|null-byte)$', word): + r'(yes|on|no|off|true|false|tab|cr|lf|newline|escape|slash|sp|space|null|' + r'none|crlf|dot|null-byte)$', word): yield match.start(), Name.Builtin.Pseudo, word elif re.match( - r'(#system-global|#include|#enum|#define|#either|#if|#import|#export|#switch|#default|#get-definition)$', word): + r'(#system-global|#include|#enum|#define|#either|#if|#import|#export|' + r'#switch|#default|#get-definition)$', word): yield match.start(), Keyword.Namespace, word elif re.match( - r'(system|halt|quit|quit-return|do|load|q|recycle|call|run|ask|parse|raise-error|' - r'return|exit|break|alias|push|pop|probe|\?\?|spec-of|body-of|quote|forever)$', word): + r'(system|halt|quit|quit-return|do|load|q|recycle|call|run|ask|parse|' + r'raise-error|return|exit|break|alias|push|pop|probe|\?\?|spec-of|body-of|' + r'quote|forever)$', word): yield match.start(), Name.Exception, word elif re.match( - r'(action\?|block\?|char\?|datatype\?|file\?|function\?|get-path\?|zero\?|any-struct\?|' - r'get-word\?|integer\?|issue\?|lit-path\?|lit-word\?|logic\?|native\?|none\?|' - r'op\?|paren\?|path\?|refinement\?|set-path\?|set-word\?|string\?|unset\?|word\?|any-series\?)$', word): + r'(action\?|block\?|char\?|datatype\?|file\?|function\?|get-path\?|zero\?|' + r'get-word\?|integer\?|issue\?|lit-path\?|lit-word\?|logic\?|native\?|' + r'op\?|paren\?|path\?|refinement\?|set-path\?|set-word\?|string\?|unset\?|' + r'any-struct\?|none\?|word\?|any-series\?)$', word): yield match.start(), Keyword, word elif re.match(r'(JNICALL|stdcall|cdecl|infix)$', word): yield match.start(), Keyword.Namespace, word @@ -4320,10 +4326,11 @@ class RedLexer(RegexLexer): 'script': [ (r'\s+', Text), (r'#"', String.Char, 'char'), - (r'#{[0-9a-fA-F\s]*}', Number.Hex), + (r'#{[0-9a-f\s]*}', Number.Hex), (r'2#{', Number.Hex, 'bin2'), - (r'64#{[0-9a-zA-Z+/=\s]*}', Number.Hex), - (r'([0-9a-fA-F]+)(h)((\s)|(?=[\[\]{}""\(\)]))', bygroups(Number.Hex, Name.Variable, Whitespace)), + (r'64#{[0-9a-z+/=\s]*}', Number.Hex), + (r'([0-9a-f]+)(h)((\s)|(?=[\[\]{}""\(\)]))', + bygroups(Number.Hex, Name.Variable, Whitespace)), (r'"', String, 'string'), (r'{', String, 'string2'), (r';#+.*\n', Comment.Special), @@ -4331,9 +4338,9 @@ class RedLexer(RegexLexer): (r';.*\n', Comment), (r'%"', Name.Decorator, 'stringFile'), (r'%[^(\^{^")\s\[\]]+', Name.Decorator), - (r'[+-]?([a-zA-Z]{1,3})?\$\d+(\.\d+)?', Number.Float), # money + (r'[+-]?([a-z]{1,3})?\$\d+(\.\d+)?', Number.Float), # money (r'[+-]?\d+\:\d+(\:\d+)?(\.\d+)?', String.Other), # time - (r'\d+[\-\/][0-9a-zA-Z]+[\-\/]\d+(\/\d+\:\d+((\:\d+)?' + (r'\d+[\-\/][0-9a-z]+[\-\/]\d+(\/\d+\:\d+((\:\d+)?' r'([\.\d+]?([+-]?\d+:\d+)?)?)?)?', String.Other), # date (r'\d+(\.\d+)+\.\d+', Keyword.Constant), # tuple (r'\d+[xX]\d+', Keyword.Constant), # pair @@ -4341,13 +4348,16 @@ class RedLexer(RegexLexer): (r'[+-]?\d+(\'\d+)?[\.,]\d*', Number.Float), (r'[+-]?\d+(\'\d+)?', Number), (r'[\[\]\(\)]', Generic.Strong), - (r'[a-zA-Z]+[^(\^{"\s:)]*://[^(\^{"\s)]*', Name.Decorator), # url + (r'[a-z]+[^(\^{"\s:)]*://[^(\^{"\s)]*', Name.Decorator), # url (r'mailto:[^(\^{"@\s)]+@[^(\^{"@\s)]+', Name.Decorator), # url (r'[^(\^{"@\s)]+@[^(\^{"@\s)]+', Name.Decorator), # email - (r'comment\s', Comment, 'comment'), + (r'comment\s"', Comment, 'commentString1'), + (r'comment\s{', Comment, 'commentString2'), + (r'comment\s\[', Comment, 'commentBlock'), + (r'comment\s[^(\s{\"\[]+', Comment), (r'/[^(\^{^")\s/[\]]*', Name.Attribute), (r'([^(\^{^")\s/[\]]+)(?=[:({"\s/\[\]])', word_callback), - (r'<[a-zA-Z0-9:._-]*>', Name.Tag), + (r'<[\w:.-]*>', Name.Tag), (r'<[^(<>\s")]+', Name.Tag, 'tag'), (r'([^(\^{^")\s]+)', Text), ], @@ -4399,12 +4409,6 @@ class RedLexer(RegexLexer): (r'([0-1]\s*){8}', Number.Hex), (r'}', Number.Hex, '#pop'), ], - 'comment': [ - (r'"', Comment, 'commentString1'), - (r'{', Comment, 'commentString2'), - (r'\[', Comment, 'commentBlock'), - (r'[^(\s{\"\[]+', Comment, '#pop'), - ], 'commentString1': [ (r'[^(\^")]+', Comment), (escape_re, Comment), @@ -4423,6 +4427,8 @@ class RedLexer(RegexLexer): 'commentBlock': [ (r'\[', Comment, '#push'), (r'\]', Comment, '#pop'), - (r'[^(\[\])]+', Comment), + (r'"', Comment, "commentString1"), + (r'{', Comment, "commentString2"), + (r'[^(\[\]\"{)]+', Comment), ], } diff --git a/pygments/lexers/parsers.py b/pygments/lexers/parsers.py index fc8cbb6f..4c23c760 100644 --- a/pygments/lexers/parsers.py +++ b/pygments/lexers/parsers.py @@ -102,7 +102,7 @@ class RagelLexer(RegexLexer): 'host': [ (r'(' + r'|'.join(( # keep host code in largest possible chunks r'[^{}\'"/#]+', # exclude unsafe characters - r'[^\\][\\][{}]', # allow escaped { or } + r'[^\\]\\[{}]', # allow escaped { or } # strings and comments may safely contain unsafe characters r'"(\\\\|\\"|[^"])*"', # double quote string @@ -173,7 +173,7 @@ class RagelEmbeddedLexer(RegexLexer): r'[^}\'"\[/#]', # exclude unsafe characters r'}(?=[^%]|$)', # } is okay as long as it's not followed by % r'}%(?=[^%]|$)', # ...well, one %'s okay, just not two... - r'[^\\][\\][{}]', # ...and } is okay if it's escaped + r'[^\\]\\[{}]', # ...and } is okay if it's escaped # allow / if it's preceded with one of these symbols # (ragel EOF actions) diff --git a/pygments/lexers/shell.py b/pygments/lexers/shell.py index f809dae9..df9b56f4 100644 --- a/pygments/lexers/shell.py +++ b/pygments/lexers/shell.py @@ -79,7 +79,7 @@ class BashLexer(RegexLexer): 'curly': [ (r'}', Keyword, '#pop'), (r':-', Keyword), - (r'[a-zA-Z0-9_]+', Name.Variable), + (r'\w+', Name.Variable), (r'[^}:"\'`$]+', Punctuation), (r':', Punctuation), include('root'), @@ -314,7 +314,7 @@ class TcshLexer(RegexLexer): 'curly': [ (r'}', Keyword, '#pop'), (r':-', Keyword), - (r'[a-zA-Z0-9_]+', Name.Variable), + (r'\w+', Name.Variable), (r'[^}:"\'`$]+', Punctuation), (r':', Punctuation), include('root'), @@ -390,13 +390,13 @@ class PowerShellLexer(RegexLexer): (r'`[\'"$@-]', Punctuation), (r'"', String.Double, 'string'), (r"'([^']|'')*'", String.Single), - (r'(\$|@@|@)((global|script|private|env):)?[a-z0-9_]+', + (r'(\$|@@|@)((global|script|private|env):)?\w+', Name.Variable), (r'(%s)\b' % '|'.join(keywords), Keyword), (r'-(%s)\b' % '|'.join(operators), Operator), - (r'(%s)-[a-z_][a-z0-9_]*\b' % '|'.join(verbs), Name.Builtin), - (r'\[[a-z_\[][a-z0-9_. `,\[\]]*\]', Name.Constant), # .net [type]s - (r'-[a-z_][a-z0-9_]*', Name), + (r'(%s)-[a-z_]\w*\b' % '|'.join(verbs), Name.Builtin), + (r'\[[a-z_\[][\w. `,\[\]]*\]', Name.Constant), # .net [type]s + (r'-[a-z_]\w*', Name), (r'\w+', Name), (r'[.,;@{}\[\]$()=+*/\\&%!~?^`|<>-]|::', Punctuation), ], diff --git a/pygments/lexers/sql.py b/pygments/lexers/sql.py index 9a3dcb8d..7540f079 100644 --- a/pygments/lexers/sql.py +++ b/pygments/lexers/sql.py @@ -150,10 +150,10 @@ class PostgresLexer(PostgresBase, RegexLexer): (r"(E|U&)?'(''|[^'])*'", String.Single), (r'(U&)?"(""|[^"])*"', String.Name), # quoted identifier (r'(?s)(\$[^\$]*\$)(.*?)(\1)', language_callback), - (r'[a-zA-Z_][a-zA-Z0-9_]*', Name), + (r'[a-z_]\w*', Name), # psql variable in SQL - (r""":(['"]?)[a-z][a-z0-9_]*\b\1""", Name.Variable), + (r""":(['"]?)[a-z]\w*\b\1""", Name.Variable), (r'[;:()\[\]\{\},\.]', Punctuation), ], @@ -192,10 +192,10 @@ class PlPgsqlLexer(PostgresBase, RegexLexer): # Add specific PL/pgSQL rules (before the SQL ones) tokens['root'][:0] = [ - (r'\%[a-z][a-z0-9_]*\b', Name.Builtin), # actually, a datatype + (r'\%[a-z]\w*\b', Name.Builtin), # actually, a datatype (r':=', Operator), - (r'\<\<[a-z][a-z0-9_]*\>\>', Name.Label), - (r'\#[a-z][a-z0-9_]*\b', Keyword.Pseudo), # #variable_conflict + (r'\<\<[a-z]\w*\>\>', Name.Label), + (r'\#[a-z]\w*\b', Keyword.Pseudo), # #variable_conflict ] @@ -219,7 +219,7 @@ class PsqlRegexLexer(PostgresBase, RegexLexer): (r'\n', Text, 'root'), (r'\s+', Text), (r'\\[^\s]+', Keyword.Pseudo), - (r""":(['"]?)[a-z][a-z0-9_]*\b\1""", Name.Variable), + (r""":(['"]?)[a-z]\w*\b\1""", Name.Variable), (r"'(''|[^'])*'", String.Single), (r"`([^`])*`", String.Backtick), (r"[^\s]+", String.Symbol), @@ -435,7 +435,7 @@ class SqlLexer(RegexLexer): # TODO: Backslash escapes? (r"'(''|[^'])*'", String.Single), (r'"(""|[^"])*"', String.Symbol), # not a real string literal in ANSI SQL - (r'[a-zA-Z_][a-zA-Z0-9_]*', Name), + (r'[a-z_]\w*', Name), (r'[;:()\[\],\.]', Punctuation) ], 'multiline-comments': [ @@ -506,10 +506,10 @@ class MySqlLexer(RegexLexer): # TODO: this list is not complete (r'\b(auto_increment|engine|charset|tables)\b', Keyword.Pseudo), (r'(true|false|null)', Name.Constant), - (r'([a-zA-Z_][a-zA-Z0-9_]*)(\s*)(\()', + (r'([a-z_]\w*)(\s*)(\()', bygroups(Name.Function, Text, Punctuation)), - (r'[a-zA-Z_][a-zA-Z0-9_]*', Name), - (r'@[A-Za-z0-9]*[._]*[A-Za-z0-9]*', Name.Variable), + (r'[a-z_]\w*', Name), + (r'@[a-z0-9]*[._]*[a-z0-9]*', Name.Variable), (r'[;:()\[\],\.]', Punctuation) ], 'multiline-comments': [ @@ -584,7 +584,7 @@ class RqlLexer(RegexLexer): (r'[+*/<>=%-]', Operator), (r'(Any|is|instance_of|CWEType|CWRelation)\b', Name.Builtin), (r'[0-9]+', Number.Integer), - (r'[A-Z_][A-Z0-9_]*\??', Name), + (r'[A-Z_]\w*\??', Name), (r"'(''|[^'])*'", String.Single), (r'"(""|[^"])*"', String.Single), (r'[;:()\[\],\.]', Punctuation) diff --git a/pygments/lexers/templates.py b/pygments/lexers/templates.py index d06ea288..4d53dca6 100644 --- a/pygments/lexers/templates.py +++ b/pygments/lexers/templates.py @@ -161,22 +161,22 @@ class SmartyLexer(RegexLexer): (r'(\{php\})(.*?)(\{/php\})', bygroups(Comment.Preproc, using(PhpLexer, startinline=True), Comment.Preproc)), - (r'(\{)(/?[a-zA-Z_][a-zA-Z0-9_]*)(\s*)', + (r'(\{)(/?[a-zA-Z_]\w*)(\s*)', bygroups(Comment.Preproc, Name.Function, Text), 'smarty'), (r'\{', Comment.Preproc, 'smarty') ], 'smarty': [ (r'\s+', Text), (r'\}', Comment.Preproc, '#pop'), - (r'#[a-zA-Z_][a-zA-Z0-9_]*#', Name.Variable), - (r'\$[a-zA-Z_][a-zA-Z0-9_]*(\.[a-zA-Z0-9_]+)*', Name.Variable), + (r'#[a-zA-Z_]\w*#', Name.Variable), + (r'\$[a-zA-Z_]\w*(\.\w+)*', Name.Variable), (r'[~!%^&*()+=|\[\]:;,.<>/?{}@-]', Operator), (r'(true|false|null)\b', Keyword.Constant), (r"[0-9](\.[0-9]*)?(eE[+-][0-9])?[flFLdD]?|" r"0[xX][0-9a-fA-F]+[Ll]?", Number), (r'"(\\\\|\\"|[^"])*"', String.Double), (r"'(\\\\|\\'|[^'])*'", String.Single), - (r'[a-zA-Z_][a-zA-Z0-9_]*', Name.Attribute) + (r'[a-zA-Z_]\w*', Name.Attribute) ] } @@ -207,7 +207,7 @@ class VelocityLexer(RegexLexer): flags = re.MULTILINE | re.DOTALL - identifier = r'[a-zA-Z_][a-zA-Z0-9_]*' + identifier = r'[a-zA-Z_]\w*' tokens = { 'root': [ @@ -267,8 +267,8 @@ class VelocityLexer(RegexLexer): rv += 0.15 if re.search(r'#\{?foreach\}?\(.+?\).*?#\{?end\}?', text): rv += 0.15 - if re.search(r'\$\{?[a-zA-Z_][a-zA-Z0-9_]*(\([^)]*\))?' - r'(\.[a-zA-Z0-9_]+(\([^)]*\))?)*\}?', text): + if re.search(r'\$\{?[a-zA-Z_]\w*(\([^)]*\))?' + r'(\.\w+(\([^)]*\))?)*\}?', text): rv += 0.01 return rv @@ -347,25 +347,25 @@ class DjangoLexer(RegexLexer): Text, Comment.Preproc, Text, Keyword, Text, Comment.Preproc)), # filter blocks - (r'(\{%)(-?\s*)(filter)(\s+)([a-zA-Z_][a-zA-Z0-9_]*)', + (r'(\{%)(-?\s*)(filter)(\s+)([a-zA-Z_]\w*)', bygroups(Comment.Preproc, Text, Keyword, Text, Name.Function), 'block'), - (r'(\{%)(-?\s*)([a-zA-Z_][a-zA-Z0-9_]*)', + (r'(\{%)(-?\s*)([a-zA-Z_]\w*)', bygroups(Comment.Preproc, Text, Keyword), 'block'), (r'\{', Other) ], 'varnames': [ - (r'(\|)(\s*)([a-zA-Z_][a-zA-Z0-9_]*)', + (r'(\|)(\s*)([a-zA-Z_]\w*)', bygroups(Operator, Text, Name.Function)), - (r'(is)(\s+)(not)?(\s+)?([a-zA-Z_][a-zA-Z0-9_]*)', + (r'(is)(\s+)(not)?(\s+)?([a-zA-Z_]\w*)', bygroups(Keyword, Text, Keyword, Text, Name.Function)), (r'(_|true|false|none|True|False|None)\b', Keyword.Pseudo), (r'(in|as|reversed|recursive|not|and|or|is|if|else|import|' r'with(?:(?:out)?\s*context)?|scoped|ignore\s+missing)\b', Keyword), (r'(loop|block|super|forloop)\b', Name.Builtin), - (r'[a-zA-Z][a-zA-Z0-9_-]*', Name.Variable), - (r'\.[a-zA-Z0-9_]+', Name.Variable), + (r'[a-zA-Z][\w-]*', Name.Variable), + (r'\.\w+', Name.Variable), (r':?"(\\\\|\\"|[^"])*"', String.Double), (r":?'(\\\\|\\'|[^'])*'", String.Single), (r'([{}()\[\]+\-*/,:~]|[><=]=?)', Operator), @@ -745,7 +745,7 @@ class CheetahLexer(RegexLexer): (bygroups(Comment.Preproc, using(CheetahPythonLexer), Comment.Preproc))), # TODO support other Python syntax like $foo['bar'] - (r'(\$)([a-zA-Z_][a-zA-Z0-9_\.]*[a-zA-Z0-9_])', + (r'(\$)([a-zA-Z_][\w\.]*\w)', bygroups(Comment.Preproc, using(CheetahPythonLexer))), (r'(\$\{!?)(.*?)(\})(?s)', bygroups(Comment.Preproc, using(CheetahPythonLexer), @@ -753,7 +753,7 @@ class CheetahLexer(RegexLexer): (r'''(?sx) (.+?) # anything, followed by: (?: - (?=[#][#a-zA-Z]*) | # an eval comment + (?=\#[#a-zA-Z]*) | # an eval comment (?=\$[a-zA-Z_{]) | # a substitution \Z # end of string ) @@ -843,7 +843,7 @@ class GenshiTextLexer(RegexLexer): 'variable': [ (r'(?<!\$)(\$\{)(.+?)(\})', bygroups(Comment.Preproc, using(PythonLexer), Comment.Preproc)), - (r'(?<!\$)(\$)([a-zA-Z_][a-zA-Z0-9_\.]*)', + (r'(?<!\$)(\$)([a-zA-Z_][\w\.]*)', Name.Variable), ] } @@ -871,7 +871,7 @@ class GenshiMarkupLexer(RegexLexer): ], 'pytag': [ (r'\s+', Text), - (r'[a-zA-Z0-9_:-]+\s*=', Name.Attribute, 'pyattr'), + (r'[\w:-]+\s*=', Name.Attribute, 'pyattr'), (r'/?\s*>', Name.Tag, '#pop'), ], 'pyattr': [ @@ -881,8 +881,8 @@ class GenshiMarkupLexer(RegexLexer): ], 'tag': [ (r'\s+', Text), - (r'py:[a-zA-Z0-9_-]+\s*=', Name.Attribute, 'pyattr'), - (r'[a-zA-Z0-9_:-]+\s*=', Name.Attribute, 'attr'), + (r'py:[\w-]+\s*=', Name.Attribute, 'pyattr'), + (r'[\w:-]+\s*=', Name.Attribute, 'attr'), (r'/?\s*>', Name.Tag, '#pop'), ], 'attr': [ @@ -907,7 +907,7 @@ class GenshiMarkupLexer(RegexLexer): 'variable': [ (r'(?<!\$)(\$\{)(.+?)(\})', bygroups(Comment.Preproc, using(PythonLexer), Comment.Preproc)), - (r'(?<!\$)(\$)([a-zA-Z_][a-zA-Z0-9_\.]*)', + (r'(?<!\$)(\$)([a-zA-Z_][\w\.]*)', Name.Variable), ] } @@ -1498,13 +1498,15 @@ class ColdfusionLexer(RegexLexer): # strings, evidently. (r"'.*?'", String.Single), (r'\d+', Number), - (r'(if|else|len|var|case|default|break|switch|component|property|function|do|try|catch|in|continue|for|return|while)\b', Keyword), - (r'(required|any|array|binary|boolean|component|date|guid|numeric|query|string|struct|uuid|xml)\b', Keyword), + (r'(if|else|len|var|xml|default|break|switch|component|property|function|do|' + r'try|catch|in|continue|for|return|while|required|any|array|binary|boolean|' + r'component|date|guid|numeric|query|string|struct|uuid|case)\b', Keyword), (r'(true|false|null)\b', Keyword.Constant), - (r'(application|session|client|cookie|super|this|variables|arguments)\b', Name.Constant), - (r'([A-Za-z_$][A-Za-z0-9_.]*)(\s*)(\()', + (r'(application|session|client|cookie|super|this|variables|arguments)\b', + Name.Constant), + (r'([a-z_$][\w.]*)(\s*)(\()', bygroups(Name.Function, Text, Punctuation)), - (r'[A-Za-z_$][A-Za-z0-9_.]*', Name.Variable), + (r'[a-z_$][\w.]*', Name.Variable), (r'[()\[\]{};:,.\\]', Punctuation), (r'\s+', Text), ], @@ -1534,7 +1536,7 @@ class ColdfusionMarkupLexer(RegexLexer): (r'<[^<>]*', Other), ], 'tags': [ - (r'(?s)<!---.*?--->', Comment.Multiline), + (r'<!---', Comment.Multiline, 'cfcomment'), (r'(?s)<!--.*?-->', Comment), (r'<cfoutput.*?>', Name.Builtin, 'cfoutput'), (r'(?s)(<cfscript.*?>)(.+?)(</cfscript.*?>)', @@ -1556,6 +1558,12 @@ class ColdfusionMarkupLexer(RegexLexer): (r'(?s)<[^<>]*', Other), (r'#', Other), ], + 'cfcomment': [ + (r'(?s)(.*?)(<!---)', + bygroups(Comment.Multiline, Comment.Multiline), '#push'), + (r'(?s)(.*?)(--->)', + bygroups(Comment.Multiline, Comment.Multiline), '#pop'), + ], } @@ -1803,8 +1811,8 @@ class HandlebarsLexer(RegexLexer): # borrowed from DjangoLexer (r':?"(\\\\|\\"|[^"])*"', String.Double), (r":?'(\\\\|\\'|[^'])*'", String.Single), - (r'[a-zA-Z][a-zA-Z0-9_-]*', Name.Variable), - (r'\.[a-zA-Z0-9_]+', Name.Variable), + (r'[a-zA-Z][\w-]*', Name.Variable), + (r'\.\w+', Name.Variable), (r"[0-9](\.[0-9]*)?(eE[+-][0-9])?[flFLdD]?|" r"0[xX][0-9a-fA-F]+[Ll]?", Number), ] diff --git a/pygments/lexers/text.py b/pygments/lexers/text.py index fc1cc6a6..10675654 100644 --- a/pygments/lexers/text.py +++ b/pygments/lexers/text.py @@ -17,6 +17,7 @@ from pygments.lexer import Lexer, LexerContext, RegexLexer, ExtendedRegexLexer, from pygments.token import Punctuation, Text, Comment, Keyword, Name, String, \ Generic, Operator, Number, Whitespace, Literal from pygments.util import get_bool_opt, ClassNotFound +from pygments.lexers.agile import PythonLexer from pygments.lexers.other import BashLexer __all__ = ['IniLexer', 'PropertiesLexer', 'SourcesListLexer', 'BaseMakefileLexer', @@ -234,11 +235,11 @@ class BaseMakefileLexer(RegexLexer): (r'\$[<@$+%?|*]', Keyword), (r'\s+', Text), (r'#.*?\n', Comment), - (r'(export)(\s+)(?=[a-zA-Z0-9_${}\t -]+\n)', + (r'(export)(\s+)(?=[\w${}\t -]+\n)', bygroups(Keyword, Text), 'export'), (r'export\s+', Keyword), # assignment - (r'([a-zA-Z0-9_${}.-]+)(\s*)([!?:+]?=)([ \t]*)((?:.*\\\n)+|.*\n)', + (r'([\w${}.-]+)(\s*)([!?:+]?=)([ \t]*)((?:.*\\\n)+|.*\n)', bygroups(Name.Variable, Text, Operator, Text, using(BashLexer))), # strings (r'(?s)"(\\\\|\\.|[^"\\])*"', String.Double), @@ -257,7 +258,7 @@ class BaseMakefileLexer(RegexLexer): (r'\)', Keyword, '#pop'), ], 'export': [ - (r'[a-zA-Z0-9_${}-]+', Name.Variable), + (r'[\w${}-]+', Name.Variable), (r'\n', Text, '#pop'), (r'\s+', Text), ], @@ -588,7 +589,7 @@ class ApacheConfLexer(RegexLexer): (r'(#.*?)$', Comment), (r'(<[^\s>]+)(?:(\s+)(.*?))?(>)', bygroups(Name.Tag, Text, String, Name.Tag)), - (r'([a-zA-Z][a-zA-Z0-9_]*)(\s+)', + (r'([a-z]\w*)(\s+)', bygroups(Name.Builtin, Text), 'value'), (r'\.+', Text), ], @@ -597,7 +598,7 @@ class ApacheConfLexer(RegexLexer): (r'[^\S\n]+', Text), (r'\d+\.\d+\.\d+\.\d+(?:/\d+)?', Number), (r'\d+', Number), - (r'/([a-zA-Z0-9][a-zA-Z0-9_./-]+)', String.Other), + (r'/([a-z0-9][\w./-]+)', String.Other), (r'(on|off|none|any|all|double|email|dns|min|minimal|' r'os|productonly|full|emerg|alert|crit|error|warn|' r'notice|info|debug|registry|script|inetd|standalone|' @@ -775,7 +776,7 @@ class RstLexer(RegexLexer): (r'^( *)(:.*?:)([ \t]+)(.*?)$', bygroups(Text, Name.Class, Text, Name.Function)), # Definition list - (r'^([^\s].*(?<!::)\n)((?:(?: +.*)\n)+)', + (r'^(\S.*(?<!::)\n)((?:(?: +.*)\n)+)', bygroups(using(this, state='inline'), using(this, state='inline'))), # Code blocks (r'(::)(\n[ \t]*\n)([ \t]+)(.*)(\n)((?:(?:\3.*|)\n)+)', @@ -835,8 +836,16 @@ class VimLexer(RegexLexer): mimetypes = ['text/x-vim'] flags = re.MULTILINE + _python = r'py(?:t(?:h(?:o(?:n)?)?)?)?' + tokens = { 'root': [ + (r'^([ \t:]*)(' + _python + r')([ \t]*)(<<)([ \t]*)(.*)((?:\n|.)*)(\6)', + bygroups(using(this), Keyword, Text, Operator, Text, Text, + using(PythonLexer), Text)), + (r'^([ \t:]*)(' + _python + r')([ \t])(.*)', + bygroups(using(this), Keyword, Text, using(PythonLexer))), + (r'^\s*".*', Comment), (r'[ \t]+', Text), diff --git a/pygments/lexers/web.py b/pygments/lexers/web.py index 180c8e8a..5260b9f5 100644 --- a/pygments/lexers/web.py +++ b/pygments/lexers/web.py @@ -81,7 +81,7 @@ class JavascriptLexer(RegexLexer): r'decodeURIComponent|encodeURI|encodeURIComponent|' r'Error|eval|isFinite|isNaN|parseFloat|parseInt|document|this|' r'window)\b', Name.Builtin), - (r'[$a-zA-Z_][a-zA-Z0-9_]*', Name.Other), + (r'[$a-zA-Z_]\w*', Name.Other), (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float), (r'0x[0-9a-fA-F]+', Number.Hex), (r'[0-9]+', Number.Integer), @@ -248,7 +248,7 @@ class ActionScriptLexer(RegexLexer): r'isXMLName|clearInterval|fscommand|getTimer|getURL|getVersion|' r'isFinite|parseFloat|parseInt|setInterval|trace|updateAfterEvent|' r'unescape)\b',Name.Function), - (r'[$a-zA-Z_][a-zA-Z0-9_]*', Name.Other), + (r'[$a-zA-Z_]\w*', Name.Other), (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float), (r'0x[0-9a-f]+', Number.Hex), (r'[0-9]+', Number.Integer), @@ -271,7 +271,7 @@ class ActionScript3Lexer(RegexLexer): mimetypes = ['application/x-actionscript', 'text/x-actionscript', 'text/actionscript'] - identifier = r'[$a-zA-Z_][a-zA-Z0-9_]*' + identifier = r'[$a-zA-Z_]\w*' typeidentifier = identifier + '(?:\.<\w+>)?' flags = re.DOTALL | re.MULTILINE @@ -359,11 +359,11 @@ class CssLexer(RegexLexer): (r'\s+', Text), (r'/\*(?:.|\n)*?\*/', Comment), (r'{', Punctuation, 'content'), - (r'\:[a-zA-Z0-9_-]+', Name.Decorator), - (r'\.[a-zA-Z0-9_-]+', Name.Class), - (r'\#[a-zA-Z0-9_-]+', Name.Function), - (r'@[a-zA-Z0-9_-]+', Keyword, 'atrule'), - (r'[a-zA-Z0-9_-]+', Name.Tag), + (r'\:[\w-]+', Name.Decorator), + (r'\.[\w-]+', Name.Class), + (r'\#[\w-]+', Name.Function), + (r'@[\w-]+', Keyword, 'atrule'), + (r'[\w-]+', Name.Tag), (r'[~\^\*!%&$\[\]\(\)<>\|+=@:;,./?-]', Operator), (r'"(\\\\|\\"|[^"])*"', String.Double), (r"'(\\\\|\\'|[^'])*'", String.Single) @@ -475,7 +475,7 @@ class CssLexer(RegexLexer): (r'[\[\]();]+', Punctuation), (r'"(\\\\|\\"|[^"])*"', String.Double), (r"'(\\\\|\\'|[^'])*'", String.Single), - (r'[a-zA-Z_][a-zA-Z0-9_]*', Name) + (r'[a-zA-Z_]\w*', Name) ] } @@ -595,26 +595,26 @@ class ObjectiveJLexer(RegexLexer): r'Error|eval|isFinite|isNaN|parseFloat|parseInt|document|this|' r'window)\b', Name.Builtin), - (r'([$a-zA-Z_][a-zA-Z0-9_]*)(' + _ws + r')(?=\()', + (r'([$a-zA-Z_]\w*)(' + _ws + r')(?=\()', bygroups(Name.Function, using(this))), - (r'[$a-zA-Z_][a-zA-Z0-9_]*', Name), + (r'[$a-zA-Z_]\w*', Name), ], 'classname' : [ # interface definition that inherits - (r'([a-zA-Z_][a-zA-Z0-9_]*)(' + _ws + r':' + _ws + - r')([a-zA-Z_][a-zA-Z0-9_]*)?', + (r'([a-zA-Z_]\w*)(' + _ws + r':' + _ws + + r')([a-zA-Z_]\w*)?', bygroups(Name.Class, using(this), Name.Class), '#pop'), # interface definition for a category - (r'([a-zA-Z_][a-zA-Z0-9_]*)(' + _ws + r'\()([a-zA-Z_][a-zA-Z0-9_]*)(\))', + (r'([a-zA-Z_]\w*)(' + _ws + r'\()([a-zA-Z_]\w*)(\))', bygroups(Name.Class, using(this), Name.Label, Text), '#pop'), # simple interface / implementation - (r'([a-zA-Z_][a-zA-Z0-9_]*)', Name.Class, '#pop'), + (r'([a-zA-Z_]\w*)', Name.Class, '#pop'), ], 'forward_classname' : [ - (r'([a-zA-Z_][a-zA-Z0-9_]*)(\s*,\s*)', + (r'([a-zA-Z_]\w*)(\s*,\s*)', bygroups(Name.Class, Text), '#push'), - (r'([a-zA-Z_][a-zA-Z0-9_]*)(\s*;?)', + (r'([a-zA-Z_]\w*)(\s*;?)', bygroups(Name.Class, Text), '#pop'), ], 'function_signature': [ @@ -622,26 +622,26 @@ class ObjectiveJLexer(RegexLexer): # start of a selector w/ parameters (r'(\(' + _ws + r')' # open paren - r'([a-zA-Z_][a-zA-Z0-9_]+)' # return type + r'([a-zA-Z_]\w+)' # return type r'(' + _ws + r'\)' + _ws + r')' # close paren - r'([$a-zA-Z_][a-zA-Z0-9_]+' + _ws + r':)', # function name + r'([$a-zA-Z_]\w+' + _ws + r':)', # function name bygroups(using(this), Keyword.Type, using(this), Name.Function), 'function_parameters'), # no-param function (r'(\(' + _ws + r')' # open paren - r'([a-zA-Z_][a-zA-Z0-9_]+)' # return type + r'([a-zA-Z_]\w+)' # return type r'(' + _ws + r'\)' + _ws + r')' # close paren - r'([$a-zA-Z_][a-zA-Z0-9_]+)', # function name + r'([$a-zA-Z_]\w+)', # function name bygroups(using(this), Keyword.Type, using(this), Name.Function), "#pop"), # no return type given, start of a selector w/ parameters - (r'([$a-zA-Z_][a-zA-Z0-9_]+' + _ws + r':)', # function name + (r'([$a-zA-Z_]\w+' + _ws + r':)', # function name bygroups (Name.Function), 'function_parameters'), # no return type given, no-param function - (r'([$a-zA-Z_][a-zA-Z0-9_]+)', # function name + (r'([$a-zA-Z_]\w+)', # function name bygroups(Name.Function), "#pop"), ('', Text, '#pop'), @@ -653,11 +653,11 @@ class ObjectiveJLexer(RegexLexer): (r'(\(' + _ws + ')' # open paren r'([^\)]+)' # type r'(' + _ws + r'\)' + _ws + r')' # close paren - r'([$a-zA-Z_][a-zA-Z0-9_]+)', # param name + r'([$a-zA-Z_]\w+)', # param name bygroups(using(this), Keyword.Type, using(this), Text)), # one piece of a selector name - (r'([$a-zA-Z_][a-zA-Z0-9_]+' + _ws + r':)', # function name + (r'([$a-zA-Z_]\w+' + _ws + r':)', # function name Name.Function), # smallest possible selector piece @@ -667,10 +667,10 @@ class ObjectiveJLexer(RegexLexer): (r'(,' + _ws + r'\.\.\.)', using(this)), # param name - (r'([$a-zA-Z_][a-zA-Z0-9_]+)', Text), + (r'([$a-zA-Z_]\w+)', Text), ], 'expression' : [ - (r'([$a-zA-Z_][a-zA-Z0-9_]*)(\()', bygroups(Name.Function, + (r'([$a-zA-Z_]\w*)(\()', bygroups(Name.Function, Punctuation)), (r'(\))', Punctuation, "#pop"), ], @@ -737,8 +737,8 @@ class HtmlLexer(RegexLexer): ], 'tag': [ (r'\s+', Text), - (r'[a-zA-Z0-9_:-]+\s*=', Name.Attribute, 'attr'), - (r'[a-zA-Z0-9_:-]+', Name.Attribute), + (r'[\w:-]+\s*=', Name.Attribute, 'attr'), + (r'[\w:-]+', Name.Attribute), (r'/?\s*>', Name.Tag, '#pop'), ], 'script-content': [ @@ -801,8 +801,8 @@ class PhpLexer(RegexLexer): # Note that a backslash is included in the following two patterns # PHP uses a backslash as a namespace separator - _ident_char = r'[\\_a-zA-Z0-9]|[^\x00-\x7f]' - _ident_begin = r'(?:[\\_a-zA-Z]|[^\x00-\x7f])' + _ident_char = r'[\\\w]|[^\x00-\x7f]' + _ident_begin = r'(?:[\\_a-z]|[^\x00-\x7f])' _ident_end = r'(?:' + _ident_char + ')*' _ident_inner = _ident_begin + _ident_end @@ -852,7 +852,7 @@ class PhpLexer(RegexLexer): (r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?', Number.Float), (r'\d+[eE][+-]?[0-9]+', Number.Float), (r'0[0-7]+', Number.Oct), - (r'0[xX][a-fA-F0-9]+', Number.Hex), + (r'0[xX][a-f0-9]+', Number.Hex), (r'\d+', Number.Integer), (r'0b[01]+', Number.Binary), (r"'([^'\\]*(?:\\.[^'\\]*)*)'", String.Single), @@ -868,7 +868,7 @@ class PhpLexer(RegexLexer): 'string': [ (r'"', String.Double, '#pop'), (r'[^{$"\\]+', String.Double), - (r'\\([nrt\"$\\]|[0-7]{1,3}|x[0-9A-Fa-f]{1,2})', String.Escape), + (r'\\([nrt\"$\\]|[0-7]{1,3}|x[0-9a-f]{1,2})', String.Escape), (r'\$' + _ident_inner + '(\[\S+?\]|->' + _ident_inner + ')?', String.Interpol), (r'(\{\$\{)(.*?)(\}\})', @@ -1118,8 +1118,8 @@ class MxmlLexer(RegexLexer): ('<!--', Comment, 'comment'), (r'<\?.*?\?>', Comment.Preproc), ('<![^>]*>', Comment.Preproc), - (r'<\s*[a-zA-Z0-9:._-]+', Name.Tag, 'tag'), - (r'<\s*/\s*[a-zA-Z0-9:._-]+\s*>', Name.Tag), + (r'<\s*[\w:.-]+', Name.Tag, 'tag'), + (r'<\s*/\s*[\w:.-]+\s*>', Name.Tag), ], 'comment': [ ('[^-]+', Comment), @@ -1128,7 +1128,7 @@ class MxmlLexer(RegexLexer): ], 'tag': [ (r'\s+', Text), - (r'[a-zA-Z0-9_.:-]+\s*=', Name.Attribute, 'attr'), + (r'[\w.:-]+\s*=', Name.Attribute, 'attr'), (r'/?\s*>', Name.Tag, '#pop'), ], 'attr': [ @@ -1161,11 +1161,10 @@ class HaxeLexer(ExtendedRegexLexer): r'inline|using|null|true|false|abstract)\b') # idtype in lexer.mll - typeid = r'_*[A-Z][_a-zA-Z0-9]*' + typeid = r'_*[A-Z]\w*' # combined ident and dollar and idtype - ident = r'(?:_*[a-z][_a-zA-Z0-9]*|_+[0-9][_a-zA-Z0-9]*|' + typeid + \ - '|_+|\$[_a-zA-Z0-9]+)' + ident = r'(?:_*[a-z]\w*|_+[0-9]\w*|' + typeid + '|_+|\$\w+)' binop = (r'(?:%=|&=|\|=|\^=|\+=|\-=|\*=|/=|<<=|>\s*>\s*=|>\s*>\s*>\s*=|==|' r'!=|<=|>\s*=|&&|\|\||<<|>>>|>\s*>|\.\.\.|<|>|%|&|\||\^|\+|\*|' @@ -1297,6 +1296,19 @@ class HaxeLexer(ExtendedRegexLexer): (r'\(', Comment.Preproc, ('#pop', 'preproc-parenthesis')), (ident, Comment.Preproc, '#pop'), + + # Float + (r'\.[0-9]+', Number.Float), + (r'[0-9]+[eE][\+\-]?[0-9]+', Number.Float), + (r'[0-9]+\.[0-9]*[eE][\+\-]?[0-9]+', Number.Float), + (r'[0-9]+\.[0-9]+', Number.Float), + (r'[0-9]+\.(?!' + ident + '|\.\.)', Number.Float), + + # Int + (r'0x[0-9a-fA-F]+', Number.Hex), + (r'[0-9]+', Number.Integer), + + # String (r"'", String.Single, ('#pop', 'string-single')), (r'"', String.Double, ('#pop', 'string-double')), ], @@ -1321,6 +1333,19 @@ class HaxeLexer(ExtendedRegexLexer): ('#pop', 'preproc-expr-chain', 'preproc-parenthesis')), (ident, Comment.Preproc, ('#pop', 'preproc-expr-chain')), + + # Float + (r'\.[0-9]+', Number.Float, ('#pop', 'preproc-expr-chain')), + (r'[0-9]+[eE][\+\-]?[0-9]+', Number.Float, ('#pop', 'preproc-expr-chain')), + (r'[0-9]+\.[0-9]*[eE][\+\-]?[0-9]+', Number.Float, ('#pop', 'preproc-expr-chain')), + (r'[0-9]+\.[0-9]+', Number.Float, ('#pop', 'preproc-expr-chain')), + (r'[0-9]+\.(?!' + ident + '|\.\.)', Number.Float, ('#pop', 'preproc-expr-chain')), + + # Int + (r'0x[0-9a-fA-F]+', Number.Hex, ('#pop', 'preproc-expr-chain')), + (r'[0-9]+', Number.Integer, ('#pop', 'preproc-expr-chain')), + + # String (r"'", String.Single, ('#pop', 'preproc-expr-chain', 'string-single')), (r'"', String.Double, @@ -1454,7 +1479,7 @@ class HaxeLexer(ExtendedRegexLexer): 'class-member': [ include('spaces'), (r'(var)\b', Keyword.Declaration, - ('#pop', 'optional-semicolon', 'prop')), + ('#pop', 'optional-semicolon', 'var')), (r'(function)\b', Keyword.Declaration, ('#pop', 'optional-semicolon', 'class-method')), ], @@ -1463,7 +1488,7 @@ class HaxeLexer(ExtendedRegexLexer): 'function-local': [ include('spaces'), (r'(' + ident_no_keyword + ')?', Name.Function, - ('#pop', 'expr', 'flag', 'function-param', + ('#pop', 'optional-expr', 'flag', 'function-param', 'parenthesis-open', 'type-param-constraint')), ], @@ -1495,13 +1520,6 @@ class HaxeLexer(ExtendedRegexLexer): (r',', Punctuation, ('#pop', 'function-param')), ], - # class property - # eg. var prop(default, null):String; - 'prop': [ - include('spaces'), - (ident_no_keyword, Name, ('#pop', 'assign', 'flag', 'prop-get-set')), - ], - 'prop-get-set': [ include('spaces'), (r'\(', Punctuation, ('#pop', 'parenthesis-close', @@ -1528,7 +1546,8 @@ class HaxeLexer(ExtendedRegexLexer): 'meta-ident', 'meta-colon')), (r'(?:\+\+|\-\-|~(?!/)|!|\-)', Operator), (r'\(', Punctuation, ('#pop', 'expr-chain', 'parenthesis')), - (r'(?:inline)\b', Keyword.Declaration), + (r'(?:static|public|private|override|dynamic|inline)\b', + Keyword.Declaration), (r'(?:function)\b', Keyword.Declaration, ('#pop', 'expr-chain', 'function-local')), (r'\{', Punctuation, ('#pop', 'expr-chain', 'bracket')), @@ -1587,7 +1606,15 @@ class HaxeLexer(ExtendedRegexLexer): # macro reification 'macro': [ include('spaces'), + include('meta'), (r':', Punctuation, ('#pop', 'type')), + + (r'(?:extern|private)\b', Keyword.Declaration), + (r'(?:abstract)\b', Keyword.Declaration, ('#pop', 'optional-semicolon', 'abstract')), + (r'(?:class|interface)\b', Keyword.Declaration, ('#pop', 'optional-semicolon', 'class')), + (r'(?:enum)\b', Keyword.Declaration, ('#pop', 'optional-semicolon', 'enum')), + (r'(?:typedef)\b', Keyword.Declaration, ('#pop', 'optional-semicolon', 'typedef')), + (r'', Text, ('#pop', 'expr')), ], @@ -1740,7 +1767,7 @@ class HaxeLexer(ExtendedRegexLexer): 'dollar': [ include('spaces'), - (r'\{', Keyword, ('#pop', 'bracket-close', 'expr')), + (r'\{', Punctuation, ('#pop', 'expr-chain', 'bracket-close', 'expr')), (r'', Text, ('#pop', 'expr-chain')), ], @@ -1868,7 +1895,7 @@ class HaxeLexer(ExtendedRegexLexer): # a parenthesis expr that contain exactly one expr 'parenthesis': [ include('spaces'), - (r'', Text, ('#pop', 'parenthesis-close', 'expr')), + (r'', Text, ('#pop', 'parenthesis-close', 'flag', 'expr')), ], 'parenthesis-open': [ @@ -1883,7 +1910,7 @@ class HaxeLexer(ExtendedRegexLexer): 'var': [ include('spaces'), - (ident_no_keyword, Text, ('#pop', 'var-sep', 'assign', 'flag')), + (ident_no_keyword, Text, ('#pop', 'var-sep', 'assign', 'flag', 'prop-get-set')), ], # optional more var decl. @@ -2038,8 +2065,8 @@ class HamlLexer(ExtendedRegexLexer): ], 'css': [ - (r'\.[a-z0-9_:-]+', Name.Class, 'tag'), - (r'\#[a-z0-9_:-]+', Name.Function, 'tag'), + (r'\.[\w:-]+', Name.Class, 'tag'), + (r'\#[\w:-]+', Name.Function, 'tag'), ], 'eval-or-plain': [ @@ -2052,7 +2079,7 @@ class HamlLexer(ExtendedRegexLexer): 'content': [ include('css'), - (r'%[a-z0-9_:-]+', Name.Tag, 'tag'), + (r'%[\w:-]+', Name.Tag, 'tag'), (r'!!!' + _dot + r'*\n', Name.Namespace, '#pop'), (r'(/)(\[' + _dot + '*?\])(' + _dot + r'*\n)', bygroups(Comment, Comment.Special, Comment), @@ -2088,16 +2115,16 @@ class HamlLexer(ExtendedRegexLexer): 'html-attributes': [ (r'\s+', Text), - (r'[a-z0-9_:-]+[ \t]*=', Name.Attribute, 'html-attribute-value'), - (r'[a-z0-9_:-]+', Name.Attribute), + (r'[\w:-]+[ \t]*=', Name.Attribute, 'html-attribute-value'), + (r'[\w:-]+', Name.Attribute), (r'\)', Text, '#pop'), ], 'html-attribute-value': [ (r'[ \t]+', Text), - (r'[a-z0-9_]+', Name.Variable, '#pop'), - (r'@[a-z0-9_]+', Name.Variable.Instance, '#pop'), - (r'\$[a-z0-9_]+', Name.Variable.Global, '#pop'), + (r'\w+', Name.Variable, '#pop'), + (r'@\w+', Name.Variable.Instance, '#pop'), + (r'\$\w+', Name.Variable.Global, '#pop'), (r"'(\\\\|\\'|[^'\n])*'", String, '#pop'), (r'"(\\\\|\\"|[^"\n])*"', String, '#pop'), ], @@ -2237,7 +2264,7 @@ common_sass_tokens = { (r'\:', Name.Decorator, 'pseudo-class'), (r'\.', Name.Class, 'class'), (r'\#', Name.Namespace, 'id'), - (r'[a-zA-Z0-9_-]+', Name.Tag), + (r'[\w-]+', Name.Tag), (r'#\{', String.Interpol, 'interpolation'), (r'&', Keyword), (r'[~\^\*!&\[\]\(\)<>\|+=@:;,./?-]', Operator), @@ -2317,7 +2344,7 @@ class SassLexer(ExtendedRegexLexer): (r'(@mixin)( [\w-]+)', bygroups(Keyword, Name.Function), 'value'), (r'(@include)( [\w-]+)', bygroups(Keyword, Name.Decorator), 'value'), (r'@extend', Keyword, 'selector'), - (r'@[a-z0-9_-]+', Keyword, 'selector'), + (r'@[\w-]+', Keyword, 'selector'), (r'=[\w-]+', Name.Function, 'value'), (r'\+[\w-]+', Name.Decorator, 'value'), (r'([!$][\w-]\w*)([ \t]*(?:(?:\|\|)?=|:))', @@ -2390,7 +2417,7 @@ class ScssLexer(RegexLexer): (r'(@mixin)( [\w-]+)', bygroups(Keyword, Name.Function), 'value'), (r'(@include)( [\w-]+)', bygroups(Keyword, Name.Decorator), 'value'), (r'@extend', Keyword, 'selector'), - (r'@[a-z0-9_-]+', Keyword, 'selector'), + (r'@[\w-]+', Keyword, 'selector'), (r'(\$[\w-]*\w)([ \t]*:)', bygroups(Name.Variable, Operator), 'value'), (r'(?=[^;{}][;}])', Name.Attribute, 'attr'), (r'(?=[^;{}:]+:[^a-z])', Name.Attribute, 'attr'), @@ -2473,12 +2500,12 @@ class CoffeeScriptLexer(RegexLexer): r'decodeURIComponent|encodeURI|encodeURIComponent|' r'eval|isFinite|isNaN|parseFloat|parseInt|document|window)\b', Name.Builtin), - (r'[$a-zA-Z_][a-zA-Z0-9_\.:\$]*\s*[:=]\s', Name.Variable, + (r'[$a-zA-Z_][\w\.:\$]*\s*[:=]\s', Name.Variable, 'slashstartsregex'), - (r'@[$a-zA-Z_][a-zA-Z0-9_\.:\$]*\s*[:=]\s', Name.Variable.Instance, + (r'@[$a-zA-Z_][\w\.:\$]*\s*[:=]\s', Name.Variable.Instance, 'slashstartsregex'), (r'@', Name.Other, 'slashstartsregex'), - (r'@?[$a-zA-Z_][a-zA-Z0-9_\$]*', Name.Other, 'slashstartsregex'), + (r'@?[$a-zA-Z_][\w\$]*', Name.Other, 'slashstartsregex'), (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float), (r'0x[0-9a-fA-F]+', Number.Hex), (r'[0-9]+', Number.Integer), @@ -2500,6 +2527,7 @@ class CoffeeScriptLexer(RegexLexer): (r'"', String, '#pop'), (r'\\.|\'', String), # double-quoted string don't need ' escapes (r'#{', String.Interpol, "interpoling_string"), + (r'#', String), include('strings') ], 'sqs': [ @@ -2511,6 +2539,7 @@ class CoffeeScriptLexer(RegexLexer): (r'"""', String, '#pop'), (r'\\.|\'|"', String), # no need to escape quotes in triple-string (r'#{', String.Interpol, "interpoling_string"), + (r'#', String), include('strings'), ], 'tsqs': [ @@ -2544,13 +2573,13 @@ class KalLexer(RegexLexer): (r'#(?!##[^#]).*?\n', Comment.Single), ], 'functiondef': [ - (r'[$a-zA-Z_][a-zA-Z0-9_\$]*\s*', Name.Function, '#pop'), + (r'[$a-zA-Z_][\w\$]*\s*', Name.Function, '#pop'), include('commentsandwhitespace'), ], 'classdef': [ (r'\binherits\s+from\b', Keyword), - (r'[$a-zA-Z_][a-zA-Z0-9_\$]*\s*\n', Name.Class, '#pop'), - (r'[$a-zA-Z_][a-zA-Z0-9_\$]*\s*', Name.Class), + (r'[$a-zA-Z_][\w\$]*\s*\n', Name.Class, '#pop'), + (r'[$a-zA-Z_][\w\$]*\s*', Name.Class), include('commentsandwhitespace'), ], 'listcomprehension': [ @@ -2579,7 +2608,7 @@ class KalLexer(RegexLexer): (r'\b(function|method|task)\b', Keyword.Declaration, 'functiondef'), (r'\bclass\b', Keyword.Declaration, 'classdef'), (r'\b(safe\s+)?wait\s+for\b', Keyword, 'waitfor'), - (r'\b(me|this)(\.[$a-zA-Z_][a-zA-Z0-9_\.\$]*)?\b', Name.Variable.Instance), + (r'\b(me|this)(\.[$a-zA-Z_][\w\.\$]*)?\b', Name.Variable.Instance), (r'(?<![\.\$])(for(\s+(parallel|series))?|in|of|while|until|' r'break|return|continue|' r'when|if|unless|else|otherwise|except\s+when|' @@ -2595,7 +2624,7 @@ class KalLexer(RegexLexer): r'eval|isFinite|isNaN|parseFloat|parseInt|document|window|' r'print)\b', Name.Builtin), - (r'[$a-zA-Z_][a-zA-Z0-9_\.\$]*\s*(:|[\+\-\*\/]?\=)?\b', Name.Variable), + (r'[$a-zA-Z_][\w\.\$]*\s*(:|[\+\-\*\/]?\=)?\b', Name.Variable), (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float), (r'0x[0-9a-fA-F]+', Number.Hex), (r'[0-9]+', Number.Integer), @@ -2698,12 +2727,12 @@ class LiveScriptLexer(RegexLexer): r'decodeURIComponent|encodeURI|encodeURIComponent|' r'eval|isFinite|isNaN|parseFloat|parseInt|document|window)\b', Name.Builtin), - (r'[$a-zA-Z_][a-zA-Z0-9_\.\-:\$]*\s*[:=]\s', Name.Variable, + (r'[$a-zA-Z_][\w\.\-:\$]*\s*[:=]\s', Name.Variable, 'slashstartsregex'), - (r'@[$a-zA-Z_][a-zA-Z0-9_\.\-:\$]*\s*[:=]\s', Name.Variable.Instance, + (r'@[$a-zA-Z_][\w\.\-:\$]*\s*[:=]\s', Name.Variable.Instance, 'slashstartsregex'), (r'@', Name.Other, 'slashstartsregex'), - (r'@?[$a-zA-Z_][a-zA-Z0-9_\-]*', Name.Other, 'slashstartsregex'), + (r'@?[$a-zA-Z_][\w\-]*', Name.Other, 'slashstartsregex'), (r'[0-9]+\.[0-9]+([eE][0-9]+)?[fd]?(?:[a-zA-Z_]+)?', Number.Float), (r'[0-9]+(~[0-9a-z]+)?(?:[a-zA-Z_]+)?', Number.Integer), ('"""', String, 'tdqs'), @@ -2808,8 +2837,8 @@ class ScamlLexer(ExtendedRegexLexer): ], 'css': [ - (r'\.[a-z0-9_:-]+', Name.Class, 'tag'), - (r'\#[a-z0-9_:-]+', Name.Function, 'tag'), + (r'\.[\w:-]+', Name.Class, 'tag'), + (r'\#[\w:-]+', Name.Function, 'tag'), ], 'eval-or-plain': [ @@ -2822,7 +2851,7 @@ class ScamlLexer(ExtendedRegexLexer): 'content': [ include('css'), - (r'%[a-z0-9_:-]+', Name.Tag, 'tag'), + (r'%[\w:-]+', Name.Tag, 'tag'), (r'!!!' + _dot + r'*\n', Name.Namespace, '#pop'), (r'(/)(\[' + _dot + '*?\])(' + _dot + r'*\n)', bygroups(Comment, Comment.Special, Comment), @@ -2861,16 +2890,16 @@ class ScamlLexer(ExtendedRegexLexer): 'html-attributes': [ (r'\s+', Text), - (r'[a-z0-9_:-]+[ \t]*=', Name.Attribute, 'html-attribute-value'), - (r'[a-z0-9_:-]+', Name.Attribute), + (r'[\w:-]+[ \t]*=', Name.Attribute, 'html-attribute-value'), + (r'[\w:-]+', Name.Attribute), (r'\)', Text, '#pop'), ], 'html-attribute-value': [ (r'[ \t]+', Text), - (r'[a-z0-9_]+', Name.Variable, '#pop'), - (r'@[a-z0-9_]+', Name.Variable.Instance, '#pop'), - (r'\$[a-z0-9_]+', Name.Variable.Global, '#pop'), + (r'\w+', Name.Variable, '#pop'), + (r'@\w+', Name.Variable.Instance, '#pop'), + (r'\$\w+', Name.Variable.Global, '#pop'), (r"'(\\\\|\\'|[^'\n])*'", String, '#pop'), (r'"(\\\\|\\"|[^"\n])*"', String, '#pop'), ], @@ -2918,8 +2947,8 @@ class JadeLexer(ExtendedRegexLexer): ], 'css': [ - (r'\.[a-z0-9_:-]+', Name.Class, 'tag'), - (r'\#[a-z0-9_:-]+', Name.Function, 'tag'), + (r'\.[\w:-]+', Name.Class, 'tag'), + (r'\#[\w:-]+', Name.Function, 'tag'), ], 'eval-or-plain': [ @@ -2947,7 +2976,7 @@ class JadeLexer(ExtendedRegexLexer): '#pop'), (r':' + _dot + r'*\n', _starts_block(Name.Decorator, 'filter-block'), '#pop'), - (r'[a-z0-9_:-]+', Name.Tag, 'tag'), + (r'[\w:-]+', Name.Tag, 'tag'), (r'\|', Text, 'eval-or-plain'), ], @@ -2970,16 +2999,16 @@ class JadeLexer(ExtendedRegexLexer): 'html-attributes': [ (r'\s+', Text), - (r'[a-z0-9_:-]+[ \t]*=', Name.Attribute, 'html-attribute-value'), - (r'[a-z0-9_:-]+', Name.Attribute), + (r'[\w:-]+[ \t]*=', Name.Attribute, 'html-attribute-value'), + (r'[\w:-]+', Name.Attribute), (r'\)', Text, '#pop'), ], 'html-attribute-value': [ (r'[ \t]+', Text), - (r'[a-z0-9_]+', Name.Variable, '#pop'), - (r'@[a-z0-9_]+', Name.Variable.Instance, '#pop'), - (r'\$[a-z0-9_]+', Name.Variable.Global, '#pop'), + (r'\w+', Name.Variable, '#pop'), + (r'@\w+', Name.Variable.Instance, '#pop'), + (r'\$\w+', Name.Variable.Global, '#pop'), (r"'(\\\\|\\'|[^'\n])*'", String, '#pop'), (r'"(\\\\|\\"|[^"\n])*"', String, '#pop'), ], @@ -3699,8 +3728,8 @@ class DartLexer(RegexLexer): (r'\b(bool|double|Dynamic|int|num|Object|String|void)\b', Keyword.Type), (r'\b(false|null|true)\b', Keyword.Constant), (r'[~!%^&*+=|?:<>/-]|as\b', Operator), - (r'[a-zA-Z_$][a-zA-Z0-9_]*:', Name.Label), - (r'[a-zA-Z_$][a-zA-Z0-9_]*', Name), + (r'[a-zA-Z_$]\w*:', Name.Label), + (r'[a-zA-Z_$]\w*', Name), (r'[(){}\[\],.;]', Punctuation), (r'0[xX][0-9a-fA-F]+', Number.Hex), # DIGIT+ (‘.’ DIGIT*)? EXPONENT? @@ -3710,13 +3739,13 @@ class DartLexer(RegexLexer): # pseudo-keyword negate intentionally left out ], 'class': [ - (r'[a-zA-Z_$][a-zA-Z0-9_]*', Name.Class, '#pop') + (r'[a-zA-Z_$]\w*', Name.Class, '#pop') ], 'import_decl': [ include('string_literal'), (r'\s+', Text), (r'\b(as|show|hide)\b', Keyword), - (r'[a-zA-Z_$][a-zA-Z0-9_]*', Name), + (r'[a-zA-Z_$]\w*', Name), (r'\,', Punctuation), (r'\;', Punctuation, '#pop') ], @@ -3735,7 +3764,7 @@ class DartLexer(RegexLexer): 'string_common': [ (r"\\(x[0-9A-Fa-f]{2}|u[0-9A-Fa-f]{4}|u\{[0-9A-Fa-f]*\}|[a-z\'\"$\\])", String.Escape), - (r'(\$)([a-zA-Z_][a-zA-Z0-9_]*)', bygroups(String.Interpol, Name)), + (r'(\$)([a-zA-Z_]\w*)', bygroups(String.Interpol, Name)), (r'(\$\{)(.*?)(\})', bygroups(String.Interpol, using(this), String.Interpol)) ], @@ -3818,7 +3847,7 @@ class TypeScriptLexer(RegexLexer): r'Error|eval|isFinite|isNaN|parseFloat|parseInt|document|this|' r'window)\b', Name.Builtin), # Match stuff like: module name {...} - (r'\b(module)(\s*)(\s*[a-zA-Z0-9_?.$][\w?.$]*)(\s*)', + (r'\b(module)(\s*)(\s*[\w?.$][\w?.$]*)(\s*)', bygroups(Keyword.Reserved, Text, Name.Other, Text), 'slashstartsregex'), # Match variable type keywords (r'\b(string|bool|number)\b', Keyword.Type), @@ -3830,9 +3859,9 @@ class TypeScriptLexer(RegexLexer): # Match stuff like: function() {...} (r'([a-zA-Z_?.$][\w?.$]*)\(\) \{', Name.Other, 'slashstartsregex'), # Match stuff like: (function: return type) - (r'([a-zA-Z0-9_?.$][\w?.$]*)(\s*:\s*)([a-zA-Z0-9_?.$][\w?.$]*)', + (r'([\w?.$][\w?.$]*)(\s*:\s*)([\w?.$][\w?.$]*)', bygroups(Name.Other, Text, Keyword.Type)), - (r'[$a-zA-Z_][a-zA-Z0-9_]*', Name.Other), + (r'[$a-zA-Z_]\w*', Name.Other), (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float), (r'0x[0-9a-fA-F]+', Number.Hex), (r'[0-9]+', Number.Integer), @@ -4165,7 +4194,7 @@ class QmlLexer(RegexLexer): r'decodeURIComponent|encodeURI|encodeURIComponent|' r'Error|eval|isFinite|isNaN|parseFloat|parseInt|document|this|' r'window)\b', Name.Builtin), - (r'[$a-zA-Z_][a-zA-Z0-9_]*', Name.Other), + (r'[$a-zA-Z_]\w*', Name.Other), (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float), (r'0x[0-9a-fA-F]+', Number.Hex), (r'[0-9]+', Number.Integer), @@ -4256,8 +4285,8 @@ class MaskLexer(RegexLexer): (r'"', String, 'string-double'), (r'([\w-]+)', Name.Tag, 'node'), (r'([^\.#;{>\s]+)', Name.Class, 'node'), - (r'(#[\w_-]+)', Name.Function, 'node'), - (r'(\.[\w_-]+)', Name.Variable.Class, 'node') + (r'(#[\w-]+)', Name.Function, 'node'), + (r'(\.[\w-]+)', Name.Variable.Class, 'node') ], 'string-base': [ (r'\\.', String.Escape), @@ -4294,8 +4323,8 @@ class MaskLexer(RegexLexer): (r'\.', Name.Variable.Class, 'node-class'), (r'\#', Name.Function, 'node-id'), (r'style[ \t]*=', Name.Attribute, 'node-attr-style-value'), - (r'[\w_:-]+[ \t]*=', Name.Attribute, 'node-attr-value'), - (r'[\w_:-]+', Name.Attribute), + (r'[\w:-]+[ \t]*=', Name.Attribute, 'node-attr-value'), + (r'[\w:-]+', Name.Attribute), (r'[>{;]', Punctuation, '#pop') ], 'node-class': [ @@ -4310,7 +4339,7 @@ class MaskLexer(RegexLexer): ], 'node-attr-value':[ (r'\s+', Text), - (r'[\w_]+', Name.Variable, '#pop'), + (r'\w+', Name.Variable, '#pop'), (r"'", String, 'string-single-pop2'), (r'"', String, 'string-double-pop2'), (r'', Text, '#pop') @@ -4323,8 +4352,8 @@ class MaskLexer(RegexLexer): ], 'css-base': [ (r'\s+', Text), - (r"[;]", Punctuation), - (r"[\w\-_]+\s*:", Name.Builtin) + (r";", Punctuation), + (r"[\w\-]+\s*:", Name.Builtin) ], 'css-single-end': [ include('css-base'), @@ -4388,19 +4417,20 @@ class ZephirLexer(RegexLexer): r'(<<|>>>?|==?|!=?|->|[-<>+*%&\|\^/])=?', Operator, 'slashstartsregex'), (r'[{(\[;,]', Punctuation, 'slashstartsregex'), (r'[})\].]', Punctuation), - (r'(for|in|while|do|break|return|continue|switch|case|default|if|else|loop|require|inline|' - r'throw|try|catch|finally|new|delete|typeof|instanceof|void|namespace|use|extends|' - r'this|fetch|isset|unset|echo|fetch|likely|unlikely|empty)\b', Keyword, 'slashstartsregex'), + (r'(for|in|while|do|break|return|continue|switch|case|default|if|else|loop|' + r'require|inline|throw|try|catch|finally|new|delete|typeof|instanceof|void|' + r'namespace|use|extends|this|fetch|isset|unset|echo|fetch|likely|unlikely|' + r'empty)\b', Keyword, 'slashstartsregex'), (r'(var|let|with|function)\b', Keyword.Declaration, 'slashstartsregex'), - (r'(abstract|boolean|bool|char|class|const|double|enum|export|' - r'extends|final|float|goto|implements|import|int|string|interface|long|ulong|char|uchar|native|unsigned|' - r'private|protected|public|short|static|self|throws|reverse|' + (r'(abstract|boolean|bool|char|class|const|double|enum|export|extends|final|' + r'native|goto|implements|import|int|string|interface|long|ulong|char|uchar|' + r'float|unsigned|private|protected|public|short|static|self|throws|reverse|' r'transient|volatile)\b', Keyword.Reserved), (r'(true|false|null|undefined)\b', Keyword.Constant), (r'(Array|Boolean|Date|_REQUEST|_COOKIE|_SESSION|' r'_GET|_POST|_SERVER|this|stdClass|range|count|iterator|' r'window)\b', Name.Builtin), - (r'[$a-zA-Z_][a-zA-Z0-9_\\]*', Name.Other), + (r'[$a-zA-Z_][\w\\]*', Name.Other), (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float), (r'0x[0-9a-fA-F]+', Number.Hex), (r'[0-9]+', Number.Integer), diff --git a/pygments/util.py b/pygments/util.py index c302900f..5dc6981f 100644 --- a/pygments/util.py +++ b/pygments/util.py @@ -208,6 +208,11 @@ def looks_like_xml(text): # Python narrow build compatibility def _surrogatepair(c): + # Given a unicode character code + # with length greater than 16 bits, + # return the two 16 bit surrogate pair. + # From example D28 of: + # http://www.unicode.org/book/ch03.pdf return (0xd7c0 + (c >> 10), (0xdc00 + (c & 0x3ff))) def unirange(a, b): diff --git a/tests/examplefiles/demo.cfm b/tests/examplefiles/demo.cfm index d94a06a0..49690484 100644 --- a/tests/examplefiles/demo.cfm +++ b/tests/examplefiles/demo.cfm @@ -1,4 +1,11 @@ <!--- cfcomment ---> +<!--- nested <!--- cfcomment ---> ---> +<!--- multi-line +nested +<!--- +cfcomment +---> +---> <!-- html comment --> <html> <head> @@ -17,6 +24,9 @@ #IsDate("foo")#<br /> #DaysInMonth(RightNow)# </cfoutput> +<cfset x="x"> +<cfset y="y"> +<cfset z="z"> <cfoutput group="x"> #x# <cfoutput>#y#</cfoutput> @@ -29,6 +39,8 @@ <cfset greeting = "Hello #person#"> <cfset greeting = "Hello" & " world!"> +<cfset a = 5> +<cfset b = 10> <cfset c = a^b> <cfset c = a MOD b> <cfset c = a / b> diff --git a/tests/examplefiles/function_arrows.coffee b/tests/examplefiles/example.coffee index cd1ef1e8..2cbd1df3 100644 --- a/tests/examplefiles/function_arrows.coffee +++ b/tests/examplefiles/example.coffee @@ -1,3 +1,5 @@ +# function arrows + methodA:-> 'A' methodB:=> 'B' methodC:()=> 'C' @@ -9,3 +11,17 @@ methodF:(c,d)-> 'F' (-> 'I') (=> 'J') + +# strings + +"#{wow}" +"w#{wow}w" +"#wow" +"wow#" +"w#ow" + +'#{wow}' +'w#{wow}w' +'#wow' +'wow#' +'w#ow' diff --git a/tests/examplefiles/import.hs b/tests/examplefiles/example.hs index f266f62e..9efd3364 100644 --- a/tests/examplefiles/import.hs +++ b/tests/examplefiles/example.hs @@ -1,3 +1,5 @@ +module ĈrazyThings where + import "base" Data.Char import "base" Data.Char (isControl, isSpace) import "base" Data.Char (isControl, --isSpace) @@ -13,3 +15,13 @@ import "base" Data.Char (isControl, -- isSpace) main :: IO () main = putStrLn "hello world" + +gádd x y = x + y +ádd x y = x + y + + +data ĈrazyThings = + Ĉar | + House | + Peár + deriving (Show, Eq) diff --git a/tests/examplefiles/example.hx b/tests/examplefiles/example.hx index fd93bb49..381cf825 100644 --- a/tests/examplefiles/example.hx +++ b/tests/examplefiles/example.hx @@ -139,4 +139,47 @@ typedef Pt2 = { y:Float, ?z:Float, //optional z add : Point -> Void, -}
\ No newline at end of file +} + + +//top-level class members +public function test(); +private var attr(get, set) = 1; + + +//pre-proc number +public static inline function indexOf<T>(arr:Array<T>, v:T) : Int +{ + #if (haxe_ver >= 3.1) + return arr.indexOf(v); + #else + #if (flash || js) + return untyped arr.indexOf(v); + #else + return std.Lambda.indexOf(arr, v); + #end + #end +} + +//macro reification +var e = macro var $myVar = 0; +var e = macro ${v}.toLowerCase(); +var e = macro o.$myField; +var e = macro { $myField : 0 }; +var e = macro $i{varName}++; +var e = macro $v{myStr}; +var args = [macro "sub", macro 3]; +var e = macro "Hello".toLowerCase($a{args}); +(macro $i{tmp}.addAtom($v{name}, $atom)).finalize(op.pos); + +var c = macro class MyClass { + public function new() { } + public function $funcName() { + trace($v{funcName} + " was called"); + } +} + +var c = macro interface IClass {}; + +//ECheckType +var f = (123:Float);
\ No newline at end of file diff --git a/tests/examplefiles/example.j b/tests/examplefiles/example.j new file mode 100644 index 00000000..16cdde86 --- /dev/null +++ b/tests/examplefiles/example.j @@ -0,0 +1,564 @@ +; Example JVM assembly +; Tested with JasminXT 2.4 + +.bytecode 49.0 +.source HelloWorld.java +.class public final enum HelloWorld +.super java/lang/Object +.implements java/io/Serializable +.signature "Ljava/lang/Object;Ljava/io/Serializable;" +.enclosing method hw/jasmin.HelloWorldRunner.run()V +.deprecated +.annotation visible HelloWorld + I I = 0 +.end annotation +.debug "Happy debugging!" + +.inner interface public InnerInterface inner 'HelloWorld$InnerInterface' outer HelloWorld +.inner class public InnerClass inner HelloWorld$InnerClass outer 'HelloWorld' + +.field public volatile transient I I +.field static protected final serialVersionUID 'J' signature "TJ;" = 2147483648 +.field annotation protected 'protected' [[[Lcom/oracle/util/Checksums; + .deprecated + .signature "[[[Lcom/oracle/util/Checksums;" + .attribute foo "foo.txt" + .attribute 'foo' "foo.txt" +.end field +.field public newline I +.field public static defaultString 'Ljava/lang/String;' + +.method public <init>()V + .limit stack 3 +.line 7 + .var 0 is self LHelloWorld; from 0 to 1 + aload_0 + invokenonvirtual java/lang/Object/<init>()V + return +.end method + +.method static public main([Ljava/lang/String;)V + .limit locals 7 + .limit stack 10 + .throws java.lang/RuntimeException + .catch java/lang.ClassCastException from cast to 'extra_l' using /extra + .signature "([Ljava/lang/String;)V" + .stack + offset /Input + locals Object java/lang/String + locals Uninitialized 'End' + locals Uninitialized 0 + locals Top + locals Integer + locals Float + locals Long + locals Double + locals Null + locals UninitializedThis + stack Object java/lang/String + stack Uninitialized End + stack 'Uninitialized' 0 + stack 'Top' + stack Integer + stack Float + stack Long + stack Double + stack Null + stack UninitializedThis + .end stack + .stack use 1 locals + offset 'extra' + .end stack + .stack use locals + .end stack +.line 0xd + .var 0 is args [Ljava/lang/String; + aload_w 0 + arraylength + ifne /Input + iconst_1 + anewarray java/lang/String + checkcast [Ljava/lang/String; + astore_0 + aload_0 + iconst_0 + ldc "World" + dup + putstatic HelloWorld.defaultString Ljava/lang/String; + aastore +/Input: + iconst_2 + iconst_3 + multianewarray [[C 2 + astore_1 + aload_1 + iconst_0 + aaload + astore_2 + aload_1 + iconst_1 + aaload + astore_3 + +<<o: + aload_3 + iconst_0 + invokestatic HelloWorld/int()I + castore + +<<\u0020: + aload_3 + dconst_1 + dconst_0 + dsub + d2i + invokestatic HelloWorld/double()D + d2i + castore + +<<!: + aload_3 + lconst_0 + dup2 + lxor + lconst_1 + dup2 + ladd + lsub + lneg + l2i + invokestatic HelloWorld/long()J + l2i + castore + +<<H: + aload_2 + fconst_0 + fconst_1 + fconst_2 + dup_x2 + fdiv + fmul + f2l + l2i + swap + invokestatic HelloWorld/float(F)F + f2i + castore + +<<e : + aload_2 + iconst_1 + i2s + i2c + i2b + iconst_1 + newarray short + dup + iconst_0 + iconst_1 + newarray byte + dup + iconst_0 + sipush 0x65 + bastore + iconst_0 + baload + sastore + iconst_0 + saload + int2short + int2char + int2byte + castore + + <<l : + aload_2 + iconst_2 + bipush 0x1b +*2: + iconst_1 + ishl + dup + lookupswitch + 0: '/lookupswitch' + 0x6c: /lookupswitch + default: *2 +/lookupswitch: + castore + + ldc2_w 2 + dup2 + lcmp + .set i 4 + .set 'j' 5 + .var 4 is i I from 'i++' to End + .var 5 is j I signature "I" from i++ to End + istore 4 + goto 1 +i++: + iinc 4 1 +1: iconst_0 + istore_w 5 + goto_w 2 +j++: + iinc_w 5 1 +2: getstatic java/lang/System/out Ljava/io/PrintStream; + aload_1 + iload 4 + aaload + iload_w 5 + caload + invokevirtual java/io/PrintStream/print(C)V + iload 5 + iconst_1 + if_icmpne $+6 + jsr extra + iload 5 + iconst_2 + if_icmplt j++ + iconst_1 + iload 4 + if_icmpgt i++ + +<<\u00a0: + getstatic java/lang/System/out Ljava/io/PrintStream; + invokestatic HelloWorld/get"example()LHelloWorld; + getfield HelloWorld/newline I + invokevirtual java/io/PrintStream/print(C)V +End: + return + +extra: + astore 6 + iload 4 + tableswitch 0 1 + extra_l + extra_string + default: 'End' + nop +extra_string: + getstatic java/lang/System/out Ljava/io/PrintStream; + aload 0 + iconst_0 + aaload + invokevirtual java/io/PrintStream/print(Ljava/lang/String;)V +cast: + ldc java/lang/String + checkcast java/lang/Class + pop + ldc Ljava/lang/String; + checkcast Ljava/lang/Class; + pop + iconst_1 + dup + newarray boolean + checkcast [Z + pop + newarray 'int' + checkcast HelloWorld + checkcast LHelloWorld; + pop +extra_l: + getstatic java/lang/System/out Ljava/io/PrintStream; + dup + ldc "\123.\456.\u006c.\n.\r.\t.\f.\b.\".\'.\\" + iconst_5 + invokeinterface java/lang/CharSequence/charAt(I)C 2 + invokevirtual java/io/PrintStream/print(C)V +/extra: + pop + ret 6 +.end method + +.method private static get"example()LHelloWorld; + .limit locals 3 + .limit stack 4 + .catch all from 7 to 53 using 59 + aconst_null + dup + dup + astore_w 0 +try: + goto $+0x11 +finally: + astore_w 2 + putfield HelloWorld/newline I + ret_w 2 + nop + aload_0 + areturn + ifnonnull $-2 + ifnull $+3 + new HelloWorld + dup + dup + invokespecial HelloWorld/<init>()V + astore 0 + aload 0 + monitorenter + monitorexit + new java/lang/RuntimeException + dup + invokespecial java/lang/RuntimeException/<init>()V + athrow + aconst_null +/try: + dup + aconst_null + if_acmpeq $+3 + areturn +catch: + jsr $+10 + aload_0 + dup + aconst_null + if_acmpne /try + areturn + astore_1 + aload_0 + ldc 10 + jsr_w finally + ret 1 +'single\u0020quoted\u0020label': ; Messes up [@ below if lexed sloppily +.end method + +.method varargs private static int()I + .annotation invisible HelloWorld + [@ [@ WhatIsThis??? = .annotation ; name, type, exttype + I I = 1 ; name, type + another-I I = 2 + Enum e Ljava/util/logging/Level; = FINE + .end annotation + .annotation + s s = "foo" + another-s s = "bar" + Enum [e Ljava/util/logging/Level; = FINE FINE 'FINE' FINE + .end annotation + float F = 123.456 + .end annotation + .annotation visibleparam 1 LHelloWorld; + x [I = 0x01 0x02 0x03 + y I = 2 + .end annotation + .annotation invisibleparam 255 HelloWorld + a F = 1.2 + b D = 3.4 + .end annotation + .annotation default + I = 0 + .end annotation + .limit locals 4 + .limit stack 20 + iconst_1 + newarray int + dup + dup + instanceof [Z + bipush 0x9 + bipush 0xB + iand + iconst_5 + iconst_4 + dup_x1 + iconst_m1 + iadd + bipush +-111 + ineg + swap + idiv + dup_x2 + dup + ishr + ishl + imul + ior + bipush -73 + ixor + isub + dup + iconst_1 + iadd + irem + iastore + iconst_0 + iaload + istore_0 + iload_0 + istore_1 + iload_1 + istore_2 + iload_2 + istore_3 + iload_3 + dup + dup + dup2_x1 + if_icmpeq $+33 + dup + dup + if_icmpge $+28 + dup + dup + if_icmple $+23 + dup + ifle $+19 + dup + ifeq $+15 + dup + iflt $+11 + dup + ifgt $+7 + dup + ifge $+3 + ireturn +.end method + +.method static private fpstrict double()D + .limit locals 7 + .limit stack 11 + dconst_1 + dconst_0 + dcmpg + newarray double + dup + dconst_0 + dup2 + dcmpl + ldc2_w 128. + ldc2_w -240.221d + dneg + ldc2_w 158.d + dup2 + dadd + dup2_x2 + drem + ddiv + pop2 + dconst_1 + dmul + d2f + f2d + d2l + l2i + iconst_2 + iushr + i2d + dastore + iconst_0 + daload + dstore_0 + dload_0 + dstore_1 + dload_1 + dstore_2 + dload_2 + dstore_3 + dload_3 + dstore 4 + dload 4 + dstore_w 5 + dload_w 5 + dreturn +.end method + +.method static long()J + .limit locals 7 + .limit stack 11 + iconst_1 + newarray long + dup + iconst_0 + ldc2_w 5718613688 + ldc2_w 3143486100 + ldc2_w 0x3 + ldiv + lmul + ldc2_w -10000000000 + lrem + ldc_w 0x60 + i2l + lor + ldc 0x33 + i2l + land + dup2 + iconst_1 + lshl + iconst_3 + lshr + iconst_3 + lushr + ladd + l2d + d2l + l2f + f2l + lastore + iconst_0 + laload + lstore_0 + lload_0 + lstore_1 + lload_1 + lstore_2 + lload_2 + lstore_3 + lload_3 + lstore 4 + lload 4 + lstore_w 5 + lload_w 5 + lreturn +.end method + +.method private static float(F)F + .limit locals 6 + .limit stack 9 + iconst_1 + newarray float + dup + fload_0 + dup + fcmpg + fload_0 + dup + dup + dup + dup2_x2 + fadd + fsub + fneg + frem + ldc 70 + i2f + fadd + fadd + swap + pop + fastore + fload_0 + dup + fcmpl + faload + fstore_0 + fload_0 + fstore_1 + fload_1 + fstore_2 + fload_2 + fstore_3 + fload_3 + fstore 4 + fload 4 + fstore_w 5 + fload_w 5 + freturn +.end method + +.method abstract bridge synthetic 'acc1()V' + breakpoint +.end method + +.method native synchronized acc2()V +.end method diff --git a/tests/examplefiles/example.java b/tests/examplefiles/example.java new file mode 100644 index 00000000..f2e94322 --- /dev/null +++ b/tests/examplefiles/example.java @@ -0,0 +1,16 @@ +class _PostUnico$deClassá +{void fo$o() {} + + void PostUnicodeFunctioná() { + láb$el: + break láb$el; + + } +} + +class áPreUnicode$Class +{ + public int $foo; + public int á$foo; + _PostUnico$deClassá áPreUnicodeFunction() { return null; } +} diff --git a/tests/examplefiles/example.reds b/tests/examplefiles/example.reds index dd4ad0f9..eb92310d 100644 --- a/tests/examplefiles/example.reds +++ b/tests/examplefiles/example.reds @@ -109,6 +109,32 @@ foo char ;lit-word: 'foo 'foo +;multiple comment tests... +1 + 1 +comment "aa" +2 + 2 +comment {aa} +3 + 3 +comment {a^{} +4 + 4 +comment {{}} +5 + 5 +comment { + foo: 6 +} +6 + 6 +comment [foo: 6] +7 + 7 +comment [foo: "[" ] +8 + 8 +comment [foo: {^{} ] +9 + 9 +comment [foo: {boo} ] +10 + 10 +comment 5-May-2014/11:17:34+2:00 +11 + 11 + + to-integer foo foo/(a + 1)/b diff --git a/tests/examplefiles/example.rkt b/tests/examplefiles/example.rkt index a3e4a29e..acc0328e 100644 --- a/tests/examplefiles/example.rkt +++ b/tests/examplefiles/example.rkt @@ -1,5 +1,7 @@ #lang racket +(require (only-in srfi/13 string-contains)) + ; Single-line comment style. ;; Single-line comment style. @@ -8,45 +10,259 @@ #| Multi-line comment style ... +#|### #| nested |#||| |# ... on multiple lines |# -(define (a-function x #:keyword [y 0]) +#;(s-expression comment (one line)) + +#; +(s-expression comment + (multiple lines)) + +#! shebang comment + +#!/shebang comment + +#! shebang \ +comment + +#!/shebang \ +comment + +;; Uncommented numbers after single-line comments +;; NEL
133 +;; LS
8232 +;; PS
8233 + +#reader racket +(define(a-function x #:keyword [y 0]) (define foo0 'symbol) ; () [define foo1 'symbol] ; [] {define foo2 'symbol} ; {} - (and (append (car '(1 2 3)))) + (define 100-Continue 'symbol) + (and (append (car'(1 2 3)))) (regexp-match? #rx"foobar" "foobar") - (regexp-match? #px"foobar" "foobar") - (define a 1)) - (let ([b "foo"]) - (displayln b)) + (regexp-match? #px"\"foo\\(bar\\)?\"" "foobar") + (regexp-match? #rx#"foobar" "foobar") + (regexp-match? #px#"foobar" "foobar") + (define #csa 1) + #Ci (let ([#%A|||b #true C +\|d "foo"]) + (displayln #cS #%\ab\ #true\ C\ +\\d||)) (for/list ([x (in-list (list 1 2 (list 3 4)))]) - (cond - [(pair? x) (car x)] - [else x]))) + (cond + [(pair? x) (car x)] + [else x]))) -;; Literal number examples +;; Literals (values ;; #b - #b1.1 - #b-1.1 - #b1e1 - #b0/1 - #b1/1 - #b1e-1 - #b101 - + #b1 + #b+1 + #b-1 + #b.1 + #b1. + #b0.1 + #b+0.1 + #b-0.1 + #b1/10 + #b+1/10 + #b-1/10 + #b1e11 + #b+1e11 + #b-1e11 + #b.1e11 + #b1.e11 + #b0.1e11 + #b+0.1e11 + #b-0.1e11 + #b1/10e11 + #b+1/10e11 + #b-1/10e11 + #b+i + #b1+i + #b+1+i + #b-1+i + #b.1+i + #b1.+i + #b0.1+i + #b+0.1+i + #b-0.1+i + #b1/10+i + #b+1/10+i + #b-1/10+i + #b1e11+i + #b+1e11+i + #b-1e11+i + #b1.e11+i + #b.1e11+i + #b0.1e11+i + #b+0.1e11+i + #b-0.1e11+i + #b1/10e11+i + #b+1/10e11+i + #b-1/10e11+i + #b+1i + #b1+1i + #b+1+1i + #b-1+1i + #b1.+1i + #b.1+1i + #b0.1+1i + #b+0.1+1i + #b-0.1+1i + #b1/10+1i + #b+1/10+1i + #b-1/10+1i + #b1e11+1i + #b+1e11+1i + #b-1e11+1i + #b.1e11+1i + #b0.1e11+1i + #b+0.1e11+1i + #b-0.1e11+1i + #b1/10e11+1i + #b+1/10e11+1i + #b-1/10e11+1i + #b+1/10e11i + #b1+1/10e11i + #b+1+1/10e11i + #b-1+1/10e11i + #b.1+1/10e11i + #b0.1+1/10e11i + #b+0.1+1/10e11i + #b-0.1+1/10e11i + #b1/10+1/10e11i + #b+1/10+1/10e11i + #b-1/10+1/10e11i + #b1e11+1/10e11i + #b+1e11+1/10e11i + #b-1e11+1/10e11i + #b.1e11+1/10e11i + #b0.1e11+1/10e11i + #b+0.1e11+1/10e11i + #b-0.1e11+1/10e11i + #b1/10e11+1/10e11i + #b+1/10e11+1/10e11i + #b-1/10e11+1/10e11i ;; #d - #d-1.23 - #d1.123 - #d1e3 - #d1e-22 - #d1/2 - #d-1/2 #d1 + #d+1 #d-1 - + #d.1 + #d1. + #d1.2 + #d+1.2 + #d-1.2 + #d1/2 + #d+1/2 + #d-1/2 + #d1e3 + #d+1e3 + #d-1e3 + #d.1e3 + #d1.e3 + #d1.2e3 + #d+1.2e3 + #d-1.2e3 + #d1/2e3 + #d+1/2e3 + #d-1/2e3 + #d+i + #d1+i + #d+1+i + #d-1+i + #d.1+i + #d1.+i + #d1.2+i + #d+1.2+i + #d-1.2+i + #d1/2+i + #d+1/2+i + #d-1/2+i + #d1e3+i + #d+1e3+i + #d-1e3+i + #d1.e3+i + #d.1e3+i + #d1.2e3+i + #d+1.2e3+i + #d-1.2e3+i + #d1/2e3+i + #d+1/2e3+i + #d-1/2e3+i + #d+1i + #d1+1i + #d+1+1i + #d-1+1i + #d1.+1i + #d.1+1i + #d1.2+1i + #d+1.2+1i + #d-1.2+1i + #d1/2+1i + #d+1/2+1i + #d-1/2+1i + #d1e3+1i + #d+1e3+1i + #d-1e3+1i + #d.1e3+1i + #d1.2e3+1i + #d+1.2e3+1i + #d-1.2e3+1i + #d1/2e3+1i + #d+1/2e3+1i + #d-1/2e3+1i + #d+1/2e3i + #d1+1/2e3i + #d+1+1/2e3i + #d-1+1/2e3i + #d.1+1/2e3i + #d1.2+1/2e3i + #d+1.2+1/2e3i + #d-1.2+1/2e3i + #d1/2+1/2e3i + #d+1/2+1/2e3i + #d-1/2+1/2e3i + #d1e3+1/2e3i + #d+1e3+1/2e3i + #d-1e3+1/2e3i + #d.1e3+1/2e3i + #d1.2e3+1/2e3i + #d+1.2e3+1/2e3i + #d-1.2e3+1/2e3i + #d1/2e3+1/2e3i + #d+1/2e3+1/2e3i + #d-1/2e3+1/2e3i + ;; Extflonums + +nan.t + 1t3 + +1t3 + -1t3 + .1t3 + 1.t3 + 1.2t3 + +1.2t3 + -1.2t3 + 1/2t3 + +1/2t3 + -1/2t3 + 1#t0 + 1.#t0 + .2#t0 + 1.2#t0 + 1#/2t0 + 1/2#t0 + 1#/2#t0 + 1#t3 + 1.#t3 + .2#t3 + 1.2#t3 + 1#/2t3 + 1/2#t3 + 1#/2#t3 ;; No # reader prefix -- same as #d -1.23 1.123 @@ -56,7 +272,6 @@ Multi-line comment style ... -1/2 1 -1 - ;; #e #e-1.23 #e1.123 @@ -66,7 +281,24 @@ Multi-line comment style ... #e-1 #e1/2 #e-1/2 - + ;; #d#e + #d#e-1.23 + #d#e1.123 + #d#e1e3 + #d#e1e-22 + #d#e1 + #d#e-1 + #d#e1/2 + #d#e-1/2 + ;; #e#d + #e#d-1.23 + #e#d1.123 + #e#d1e3 + #e#d1e-22 + #e#d1 + #e#d-1 + #e#d1/2 + #e#d-1/2 ;; #i always float #i-1.23 #i1.123 @@ -76,7 +308,126 @@ Multi-line comment style ... #i-1/2 #i1 #i-1 - + ;; Implicitly inexact numbers + +nan.0 + 1# + 1.# + .2# + 1.2# + 1#/2 + 1/2# + 1#/2# + 1#e3 + 1.#e3 + .2#e3 + 1.2#e3 + 1#/2e3 + 1/2#e3 + 1#/2#e3 + +nan.0+i + 1#+i + 1.#+i + .2#+i + 1.2#+i + 1#/2+i + 1/2#+i + 1#/2#+i + 1#e3+i + 1.#e3+i + .2#e3+i + 1.2#e3+i + 1#/2e3+i + 1/2#e3+i + 1#/2#e3+i + +nan.0i + +1#i + +1.#i + +.2#i + +1.2#i + +1#/2i + +1/2#i + +1#/2#i + +1#e3i + +1.#e3i + +.2#e3i + +1.2#e3i + +1#/2e3i + +1/2#e3i + +1#/2#e3i + 0+nan.0i + 0+1#i + 0+1.#i + 0+.2#i + 0+1.2#i + 0+1#/2i + 0+1/2#i + 0+1#/2#i + 0+1#e3i + 0+1.#e3i + 0+.2#e3i + 0+1.2#e3i + 0+1#/2e3i + 0+1/2#e3i + 0+1#/2#e3i + 1#/2#e3+nan.0i + 1#/2#e3+1#i + 1#/2#e3+1.#i + 1#/2#e3+.2#i + 1#/2#e3+1.2#i + 1#/2#e3+1#/2i + 1#/2#e3+1/2#i + 1#/2#e3+1#/2#i + 1#/2#e3+1#e3i + 1#/2#e3+1.#e3i + 1#/2#e3+.2#e3i + 1#/2#e3+1.2#e3i + 1#/2#e3+1#/2e3i + 1#/2#e3+1/2#e3i + 1#/2#e3+1#/2#e3i + +nan.0@1 + 1#@1 + 1.#@1 + .2#@1 + 1.2#@1 + 1#/2@1 + 1/2#@1 + 1#/2#@1 + 1#e3@1 + 1.#e3@1 + .2#e3@1 + 1.2#e3@1 + 1#/2e3@1 + 1/2#e3@1 + 1#/2#e3@1 + 1@+nan.0 + 1@1# + 1@1.# + 1@.2# + 1@1.2# + 1@1#/2 + 1@1/2# + 1@1#/2# + 1@1#e3 + 1@1.#e3 + 1@.2#e3 + 1@1.2#e3 + 1@1#/2e3 + 1@1/2#e3 + 1@1#/2#e3 + 1#/2#e3@1# + 1#/2#e3@1.# + 1#/2#e3@.2# + 1#/2#e3@1.2# + 1#/2#e3@1#/2 + 1#/2#e3@1/2# + 1#/2#e3@1#/2# + 1#/2#e3@1#e3 + 1#/2#e3@1.#e3 + 1#/2#e3@.2#e3 + 1#/2#e3@1.2#e3 + 1#/2#e3@1#/2e3 + 1#/2#e3@1/2#e3 + 1#/2#e3@1#/2#e3 ;; #o #o777.777 #o-777.777 @@ -86,10 +437,307 @@ Multi-line comment style ... #o-3/7 #o777 #o-777 - + #e#o777.777 + #e#o-777.777 + #e#o777e777 + #e#o777e-777 + #e#o3/7 + #e#o-3/7 + #e#o777 + #e#o-777 + #i#o777.777 + #i#o-777.777 + #i#o777e777 + #i#o777e-777 + #i#o3/7 + #i#o-3/7 + #i#o777 + #i#o-777 ;; #x #x-f.f #xf.f + #xfsf + #xfs-f + #x7/f + #x-7/f #x-f #xf + #e#x-f.f + #e#xf.f + #e#xfsf + #e#xfs-f + #e#x7/f + #e#x-7/f + #e#x-f + #e#xf + #i#x-f.f + #i#xf.f + #i#xfsf + #i#xfs-f + #i#x7/f + #i#x-7/f + #i#x-f + #i#xf + ;; Not numbers + '-1.23x + '1.123x + '1e3x + '1e-22x + '1/2x + '-1/2x + '1x + '-1x + '/ + '1/ + '/2 + '1//2 + '1e3. + '1e + 'e3 + '.i + '1.2.3 + '1..2 + '.1. + '@ + '1@ + '@2 + '1@@2 + '1@2@3 + '1@2i + '1+-2i + '1i+2 + '1i+2i + '1+2i+3i + '- + '--1 + '+ + '++1 + '1/2.3 + '1#2 + '1#.2 + '1.#2 + '.#2 + '+nan.t+nan.ti + '+nan.t@nan.t + ;; Booleans + #t + #T + #true + #f + #F + #false + ;; Characters, strings, and byte strings + #\ + #\Null9 + #\n9 + #\99 + #\0009 + #\u3BB + #\u03BB9 + #\U3BB + #\U000003BB9 + #\λ9 + "string\ + \a.\b.\t.\n.\v.\f.\r.\e.\".\'.\\.\1.\123.\1234.\x9.\x30.\x303" + "\u9.\u1234.\u12345.\U9.\U00100000.\U001000000" + #"byte-string\7\xff\t" + #<<HERE STRING +lorem ipsum +dolor sit amet +consectetur HERE STRING +HERE STRING adipisicing elit +HERE STRING + #| +HERE STRING +|# + ;; Other literals + #(vector) + #20() + #s[prefab-structure 1 2 3] + #&{box} + #hash(("a" . 5)) + #hasheq((a . 5) (b . 7)) + #hasheqv((a . 5) (b . 7)) + #'(define x 1) + #`(define x #,pi) + ;; quote, quasiquote, and unquote + 'pi + ' pi + ''pi + '`pi + '`,pi + ',pi + `pi + ` pi + `'pi + ``pi + `,pi + ` , pi + `,'pi + `,`pi + `,`,pi + '(+) + ' (+) + ''(+) + '`(+) + ',(+) + `(+) + ` (+) + `'(+) + ``(+) + `,(+) + ` , (+) + `,'(+) + `,`(+) + `,`,(+) + #readerracket/base'pi.f + '#readerracket/base pi.f + #readerracket/base`pi.f + `#readerracket/base pi.f + #readerracket/base`,pi.f + `#readerracket/base,pi.f + `,#readerracket/base pi.f + #readerracket/base'`,pi.f + '#readerracket/base`,pi.f + '`#readerracket/base,pi.f + '`,#readerracket/base pi.f + #readerracket/base'(*) + '#readerracket/base(*) + #readerracket/base`(*) + `#readerracket/base(*) + #readerracket/base`,(*) + `#readerracket/base,(*) + `,#readerracket/base(*) + #readerracket/base'`,(*) + '#readerracket/base`,(*) + '`#readerracket/base,(*) + '`,#readerracket/base(*) + (quote pi) + (quote (quote pi)) + (quote (quasiquote pi)) + (quote (quasiquote (unquote pi))) + (quote (unquote pi)) + (quasiquote pi) + (quasiquote (quote pi)) + (quasiquote (quasiquote pi)) + (quasiquote (unquote pi)) + (quasiquote (unquote (quote pi))) + (quasiquote (unquote (quasiquote pi))) + (quasiquote (unquote (quasiquote (unquote pi)))) + (quote (+)) + (quote (quote (+))) + (quote (quasiquote (+))) + (quote (unquote (+))) + (quasiquote (+)) + (quasiquote (quote (+))) + (quasiquote (quasiquote (+))) + (quasiquote (unquote (+))) + (quasiquote (unquote (quote (+)))) + (quasiquote (unquote (quasiquote (+)))) + (quasiquote (unquote (quasiquote (unquote (+))))) + #reader racket/base (quote pi.f) + (quote #reader racket/base pi.f) + #reader racket/base (quasiquote pi.f) + (quasiquote #reader racket/base pi.f) + #reader racket/base (quasiquote (unquote pi.f)) + (quasiquote #reader racket/base (unquote pi.f)) + (quasiquote (unquote #reader racket/base pi.f)) + #reader racket/base (quote (quasiquote (unquote pi.f))) + (quote #reader racket/base (quasiquote (unquote pi.f))) + (quote (quasiquote #reader racket/base (unquote pi.f))) + (quote (quasiquote (unquote #reader racket/base pi.f))) + #reader racket/base (quote (*)) + (quote #reader racket/base (*)) + #reader racket/base (quasiquote (*)) + (quasiquote #reader racket/base (*)) + #reader racket/base (quasiquote (unquote (*))) + (quasiquote #reader racket/base (unquote (*))) + (quasiquote (unquote #reader racket/base (*))) + #reader racket/base (quote (quasiquote (unquote (*)))) + (quote #reader racket/base (quasiquote (unquote (*)))) + (quote (quasiquote #reader racket/base (unquote (*)))) + (quote (quasiquote (unquote #reader racket/base (*)))) + ;; Make sure non-identifiers work with quotes + ' "" pi + ' #t pi + ' #() pi + ' #s(s) pi + ' #\u3BB pi + ' #\U000003BB pi + ' #\space pi + ' #\. pi + ' #"" pi + ' #:kw pi + ' #&b pi + ' #'(define x 1) pi + ' #`(define x #,pi) pi + ' #I0 pi + ' #E0 pi + ' #X0 pi + ' #O0 pi + ' #D0 pi + ' #B0 pi + ' #<<EOF +EOF + pi + ' #rx"" pi + ' #rx#"" pi + ' #px"" pi + ' #px#"" pi + ' #hash() pi + ' #hasheq[] pi + ' #hasheqv{} pi + ' #1(v) pi ) + +;; Use the following to generate lists of built-ins and keywords. +;; Run +;; (displayln (wrap-lines KEYWORDS)) +;; (displayln (wrap-lines BUILTINS)) +;; and copy the results into RacketLexer._keywords and RacketLexer._builtins. + +;; (-> (listof string?) string?) +;; Appends all the strings together, quoting them as appropriate for Python, +;; with commas and spaces between them, wrapping at 80 characters, with an +;; indentation of 8 spaces. +(define (wrap-lines lst) + (define INDENTATION '" ") + (define WIDTH '80) + (define (wrap-lines* lst done-lines current-line) + (if (null? lst) + (string-append (foldr string-append "" done-lines) current-line) + (let* ([str (first lst)] + [wrapped-str (if (regexp-match-exact? '#px"[[:ascii:]]+" str) + (string-append "'" str "',") + (string-append "u'" str "',"))] + [new-line (string-append current-line " " wrapped-str)]) + (if ((string-length new-line) . >= . WIDTH) + (wrap-lines* (rest lst) + (append done-lines + `(,(string-append current-line "\n"))) + (string-append INDENTATION wrapped-str)) + (wrap-lines* (rest lst) + done-lines + new-line))))) + (wrap-lines* lst '() INDENTATION)) + +;; (-> string? boolean?) +;; Returns #t if str represents a syntax identifier in the current namespace, +;; otherwise #f. +(define (syntax-identifier? str) + (with-handlers ([exn? exn?]) + (not (eval (call-with-input-string str read))))) + +(define RACKET-NAMESPACE + (parameterize ([current-namespace (make-base-namespace)]) + (namespace-require 'racket) + (current-namespace))) + +(define BOUND-IDENTIFIERS + (parameterize ([current-namespace RACKET-NAMESPACE]) + (sort (map symbol->string (namespace-mapped-symbols)) + string<=?))) + +(define-values (KEYWORDS BUILTINS) + (parameterize ([current-namespace RACKET-NAMESPACE]) + (partition syntax-identifier? BOUND-IDENTIFIERS))) diff --git a/tests/examplefiles/test.r3 b/tests/examplefiles/test.r3 index cad12a8d..707102db 100644 --- a/tests/examplefiles/test.r3 +++ b/tests/examplefiles/test.r3 @@ -1,3 +1,9 @@ +preface.... everything what is before header is not evaluated +so this should not be colorized: +1 + 2 + +REBOL [] ;<- this is minimal header, everything behind it must be colorized + ;## String tests ## print "Hello ^"World" ;<- with escaped char multiline-string: { @@ -52,15 +58,29 @@ type? #ff0000 ;== issue! to integer! (1 + (x / 4.5) * 1E-4) ;## some spec comments -comment now -comment 10 +1 + 1 +comment "aa" +2 + 2 +comment {aa} +3 + 3 +comment {a^{} +4 + 4 +comment {{}} +5 + 5 comment { - bla - bla + foo: 6 } -comment [ - quit -] +6 + 6 +comment [foo: 6] +7 + 7 +comment [foo: "[" ] +8 + 8 +comment [foo: {^{} ] +9 + 9 +comment [foo: {boo} ] +10 + 10 +comment 5-May-2014/11:17:34+2:00 +5-May-2014/11:17:34+2:00 11 + 11 ;## other tests ## ---: 1 diff --git a/tests/examplefiles/vimrc b/tests/examplefiles/vimrc new file mode 100644 index 00000000..d2f9cd1b --- /dev/null +++ b/tests/examplefiles/vimrc @@ -0,0 +1,21 @@ +" A comment + +:py print "py" +::pyt print 'pyt' + pyth print '''pyth''' + : pytho print "pytho" +python print """python""" + + : : python<<E OF +print """my script""" + +def MyFunc(str): + """ My Function """ + print str +E OF + +let py = 42 +echo py + +let foo = 42 +echo foo diff --git a/tests/string_asserts.py b/tests/string_asserts.py new file mode 100644 index 00000000..025a5281 --- /dev/null +++ b/tests/string_asserts.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +""" + Pygments string assert utility + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +class StringTests(object): + + def assertStartsWith(self, haystack, needle, msg=None): + if msg is None: + msg = "'{}' does not start with '{}'".format(haystack, needle) + if not haystack.startswith(needle): + raise(AssertionError(msg)) + + def assertEndsWith(self, haystack, needle, msg=None): + if msg is None: + msg = "'{}' does not end with '{}'".format(haystack, needle) + if not haystack.endswith(needle): + raise(AssertionError(msg)) diff --git a/tests/test_cfm.py b/tests/test_cfm.py new file mode 100644 index 00000000..2ff25bd6 --- /dev/null +++ b/tests/test_cfm.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- +""" + Basic ColdfusionHtmlLexer Test + ~~~~~~~~~~~~~~~~~ + + :copyright: Copyright 2014 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import unittest +import os + +from pygments.token import Token +from pygments.lexers import ColdfusionHtmlLexer + + +class ColdfusionHtmlLexerTest(unittest.TestCase): + + def setUp(self): + self.lexer = ColdfusionHtmlLexer() + + def testBasicComment(self): + fragment = u'<!--- cfcomment --->' + expected = [ + (Token.Text, u''), + (Token.Comment.Multiline, u'<!---'), + (Token.Comment.Multiline, u' cfcomment '), + (Token.Comment.Multiline, u'--->'), + (Token.Text, u'\n'), + ] + self.assertEqual(expected, list(self.lexer.get_tokens(fragment))) + + def testNestedComment(self): + fragment = u'<!--- nested <!--- cfcomment ---> --->' + expected = [ + (Token.Text, u''), + (Token.Comment.Multiline, u'<!---'), + (Token.Comment.Multiline, u' nested '), + (Token.Comment.Multiline, u'<!---'), + (Token.Comment.Multiline, u' cfcomment '), + (Token.Comment.Multiline, u'--->'), + (Token.Comment.Multiline, u' '), + (Token.Comment.Multiline, u'--->'), + (Token.Text, u'\n'), + ] + self.assertEqual(expected, list(self.lexer.get_tokens(fragment))) diff --git a/tests/test_clexer.py b/tests/test_clexer.py index e359c44e..4aac6d39 100644 --- a/tests/test_clexer.py +++ b/tests/test_clexer.py @@ -43,7 +43,7 @@ class CLexerTest(unittest.TestCase): } } ''' - expected = [ + tokens = [ (Token.Keyword.Type, u'int'), (Token.Text, u' '), (Token.Name.Function, u'main'), @@ -81,7 +81,7 @@ class CLexerTest(unittest.TestCase): (Token.Punctuation, u'}'), (Token.Text, u'\n'), ] - self.assertEqual(expected, list(self.lexer.get_tokens(textwrap.dedent(fragment)))) + self.assertEqual(tokens, list(self.lexer.get_tokens(textwrap.dedent(fragment)))) def testSwitchSpaceBeforeColon(self): fragment = u'''\ @@ -95,7 +95,7 @@ class CLexerTest(unittest.TestCase): } } ''' - expected = [ + tokens = [ (Token.Keyword.Type, u'int'), (Token.Text, u' '), (Token.Name.Function, u'main'), @@ -135,7 +135,7 @@ class CLexerTest(unittest.TestCase): (Token.Punctuation, u'}'), (Token.Text, u'\n'), ] - self.assertEqual(expected, list(self.lexer.get_tokens(textwrap.dedent(fragment)))) + self.assertEqual(tokens, list(self.lexer.get_tokens(textwrap.dedent(fragment)))) def testLabel(self): fragment = u'''\ @@ -145,7 +145,7 @@ class CLexerTest(unittest.TestCase): goto foo; } ''' - expected = [ + tokens = [ (Token.Keyword.Type, u'int'), (Token.Text, u' '), (Token.Name.Function, u'main'), @@ -166,7 +166,7 @@ class CLexerTest(unittest.TestCase): (Token.Punctuation, u'}'), (Token.Text, u'\n'), ] - self.assertEqual(expected, list(self.lexer.get_tokens(textwrap.dedent(fragment)))) + self.assertEqual(tokens, list(self.lexer.get_tokens(textwrap.dedent(fragment)))) def testLabelSpaceBeforeColon(self): fragment = u'''\ @@ -176,7 +176,7 @@ class CLexerTest(unittest.TestCase): goto foo; } ''' - expected = [ + tokens = [ (Token.Keyword.Type, u'int'), (Token.Text, u' '), (Token.Name.Function, u'main'), @@ -198,7 +198,7 @@ class CLexerTest(unittest.TestCase): (Token.Punctuation, u'}'), (Token.Text, u'\n'), ] - self.assertEqual(expected, list(self.lexer.get_tokens(textwrap.dedent(fragment)))) + self.assertEqual(tokens, list(self.lexer.get_tokens(textwrap.dedent(fragment)))) def testLabelFollowedByStatement(self): fragment = u'''\ @@ -208,7 +208,7 @@ class CLexerTest(unittest.TestCase): goto foo; } ''' - expected = [ + tokens = [ (Token.Keyword.Type, u'int'), (Token.Text, u' '), (Token.Name.Function, u'main'), @@ -233,4 +233,4 @@ class CLexerTest(unittest.TestCase): (Token.Punctuation, u'}'), (Token.Text, u'\n'), ] - self.assertEqual(expected, list(self.lexer.get_tokens(textwrap.dedent(fragment)))) + self.assertEqual(tokens, list(self.lexer.get_tokens(textwrap.dedent(fragment)))) diff --git a/tests/test_rtf_formatter.py b/tests/test_rtf_formatter.py new file mode 100644 index 00000000..30b136fd --- /dev/null +++ b/tests/test_rtf_formatter.py @@ -0,0 +1,109 @@ +# -*- coding: utf-8 -*- +""" + Pygments RTF formatter tests + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import unittest +from string_asserts import StringTests + +from pygments.util import StringIO +from pygments.formatters import RtfFormatter +from pygments.lexers.special import TextLexer + +class RtfFormatterTest(StringTests, unittest.TestCase): + foot = (r'\par' '\n' r'}') + + def _escape(self, string): + return(string.replace("\n", r"\n")) + + def _build_message(self, *args, **kwargs): + string = kwargs.get('string', None) + t = self._escape(kwargs.get('t', '')) + expected = self._escape(kwargs.get('expected', '')) + result = self._escape(kwargs.get('result', '')) + + if string is None: + string = (u"The expected output of '{t}'\n" + u"\t\tShould be '{expected}'\n" + u"\t\tActually outputs '{result}'\n" + u"\t(WARNING: Partial Output of Result!)") + + end = -(len(self._escape(self.foot))) + start = end-len(expected) + + return string.format(t=t, + result = result[start:end], + expected = expected) + + def format_rtf(self, t): + tokensource = list(TextLexer().get_tokens(t)) + fmt = RtfFormatter() + buf = StringIO() + fmt.format(tokensource, buf) + result = buf.getvalue() + buf.close() + return result + + def test_rtf_header(self): + t = u'' + result = self.format_rtf(t) + expected = r'{\rtf1\ansi\uc0' + msg = (u"RTF documents are expected to start with '{expected}'\n" + u"\t\tStarts intead with '{result}'\n" + u"\t(WARNING: Partial Output of Result!)".format( + expected = expected, + result = result[:len(expected)])) + self.assertStartsWith(result, expected, msg) + + def test_rtf_footer(self): + t = u'' + result = self.format_rtf(t) + expected = self.foot + msg = (u"RTF documents are expected to end with '{expected}'\n" + u"\t\tEnds intead with '{result}'\n" + u"\t(WARNING: Partial Output of Result!)".format( + expected = self._escape(expected), + result = self._escape(result[-len(expected):]))) + self.assertEndsWith(result, expected, msg) + + def test_ascii_characters(self): + t = u'a b c d ~' + result = self.format_rtf(t) + expected = (r'a b c d ~') + if not result.endswith(self.foot): + return(unittest.skip('RTF Footer incorrect')) + msg = self._build_message(t=t, result=result, expected=expected) + self.assertEndsWith(result, expected+self.foot, msg) + + def test_escape_characters(self): + t = u'\ {{' + result = self.format_rtf(t) + expected = (r'\\ \{\{') + if not result.endswith(self.foot): + return(unittest.skip('RTF Footer incorrect')) + msg = self._build_message(t=t, result=result, expected=expected) + self.assertEndsWith(result, expected+self.foot, msg) + + def test_single_characters(self): + t = u'â € ¤ каждой' + result = self.format_rtf(t) + expected = (r'{\u226} {\u8364} {\u164} ' + r'{\u1082}{\u1072}{\u1078}{\u1076}{\u1086}{\u1081}') + if not result.endswith(self.foot): + return(unittest.skip('RTF Footer incorrect')) + msg = self._build_message(t=t, result=result, expected=expected) + self.assertEndsWith(result, expected+self.foot, msg) + + def test_double_characters(self): + t = u'က 힣 ↕ ↕︎ 鼖' + result = self.format_rtf(t) + expected = (r'{\u4096} {\u55203} {\u8597} ' + r'{\u8597}{\u65038} {\u55422}{\u56859}') + if not result.endswith(self.foot): + return(unittest.skip('RTF Footer incorrect')) + msg = self._build_message(t=t, result=result, expected=expected) + self.assertEndsWith(result, expected+self.foot, msg) diff --git a/tests/test_string_asserts.py b/tests/test_string_asserts.py new file mode 100644 index 00000000..0beed15c --- /dev/null +++ b/tests/test_string_asserts.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- +""" + Pygments string assert utility tests + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import unittest +from string_asserts import StringTests + +class TestStringTests(StringTests, unittest.TestCase): + + def test_startswith_correct(self): + self.assertStartsWith("AAA", "A") + + # @unittest.expectedFailure not supported by nose + def test_startswith_incorrect(self): + with self.assertRaises(AssertionError): + self.assertStartsWith("AAA", "B") + + # @unittest.expectedFailure not supported by nose + def test_startswith_short(self): + with self.assertRaises(AssertionError): + self.assertStartsWith("A", "AA") + + def test_endswith_correct(self): + self.assertEndsWith("AAA", "A") + + # @unittest.expectedFailure not supported by nose + def test_endswith_incorrect(self): + with self.assertRaises(AssertionError): + self.assertEndsWith("AAA", "B") + + # @unittest.expectedFailure not supported by nose + def test_endswith_short(self): + with self.assertRaises(AssertionError): + self.assertEndsWith("A", "AA") |