diff options
Diffstat (limited to 'pygments/lexers')
30 files changed, 1221 insertions, 204 deletions
diff --git a/pygments/lexers/__init__.py b/pygments/lexers/__init__.py index 1cdf25d2..100d8076 100644 --- a/pygments/lexers/__init__.py +++ b/pygments/lexers/__init__.py @@ -147,8 +147,8 @@ def load_lexer_from_file(filename, lexername="CustomLexer", **options): # And finally instantiate it with the options return lexer_class(**options) except IOError as err: - raise ClassNotFound('cannot read %s' % filename) - except ClassNotFound as err: + raise ClassNotFound('cannot read %s: %s' % (filename, err)) + except ClassNotFound: raise except Exception as err: raise ClassNotFound('error when loading custom lexer: %s' % err) diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py index 22c0e32e..777572a0 100644 --- a/pygments/lexers/_mapping.py +++ b/pygments/lexers/_mapping.py @@ -233,6 +233,7 @@ LEXERS = { 'JuttleLexer': ('pygments.lexers.javascript', 'Juttle', ('juttle', 'juttle'), ('*.juttle',), ('application/juttle', 'application/x-juttle', 'text/x-juttle', 'text/juttle')), 'KalLexer': ('pygments.lexers.javascript', 'Kal', ('kal',), ('*.kal',), ('text/kal', 'application/kal')), 'KconfigLexer': ('pygments.lexers.configs', 'Kconfig', ('kconfig', 'menuconfig', 'linux-config', 'kernel-config'), ('Kconfig', '*Config.in*', 'external.in*', 'standard-modules.in'), ('text/x-kconfig',)), + 'KernelLogLexer': ('pygments.lexers.textfmts', 'Kernel log', ('kmsg', 'dmesg'), ('*.kmsg', '*.dmesg'), ()), 'KokaLexer': ('pygments.lexers.haskell', 'Koka', ('koka',), ('*.kk', '*.kki'), ('text/x-koka',)), 'KotlinLexer': ('pygments.lexers.jvm', 'Kotlin', ('kotlin',), ('*.kt',), ('text/x-kotlin',)), 'LSLLexer': ('pygments.lexers.scripting', 'LSL', ('lsl',), ('*.lsl',), ('text/x-lsl',)), @@ -278,6 +279,7 @@ LEXERS = { 'MonkeyLexer': ('pygments.lexers.basic', 'Monkey', ('monkey',), ('*.monkey',), ('text/x-monkey',)), 'MonteLexer': ('pygments.lexers.monte', 'Monte', ('monte',), ('*.mt',), ()), 'MoonScriptLexer': ('pygments.lexers.scripting', 'MoonScript', ('moon', 'moonscript'), ('*.moon',), ('text/x-moonscript', 'application/x-moonscript')), + 'MoselLexer': ('pygments.lexers.mosel', 'Mosel', ('mosel',), ('*.mos',), ()), 'MozPreprocCssLexer': ('pygments.lexers.markup', 'CSS+mozpreproc', ('css+mozpreproc',), ('*.css.in',), ()), 'MozPreprocHashLexer': ('pygments.lexers.markup', 'mozhashpreproc', ('mozhashpreproc',), (), ()), 'MozPreprocJavascriptLexer': ('pygments.lexers.markup', 'Javascript+mozpreproc', ('javascript+mozpreproc',), ('*.js.in',), ()), @@ -322,7 +324,7 @@ LEXERS = { 'PanLexer': ('pygments.lexers.dsls', 'Pan', ('pan',), ('*.pan',), ()), 'ParaSailLexer': ('pygments.lexers.parasail', 'ParaSail', ('parasail',), ('*.psi', '*.psl'), ('text/x-parasail',)), 'PawnLexer': ('pygments.lexers.pawn', 'Pawn', ('pawn',), ('*.p', '*.pwn', '*.inc'), ('text/x-pawn',)), - 'Perl6Lexer': ('pygments.lexers.perl', 'Perl6', ('perl6', 'pl6'), ('*.pl', '*.pm', '*.nqp', '*.p6', '*.6pl', '*.p6l', '*.pl6', '*.6pm', '*.p6m', '*.pm6', '*.t'), ('text/x-perl6', 'application/x-perl6')), + 'Perl6Lexer': ('pygments.lexers.perl', 'Perl6', ('perl6', 'pl6', 'raku'), ('*.pl', '*.pm', '*.nqp', '*.p6', '*.6pl', '*.p6l', '*.pl6', '*.6pm', '*.p6m', '*.pm6', '*.t', '*.raku', '*.rakumod', '*.rakutest', '*.rakudoc'), ('text/x-perl6', 'application/x-perl6')), 'PerlLexer': ('pygments.lexers.perl', 'Perl', ('perl', 'pl'), ('*.pl', '*.pm', '*.t'), ('text/x-perl', 'application/x-perl')), 'PhpLexer': ('pygments.lexers.php', 'PHP', ('php', 'php3', 'php4', 'php5'), ('*.php', '*.php[345]', '*.inc'), ('text/x-php',)), 'PigLexer': ('pygments.lexers.jvm', 'Pig', ('pig',), ('*.pig',), ('text/x-pig',)), @@ -372,6 +374,7 @@ LEXERS = { 'ResourceLexer': ('pygments.lexers.resource', 'ResourceBundle', ('resource', 'resourcebundle'), (), ()), 'RexxLexer': ('pygments.lexers.scripting', 'Rexx', ('rexx', 'arexx'), ('*.rexx', '*.rex', '*.rx', '*.arexx'), ('text/x-rexx',)), 'RhtmlLexer': ('pygments.lexers.templates', 'RHTML', ('rhtml', 'html+erb', 'html+ruby'), ('*.rhtml',), ('text/html+ruby',)), + 'RideLexer': ('pygments.lexers.ride', 'Ride', ('ride',), ('*.ride',), ('text/x-ride',)), 'RoboconfGraphLexer': ('pygments.lexers.roboconf', 'Roboconf Graph', ('roboconf-graph',), ('*.graph',), ()), 'RoboconfInstancesLexer': ('pygments.lexers.roboconf', 'Roboconf Instances', ('roboconf-instances',), ('*.instances',), ()), 'RobotFrameworkLexer': ('pygments.lexers.robotframework', 'RobotFramework', ('robotframework',), ('*.robot',), ('text/x-robotframework',)), @@ -381,7 +384,7 @@ LEXERS = { 'RtsLexer': ('pygments.lexers.trafficscript', 'TrafficScript', ('rts', 'trafficscript'), ('*.rts',), ()), 'RubyConsoleLexer': ('pygments.lexers.ruby', 'Ruby irb session', ('rbcon', 'irb'), (), ('text/x-ruby-shellsession',)), 'RubyLexer': ('pygments.lexers.ruby', 'Ruby', ('rb', 'ruby', 'duby'), ('*.rb', '*.rbw', 'Rakefile', '*.rake', '*.gemspec', '*.rbx', '*.duby', 'Gemfile'), ('text/x-ruby', 'application/x-ruby')), - 'RustLexer': ('pygments.lexers.rust', 'Rust', ('rust', 'rs'), ('*.rs', '*.rs.in'), ('text/rust',)), + 'RustLexer': ('pygments.lexers.rust', 'Rust', ('rust', 'rs'), ('*.rs', '*.rs.in'), ('text/rust', 'text/x-rust')), 'SASLexer': ('pygments.lexers.sas', 'SAS', ('sas',), ('*.SAS', '*.sas'), ('text/x-sas', 'text/sas', 'application/x-sas')), 'SLexer': ('pygments.lexers.r', 'S', ('splus', 's', 'r'), ('*.S', '*.R', '.Rhistory', '.Rprofile', '.Renviron'), ('text/S-plus', 'text/S', 'text/x-r-source', 'text/x-r', 'text/x-R', 'text/x-r-history', 'text/x-r-profile')), 'SMLLexer': ('pygments.lexers.ml', 'Standard ML', ('sml',), ('*.sml', '*.sig', '*.fun'), ('text/x-standardml', 'application/x-standardml')), @@ -395,6 +398,7 @@ LEXERS = { 'ScssLexer': ('pygments.lexers.css', 'SCSS', ('scss',), ('*.scss',), ('text/x-scss',)), 'ShExCLexer': ('pygments.lexers.rdf', 'ShExC', ('shexc', 'shex'), ('*.shex',), ('text/shex',)), 'ShenLexer': ('pygments.lexers.lisp', 'Shen', ('shen',), ('*.shen',), ('text/x-shen', 'application/x-shen')), + 'SieveLexer': ('pygments.lexers.sieve', 'Sieve', ('sieve',), ('*.siv', '*.sieve'), ()), 'SilverLexer': ('pygments.lexers.verification', 'Silver', ('silver',), ('*.sil', '*.vpr'), ()), 'SlashLexer': ('pygments.lexers.slash', 'Slash', ('slash',), ('*.sl',), ()), 'SlimLexer': ('pygments.lexers.webmisc', 'Slim', ('slim',), ('*.slim',), ('text/x-slim',)), @@ -447,6 +451,7 @@ LEXERS = { 'UcodeLexer': ('pygments.lexers.unicon', 'ucode', ('ucode',), ('*.u', '*.u1', '*.u2'), ()), 'UniconLexer': ('pygments.lexers.unicon', 'Unicon', ('unicon',), ('*.icn',), ('text/unicon',)), 'UrbiscriptLexer': ('pygments.lexers.urbi', 'UrbiScript', ('urbiscript',), ('*.u',), ('application/x-urbiscript',)), + 'UsdLexer': ('pygments.lexers.usd', 'USD', ('usd', 'usda'), ('*.usd', '*.usda'), ()), 'VBScriptLexer': ('pygments.lexers.basic', 'VBScript', ('vbscript',), ('*.vbs', '*.VBS'), ()), 'VCLLexer': ('pygments.lexers.varnish', 'VCL', ('vcl',), ('*.vcl',), ('text/x-vclsrc',)), 'VCLSnippetLexer': ('pygments.lexers.varnish', 'VCLSnippets', ('vclsnippets', 'vclsnippet'), (), ('text/x-vclsnippet',)), diff --git a/pygments/lexers/_usd_builtins.py b/pygments/lexers/_usd_builtins.py new file mode 100644 index 00000000..0c7316a6 --- /dev/null +++ b/pygments/lexers/_usd_builtins.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +"""A collection of known USD-related keywords, attributes, and types.""" + +COMMON_ATTRIBUTES = [ + "extent", + "xformOpOrder", +] + +KEYWORDS = [ + "class", + "clips", + "custom", + "customData", + "def", + "dictionary", + "inherits", + "over", + "payload", + "references", + "rel", + "subLayers", + "timeSamples", + "uniform", + "variantSet", + "variantSets", + "variants", +] + +OPERATORS = [ + "add", + "append", + "delete", + "prepend", + "reorder", +] + +SPECIAL_NAMES = [ + "active", + "apiSchemas", + "defaultPrim", + "elementSize", + "endTimeCode", + "hidden", + "instanceable", + "interpolation", + "kind", + "startTimeCode", + "upAxis", +] + +TYPES = [ + "asset", + "bool", + "color3d", + "color3f", + "color3h", + "color4d", + "color4f", + "color4h", + "double", + "double2", + "double3", + "double4", + "float", + "float2", + "float3", + "float4", + "frame4d", + "half", + "half2", + "half3", + "half4", + "int", + "int2", + "int3", + "int4", + "keyword", + "matrix2d", + "matrix3d", + "matrix4d", + "normal3d", + "normal3f", + "normal3h", + "point3d", + "point3f", + "point3h", + "quatd", + "quatf", + "quath", + "string", + "syn", + "token", + "uchar", + "uchar2", + "uchar3", + "uchar4", + "uint", + "uint2", + "uint3", + "uint4", + "usdaType", + "vector3d", + "vector3f", + "vector3h", +] diff --git a/pygments/lexers/asm.py b/pygments/lexers/asm.py index 32ac9361..72a68a57 100644 --- a/pygments/lexers/asm.py +++ b/pygments/lexers/asm.py @@ -55,7 +55,9 @@ class GasLexer(RegexLexer): (number, Number.Integer), (register, Name.Variable), (r'[\r\n]+', Text, '#pop'), - (r'[;#].*?\n', Comment, '#pop'), + (r'([;#]|//).*?\n', Comment.Single, '#pop'), + (r'/[*].*?[*]/', Comment.Multiline), + (r'/[*].*?\n[\w\W]*?[*]/', Comment.Multiline, '#pop'), include('punctuation'), include('whitespace') @@ -79,7 +81,9 @@ class GasLexer(RegexLexer): ('$'+number, Number.Integer), (r"$'(.|\\')'", String.Char), (r'[\r\n]+', Text, '#pop'), - (r'[;#].*?\n', Comment, '#pop'), + (r'([;#]|//).*?\n', Comment.Single, '#pop'), + (r'/[*].*?[*]/', Comment.Multiline), + (r'/[*].*?\n[\w\W]*?[*]/', Comment.Multiline, '#pop'), include('punctuation'), include('whitespace') @@ -87,7 +91,8 @@ class GasLexer(RegexLexer): 'whitespace': [ (r'\n', Text), (r'\s+', Text), - (r'[;#].*?\n', Comment) + (r'([;#]|//).*?\n', Comment.Single), + (r'/[*][\w\W]*?[*]/', Comment.Multiline) ], 'punctuation': [ (r'[-*,.()\[\]!:]+', Punctuation) diff --git a/pygments/lexers/business.py b/pygments/lexers/business.py index 37f161a0..5eeaba56 100644 --- a/pygments/lexers/business.py +++ b/pygments/lexers/business.py @@ -504,7 +504,7 @@ class OpenEdgeLexer(RegexLexer): class GoodDataCLLexer(RegexLexer): """ Lexer for `GoodData-CL - <http://github.com/gooddata/GoodData-CL/raw/master/cli/src/main/resources/\ + <https://github.com/gooddata/GoodData-CL/raw/master/cli/src/main/resources/\ com/gooddata/processor/COMMANDS.txt>`_ script files. diff --git a/pygments/lexers/configs.py b/pygments/lexers/configs.py index 0911b6e2..ad724b22 100644 --- a/pygments/lexers/configs.py +++ b/pygments/lexers/configs.py @@ -303,8 +303,7 @@ class ApacheConfLexer(RegexLexer): (r'#(.*\\\n)+.*$|(#.*?)$', Comment), (r'(<[^\s>]+)(?:(\s+)(.*))?(>)', bygroups(Name.Tag, Text, String, Name.Tag)), - (r'([a-z]\w*)(\s+)', - bygroups(Name.Builtin, Text), 'value'), + (r'[a-z]\w*', Name.Builtin, 'value'), (r'\.+', Text), ], 'value': [ diff --git a/pygments/lexers/dsls.py b/pygments/lexers/dsls.py index 0af3c6c2..7174c16e 100644 --- a/pygments/lexers/dsls.py +++ b/pygments/lexers/dsls.py @@ -640,7 +640,7 @@ class AlloyLexer(RegexLexer): class PanLexer(RegexLexer): """ - Lexer for `pan <http://github.com/quattor/pan/>`_ source files. + Lexer for `pan <https://github.com/quattor/pan/>`_ source files. Based on tcsh lexer. diff --git a/pygments/lexers/haskell.py b/pygments/lexers/haskell.py index 0c0917e7..edc506cf 100644 --- a/pygments/lexers/haskell.py +++ b/pygments/lexers/haskell.py @@ -66,6 +66,7 @@ class HaskellLexer(RegexLexer): (r"(')[" + uni.Lu + r"][\w\']*", Keyword.Type), (r"(')\[[^\]]*\]", Keyword.Type), # tuples and lists get special treatment in GHC (r"(')\([^)]*\)", Keyword.Type), # .. + (r"(')[:!#$%&*+.\\/<=>?@^|~-]+", Keyword.Type), # promoted type operators # Operators (r'\\(?![:!#$%&*+.\\/<=>?@^|~-]+)', Name.Function), # lambda operator (r'(<-|::|->|=>|=)(?![:!#$%&*+.\\/<=>?@^|~-]+)', Operator.Word), # specials diff --git a/pygments/lexers/javascript.py b/pygments/lexers/javascript.py index e9cf6722..665fc28e 100644 --- a/pygments/lexers/javascript.py +++ b/pygments/lexers/javascript.py @@ -259,11 +259,11 @@ class LiveScriptLexer(RegexLexer): (r'//', String.Regex, ('#pop', 'multilineregex')), (r'/(?! )(\\.|[^[/\\\n]|\[(\\.|[^\]\\\n])*])+/' r'([gim]+\b|\B)', String.Regex, '#pop'), + (r'/', Operator, '#pop'), default('#pop'), ], 'root': [ - # this next expr leads to infinite loops root -> slashstartsregex - # (r'^(?=\s|/|<!--)', Text, 'slashstartsregex'), + (r'^(?=\s|/)', Text, 'slashstartsregex'), include('commentsandwhitespace'), (r'(?:\([^()]+\))?[ ]*[~-]{1,2}>|' r'(?:\(?[^()\n]+\)?)?[ ]*<[~-]{1,2}', Name.Function), @@ -1060,7 +1060,7 @@ class CoffeeScriptLexer(RegexLexer): # This isn't really guarding against mishighlighting well-formed # code, just the ability to infinite-loop between root and # slashstartsregex. - (r'/', Operator), + (r'/', Operator, '#pop'), default('#pop'), ], 'root': [ @@ -1135,7 +1135,7 @@ class CoffeeScriptLexer(RegexLexer): class MaskLexer(RegexLexer): """ - For `Mask <http://github.com/atmajs/MaskJS>`__ markup. + For `Mask <https://github.com/atmajs/MaskJS>`__ markup. .. versionadded:: 2.0 """ diff --git a/pygments/lexers/jvm.py b/pygments/lexers/jvm.py index 5728e7c5..5588b796 100644 --- a/pygments/lexers/jvm.py +++ b/pygments/lexers/jvm.py @@ -271,8 +271,7 @@ class ScalaLexer(RegexLexer): # method names (r'(class|trait|object)(\s+)', bygroups(Keyword, Text), 'class'), (r'[^\S\n]+', Text), - (r'//.*?\n', Comment.Single), - (r'/\*', Comment.Multiline, 'comment'), + include('comments'), (u'@%s' % idrest, Name.Decorator), (u'(abstract|ca(?:se|tch)|d(?:ef|o)|e(?:lse|xtends)|' u'f(?:inal(?:ly)?|or(?:Some)?)|i(?:f|mplicit)|' @@ -307,16 +306,17 @@ class ScalaLexer(RegexLexer): ], 'class': [ (u'(%s|%s|`[^`]+`)(\\s*)(\\[)' % (idrest, op), - bygroups(Name.Class, Text, Operator), 'typeparam'), + bygroups(Name.Class, Text, Operator), ('#pop', 'typeparam')), (r'\s+', Text), + include('comments'), (r'\{', Operator, '#pop'), (r'\(', Operator, '#pop'), - (r'//.*?\n', Comment.Single, '#pop'), (u'%s|%s|`[^`]+`' % (idrest, op), Name.Class, '#pop'), ], 'type': [ (r'\s+', Text), - (r'<[%:]|>:|[#_]|forSome|type', Keyword), + include('comments'), + (r'<[%:]|>:|[#_]|\bforSome\b|\btype\b', Keyword), (u'([,);}]|=>|=|\u21d2)(\\s*)', bygroups(Operator, Text), '#pop'), (r'[({]', Operator, '#push'), (u'((?:%s|%s|`[^`]+`)(?:\\.(?:%s|%s|`[^`]+`))*)(\\s*)(\\[)' % @@ -325,16 +325,21 @@ class ScalaLexer(RegexLexer): (u'((?:%s|%s|`[^`]+`)(?:\\.(?:%s|%s|`[^`]+`))*)(\\s*)$' % (idrest, op, idrest, op), bygroups(Keyword.Type, Text), '#pop'), - (r'//.*?\n', Comment.Single, '#pop'), (u'\\.|%s|%s|`[^`]+`' % (idrest, op), Keyword.Type) ], 'typeparam': [ - (r'[\s,]+', Text), - (u'<[%:]|=>|>:|[#_\u21D2]|forSome|type', Keyword), + (r'\s+', Text), + include('comments'), + (r',+', Punctuation), + (u'<[%:]|=>|>:|[#_\u21D2]|\bforSome\b|\btype\b', Keyword), (r'([\])}])', Operator, '#pop'), (r'[(\[{]', Operator, '#push'), (u'\\.|%s|%s|`[^`]+`' % (idrest, op), Keyword.Type) ], + 'comments': [ + (r'//.*?\n', Comment.Single), + (r'/\*', Comment.Multiline, 'comment'), + ], 'comment': [ (r'[^/*]+', Comment.Multiline), (r'/\*', Comment.Multiline, '#push'), diff --git a/pygments/lexers/matlab.py b/pygments/lexers/matlab.py index 64dd3a3c..16df0398 100644 --- a/pygments/lexers/matlab.py +++ b/pygments/lexers/matlab.py @@ -104,8 +104,8 @@ class MatlabLexer(RegexLexer): # is recognized if it is either surrounded by spaces or by no # spaces on both sides; only the former case matters for us. (This # allows distinguishing `cd ./foo` from `cd ./ foo`.) - (r'(?:^|(?<=;))\s*\w+\s+(?!=|\(|(%s)\s+)' % _operators, Name, - 'commandargs'), + (r'(?:^|(?<=;))(\s*)(\w+)(\s+)(?!=|\(|(%s)\s+)' % _operators, + bygroups(Text, Name, Text), 'commandargs'), # operators: (_operators, Operator), @@ -147,9 +147,10 @@ class MatlabLexer(RegexLexer): (r"[^']*'", String, '#pop'), ], 'commandargs': [ + (r"[ \t]+", Text), ("'[^']*'", String), - ("[^';\n]+", String), - (";?\n?", Punctuation, '#pop'), + (r"[^';\s]+", String), + (";?", Punctuation, '#pop'), ] } diff --git a/pygments/lexers/mosel.py b/pygments/lexers/mosel.py new file mode 100644 index 00000000..1dbda1dc --- /dev/null +++ b/pygments/lexers/mosel.py @@ -0,0 +1,448 @@ +# -*- coding: utf-8 -*- +""" + pygments.lexers.mosel + ~~~~~~~~~~~~~~~~~~~~~ + + Lexers for the mosel language. + http://www.fico.com/en/products/fico-xpress-optimization + + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.lexer import RegexLexer, words +from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ + Number, Punctuation + +__all__ = ['MoselLexer'] + +FUNCTIONS = ( + # core functions + '_', + 'abs', + 'arctan', + 'asproc', + 'assert', + 'bitflip', + 'bitneg', + 'bitset', + 'bitshift', + 'bittest', + 'bitval', + 'ceil', + 'cos', + 'create', + 'currentdate', + 'currenttime', + 'cutelt', + 'cutfirst', + 'cuthead', + 'cutlast', + 'cuttail', + 'datablock', + 'delcell', + 'exists', + 'exit', + 'exp', + 'exportprob', + 'fclose', + 'fflush', + 'finalize', + 'findfirst', + 'findlast', + 'floor', + 'fopen', + 'fselect', + 'fskipline', + 'fwrite', + 'fwrite_', + 'fwriteln', + 'fwriteln_', + 'getact', + 'getcoeff', + 'getcoeffs', + 'getdual', + 'getelt', + 'getfid', + 'getfirst', + 'getfname', + 'gethead', + 'getlast', + 'getobjval', + 'getparam', + 'getrcost', + 'getreadcnt', + 'getreverse', + 'getsize', + 'getslack', + 'getsol', + 'gettail', + 'gettype', + 'getvars', + 'isdynamic', + 'iseof', + 'isfinite', + 'ishidden', + 'isinf', + 'isnan', + 'isodd', + 'ln', + 'localsetparam', + 'log', + 'makesos1', + 'makesos2', + 'maxlist', + 'memoryuse', + 'minlist', + 'newmuid', + 'publish', + 'random', + 'read', + 'readln', + 'reset', + 'restoreparam', + 'reverse', + 'round', + 'setcoeff', + 'sethidden', + 'setioerr', + 'setmatherr', + 'setname', + 'setparam', + 'setrandseed', + 'setrange', + 'settype', + 'sin', + 'splithead', + 'splittail', + 'sqrt', + 'strfmt', + 'substr', + 'timestamp', + 'unpublish', + 'versionnum', + 'versionstr', + 'write', + 'write_', + 'writeln', + 'writeln_', + + # mosel exam mmxprs | sed -n -e "s/ [pf][a-z]* \([a-zA-Z0-9_]*\).*/'\1',/p" | sort -u + 'addcut', + 'addcuts', + 'addmipsol', + 'basisstability', + 'calcsolinfo', + 'clearmipdir', + 'clearmodcut', + 'command', + 'copysoltoinit', + 'crossoverlpsol', + 'defdelayedrows', + 'defsecurevecs', + 'delcuts', + 'dropcuts', + 'estimatemarginals', + 'fixglobal', + 'flushmsgq', + 'getbstat', + 'getcnlist', + 'getcplist', + 'getdualray', + 'getiis', + 'getiissense', + 'getiistype', + 'getinfcause', + 'getinfeas', + 'getlb', + 'getlct', + 'getleft', + 'getloadedlinctrs', + 'getloadedmpvars', + 'getname', + 'getprimalray', + 'getprobstat', + 'getrange', + 'getright', + 'getsensrng', + 'getsize', + 'getsol', + 'gettype', + 'getub', + 'getvars', + 'gety', + 'hasfeature', + 'implies', + 'indicator', + 'initglobal', + 'ishidden', + 'isiisvalid', + 'isintegral', + 'loadbasis', + 'loadcuts', + 'loadlpsol', + 'loadmipsol', + 'loadprob', + 'maximise', + 'maximize', + 'minimise', + 'minimize', + 'postsolve', + 'readbasis', + 'readdirs', + 'readsol', + 'refinemipsol', + 'rejectintsol', + 'repairinfeas', + 'repairinfeas_deprec', + 'resetbasis', + 'resetiis', + 'resetsol', + 'savebasis', + 'savemipsol', + 'savesol', + 'savestate', + 'selectsol', + 'setarchconsistency', + 'setbstat', + 'setcallback', + 'setcbcutoff', + 'setgndata', + 'sethidden', + 'setlb', + 'setmipdir', + 'setmodcut', + 'setsol', + 'setub', + 'setucbdata', + 'stopoptimise', + 'stopoptimize', + 'storecut', + 'storecuts', + 'unloadprob', + 'uselastbarsol', + 'writebasis', + 'writedirs', + 'writeprob', + 'writesol', + 'xor', + 'xprs_addctr', + 'xprs_addindic', + + # mosel exam mmsystem | sed -n -e "s/ [pf][a-z]* \([a-zA-Z0-9_]*\).*/'\1',/p" | sort -u + 'addmonths', + 'copytext', + 'cuttext', + 'deltext', + 'endswith', + 'erase', + 'expandpath', + 'fcopy', + 'fdelete', + 'findfiles', + 'findtext', + 'fmove', + 'formattext', + 'getasnumber', + 'getchar', + 'getcwd', + 'getdate', + 'getday', + 'getdaynum', + 'getdays', + 'getdirsep', + 'getdsoparam', + 'getendparse', + 'getenv', + 'getfsize', + 'getfstat', + 'getftime', + 'gethour', + 'getminute', + 'getmonth', + 'getmsec', + 'getoserrmsg', + 'getoserror', + 'getpathsep', + 'getqtype', + 'getsecond', + 'getsepchar', + 'getsize', + 'getstart', + 'getsucc', + 'getsysinfo', + 'getsysstat', + 'gettime', + 'gettmpdir', + 'gettrim', + 'getweekday', + 'getyear', + 'inserttext', + 'isvalid', + 'jointext', + 'makedir', + 'makepath', + 'newtar', + 'newzip', + 'nextfield', + 'openpipe', + 'parseextn', + 'parseint', + 'parsereal', + 'parsetext', + 'pastetext', + 'pathmatch', + 'pathsplit', + 'qsort', + 'quote', + 'readtextline', + 'regmatch', + 'regreplace', + 'removedir', + 'removefiles', + 'setchar', + 'setdate', + 'setday', + 'setdsoparam', + 'setendparse', + 'setenv', + 'sethour', + 'setminute', + 'setmonth', + 'setmsec', + 'setoserror', + 'setqtype', + 'setsecond', + 'setsepchar', + 'setstart', + 'setsucc', + 'settime', + 'settrim', + 'setyear', + 'sleep', + 'splittext', + 'startswith', + 'system', + 'tarlist', + 'textfmt', + 'tolower', + 'toupper', + 'trim', + 'untar', + 'unzip', + 'ziplist', + + # mosel exam mmjobs | sed -n -e "s/ [pf][a-z]* \([a-zA-Z0-9_]*\).*/'\1',/p" | sort -u + 'canceltimer', + 'clearaliases', + 'compile', + 'connect', + 'detach', + 'disconnect', + 'dropnextevent', + 'findxsrvs', + 'getaliases', + 'getannidents', + 'getannotations', + 'getbanner', + 'getclass', + 'getdsoprop', + 'getdsopropnum', + 'getexitcode', + 'getfromgid', + 'getfromid', + 'getfromuid', + 'getgid', + 'gethostalias', + 'getid', + 'getmodprop', + 'getmodpropnum', + 'getnextevent', + 'getnode', + 'getrmtid', + 'getstatus', + 'getsysinfo', + 'gettimer', + 'getuid', + 'getvalue', + 'isqueueempty', + 'load', + 'nullevent', + 'peeknextevent', + 'resetmodpar', + 'run', + 'send', + 'setcontrol', + 'setdefstream', + 'setgid', + 'sethostalias', + 'setmodpar', + 'settimer', + 'setuid', + 'setworkdir', + 'stop', + 'unload', + 'wait', + 'waitexpired', + 'waitfor', + 'waitforend', +) + + +class MoselLexer(RegexLexer): + """ + For the Mosel optimization language. + + .. versionadded:: 2.6 + """ + name = 'Mosel' + aliases = ['mosel'] + filenames = ['*.mos'] + + tokens = { + 'root': [ + (r'\n', Text), + (r'\s+', Text.Whitespace), + (r'!.*?\n', Comment.Single), + (r'\(!(.|\n)*?!\)', Comment.Multiline), + (words(( + 'and', 'as', 'break', 'case', 'count', 'declarations', 'do', + 'dynamic', 'elif', 'else', 'end-', 'end', 'evaluation', 'false', + 'forall', 'forward', 'from', 'function', 'hashmap', 'if', + 'imports', 'include', 'initialisations', 'initializations', 'inter', + 'max', 'min', 'model', 'namespace', 'next', 'not', 'nsgroup', + 'nssearch', 'of', 'options', 'or', 'package', 'parameters', + 'procedure', 'public', 'prod', 'record', 'repeat', 'requirements', + 'return', 'sum', 'then', 'to', 'true', 'union', 'until', 'uses', + 'version', 'while', 'with'), prefix=r'\b', suffix=r'\b'), + Keyword.Builtin), + (words(( + 'range', 'array', 'set', 'list', 'mpvar', 'mpproblem', 'linctr', + 'nlctr', 'integer', 'string', 'real', 'boolean', 'text', 'time', + 'date', 'datetime', 'returned', 'Model', 'Mosel', 'counter', + 'xmldoc', 'is_sos1', 'is_sos2', 'is_integer', 'is_binary', + 'is_continuous', 'is_free', 'is_semcont', 'is_semint', + 'is_partint'), prefix=r'\b', suffix=r'\b'), + Keyword.Type), + (r'(\+|\-|\*|/|=|<=|>=|\||\^|<|>|<>|\.\.|\.|:=|::|:|in|mod|div)', + Operator), + (r'[()\[\]{},;]+', Punctuation), + (words(FUNCTIONS, prefix=r'\b', suffix=r'\b'), Name.Function), + (r'(\d+\.(?!\.)\d*|\.(?!.)\d+)([eE][+-]?\d+)?', Number.Float), + (r'\d+([eE][+-]?\d+)?', Number.Integer), + (r'[+-]?Infinity', Number.Integer), + (r'0[xX][0-9a-fA-F]+', Number), + (r'"', String.Double, 'double_quote'), + (r'\'', String.Single, 'single_quote'), + (r'(\w+|(\.(?!\.)))', Text), + ], + 'single_quote': [ + (r'\'', String.Single, '#pop'), + (r'[^\']+', String.Single), + ], + 'double_quote': [ + (r'(\\"|\\[0-7]{1,3}\D|\\[abfnrtv]|\\\\)', String.Escape), + (r'\"', String.Double, '#pop'), + (r'[^"\\]+', String.Double), + ], + } diff --git a/pygments/lexers/pascal.py b/pygments/lexers/pascal.py index 0a8dd7df..d4b43fd3 100644 --- a/pygments/lexers/pascal.py +++ b/pygments/lexers/pascal.py @@ -577,7 +577,7 @@ class AdaLexer(RegexLexer): (r'\n+', Text), ], 'numbers': [ - (r'[0-9_]+#[0-9a-f]+#', Number.Hex), + (r'[0-9_]+#[0-9a-f_\.]+#', Number.Hex), (r'[0-9_]+\.[0-9_]*', Number.Float), (r'[0-9_]+', Number.Integer), ], diff --git a/pygments/lexers/perl.py b/pygments/lexers/perl.py index 8f30c186..81be0981 100644 --- a/pygments/lexers/perl.py +++ b/pygments/lexers/perl.py @@ -3,7 +3,7 @@ pygments.lexers.perl ~~~~~~~~~~~~~~~~~~~~ - Lexers for Perl and related languages. + Lexers for Perl, Raku and related languages. :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. @@ -22,7 +22,7 @@ __all__ = ['PerlLexer', 'Perl6Lexer'] class PerlLexer(RegexLexer): """ - For `Perl <http://www.perl.org>`_ source code. + For `Perl <https://www.perl.org>`_ source code. """ name = 'Perl' @@ -214,91 +214,189 @@ class PerlLexer(RegexLexer): class Perl6Lexer(ExtendedRegexLexer): """ - For `Perl 6 <http://www.perl6.org>`_ source code. + For `Raku <https://www.raku.org>`_ (a.k.a. Perl 6) source code. .. versionadded:: 2.0 """ name = 'Perl6' - aliases = ['perl6', 'pl6'] + aliases = ['perl6', 'pl6', 'raku'] filenames = ['*.pl', '*.pm', '*.nqp', '*.p6', '*.6pl', '*.p6l', '*.pl6', - '*.6pm', '*.p6m', '*.pm6', '*.t'] + '*.6pm', '*.p6m', '*.pm6', '*.t', '*.raku', '*.rakumod', + '*.rakutest', '*.rakudoc'] mimetypes = ['text/x-perl6', 'application/x-perl6'] flags = re.MULTILINE | re.DOTALL | re.UNICODE PERL6_IDENTIFIER_RANGE = r"['\w:-]" PERL6_KEYWORDS = ( - 'BEGIN', 'CATCH', 'CHECK', 'CONTROL', 'END', 'ENTER', 'FIRST', 'INIT', - 'KEEP', 'LAST', 'LEAVE', 'NEXT', 'POST', 'PRE', 'START', 'TEMP', - 'UNDO', 'as', 'assoc', 'async', 'augment', 'binary', 'break', 'but', - 'cached', 'category', 'class', 'constant', 'contend', 'continue', - 'copy', 'deep', 'default', 'defequiv', 'defer', 'die', 'do', 'else', - 'elsif', 'enum', 'equiv', 'exit', 'export', 'fail', 'fatal', 'for', - 'gather', 'given', 'goto', 'grammar', 'handles', 'has', 'if', 'inline', - 'irs', 'is', 'last', 'leave', 'let', 'lift', 'loop', 'looser', 'macro', - 'make', 'maybe', 'method', 'module', 'multi', 'my', 'next', 'of', - 'ofs', 'only', 'oo', 'ors', 'our', 'package', 'parsed', 'prec', - 'proto', 'readonly', 'redo', 'ref', 'regex', 'reparsed', 'repeat', - 'require', 'required', 'return', 'returns', 'role', 'rule', 'rw', - 'self', 'slang', 'state', 'sub', 'submethod', 'subset', 'supersede', - 'take', 'temp', 'tighter', 'token', 'trusts', 'try', 'unary', - 'unless', 'until', 'use', 'warn', 'when', 'where', 'while', 'will', + #Phasers + 'BEGIN','CATCH','CHECK','CLOSE','CONTROL','DOC','END','ENTER','FIRST', + 'INIT','KEEP','LAST','LEAVE','NEXT','POST','PRE','QUIT','UNDO', + #Keywords + 'anon','augment','but','class','constant','default','does','else', + 'elsif','enum','for','gather','given','grammar','has','if','import', + 'is','let','loop','made','make','method','module','multi','my','need', + 'orwith','our','proceed','proto','repeat','require','return', + 'return-rw','returns','role','rule','state','sub','submethod','subset', + 'succeed','supersede','token','try','unit','unless','until','use', + 'when','while','with','without', + #Traits + 'export','native','repr','required','rw','symbol', ) PERL6_BUILTINS = ( - 'ACCEPTS', 'HOW', 'REJECTS', 'VAR', 'WHAT', 'WHENCE', 'WHERE', 'WHICH', - 'WHO', 'abs', 'acos', 'acosec', 'acosech', 'acosh', 'acotan', 'acotanh', - 'all', 'any', 'approx', 'arity', 'asec', 'asech', 'asin', 'asinh', - 'assuming', 'atan', 'atan2', 'atanh', 'attr', 'bless', 'body', 'by', - 'bytes', 'caller', 'callsame', 'callwith', 'can', 'capitalize', 'cat', - 'ceiling', 'chars', 'chmod', 'chomp', 'chop', 'chr', 'chroot', - 'circumfix', 'cis', 'classify', 'clone', 'close', 'cmp_ok', 'codes', - 'comb', 'connect', 'contains', 'context', 'cos', 'cosec', 'cosech', - 'cosh', 'cotan', 'cotanh', 'count', 'defined', 'delete', 'diag', - 'dies_ok', 'does', 'e', 'each', 'eager', 'elems', 'end', 'eof', 'eval', - 'eval_dies_ok', 'eval_elsewhere', 'eval_lives_ok', 'evalfile', 'exists', - 'exp', 'first', 'flip', 'floor', 'flunk', 'flush', 'fmt', 'force_todo', - 'fork', 'from', 'getc', 'gethost', 'getlogin', 'getpeername', 'getpw', - 'gmtime', 'graphs', 'grep', 'hints', 'hyper', 'im', 'index', 'infix', - 'invert', 'is_approx', 'is_deeply', 'isa', 'isa_ok', 'isnt', 'iterator', - 'join', 'key', 'keys', 'kill', 'kv', 'lastcall', 'lazy', 'lc', 'lcfirst', - 'like', 'lines', 'link', 'lives_ok', 'localtime', 'log', 'log10', 'map', - 'max', 'min', 'minmax', 'name', 'new', 'nextsame', 'nextwith', 'nfc', - 'nfd', 'nfkc', 'nfkd', 'nok_error', 'nonce', 'none', 'normalize', 'not', - 'nothing', 'ok', 'once', 'one', 'open', 'opendir', 'operator', 'ord', - 'p5chomp', 'p5chop', 'pack', 'pair', 'pairs', 'pass', 'perl', 'pi', - 'pick', 'plan', 'plan_ok', 'polar', 'pop', 'pos', 'postcircumfix', - 'postfix', 'pred', 'prefix', 'print', 'printf', 'push', 'quasi', - 'quotemeta', 'rand', 're', 'read', 'readdir', 'readline', 'reduce', - 'reverse', 'rewind', 'rewinddir', 'rindex', 'roots', 'round', - 'roundrobin', 'run', 'runinstead', 'sameaccent', 'samecase', 'say', - 'sec', 'sech', 'sech', 'seek', 'shape', 'shift', 'sign', 'signature', - 'sin', 'sinh', 'skip', 'skip_rest', 'sleep', 'slurp', 'sort', 'splice', - 'split', 'sprintf', 'sqrt', 'srand', 'strand', 'subst', 'substr', 'succ', - 'sum', 'symlink', 'tan', 'tanh', 'throws_ok', 'time', 'times', 'to', - 'todo', 'trim', 'trim_end', 'trim_start', 'true', 'truncate', 'uc', - 'ucfirst', 'undef', 'undefine', 'uniq', 'unlike', 'unlink', 'unpack', - 'unpolar', 'unshift', 'unwrap', 'use_ok', 'value', 'values', 'vec', - 'version_lt', 'void', 'wait', 'want', 'wrap', 'write', 'zip', + 'ACCEPTS','abs','abs2rel','absolute','accept','accessed','acos', + 'acosec','acosech','acosh','acotan','acotanh','acquire','act','action', + 'actions','add','add_attribute','add_enum_value','add_fallback', + 'add_method','add_parent','add_private_method','add_role','add_trustee', + 'adverb','after','all','allocate','allof','allowed','alternative-names', + 'annotations','antipair','antipairs','any','anyof','app_lifetime', + 'append','arch','archname','args','arity','Array','asec','asech','asin', + 'asinh','ASSIGN-KEY','ASSIGN-POS','assuming','ast','at','atan','atan2', + 'atanh','AT-KEY','atomic-assign','atomic-dec-fetch','atomic-fetch', + 'atomic-fetch-add','atomic-fetch-dec','atomic-fetch-inc', + 'atomic-fetch-sub','atomic-inc-fetch','AT-POS','attributes','auth', + 'await','backtrace','Bag','BagHash','bail-out','base','basename', + 'base-repeating','batch','BIND-KEY','BIND-POS','bind-stderr', + 'bind-stdin','bind-stdout','bind-udp','bits','bless','block','Bool', + 'bool-only','bounds','break','Bridge','broken','BUILD','build-date', + 'bytes','cache','callframe','calling-package','CALL-ME','callsame', + 'callwith','can','cancel','candidates','cando','can-ok','canonpath', + 'caps','caption','Capture','cas','catdir','categorize','categorize-list', + 'catfile','catpath','cause','ceiling','cglobal','changed','Channel', + 'chars','chdir','child','child-name','child-typename','chmod','chomp', + 'chop','chr','chrs','chunks','cis','classify','classify-list','cleanup', + 'clone','close','closed','close-stdin','cmp-ok','code','codes','collate', + 'column','comb','combinations','command','comment','compiler','Complex', + 'compose','compose_type','composer','condition','config', + 'configure_destroy','configure_type_checking','conj','connect', + 'constraints','construct','contains','contents','copy','cos','cosec', + 'cosech','cosh','cotan','cotanh','count','count-only','cpu-cores', + 'cpu-usage','CREATE','create_type','cross','cue','curdir','curupdir','d', + 'Date','DateTime','day','daycount','day-of-month','day-of-week', + 'day-of-year','days-in-month','declaration','decode','decoder','deepmap', + 'default','defined','DEFINITE','delayed','DELETE-KEY','DELETE-POS', + 'denominator','desc','DESTROY','destroyers','devnull','diag', + 'did-you-mean','die','dies-ok','dir','dirname','dir-sep','DISTROnames', + 'do','does','does-ok','done','done-testing','duckmap','dynamic','e', + 'eager','earlier','elems','emit','enclosing','encode','encoder', + 'encoding','end','ends-with','enum_from_value','enum_value_list', + 'enum_values','enums','eof','EVAL','eval-dies-ok','EVALFILE', + 'eval-lives-ok','exception','excludes-max','excludes-min','EXISTS-KEY', + 'EXISTS-POS','exit','exitcode','exp','expected','explicitly-manage', + 'expmod','extension','f','fail','fails-like','fc','feature','file', + 'filename','find_method','find_method_qualified','finish','first','flat', + 'flatmap','flip','floor','flunk','flush','fmt','format','formatter', + 'freeze','from','from-list','from-loop','from-posix','full', + 'full-barrier','get','get_value','getc','gist','got','grab','grabpairs', + 'grep','handle','handled','handles','hardware','has_accessor','Hash', + 'head','headers','hh-mm-ss','hidden','hides','hour','how','hyper','id', + 'illegal','im','in','indent','index','indices','indir','infinite', + 'infix','infix:<+>','infix:<->','install_method_cache','Instant', + 'instead','Int','int-bounds','interval','in-timezone','invalid-str', + 'invert','invocant','IO','IO::Notification.watch-path','is_trusted', + 'is_type','isa','is-absolute','isa-ok','is-approx','is-deeply', + 'is-hidden','is-initial-thread','is-int','is-lazy','is-leap-year', + 'isNaN','isnt','is-prime','is-relative','is-routine','is-setting', + 'is-win','item','iterator','join','keep','kept','KERNELnames','key', + 'keyof','keys','kill','kv','kxxv','l','lang','last','lastcall','later', + 'lazy','lc','leading','level','like','line','lines','link','List', + 'listen','live','lives-ok','local','lock','log','log10','lookup','lsb', + 'made','MAIN','make','Map','match','max','maxpairs','merge','message', + 'method','method_table','methods','migrate','min','minmax','minpairs', + 'minute','misplaced','Mix','MixHash','mkdir','mode','modified','month', + 'move','mro','msb','multi','multiness','my','name','named','named_names', + 'narrow','nativecast','native-descriptor','nativesizeof','new','new_type', + 'new-from-daycount','new-from-pairs','next','nextcallee','next-handle', + 'nextsame','nextwith','NFC','NFD','NFKC','NFKD','nl-in','nl-out', + 'nodemap','nok','none','norm','not','note','now','nude','Num', + 'numerator','Numeric','of','offset','offset-in-hours','offset-in-minutes', + 'ok','old','on-close','one','on-switch','open','opened','operation', + 'optional','ord','ords','orig','os-error','osname','out-buffer','pack', + 'package','package-kind','package-name','packages','pair','pairs', + 'pairup','parameter','params','parent','parent-name','parents','parse', + 'parse-base','parsefile','parse-names','parts','pass','path','path-sep', + 'payload','peer-host','peer-port','periods','perl','permutations','phaser', + 'pick','pickpairs','pid','placeholder','plan','plus','polar','poll', + 'polymod','pop','pos','positional','posix','postfix','postmatch', + 'precomp-ext','precomp-target','pred','prefix','prematch','prepend', + 'print','printf','print-nl','print-to','private','private_method_table', + 'proc','produce','Promise','prompt','protect','pull-one','push', + 'push-all','push-at-least','push-exactly','push-until-lazy','put', + 'qualifier-type','quit','r','race','radix','rand','range','Rat','raw', + 're','read','readchars','readonly','ready','Real','reallocate','reals', + 'reason','rebless','receive','recv','redispatcher','redo','reduce', + 'rel2abs','relative','release','rename','repeated','replacement', + 'report','reserved','resolve','restore','result','resume','rethrow', + 'reverse','right','rindex','rmdir','role','roles_to_compose','rolish', + 'roll','rootdir','roots','rotate','rotor','round','roundrobin', + 'routine-type','run','rwx','s','samecase','samemark','samewith','say', + 'schedule-on','scheduler','scope','sec','sech','second','seek','self', + 'send','Set','set_hidden','set_name','set_package','set_rw','set_value', + 'SetHash','set-instruments','setup_finalization','shape','share','shell', + 'shift','sibling','sigil','sign','signal','signals','signature','sin', + 'sinh','sink','sink-all','skip','skip-at-least','skip-at-least-pull-one', + 'skip-one','skip-rest','sleep','sleep-timer','sleep-until','Slip','slurp', + 'slurp-rest','slurpy','snap','snapper','so','socket-host','socket-port', + 'sort','source','source-package','spawn','SPEC','splice','split', + 'splitdir','splitpath','sprintf','spurt','sqrt','squish','srand','stable', + 'start','started','starts-with','status','stderr','stdout','Str', + 'sub_signature','subbuf','subbuf-rw','subname','subparse','subst', + 'subst-mutate','substr','substr-eq','substr-rw','subtest','succ','sum', + 'Supply','symlink','t','tail','take','take-rw','tan','tanh','tap', + 'target','target-name','tc','tclc','tell','then','throttle','throw', + 'throws-like','timezone','tmpdir','to','today','todo','toggle','to-posix', + 'total','trailing','trans','tree','trim','trim-leading','trim-trailing', + 'truncate','truncated-to','trusts','try_acquire','trying','twigil','type', + 'type_captures','typename','uc','udp','uncaught_handler','unimatch', + 'uniname','uninames','uniparse','uniprop','uniprops','unique','unival', + 'univals','unlike','unlink','unlock','unpack','unpolar','unshift', + 'unwrap','updir','USAGE','use-ok','utc','val','value','values','VAR', + 'variable','verbose-config','version','VMnames','volume','vow','w','wait', + 'warn','watch','watch-path','week','weekday-of-month','week-number', + 'week-year','WHAT','when','WHERE','WHEREFORE','WHICH','WHO', + 'whole-second','WHY','wordcase','words','workaround','wrap','write', + 'write-to','x','yada','year','yield','yyyy-mm-dd','z','zip','zip-latest', + ) PERL6_BUILTIN_CLASSES = ( - 'Abstraction', 'Any', 'AnyChar', 'Array', 'Associative', 'Bag', 'Bit', - 'Blob', 'Block', 'Bool', 'Buf', 'Byte', 'Callable', 'Capture', 'Char', 'Class', - 'Code', 'Codepoint', 'Comparator', 'Complex', 'Decreasing', 'Exception', - 'Failure', 'False', 'Grammar', 'Grapheme', 'Hash', 'IO', 'Increasing', - 'Int', 'Junction', 'KeyBag', 'KeyExtractor', 'KeyHash', 'KeySet', - 'KitchenSink', 'List', 'Macro', 'Mapping', 'Match', 'Matcher', 'Method', - 'Module', 'Num', 'Object', 'Ordered', 'Ordering', 'OrderingPair', - 'Package', 'Pair', 'Positional', 'Proxy', 'Range', 'Rat', 'Regex', - 'Role', 'Routine', 'Scalar', 'Seq', 'Set', 'Signature', 'Str', 'StrLen', - 'StrPos', 'Sub', 'Submethod', 'True', 'UInt', 'Undef', 'Version', 'Void', - 'Whatever', 'bit', 'bool', 'buf', 'buf1', 'buf16', 'buf2', 'buf32', - 'buf4', 'buf64', 'buf8', 'complex', 'int', 'int1', 'int16', 'int2', - 'int32', 'int4', 'int64', 'int8', 'num', 'rat', 'rat1', 'rat16', 'rat2', - 'rat32', 'rat4', 'rat64', 'rat8', 'uint', 'uint1', 'uint16', 'uint2', - 'uint32', 'uint4', 'uint64', 'uint8', 'utf16', 'utf32', 'utf8', + #Booleans + 'False','True', + #Classes + 'Any','Array','Associative','AST','atomicint','Attribute','Backtrace', + 'Backtrace::Frame','Bag','Baggy','BagHash','Blob','Block','Bool','Buf', + 'Callable','CallFrame','Cancellation','Capture','CArray','Channel','Code', + 'compiler','Complex','ComplexStr','Cool','CurrentThreadScheduler', + 'Cursor','Date','Dateish','DateTime','Distro','Duration','Encoding', + 'Exception','Failure','FatRat','Grammar','Hash','HyperWhatever','Instant', + 'Int','int16','int32','int64','int8','IntStr','IO','IO::ArgFiles', + 'IO::CatHandle','IO::Handle','IO::Notification','IO::Path', + 'IO::Path::Cygwin','IO::Path::QNX','IO::Path::Unix','IO::Path::Win32', + 'IO::Pipe','IO::Socket','IO::Socket::Async','IO::Socket::INET','IO::Spec', + 'IO::Spec::Cygwin','IO::Spec::QNX','IO::Spec::Unix','IO::Spec::Win32', + 'IO::Special','Iterable','Iterator','Junction','Kernel','Label','List', + 'Lock','Lock::Async','long','longlong','Macro','Map','Match', + 'Metamodel::AttributeContainer','Metamodel::C3MRO','Metamodel::ClassHOW', + 'Metamodel::EnumHOW','Metamodel::Finalization','Metamodel::MethodContainer', + 'Metamodel::MROBasedMethodDispatch','Metamodel::MultipleInheritance', + 'Metamodel::Naming','Metamodel::Primitives','Metamodel::PrivateMethodContainer', + 'Metamodel::RoleContainer','Metamodel::Trusting','Method','Mix','MixHash', + 'Mixy','Mu','NFC','NFD','NFKC','NFKD','Nil','Num','num32','num64', + 'Numeric','NumStr','ObjAt','Order','Pair','Parameter','Perl','Pod::Block', + 'Pod::Block::Code','Pod::Block::Comment','Pod::Block::Declarator', + 'Pod::Block::Named','Pod::Block::Para','Pod::Block::Table','Pod::Heading', + 'Pod::Item','Pointer','Positional','PositionalBindFailover','Proc', + 'Proc::Async','Promise','Proxy','PseudoStash','QuantHash','Range','Rat', + 'Rational','RatStr','Real','Regex','Routine','Scalar','Scheduler', + 'Semaphore','Seq','Set','SetHash','Setty','Signature','size_t','Slip', + 'Stash','Str','StrDistance','Stringy','Sub','Submethod','Supplier', + 'Supplier::Preserving','Supply','Systemic','Tap','Telemetry', + 'Telemetry::Instrument::Thread','Telemetry::Instrument::Usage', + 'Telemetry::Period','Telemetry::Sampler','Thread','ThreadPoolScheduler', + 'UInt','uint16','uint32','uint64','uint8','Uni','utf8','Variable', + 'Version','VM','Whatever','WhateverCode','WrapHandle' ) PERL6_OPERATORS = ( @@ -311,7 +409,7 @@ class Perl6Lexer(ExtendedRegexLexer): '~', '&', '^', 'but', 'does', '<=>', '..', '..^', '^..', '^..^', '!=', '==', '<', '<=', '>', '>=', '~~', '===', '!eqv', '&&', '||', '^^', '//', 'min', 'max', '??', '!!', 'ff', 'fff', 'so', - 'not', '<==', '==>', '<<==', '==>>', + 'not', '<==', '==>', '<<==', '==>>','unicmp', ) # Perl 6 has a *lot* of possible bracketing characters diff --git a/pygments/lexers/php.py b/pygments/lexers/php.py index bd4a2376..680f0573 100644 --- a/pygments/lexers/php.py +++ b/pygments/lexers/php.py @@ -50,13 +50,14 @@ class ZephirLexer(RegexLexer): include('commentsandwhitespace'), (r'/(\\.|[^[/\\\n]|\[(\\.|[^\]\\\n])*])+/' r'([gim]+\b|\B)', String.Regex, '#pop'), + (r'/', Operator, '#pop'), default('#pop') ], 'badregex': [ (r'\n', Text, '#pop') ], 'root': [ - (r'^(?=\s|/|<!--)', Text, 'slashstartsregex'), + (r'^(?=\s|/)', Text, 'slashstartsregex'), include('commentsandwhitespace'), (r'\+\+|--|~|&&|\?|:|\|\||\\(?=\n)|' r'(<<|>>>?|==?|!=?|->|[-<>+*%&|^/])=?', Operator, 'slashstartsregex'), diff --git a/pygments/lexers/r.py b/pygments/lexers/r.py index 8e744599..33e57b3c 100644 --- a/pygments/lexers/r.py +++ b/pygments/lexers/r.py @@ -80,7 +80,7 @@ class SLexer(RegexLexer): mimetypes = ['text/S-plus', 'text/S', 'text/x-r-source', 'text/x-r', 'text/x-R', 'text/x-r-history', 'text/x-r-profile'] - valid_name = r'(?:`[^`\\]*(?:\\.[^`\\]*)*`)|(?:(?:[a-zA-z]|[_.][^0-9])[\w_.]*)' + valid_name = r'`[^`\\]*(?:\\.[^`\\]*)*`|(?:[a-zA-Z]|\.[A-Za-z_.])[\w_.]*|\.' tokens = { 'comments': [ (r'#.*$', Comment.Single), diff --git a/pygments/lexers/ride.py b/pygments/lexers/ride.py new file mode 100644 index 00000000..17bc2466 --- /dev/null +++ b/pygments/lexers/ride.py @@ -0,0 +1,139 @@ +# -*- coding: utf-8 -*- +""" + pygments.lexers.ride + ~~~~~~~~~~~~~~~~~~~~ + + Lexer for the Ride programming language. + + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.lexer import RegexLexer, words, include +from pygments.token import Comment, Keyword, Name, Number, Punctuation, String, Text + +__all__ = ['RideLexer'] + + +class RideLexer(RegexLexer): + """ + For `Ride <https://docs.wavesplatform.com/en/ride/about-ride.html>`_ + source code. + + .. versionadded:: 2.6 + """ + + name = 'Ride' + aliases = ['ride'] + filenames = ['*.ride'] + mimetypes = ['text/x-ride'] + + validName = r'[a-zA-Z_][a-zA-Z0-9_\']*' + + builtinOps = ( + '||', '|', '>=', '>', '==', '!', + '=', '<=', '<', '::', ':+', ':', '!=', '/', + '.', '=>', '-', '+', '*', '&&', '%', '++', + ) + + globalVariablesName = ( + 'NOALG', 'MD5', 'SHA1', 'SHA224', 'SHA256', 'SHA384', 'SHA512', + 'SHA3224', 'SHA3256', 'SHA3384', 'SHA3512', 'nil', 'this', 'unit', + 'height', 'lastBlock', 'Buy', 'Sell', 'CEILING', 'FLOOR', 'DOWN', + 'HALFDOWN', 'HALFEVEN', 'HALFUP', 'UP', + ) + + typesName = ( + 'Unit', 'Int', 'Boolean', 'ByteVector', 'String', 'Address', 'Alias', + 'Transfer', 'AssetPair', 'DataEntry', 'Order', 'Transaction', + 'GenesisTransaction', 'PaymentTransaction', 'ReissueTransaction', + 'BurnTransaction', 'MassTransferTransaction', 'ExchangeTransaction', + 'TransferTransaction', 'SetAssetScriptTransaction', + 'InvokeScriptTransaction', 'IssueTransaction', 'LeaseTransaction', + 'LeaseCancelTransaction', 'CreateAliasTransaction', + 'SetScriptTransaction', 'SponsorFeeTransaction', 'DataTransaction', + 'WriteSet', 'AttachedPayment', 'ScriptTransfer', 'TransferSet', + 'ScriptResult', 'Invocation', 'Asset', 'BlockInfo', 'Issue', 'Reissue', + 'Burn', 'NoAlg', 'Md5', 'Sha1', 'Sha224', 'Sha256', 'Sha384', 'Sha512', + 'Sha3224', 'Sha3256', 'Sha3384', 'Sha3512', 'BinaryEntry', + 'BooleanEntry', 'IntegerEntry', 'StringEntry', 'List', 'Ceiling', + 'Down', 'Floor', 'HalfDown', 'HalfEven', 'HalfUp', 'Up', + ) + + functionsName = ( + 'fraction', 'size', 'toBytes', 'take', 'drop', 'takeRight', 'dropRight', + 'toString', 'isDefined', 'extract', 'throw', 'getElement', 'value', + 'cons', 'toUtf8String', 'toInt', 'indexOf', 'lastIndexOf', 'split', + 'parseInt', 'parseIntValue', 'keccak256', 'blake2b256', 'sha256', + 'sigVerify', 'toBase58String', 'fromBase58String', 'toBase64String', + 'fromBase64String', 'transactionById', 'transactionHeightById', + 'getInteger', 'getBoolean', 'getBinary', 'getString', + 'addressFromPublicKey', 'addressFromString', 'addressFromRecipient', + 'assetBalance', 'wavesBalance', 'getIntegerValue', 'getBooleanValue', + 'getBinaryValue', 'getStringValue', 'addressFromStringValue', + 'assetInfo', 'rsaVerify', 'checkMerkleProof', 'median', + 'valueOrElse', 'valueOrErrorMessage', 'contains', 'log', 'pow', + 'toBase16String', 'fromBase16String', 'blockInfoByHeight', + 'transferTransactionById', + ) + + reservedWords = words(( + 'match', 'case', 'else', 'func', 'if', + 'let', 'then', '@Callable', '@Verifier', + ), suffix=r'\b') + + tokens = { + 'root': [ + # Comments + (r'#.*', Comment.Single), + # Whitespace + (r'\s+', Text), + # Strings + (r'"', String, 'doublequote'), + (r'utf8\'', String, 'utf8quote'), + (r'base(58|64|16)\'', String, 'singlequote'), + # Keywords + (reservedWords, Keyword.Reserved), + (r'\{-#.*?#-\}', Keyword.Reserved), + (r'FOLD<\d+>', Keyword.Reserved), + # Types + (words(typesName), Keyword.Type), + # Main + # (specialName, Keyword.Reserved), + # Prefix Operators + (words(builtinOps, prefix=r'\(', suffix=r'\)'), Name.Function), + # Infix Operators + (words(builtinOps), Name.Function), + (words(globalVariablesName), Name.Function), + (words(functionsName), Name.Function), + # Numbers + include('numbers'), + # Variable Names + (validName, Name.Variable), + # Parens + (r'[,()\[\]{}]', Punctuation), + ], + + 'doublequote': [ + (r'\\u[0-9a-fA-F]{4}', String.Escape), + (r'\\[nrfvb\\"]', String.Escape), + (r'[^"]', String), + (r'"', String, '#pop'), + ], + + 'utf8quote': [ + (r'\\u[0-9a-fA-F]{4}', String.Escape), + (r'\\[nrfvb\\\']', String.Escape), + (r'[^\']', String), + (r'\'', String, '#pop'), + ], + + 'singlequote': [ + (r'[^\']', String), + (r'\'', String, '#pop'), + ], + + 'numbers': [ + (r'_?\d+', Number.Integer), + ], + } diff --git a/pygments/lexers/rust.py b/pygments/lexers/rust.py index f731785f..77884b43 100644 --- a/pygments/lexers/rust.py +++ b/pygments/lexers/rust.py @@ -18,43 +18,42 @@ __all__ = ['RustLexer'] class RustLexer(RegexLexer): """ - Lexer for the Rust programming language (version 1.10). + Lexer for the Rust programming language (version 1.40). .. versionadded:: 1.6 """ name = 'Rust' filenames = ['*.rs', '*.rs.in'] aliases = ['rust', 'rs'] - mimetypes = ['text/rust'] + mimetypes = ['text/rust', 'text/x-rust'] - keyword_types = ( - words(('u8', 'u16', 'u32', 'u64', 'i8', 'i16', 'i32', 'i64', - 'i128', 'u128', 'usize', 'isize', 'f32', 'f64', 'str', 'bool'), - suffix=r'\b'), - Keyword.Type) + keyword_types = (words(( + 'u8', 'u16', 'u32', 'u64', 'u128', 'i8', 'i16', 'i32', 'i64', 'i128', + 'usize', 'isize', 'f32', 'f64', 'char', 'str', 'bool', + ), suffix=r'\b'), Keyword.Type) builtin_types = (words(( - # Reexported core operators - 'Copy', 'Send', 'Sized', 'Sync', + 'Send', 'Sized', 'Sync', 'Unpin', 'Drop', 'Fn', 'FnMut', 'FnOnce', - - # Reexported types and traits - 'Box', - 'ToOwned', - 'Clone', - 'PartialEq', 'PartialOrd', 'Eq', 'Ord', 'AsRef', 'AsMut', 'Into', 'From', - 'Default', - 'Iterator', 'Extend', 'IntoIterator', - 'DoubleEndedIterator', 'ExactSizeIterator', - 'Option', - 'Some', 'None', - 'Result', - 'Ok', 'Err', - 'SliceConcatExt', - 'String', 'ToString', - 'Vec'), suffix=r'\b'), - Name.Builtin) + 'Iterator', 'Extend', 'IntoIterator', 'DoubleEndedIterator', + 'ExactSizeIterator', 'Option', 'Result', + 'Box', 'ToOwned', 'String', 'ToString', 'Vec', + 'Clone', 'Copy', 'Default', 'Eq', 'Hash', 'Ord', 'PartialEq', + 'PartialOrd', 'Eq', 'Ord', + ), suffix=r'\b'), Name.Builtin) + + builtin_funcs_macros = (words(( + 'drop', 'Some', 'None', 'Ok', 'Err', + 'asm!', 'assert!', 'assert_eq!', 'assert_ne!', 'cfg!', 'column!', + 'compile_error!', 'concat!', 'concat_idents!', 'dbg!', 'debug_assert!', + 'debug_assert_eq!', 'debug_assert_ne!', 'env!', 'eprint!', 'eprintln!', + 'file!', 'format_args!', 'format_args_nl!', 'global_asm!', 'include!', + 'include_bytes!', 'include_str!', 'line!', 'log_syntax!', + 'module_path!', 'option_env!', 'panic!', 'print!', 'println!', + 'stringify!', 'thread_local!', 'todo!', 'trace_macros!', + 'unimplemented!', 'unreachable!', 'vec!', 'write!', 'writeln!', + ), suffix=r'\b'), Name.Builtin) tokens = { 'root': [ @@ -78,11 +77,11 @@ class RustLexer(RegexLexer): (r"""\$([a-zA-Z_]\w*|\(,?|\),?|,?)""", Comment.Preproc), # Keywords (words(( - 'as', 'async', 'await', 'box', 'const', 'crate', 'else', + 'as', 'async', 'await', 'box', 'const', 'crate', 'dyn', 'else', 'extern', 'for', 'if', 'impl', 'in', 'loop', 'match', 'move', 'mut', 'pub', 'ref', 'return', 'static', 'super', 'trait', - 'try', 'unsafe', 'use', 'where', 'while'), suffix=r'\b'), - Keyword), + 'try', 'unsafe', 'use', 'where', 'while', 'macro_rules!', + ), suffix=r'\b'), Keyword), (words(('abstract', 'alignof', 'become', 'do', 'final', 'macro', 'offsetof', 'override', 'priv', 'proc', 'pure', 'sizeof', 'typeof', 'unsized', 'virtual', 'yield'), suffix=r'\b'), @@ -94,9 +93,10 @@ class RustLexer(RegexLexer): (r'(struct|enum|type|union)\b', Keyword, 'typename'), (r'(default)(\s+)(type|fn)\b', bygroups(Keyword, Text, Keyword)), keyword_types, - (r'self\b', Name.Builtin.Pseudo), + (r'[sS]elf\b', Name.Builtin.Pseudo), # Prelude (taken from Rust's src/libstd/prelude.rs) builtin_types, + builtin_funcs_macros, # Path seperators, so types don't catch them. (r'::\b', Text), # Types in positions. @@ -104,49 +104,48 @@ class RustLexer(RegexLexer): # Labels (r'(break|continue)(\s*)(\'[A-Za-z_]\w*)?', bygroups(Keyword, Text.Whitespace, Name.Label)), - # Character Literal + + # Character literals (r"""'(\\['"\\nrt]|\\x[0-7][0-9a-fA-F]|\\0""" r"""|\\u\{[0-9a-fA-F]{1,6}\}|.)'""", String.Char), (r"""b'(\\['"\\nrt]|\\x[0-9a-fA-F]{2}|\\0""" r"""|\\u\{[0-9a-fA-F]{1,6}\}|.)'""", String.Char), - # Binary Literal + + # Binary literals (r'0b[01_]+', Number.Bin, 'number_lit'), - # Octal Literal + # Octal literals (r'0o[0-7_]+', Number.Oct, 'number_lit'), - # Hexadecimal Literal + # Hexadecimal literals (r'0[xX][0-9a-fA-F_]+', Number.Hex, 'number_lit'), - # Decimal Literal + # Decimal literals (r'[0-9][0-9_]*(\.[0-9_]+[eE][+\-]?[0-9_]+|' r'\.[0-9_]*(?!\.)|[eE][+\-]?[0-9_]+)', Number.Float, 'number_lit'), (r'[0-9][0-9_]*', Number.Integer, 'number_lit'), - # String Literal + + # String literals (r'b"', String, 'bytestring'), (r'"', String, 'string'), (r'b?r(#*)".*?"\1', String), - # Lifetime - (r"""'static""", Name.Builtin), - (r"""'[a-zA-Z_]\w*""", Name.Attribute), + # Lifetime names + (r"'(static|_)", Name.Builtin), + (r"'[a-zA-Z_]\w*", Name.Attribute), # Operators and Punctuation + (r'\.\.=?', Operator), (r'[{}()\[\],.;]', Punctuation), (r'[+\-*/%&|<>^!~@=:?]', Operator), - # Identifier + # Identifiers (r'[a-zA-Z_]\w*', Name), + # Raw identifiers + (r'r#[a-zA-Z_]\w*', Name), # Attributes (r'#!?\[', Comment.Preproc, 'attribute['), - # Macros - (r'([A-Za-z_]\w*)(!)(\s*)([A-Za-z_]\w*)?(\s*)(\{)', - bygroups(Comment.Preproc, Punctuation, Whitespace, Name, - Whitespace, Punctuation), 'macro{'), - (r'([A-Za-z_]\w*)(!)(\s*)([A-Za-z_]\w*)?(\()', - bygroups(Comment.Preproc, Punctuation, Whitespace, Name, - Punctuation), 'macro('), ], 'comment': [ (r'[^*/]+', Comment.Multiline), @@ -194,14 +193,6 @@ class RustLexer(RegexLexer): (r"""\\x[89a-fA-F][0-9a-fA-F]""", String.Escape), include('string'), ], - 'macro{': [ - (r'\{', Operator, '#push'), - (r'\}', Operator, '#pop'), - ], - 'macro(': [ - (r'\(', Operator, '#push'), - (r'\)', Operator, '#pop'), - ], 'attribute_common': [ (r'"', String, 'string'), (r'\[', Comment.Preproc, 'attribute['), diff --git a/pygments/lexers/sieve.py b/pygments/lexers/sieve.py new file mode 100644 index 00000000..9d7dba35 --- /dev/null +++ b/pygments/lexers/sieve.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- +""" + pygments.lexers.sieve + ~~~~~~~~~~~~~~~~~~~~~ + + Lexer for Sieve file format. + + https://tools.ietf.org/html/rfc5228 + https://tools.ietf.org/html/rfc5173 + https://tools.ietf.org/html/rfc5229 + https://tools.ietf.org/html/rfc5230 + https://tools.ietf.org/html/rfc5232 + https://tools.ietf.org/html/rfc5235 + https://tools.ietf.org/html/rfc5429 + https://tools.ietf.org/html/rfc8580 + + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.lexer import RegexLexer, bygroups +from pygments.token import Comment, Name, Literal, String, Text, Punctuation, Keyword + +__all__ = ["SieveLexer"] + + +class SieveLexer(RegexLexer): + """ + Lexer for sieve format. + """ + name = 'Sieve' + filenames = ['*.siv', '*.sieve'] + + tokens = { + 'root': [ + (r'\s+', Text), + (r'[();,{}\[\]]', Punctuation), + # import: + (r'(?i)require', + Keyword.Namespace), + # tags: + (r'(?i)(:)(addresses|all|contains|content|create|copy|comparator|count|days|detail|domain|fcc|flags|from|handle|importance|is|localpart|length|lowerfirst|lower|matches|message|mime|options|over|percent|quotewildcard|raw|regex|specialuse|subject|text|under|upperfirst|upper|value)', + bygroups(Name.Tag, Name.Tag)), + # tokens: + (r'(?i)(address|addflag|allof|anyof|body|discard|elsif|else|envelope|ereject|exists|false|fileinto|if|hasflag|header|keep|notify_method_capability|notify|not|redirect|reject|removeflag|setflag|size|spamtest|stop|string|true|vacation|virustest)', + Name.Builtin), + (r'(?i)set', + Keyword.Declaration), + # number: + (r'([0-9.]+)([kmgKMG])?', + bygroups(Literal.Number, Literal.Number)), + # comment: + (r'#.*$', + Comment.Single), + (r'/\*.*\*/', + Comment.Multiline), + # string: + (r'"[^"]*?"', + String), + # text block: + (r'text:', + Name.Tag, 'text'), + ], + 'text': [ + (r'[^.].*?\n', String), + (r'^\.', Punctuation, "#pop"), + ] + } diff --git a/pygments/lexers/smv.py b/pygments/lexers/smv.py index 2d97c2c5..6dac4af3 100644 --- a/pygments/lexers/smv.py +++ b/pygments/lexers/smv.py @@ -10,8 +10,8 @@ """ from pygments.lexer import RegexLexer, words -from pygments.token import Comment, Generic, Keyword, Name, Number, \ - Operator, Punctuation, Text +from pygments.token import Comment, Keyword, Name, Number, Operator, \ + Punctuation, Text __all__ = ['NuSMVLexer'] diff --git a/pygments/lexers/stata.py b/pygments/lexers/stata.py index b7b22edb..c6344fc6 100644 --- a/pygments/lexers/stata.py +++ b/pygments/lexers/stata.py @@ -27,8 +27,8 @@ class StataLexer(RegexLexer): """ # Syntax based on # - http://fmwww.bc.edu/RePEc/bocode/s/synlightlist.ado - # - http://github.com/isagalaev/highlight.js/blob/master/src/languages/stata.js - # - http://github.com/jpitblado/vim-stata/blob/master/syntax/stata.vim + # - https://github.com/isagalaev/highlight.js/blob/master/src/languages/stata.js + # - https://github.com/jpitblado/vim-stata/blob/master/syntax/stata.vim name = 'Stata' aliases = ['stata', 'do'] diff --git a/pygments/lexers/templates.py b/pygments/lexers/templates.py index f891242c..fce41ec1 100644 --- a/pygments/lexers/templates.py +++ b/pygments/lexers/templates.py @@ -323,7 +323,7 @@ class VelocityXmlLexer(DelegatingLexer): class DjangoLexer(RegexLexer): """ Generic `django <http://www.djangoproject.com/documentation/templates/>`_ - and `jinja <http://wsgiarea.pocoo.org/jinja/>`_ template lexer. + and `jinja <https://jinja.pocoo.org/jinja/>`_ template lexer. It just highlights django/jinja code between the preprocessor directives, other data is left untouched by the lexer. @@ -1802,22 +1802,23 @@ class HandlebarsLexer(RegexLexer): 'root': [ (r'[^{]+', Other), - # Comment start {{! }} or {{!-- + # Comment start {{! }} or {{!-- (r'\{\{!.*\}\}', Comment), - # HTML Escaping open {{{expression + # HTML Escaping open {{{expression (r'(\{\{\{)(\s*)', bygroups(Comment.Special, Text), 'tag'), - # {{blockOpen {{#blockOpen {{/blockClose with optional tilde ~ - (r'(\{\{)([#~/]+)([^\s}]*)', bygroups(Comment.Preproc, Number.Attribute,Number.Attribute), 'tag'), + # {{blockOpen {{#blockOpen {{/blockClose with optional tilde ~ + (r'(\{\{)([#~/]+)([^\s}]*)', + bygroups(Comment.Preproc, Number.Attribute, Number.Attribute), 'tag'), (r'(\{\{)(\s*)', bygroups(Comment.Preproc, Text), 'tag'), ], 'tag': [ (r'\s+', Text), - # HTML Escaping close }}} + # HTML Escaping close }}} (r'\}\}\}', Comment.Special, '#pop'), - # blockClose}}, includes optional tilde ~ + # blockClose}}, includes optional tilde ~ (r'(~?)(\}\})', bygroups(Number, Comment.Preproc), '#pop'), # {{opt=something}} diff --git a/pygments/lexers/testing.py b/pygments/lexers/testing.py index 9288b5b4..9ee8a061 100644 --- a/pygments/lexers/testing.py +++ b/pygments/lexers/testing.py @@ -17,7 +17,7 @@ __all__ = ['GherkinLexer', 'TAPLexer'] class GherkinLexer(RegexLexer): """ - For `Gherkin <http://github.com/aslakhellesoy/gherkin/>` syntax. + For `Gherkin <https://github.com/aslakhellesoy/gherkin/>` syntax. .. versionadded:: 1.2 """ diff --git a/pygments/lexers/textedit.py b/pygments/lexers/textedit.py index 66b36924..3c6fb570 100644 --- a/pygments/lexers/textedit.py +++ b/pygments/lexers/textedit.py @@ -102,9 +102,9 @@ class VimLexer(RegexLexer): (r'[ \t]+', Text), # TODO: regexes can have other delims - (r'/(\\\\|\\/|[^\n/])*/', String.Regex), - (r'"(\\\\|\\"|[^\n"])*"', String.Double), - (r"'(''|[^\n'])*'", String.Single), + (r'/[^/\\\n]*(?:\\[\s\S][^/\\\n]*)*/', String.Regex), + (r'"[^"\\\n]*(?:\\[\s\S][^"\\\n]*)*"', String.Double), + (r"'[^\n']*(?:''[^\n']*)*'", String.Single), # Who decided that doublequote was a good comment character?? (r'(?<=\s)"[^\-:.%#=*].*', Comment), diff --git a/pygments/lexers/textfmts.py b/pygments/lexers/textfmts.py index d3a191b0..bfd053d5 100644 --- a/pygments/lexers/textfmts.py +++ b/pygments/lexers/textfmts.py @@ -12,13 +12,13 @@ import re from pygments.lexers import guess_lexer, get_lexer_by_name -from pygments.lexer import RegexLexer, bygroups, default, do_insertions +from pygments.lexer import RegexLexer, bygroups, default, include from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ Number, Generic, Literal, Punctuation from pygments.util import ClassNotFound __all__ = ['IrcLogsLexer', 'TodotxtLexer', 'HttpLexer', 'GettextLexer', - 'NotmuchLexer'] + 'NotmuchLexer', 'KernelLogLexer'] class IrcLogsLexer(RegexLexer): @@ -175,13 +175,13 @@ class HttpLexer(RegexLexer): tokens = { 'root': [ (r'(GET|POST|PUT|DELETE|HEAD|OPTIONS|TRACE|PATCH)( +)([^ ]+)( +)' - r'(HTTP)(/)(1\.[01])(\r?\n|\Z)', + r'(HTTP)(/)(1\.[01]|2|3)(\r?\n|\Z)', bygroups(Name.Function, Text, Name.Namespace, Text, Keyword.Reserved, Operator, Number, Text), 'headers'), - (r'(HTTP)(/)(1\.[01])( +)(\d{3})( +)([^\r\n]+)(\r?\n|\Z)', - bygroups(Keyword.Reserved, Operator, Number, Text, Number, - Text, Name.Exception, Text), + (r'(HTTP)(/)(1\.[01]|2|3)( +)(\d{3})(?:( +)([^\r\n]+))?(\r?\n|\Z)', + bygroups(Keyword.Reserved, Operator, Number, Text, Number, Text, + Name.Exception, Text), 'headers'), ], 'headers': [ @@ -380,3 +380,52 @@ class NotmuchLexer(RegexLexer): def __init__(self, **options): self.body_lexer = options.get('body_lexer', None) RegexLexer.__init__(self, **options) + + +class KernelLogLexer(RegexLexer): + """ + For Linux Kernel log ("dmesg") output. + + .. versionadded:: 2.6 + """ + name = 'Kernel log' + aliases = ['kmsg', 'dmesg'] + filenames = ['*.kmsg', '*.dmesg'] + + tokens = { + 'root': [ + (r'^[^:]+:debug : (?=\[)', Text, 'debug'), + (r'^[^:]+:info : (?=\[)', Text, 'info'), + (r'^[^:]+:warn : (?=\[)', Text, 'warn'), + (r'^[^:]+:notice: (?=\[)', Text, 'warn'), + (r'^[^:]+:err : (?=\[)', Text, 'error'), + (r'^[^:]+:crit : (?=\[)', Text, 'error'), + (r'^(?=\[)', Text, 'unknown'), + ], + 'unknown': [ + (r'^(?=.+(warning|notice|audit|deprecated))', Text, 'warn'), + (r'^(?=.+(error|critical|fail|Bug))', Text, 'error'), + default('info'), + ], + 'base': [ + (r'\[[0-9\. ]+\] ', Number), + (r'(?<=\] ).+?:', Keyword), + (r'\n', Text, '#pop'), + ], + 'debug': [ + include('base'), + (r'.+\n', Comment, '#pop') + ], + 'info': [ + include('base'), + (r'.+\n', Text, '#pop') + ], + 'warn': [ + include('base'), + (r'.+\n', Generic.Strong, '#pop') + ], + 'error': [ + include('base'), + (r'.+\n', Generic.Error, '#pop') + ] + } diff --git a/pygments/lexers/theorem.py b/pygments/lexers/theorem.py index a26a173f..c63266dd 100644 --- a/pygments/lexers/theorem.py +++ b/pygments/lexers/theorem.py @@ -418,7 +418,7 @@ class LeanLexer(RegexLexer): u'<-', u'=', u'==', u'>', u'_', u'|', u'||', u'~', u'=>', u'<=', u'>=', u'/\\', u'\\/', u'∀', u'Π', u'λ', u'↔', u'∧', u'∨', u'≠', u'≤', u'≥', u'¬', u'⁻¹', u'⬝', u'▸', u'→', u'∃', u'ℕ', u'ℤ', u'≈', u'×', u'⌞', - u'⌟', u'≡', u'⟨', u'⟩', + u'⌟', u'≡', u'⟨', u'⟩', u'^', ) punctuation = (u'(', u')', u':', u'{', u'}', u'[', u']', u'⦃', u'⦄', diff --git a/pygments/lexers/usd.py b/pygments/lexers/usd.py new file mode 100644 index 00000000..89b0300a --- /dev/null +++ b/pygments/lexers/usd.py @@ -0,0 +1,90 @@ +# -*- coding: utf-8 -*- +""" + pygments.lexers.usd + ~~~~~~~~~~~~~~~~~~~ + + The module that parses Pixar's Universal Scene Description file format. + + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.lexer import RegexLexer, bygroups +from pygments.lexer import words as words_ +from pygments.lexers._usd_builtins import COMMON_ATTRIBUTES, KEYWORDS, \ + OPERATORS, SPECIAL_NAMES, TYPES +from pygments.token import Comment, Keyword, Name, Number, Operator, \ + Punctuation, String, Text, Whitespace + +__all__ = ["UsdLexer"] + + +def _keywords(words, type_): + return [(words_(words, prefix=r"\b", suffix=r"\b"), type_)] + + +_TYPE = r"(\w+(?:\[\])?)" +_BASE_ATTRIBUTE = r"([\w_]+(?:\:[\w_]+)*)(?:(\.)(timeSamples))?" +_WHITESPACE = r"([ \t]+)" + + +class UsdLexer(RegexLexer): + """ + A lexer that parses Pixar's Universal Scene Description file format. + + .. versionadded:: 2.6 + """ + + name = "USD" + aliases = ["usd", "usda"] + filenames = ["*.usd", "*.usda"] + + tokens = { + "root": [ + (r"(custom){_WHITESPACE}(uniform)(\s+){}(\s+){}(\s*)(=)".format( + _TYPE, _BASE_ATTRIBUTE, _WHITESPACE=_WHITESPACE), + bygroups(Keyword.Token, Whitespace, Keyword.Token, Whitespace, + Keyword.Type, Whitespace, Name.Attribute, Text, + Name.Keyword.Tokens, Whitespace, Operator)), + (r"(custom){_WHITESPACE}{}(\s+){}(\s*)(=)".format( + _TYPE, _BASE_ATTRIBUTE, _WHITESPACE=_WHITESPACE), + bygroups(Keyword.Token, Whitespace, Keyword.Type, Whitespace, + Name.Attribute, Text, Name.Keyword.Tokens, Whitespace, + Operator)), + (r"(uniform){_WHITESPACE}{}(\s+){}(\s*)(=)".format( + _TYPE, _BASE_ATTRIBUTE, _WHITESPACE=_WHITESPACE), + bygroups(Keyword.Token, Whitespace, Keyword.Type, Whitespace, + Name.Attribute, Text, Name.Keyword.Tokens, Whitespace, + Operator)), + (r"{}{_WHITESPACE}{}(\s*)(=)".format( + _TYPE, _BASE_ATTRIBUTE, _WHITESPACE=_WHITESPACE), + bygroups(Keyword.Type, Whitespace, Name.Attribute, Text, + Name.Keyword.Tokens, Whitespace, Operator)), + ] + + _keywords(KEYWORDS, Keyword.Tokens) + + _keywords(SPECIAL_NAMES, Name.Builtins) + + _keywords(COMMON_ATTRIBUTES, Name.Attribute) + + [(r"\b\w+:[\w:]+\b", Name.Attribute)] + + _keywords(OPERATORS, Operator) + # more attributes + [(type_ + r"\[\]", Keyword.Type) for type_ in TYPES] + + _keywords(TYPES, Keyword.Type) + + [ + (r"[\(\)\[\]{}]", Punctuation), + ("#.*?$", Comment.Single), + (",", Punctuation), + (";", Punctuation), # ";"s are allowed to combine separate metadata lines + ("=", Operator), + (r"[-]*([0-9]*[.])?[0-9]+(?:e[+-]*\d+)?", Number), + (r"'''(?:.|\n)*?'''", String), + (r'"""(?:.|\n)*?"""', String), + (r"'.*?'", String), + (r'".*?"', String), + (r"<(\.\./)*([\w/]+|[\w/]+\.\w+[\w:]*)>", Name.Namespace), + (r"@.*?@", String.Interpol), + (r'\(.*"[.\\n]*".*\)', String.Doc), + (r"\A#usda .+$", Comment.Hashbang), + (r"\s+", Whitespace), + (r"[\w_]+", Text), + (r"[_:\.]+", Punctuation), + ], + } diff --git a/pygments/lexers/verification.py b/pygments/lexers/verification.py index f6530726..f70a0a2c 100644 --- a/pygments/lexers/verification.py +++ b/pygments/lexers/verification.py @@ -11,7 +11,7 @@ from pygments.lexer import RegexLexer, include, words from pygments.token import Comment, Operator, Keyword, Name, Number, \ - Punctuation, Whitespace + Punctuation, Text, Generic __all__ = ['BoogieLexer', 'SilverLexer'] @@ -29,8 +29,9 @@ class BoogieLexer(RegexLexer): tokens = { 'root': [ # Whitespace and Comments - (r'\n', Whitespace), - (r'\s+', Whitespace), + (r'\n', Text), + (r'\s+', Text), + (r'\\\n', Text), # line continuation (r'//[/!](.*?)\n', Comment.Doc), (r'//(.*?)\n', Comment.Single), (r'/\*', Comment.Multiline, 'comment'), @@ -45,6 +46,7 @@ class BoogieLexer(RegexLexer): (words(('bool', 'int', 'ref'), suffix=r'\b'), Keyword.Type), include('numbers'), (r"(>=|<=|:=|!=|==>|&&|\|\||[+/\-=>*<\[\]])", Operator), + (r'\{.*?\}', Generic.Emph), #triggers (r"([{}():;,.])", Punctuation), # Identifier (r'[a-zA-Z_]\w*', Name), @@ -74,8 +76,9 @@ class SilverLexer(RegexLexer): tokens = { 'root': [ # Whitespace and Comments - (r'\n', Whitespace), - (r'\s+', Whitespace), + (r'\n', Text), + (r'\s+', Text), + (r'\\\n', Text), # line continuation (r'//[/!](.*?)\n', Comment.Doc), (r'//(.*?)\n', Comment.Single), (r'/\*', Comment.Multiline, 'comment'), @@ -83,18 +86,18 @@ class SilverLexer(RegexLexer): (words(( 'result', 'true', 'false', 'null', 'method', 'function', 'predicate', 'program', 'domain', 'axiom', 'var', 'returns', - 'field', 'define', 'requires', 'ensures', 'invariant', - 'fold', 'unfold', 'inhale', 'exhale', 'new', 'assert', + 'field', 'define', 'fold', 'unfold', 'inhale', 'exhale', 'new', 'assert', 'assume', 'goto', 'while', 'if', 'elseif', 'else', 'fresh', 'constraining', 'Seq', 'Set', 'Multiset', 'union', 'intersection', 'setminus', 'subset', 'unfolding', 'in', 'old', 'forall', 'exists', 'acc', 'wildcard', 'write', 'none', 'epsilon', 'perm', 'unique', 'apply', 'package', 'folding', 'label', 'forperm'), suffix=r'\b'), Keyword), - (words(('Int', 'Perm', 'Bool', 'Ref'), suffix=r'\b'), Keyword.Type), + (words(('requires', 'ensures', 'invariant'), suffix=r'\b'), Name.Decorator), + (words(('Int', 'Perm', 'Bool', 'Ref', 'Rational'), suffix=r'\b'), Keyword.Type), include('numbers'), - (r'[!%&*+=|?:<>/\-\[\]]', Operator), + (r'\{.*?\}', Generic.Emph), #triggers (r'([{}():;,.])', Punctuation), # Identifier (r'[\w$]\w*', Name), diff --git a/pygments/lexers/webmisc.py b/pygments/lexers/webmisc.py index b39334bc..448aff50 100644 --- a/pygments/lexers/webmisc.py +++ b/pygments/lexers/webmisc.py @@ -158,6 +158,9 @@ class XQueryLexer(ExtendedRegexLexer): # state stack if len(lexer.xquery_parse_state) == 0: ctx.stack.pop() + if not ctx.stack: + # make sure we have at least the root state on invalid inputs + ctx.stack = ['root'] elif len(ctx.stack) > 1: ctx.stack.append(lexer.xquery_parse_state.pop()) else: diff --git a/pygments/lexers/zig.py b/pygments/lexers/zig.py index 7850fdf0..c9893862 100644 --- a/pygments/lexers/zig.py +++ b/pygments/lexers/zig.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """ pygments.lexers.zig - ~~~~~~~~~~~~~~~~~~~~ + ~~~~~~~~~~~~~~~~~~~ Lexers for Zig. @@ -9,13 +9,13 @@ :license: BSD, see LICENSE for details. """ -import re -from pygments.lexer import RegexLexer, bygroups, include, words -from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ - Number, Punctuation, Error, Whitespace +from pygments.lexer import RegexLexer, words +from pygments.token import Comment, Operator, Keyword, Name, String, \ + Number, Punctuation, Whitespace __all__ = ['ZigLexer'] + class ZigLexer(RegexLexer): """ For `Zig <http://www.ziglang.org>`_ source code. @@ -28,13 +28,14 @@ class ZigLexer(RegexLexer): mimetypes = ['text/zig'] type_keywords = ( - words(('bool', 'f16', 'f32', 'f64', 'f128', 'void', 'noreturn', 'type', 'anyerror', 'promise', - 'i0', 'u0', 'isize', 'usize', 'comptime_int', 'comptime_float', - 'c_short', 'c_ushort', 'c_int', 'c_uint', 'c_long', 'c_ulong', 'c_longlong', 'c_ulonglong', 'c_longdouble', 'c_void' - 'i8', 'u8', 'i16', 'u16', 'i32', 'u32', 'i64', 'u64', 'i128', 'u128' - ), suffix=r'\b'), + words(('bool', 'f16', 'f32', 'f64', 'f128', 'void', 'noreturn', 'type', + 'anyerror', 'promise', 'i0', 'u0', 'isize', 'usize', 'comptime_int', + 'comptime_float', 'c_short', 'c_ushort', 'c_int', 'c_uint', 'c_long', + 'c_ulong', 'c_longlong', 'c_ulonglong', 'c_longdouble', 'c_void' + 'i8', 'u8', 'i16', 'u16', 'i32', 'u32', 'i64', 'u64', 'i128', + 'u128'), suffix=r'\b'), Keyword.Type) - + storage_keywords = ( words(('const', 'var', 'extern', 'packed', 'export', 'pub', 'noalias', 'inline', 'comptime', 'nakedcc', 'stdcallcc', 'volatile', 'allowzero', @@ -46,8 +47,8 @@ class ZigLexer(RegexLexer): Keyword) statement_keywords = ( - words(('break', 'return', 'continue', 'asm', 'defer', 'errdefer', - 'unreachable', 'try', 'catch', 'async', 'await', 'suspend', + words(('break', 'return', 'continue', 'asm', 'defer', 'errdefer', + 'unreachable', 'try', 'catch', 'async', 'await', 'suspend', 'resume', 'cancel'), suffix=r'\b'), Keyword) @@ -74,7 +75,7 @@ class ZigLexer(RegexLexer): (r'//.*?\n', Comment.Single), # Keywords - statement_keywords, + statement_keywords, storage_keywords, structure_keywords, repeat_keywords, @@ -96,25 +97,27 @@ class ZigLexer(RegexLexer): (r'[0-9]+', Number.Integer), # Identifier - (r'@[a-zA-Z_]\w*',Name.Builtin), + (r'@[a-zA-Z_]\w*', Name.Builtin), (r'[a-zA-Z_]\w*', Name), # Characters (r'\'\\\'\'', String.Escape), - (r'\'\\(|x[a-fA-F0-9]{2}|u[a-fA-F0-9]{4}|U[a-fA-F0-9]{6}|[nr\\t\'"])\'', String.Escape), + (r'\'\\(|x[a-fA-F0-9]{2}|u[a-fA-F0-9]{4}|U[a-fA-F0-9]{6}|[nr\\t\'"])\'', + String.Escape), (r'\'[^\\\']\'', String), # Strings (r'\\\\[^\n]*', String.Heredoc), (r'c\\\\[^\n]*', String.Heredoc), - (r'c?"',String, 'string'), + (r'c?"', String, 'string'), # Operators, Punctuation (r'[+%=><|^!?/\-*&~:]', Operator), (r'[{}()\[\],.;]', Punctuation) ], 'string': [ - (r'\\(x[a-fA-F0-9]{2}|u[a-fA-F0-9]{4}|U[a-fA-F0-9]{6}|[nr\\t\'"])', String.Escape), + (r'\\(x[a-fA-F0-9]{2}|u[a-fA-F0-9]{4}|U[a-fA-F0-9]{6}|[nr\\t\'"])', + String.Escape), (r'[^\\"\n]+', String), (r'"', String, '#pop') ] |