diff options
Diffstat (limited to 'pygments')
| -rw-r--r-- | pygments/lexers/_mapping.py | 1 | ||||
| -rw-r--r-- | pygments/lexers/compiled.py | 69 | ||||
| -rw-r--r-- | pygments/lexers/functional.py | 137 | ||||
| -rw-r--r-- | pygments/lexers/jvm.py | 1 | ||||
| -rw-r--r-- | pygments/lexers/other.py | 2 | ||||
| -rw-r--r-- | pygments/lexers/sql.py | 7 |
6 files changed, 140 insertions, 77 deletions
diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py index b028aee6..d505dcf5 100644 --- a/pygments/lexers/_mapping.py +++ b/pygments/lexers/_mapping.py @@ -239,6 +239,7 @@ LEXERS = { 'NewspeakLexer': ('pygments.lexers.other', 'Newspeak', ('newspeak',), ('*.ns2',), ('text/x-newspeak',)), 'NginxConfLexer': ('pygments.lexers.text', 'Nginx configuration file', ('nginx',), (), ('text/x-nginx-conf',)), 'NimrodLexer': ('pygments.lexers.compiled', 'Nimrod', ('nimrod', 'nim'), ('*.nim', '*.nimrod'), ('text/x-nimrod',)), + 'NitLexer': ('pygments.lexers.compiled', 'Nit', ('nit',), ('*.nit',), ()), 'NixLexer': ('pygments.lexers.functional', 'Nix', ('nixos', 'nix'), ('*.nix',), ('text/x-nix',)), 'NumPyLexer': ('pygments.lexers.math', 'NumPy', ('numpy',), (), ()), 'ObjdumpLexer': ('pygments.lexers.asm', 'objdump', ('objdump',), ('*.objdump',), ('text/x-objdump',)), diff --git a/pygments/lexers/compiled.py b/pygments/lexers/compiled.py index 38bd901f..6556b77e 100644 --- a/pygments/lexers/compiled.py +++ b/pygments/lexers/compiled.py @@ -32,7 +32,7 @@ __all__ = ['CLexer', 'CppLexer', 'DLexer', 'DelphiLexer', 'ECLexer', 'DylanLidLexer', 'DylanConsoleLexer', 'CobolLexer', 'CobolFreeformatLexer', 'LogosLexer', 'ClayLexer', 'PikeLexer', 'ChapelLexer', 'EiffelLexer', 'Inform6Lexer', 'Inform7Lexer', - 'Inform6TemplateLexer', 'MqlLexer', 'SwiftLexer'] + 'Inform6TemplateLexer', 'MqlLexer', 'SwiftLexer', 'NitLexer'] class CFamilyLexer(RegexLexer): @@ -3321,7 +3321,7 @@ class RustLexer(RegexLexer): (r'\s+', Text), (r'//[/!](.*?)\n', Comment.Doc), (r'//(.*?)\n', Comment.Single), - (r'/[*](.|\n)*?[*]/', Comment.Multiline), + (r'/\*', Comment.Multiline, 'comment'), # Keywords (r'(as|box|break|continue' @@ -3403,6 +3403,12 @@ class RustLexer(RegexLexer): (r'([A-Za-z_]\w*)!\s*([A-Za-z_]\w*)?\(', bygroups(Comment.Preproc, Name), 'macro('), ], + 'comment': [ + (r'[^*/]+', Comment.Multiline), + (r'/\*', Comment.Multiline, '#push'), + (r'\*/', Comment.Multiline, '#pop'), + (r'[*/]', Comment.Multiline), + ], 'number_lit': [ (r'(([ui](8|16|32|64)?)|(f(32|64)?))?', Keyword, '#pop'), ], @@ -3920,11 +3926,11 @@ class ChapelLexer(RegexLexer): (r'(false|nil|true)\b', Keyword.Constant), (r'(bool|complex|imag|int|opaque|range|real|string|uint)\b', Keyword.Type), - (r'(atomic|begin|break|by|cobegin|coforall|continue|iter|' + (r'(align|atomic|begin|break|by|cobegin|coforall|continue|' r'delete|dmapped|do|domain|else|enum|export|extern|for|forall|' - r'if|index|inline|label|lambda|let|local|new|on|otherwise|' - r'reduce|return|scan|select|serial|single|sparse|' - r'subdomain|sync|then|use|when|where|while|yield|zip)\b', + r'if|index|inline|iter|label|lambda|let|local|new|noinit|on|' + r'otherwise|pragma|reduce|return|scan|select|serial|single|sparse|' + r'subdomain|sync|then|use|when|where|while|with|yield|zip)\b', Keyword), (r'(proc)((?:\s|\\\s)+)', bygroups(Keyword, Text), 'procname'), (r'(class|module|record|union)(\s+)', bygroups(Keyword, Text), @@ -3946,15 +3952,17 @@ class ChapelLexer(RegexLexer): (r'0[bB][0-1]+', Number.Bin), # -- hex (r'0[xX][0-9a-fA-F]+', Number.Hex), + # -- octal + (r'0[oO][0-7]+', Number.Oct), # -- decimal - (r'(0|[1-9][0-9]*)', Number.Integer), + (r'[0-9]+', Number.Integer), # strings (r'["\'](\\\\|\\"|[^"\'])*["\']', String), # tokens (r'(=|\+=|-=|\*=|/=|\*\*=|%=|&=|\|=|\^=|&&=|\|\|=|<<=|>>=|' - r'<=>|\.\.|by|#|\.\.\.|' + r'<=>|<~>|\.\.|by|#|\.\.\.|' r'&&|\|\||!|&|\||\^|~|<<|>>|' r'==|!=|<=|>=|<|>|' r'[+\-*/%]|\*\*)', Operator), @@ -5190,3 +5198,48 @@ class SwiftLexer(ObjectiveCLexer): elif value in self.operators: token = Operator yield index, token, value + + +class NitLexer(RegexLexer): + """ + For `nit <http://nitlanguage.org>`_ source. + + .. versionadded:: 2.0 + """ + + name = 'Nit' + aliases = ['nit'] + filenames = ['*.nit'] + tokens = { + 'root': [ + (r'#.*?$', Comment.Single), + (r'(package|module|import|class|abstract|interface|' + 'universal|enum|end|fun|type|init|redef|isa|do|' + 'readable|writable|var|intern|extern|public|protected|' + 'private|intrude|if|then|else|while|loop|for|in|and|' + 'or|not|implies|return|continue|break|abort|assert|' + 'new|is|once|super|self|true|false|nullable|null|as|' + 'isset|label|__debug__)(?=( |\n|\t|\r|\())', Keyword), + (r'[A-Z][A-Za-z0-9_]*', Name.Class), + (r'"""(([^\'\\]|\\.)|\\r|\\n)*(({{?)?(""?{{?)*""""*)', String), #Simple long string + (r'\'\'\'(((\\.|[^\'\\])|\\r|\\n)|\'((\\.|[^\'\\])|\\r|\\n)|' + r'\'\'((\\.|[^\'\\])|\\r|\\n))*\'\'\'', String), #Simple long string alt + (r'"""(([^\'\\]|\\.)|\\r|\\n)*((""?)?({{?""?)*{{{{*)', String), #Start long string + (r'}}}(((\\.|[^\'\\])|\\r|\\n))*(""?)?({{?""?)*{{{{*', String), #Mid long string + (r'}}}(((\\.|[^\'\\])|\\r|\\n))*({{?)?(""?{{?)*""""*', String), #End long string + (r'"(\\.|([^"}{\\]))*"', String), #Simple String + (r'"(\\.|([^"}{\\]))*{', String), #Start string + (r'}(\\.|([^"}{\\]))*{', String), #Mid String + (r'}(\\.|([^"}{\\]))*"', String), #End String + (r'(\'[^\'\\]\')|(\'\\.\')', String.Char), + (r'[0-9]+', Number.Integer), + (r'[0-9]*.[0-9]+', Number.Float), + (r'0(x|X)[0-9A-Fa-f]+', Number.Hex), + (r'[a-z][A-Za-z0-9_]*', Name), + (r'_[A-Za-z0-9_]+', Name.Variable.Instance), + (r'==|!=|<==>|>=|>>|>|<=|<<|<|\+|-|=|/|\*|%|\+=|-=|!|@', Operator), + (r'\(|\)|\[|\]|,|\.\.\.|\.\.|\.|::|:', Punctuation), + (r'`{[^`]*`}', Text), # Extern blocks won't be Lexed by Nit + ('(\r|\n| |\t)+', Text)] + } + diff --git a/pygments/lexers/functional.py b/pygments/lexers/functional.py index a22c4f55..01ba03c6 100644 --- a/pygments/lexers/functional.py +++ b/pygments/lexers/functional.py @@ -1530,26 +1530,27 @@ class IdrisLexer(RegexLexer): 'let','proof','of','then','static','where','_','with', 'pattern', 'term', 'syntax','prefix', 'postulate','parameters','record','dsl','impossible','implicit', - 'tactics','intros','intro','compute','refine','exaxt','trivial'] + 'tactics','intros','intro','compute','refine','exact','trivial'] ascii = ['NUL','SOH','[SE]TX','EOT','ENQ','ACK', 'BEL','BS','HT','LF','VT','FF','CR','S[OI]','DLE', 'DC[1-4]','NAK','SYN','ETB','CAN', 'EM','SUB','ESC','[FGRU]S','SP','DEL'] - annotations = ['assert_total','lib','link','include','provide','access', - 'default'] + directives = ['lib','link','flag','include','hide','freeze','access', + 'default','logging','dynamic','name','error_handlers','language'] tokens = { 'root': [ + # Comments + (r'^(\s*)(%%%s)' % '|'.join(directives), + bygroups(Text, Keyword.Reserved)), + (r'(\s*)(--(?![!#$%&*+./<=>?@\^|_~:\\]).*?)$', bygroups(Text, Comment.Single)), + (r'(\s*)(\|{3}.*?)$', bygroups(Text, Comment.Single)), + (r'(\s*)({-)', bygroups(Text, Comment.Multiline), 'comment'), # Declaration (r'^(\s*)([^\s\(\)\{\}]+)(\s*)(:)(\s*)', bygroups(Text, Name.Function, Text, Operator.Word, Text)), - # Comments - (r'^(\s*)(%%%s)' % '|'.join(annotations), - bygroups(Text, Keyword.Reserved)), - (r'--(?![!#$%&*+./<=>?@\^|_~:\\]).*?$', Comment.Single), - (r'{-', Comment.Multiline, 'comment'), # Identifiers (r'\b(%s)(?!\')\b' % '|'.join(reserved), Keyword.Reserved), (r'(import|module)(\s+)', bygroups(Keyword.Reserved, Text), 'module'), @@ -3135,7 +3136,7 @@ def gen_elixir_string_rules(name, symbol, token): (r'[^#%s\\]+' % (symbol,), token), include('escapes'), (r'\\.', token), - (r'(%s)(:?)' % (symbol,), bygroups(token, Punctuation), "#pop"), + (r'(%s)' % (symbol,), bygroups(token), "#pop"), include('interpol') ] return states @@ -3169,7 +3170,7 @@ class ElixirLexer(RegexLexer): mimetypes = ['text/x-elixir'] KEYWORD = ['fn', 'do', 'end', 'after', 'else', 'rescue', 'catch'] - KEYWORD_OPERATOR = ['not', 'and', 'or', 'xor', 'when', 'in'] + KEYWORD_OPERATOR = ['not', 'and', 'or', 'when', 'in'] BUILTIN = [ 'case', 'cond', 'for', 'if', 'unless', 'try', 'receive', 'raise', 'quote', 'unquote', 'unquote_splicing', 'throw', 'super' @@ -3184,15 +3185,19 @@ class ElixirLexer(RegexLexer): PSEUDO_VAR = ['_', '__MODULE__', '__DIR__', '__ENV__', '__CALLER__'] - OPERATORS3 = ['<<<', '>>>', '|||', '&&&', '^^^', '~~~', '===', '!=='] + + OPERATORS3 = [ + '<<<', '>>>', '|||', '&&&', '^^^', '~~~', '===', '!==', + '~>>', '<~>', '|~>', '<|>', + ] OPERATORS2 = [ - '==', '!=', '<=', '>=', '&&', '||', '<>', '++', '--', '|>', '=~' + '==', '!=', '<=', '>=', '&&', '||', '<>', '++', '--', '|>', '=~', + '->', '<-', '|', '.', '=', '~>', '<~', ] OPERATORS1 = ['<', '>', '+', '-', '*', '/', '!', '^', '&'] PUNCTUATION = [ - '\\\\', '<<', '>>', '::', '->', '<-', '=>', '|', '(', ')', - '{', '}', ';', ',', '.', '[', ']', '%', '=' + '\\\\', '<<', '>>', '=>', '(', ')', ':', ';', ',', '[', ']' ] def get_tokens_unprocessed(self, text): @@ -3218,20 +3223,19 @@ class ElixirLexer(RegexLexer): yield index, token, value def gen_elixir_sigil_rules(): - # these braces are balanced inside the sigil string - braces = [ + # all valid sigil terminators (excluding heredocs) + terminators = [ (r'\{', r'\}', 'cb'), (r'\[', r'\]', 'sb'), (r'\(', r'\)', 'pa'), (r'\<', r'\>', 'ab'), + (r'/', r'/', 'slas'), + (r'\|', r'\|', 'pipe'), + ('"', '"', 'quot'), + ("'", "'", 'apos'), ] - # these are also valid sigil terminators, they are not balanced - terms = [ - (r'/', 'slas'), (r'\|', 'pipe'), ('"', 'quot'), ("'", 'apos'), - ] - - # heredocs have slightly different rules, they are not balanced + # heredocs have slightly different rules triquotes = [(r'"""', 'triquot'), (r"'''", 'triapos')] token = String.Other @@ -3255,33 +3259,14 @@ class ElixirLexer(RegexLexer): include('heredoc_no_interpol'), ] - for term, name in terms: + for lterm, rterm, name in terminators: states['sigils'] += [ - (r'~[a-z]' + term, token, name + '-intp'), - (r'~[A-Z]' + term, token, name + '-no-intp'), + (r'~[a-z]' + lterm, token, name + '-intp'), + (r'~[A-Z]' + lterm, token, name + '-no-intp'), ] - - # Similar states to the braced sigils, but no balancing of - # terminators - states[name +'-intp'] = gen_elixir_sigstr_rules(term, token) + states[name +'-intp'] = gen_elixir_sigstr_rules(rterm, token) states[name +'-no-intp'] = \ - gen_elixir_sigstr_rules(term, token, interpol=False) - - for lbrace, rbrace, name in braces: - states['sigils'] += [ - (r'~[a-z]' + lbrace, token, name + '-intp'), - (r'~[A-Z]' + lbrace, token, name + '-no-intp') - ] - - states[name +'-intp'] = [ - (r'\\.', token), - (lbrace, token, '#push'), - ] + gen_elixir_sigstr_rules(rbrace, token) - - states[name +'-no-intp'] = [ - (r'\\.', token), - (lbrace, token, '#push'), - ] + gen_elixir_sigstr_rules(rbrace, token, interpol=False) + gen_elixir_sigstr_rules(rterm, token, interpol=False) return states @@ -3290,28 +3275,35 @@ class ElixirLexer(RegexLexer): op1_re = "|".join(re.escape(s) for s in OPERATORS1) ops_re = r'(?:%s|%s|%s)' % (op3_re, op2_re, op1_re) punctuation_re = "|".join(re.escape(s) for s in PUNCTUATION) - name_re = r'[a-z_][a-zA-Z_0-9]*[!\?]?' - modname_re = r'[A-Z][A-Za-z_]*(?:\.[A-Z][A-Za-z_]*)*' + alnum = '[A-Za-z_0-9]' + name_re = r'(?:\.\.\.|[a-z_]%s*[!\?]?)' % alnum + modname_re = r'[A-Z]%(alnum)s*(?:\.[A-Z]%(alnum)s*)*' % {'alnum': alnum} complex_name_re = r'(?:%s|%s|%s)' % (name_re, modname_re, ops_re) special_atom_re = r'(?:\.\.\.|<<>>|%{}|%|{})' + long_hex_char_re = r'(\\x{)([\da-fA-F]+)(})' + hex_char_re = r'(\\x[\da-fA-F]{1,2})' + escape_char_re = r'(\\[abdefnrstv])' + tokens = { 'root': [ (r'\s+', Text), (r'#.*$', Comment.Single), # Various kinds of characters - (r'(?i)(\?)(\\x{)([\da-f]+)(})', + (r'(\?)' + long_hex_char_re, bygroups(String.Char, String.Escape, Number.Hex, String.Escape)), - (r'(?i)(\?)(\\x[\da-f]{1,2})', - bygroups(String.Char, String.Escape)), - (r'(\?)(\\[0-7]{1,3})', + (r'(\?)' + hex_char_re, bygroups(String.Char, String.Escape)), - (r'(\?)(\\[abdefnrstv])', + (r'(\?)' + escape_char_re, bygroups(String.Char, String.Escape)), (r'\?\\?.', String.Char), + # '::' has to go before atoms + (r':::', String.Symbol), + (r'::', Operator), + # atoms (r':' + special_atom_re, String.Symbol), (r':' + complex_name_re, String.Symbol), @@ -3325,6 +3317,10 @@ class ElixirLexer(RegexLexer): # @attributes (r'@' + name_re, Name.Attribute), + # identifiers + (name_re, Name), + (r'(%%?)(%s)' % (modname_re,), bygroups(Punctuation, Name.Class)), + # operators and punctuation (op3_re, Operator), (op2_re, Operator), @@ -3332,14 +3328,10 @@ class ElixirLexer(RegexLexer): (r'&\d', Name.Entity), # anon func arguments (op1_re, Operator), - # identifiers - (name_re, Name), - (modname_re, Name.Class), - # numbers - (r'0[bB][01]+', Number.Bin), - (r'0[0-7]+', Number.Oct), - (r'(?i)0x[\da-f]+', Number.Hex), + (r'0b[01]+', Number.Bin), + (r'0o[0-7]+', Number.Oct), + (r'0x[\da-fA-F]+', Number.Hex), (r'\d(_?\d)*\.\d(_?\d)*([eE][-+]?\d(_?\d)*)?', Number.Float), (r'\d(_?\d)*', Number.Integer), @@ -3350,6 +3342,9 @@ class ElixirLexer(RegexLexer): (r"'", String.Single, 'string_single'), include('sigils'), + + (r'%{', Punctuation, 'map_key'), + (r'{', Punctuation, 'tuple'), ], 'heredoc_double': [ (r'^\s*"""', String.Heredoc, '#pop'), @@ -3372,11 +3367,10 @@ class ElixirLexer(RegexLexer): (r'\n+', String.Heredoc), ], 'escapes': [ - (r'(?i)(\\x{)([\da-f]+)(})', + (long_hex_char_re, bygroups(String.Escape, Number.Hex, String.Escape)), - (r'(?i)\\x[\da-f]{1,2}', String.Escape), - (r'\\[0-7]{1,3}', String.Escape), - (r'\\[abdefnrstv]', String.Escape), + (hex_char_re, String.Escape), + (escape_char_re, String.Escape), ], 'interpol': [ (r'#{', String.Interpol, 'interpol_string'), @@ -3385,6 +3379,21 @@ class ElixirLexer(RegexLexer): (r'}', String.Interpol, "#pop"), include('root') ], + 'map_key': [ + include('root'), + (r':', Punctuation, 'map_val'), + (r'=>', Punctuation, 'map_val'), + (r'}', Punctuation, '#pop'), + ], + 'map_val': [ + include('root'), + (r',', Punctuation, '#pop'), + (r'(?=})', Punctuation, '#pop'), + ], + 'tuple': [ + include('root'), + (r'}', Punctuation, '#pop'), + ], } tokens.update(gen_elixir_string_rules('double', '"', String.Double)) tokens.update(gen_elixir_string_rules('single', "'", String.Single)) diff --git a/pygments/lexers/jvm.py b/pygments/lexers/jvm.py index fa05b7ed..0d16e2fd 100644 --- a/pygments/lexers/jvm.py +++ b/pygments/lexers/jvm.py @@ -1219,6 +1219,7 @@ class GoloLexer(RegexLexer): (r'-?\d[\d_]*', Number.Integer), ('`?[a-zA-Z_][\w$]*', Name), + (r'@[a-zA-Z_][\w$._]*', Name.Decorator), (r'"""', String, combined('stringescape', 'triplestring')), (r'"', String, combined('stringescape', 'doublestring')), diff --git a/pygments/lexers/other.py b/pygments/lexers/other.py index 608e499d..0075fc07 100644 --- a/pygments/lexers/other.py +++ b/pygments/lexers/other.py @@ -2473,7 +2473,7 @@ class ProtoBufLexer(RegexLexer): (r'/(\\\n)?/(\n|(.|\n)*?[^\\]\n)', Comment.Single), (r'/(\\\n)?\*(.|\n)*?\*(\\\n)?/', Comment.Multiline), (r'\b(import|option|optional|required|repeated|default|packed|' - r'ctype|extensions|to|max|rpc|returns)\b', Keyword), + r'ctype|extensions|to|max|rpc|returns|oneof)\b', Keyword), (r'(int32|int64|uint32|uint64|sint32|sint64|' r'fixed32|fixed64|sfixed32|sfixed64|' r'float|double|bool|string|bytes)\b', Keyword.Type), diff --git a/pygments/lexers/sql.py b/pygments/lexers/sql.py index 7540f079..5070d487 100644 --- a/pygments/lexers/sql.py +++ b/pygments/lexers/sql.py @@ -464,10 +464,9 @@ class MySqlLexer(RegexLexer): (r'/\*', Comment.Multiline, 'multiline-comments'), (r'[0-9]+', Number.Integer), (r'[0-9]*\.[0-9]+(e[+-][0-9]+)', Number.Float), - # TODO: add backslash escapes - (r"'(''|[^'])*'", String.Single), - (r'"(""|[^"])*"', String.Double), - (r"`(``|[^`])*`", String.Symbol), + (r"'(\\\\|\\'|''|[^'])*'", String.Single), + (r'"(\\\\|\\"|""|[^"])*"', String.Double), + (r"`(\\\\|\\`|``|[^`])*`", String.Symbol), (r'[+*/<>=~!@#%^&|`?-]', Operator), (r'\b(tinyint|smallint|mediumint|int|integer|bigint|date|' r'datetime|time|bit|bool|tinytext|mediumtext|longtext|text|' |