diff options
author | Georg Brandl <georg@python.org> | 2012-02-04 10:49:13 +0100 |
---|---|---|
committer | Georg Brandl <georg@python.org> | 2012-02-04 10:49:13 +0100 |
commit | 7cef32ec183e53298e76304b965f1ea2c5f3577c (patch) | |
tree | 28cec35ab3f3bcf2eb8def50c49d6610356297be /pygments | |
parent | d541282daa49002937f56da77ac5bb3a4b4f15c3 (diff) | |
parent | 2c1a1ade8db7b432ff2b5c42cb40f7a898c05f5d (diff) | |
download | pygments-7cef32ec183e53298e76304b965f1ea2c5f3577c.tar.gz |
Merge and fix ECL lexer.
Diffstat (limited to 'pygments')
-rw-r--r-- | pygments/formatters/html.py | 2 | ||||
-rw-r--r-- | pygments/lexer.py | 2 | ||||
-rw-r--r-- | pygments/lexers/_mapping.py | 9 | ||||
-rw-r--r-- | pygments/lexers/agile.py | 139 | ||||
-rw-r--r-- | pygments/lexers/compiled.py | 13 | ||||
-rw-r--r-- | pygments/lexers/dotnet.py | 17 | ||||
-rw-r--r-- | pygments/lexers/functional.py | 324 | ||||
-rw-r--r-- | pygments/lexers/other.py | 157 | ||||
-rw-r--r-- | pygments/lexers/pypylog.py | 14 | ||||
-rw-r--r-- | pygments/lexers/web.py | 81 | ||||
-rw-r--r-- | pygments/util.py | 2 |
11 files changed, 685 insertions, 75 deletions
diff --git a/pygments/formatters/html.py b/pygments/formatters/html.py index 1f0ca680..f6c6400a 100644 --- a/pygments/formatters/html.py +++ b/pygments/formatters/html.py @@ -596,7 +596,7 @@ class HtmlFormatter(Formatter): def _wrap_lineanchors(self, inner): s = self.lineanchors - i = 0 + i = self.linenostart - 1 # subtract 1 since we have to increment i *before* yielding for t, line in inner: if t: i += 1 diff --git a/pygments/lexer.py b/pygments/lexer.py index 53ea5ac1..33427ae3 100644 --- a/pygments/lexer.py +++ b/pygments/lexer.py @@ -439,7 +439,7 @@ class RegexLexerMeta(LexerMeta): def __call__(cls, *args, **kwds): """Instantiate cls after preprocessing its token definitions.""" - if not hasattr(cls, '_tokens'): + if '_tokens' not in cls.__dict__: cls._all_tokens = {} cls._tmpname = 0 if hasattr(cls, 'token_variants') and cls.token_variants: diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py index 977df8c7..49ea9728 100644 --- a/pygments/lexers/_mapping.py +++ b/pygments/lexers/_mapping.py @@ -75,8 +75,8 @@ LEXERS = { 'DtdLexer': ('pygments.lexers.web', 'DTD', ('dtd',), ('*.dtd',), ('application/xml-dtd',)), 'DuelLexer': ('pygments.lexers.web', 'Duel', ('duel', 'Duel Engine', 'Duel View', 'JBST', 'jbst', 'JsonML+BST'), ('*.duel', '*.jbst'), ('text/x-duel', 'text/x-jbst')), 'DylanLexer': ('pygments.lexers.compiled', 'Dylan', ('dylan',), ('*.dylan', '*.dyl'), ('text/x-dylan',)), + 'ECLLexer': ('pygments.lexers.other', 'ECL', ('ecl',), ('*.ecl',), ('application/x-ecl',)), 'ECLexer': ('pygments.lexers.compiled', 'eC', ('ec',), ('*.ec', '*.eh'), ('text/x-echdr', 'text/x-ecsrc')), - 'ECLLexer': ('pygments.lexers.other', 'ECL', ('ecl',), ('*.ecl'), ('text/x-ecl')), 'ErbLexer': ('pygments.lexers.templates', 'ERB', ('erb',), (), ('application/x-ruby-templating',)), 'ErlangLexer': ('pygments.lexers.functional', 'Erlang', ('erlang',), ('*.erl', '*.hrl'), ('text/x-erlang',)), 'ErlangShellLexer': ('pygments.lexers.functional', 'Erlang erl session', ('erl',), ('*.erl-sh',), ('text/x-erl-shellsession',)), @@ -109,17 +109,19 @@ LEXERS = { 'HtmlLexer': ('pygments.lexers.web', 'HTML', ('html',), ('*.html', '*.htm', '*.xhtml', '*.xslt'), ('text/html', 'application/xhtml+xml')), 'HtmlPhpLexer': ('pygments.lexers.templates', 'HTML+PHP', ('html+php',), ('*.phtml',), ('application/x-php', 'application/x-httpd-php', 'application/x-httpd-php3', 'application/x-httpd-php4', 'application/x-httpd-php5')), 'HtmlSmartyLexer': ('pygments.lexers.templates', 'HTML+Smarty', ('html+smarty',), (), ('text/html+smarty',)), + 'HttpLexer': ('pygments.lexers.other', 'HTTP', ('http',), (), ()), 'HybrisLexer': ('pygments.lexers.other', 'Hybris', ('hybris', 'hy'), ('*.hy', '*.hyb'), ('text/x-hybris', 'application/x-hybris')), 'IniLexer': ('pygments.lexers.text', 'INI', ('ini', 'cfg'), ('*.ini', '*.cfg'), ('text/x-ini',)), 'IoLexer': ('pygments.lexers.agile', 'Io', ('io',), ('*.io',), ('text/x-iosrc',)), 'IokeLexer': ('pygments.lexers.agile', 'Ioke', ('ioke', 'ik'), ('*.ik',), ('text/x-iokesrc',)), 'IrcLogsLexer': ('pygments.lexers.text', 'IRC logs', ('irc',), ('*.weechatlog',), ('text/x-irclog',)), + 'JSONLexer': ('pygments.lexers.web', 'JSON', ('json',), ('*.json',), ('application/json',)), 'JadeLexer': ('pygments.lexers.web', 'Jade', ('jade', 'JADE'), ('*.jade',), ('text/x-jade',)), 'JavaLexer': ('pygments.lexers.compiled', 'Java', ('java',), ('*.java',), ('text/x-java',)), 'JavascriptDjangoLexer': ('pygments.lexers.templates', 'JavaScript+Django/Jinja', ('js+django', 'javascript+django', 'js+jinja', 'javascript+jinja'), (), ('application/x-javascript+django', 'application/x-javascript+jinja', 'text/x-javascript+django', 'text/x-javascript+jinja', 'text/javascript+django', 'text/javascript+jinja')), 'JavascriptErbLexer': ('pygments.lexers.templates', 'JavaScript+Ruby', ('js+erb', 'javascript+erb', 'js+ruby', 'javascript+ruby'), (), ('application/x-javascript+ruby', 'text/x-javascript+ruby', 'text/javascript+ruby')), 'JavascriptGenshiLexer': ('pygments.lexers.templates', 'JavaScript+Genshi Text', ('js+genshitext', 'js+genshi', 'javascript+genshitext', 'javascript+genshi'), (), ('application/x-javascript+genshi', 'text/x-javascript+genshi', 'text/javascript+genshi')), - 'JavascriptLexer': ('pygments.lexers.web', 'JavaScript', ('js', 'javascript'), ('*.js', '*.json'), ('application/javascript', 'application/x-javascript', 'text/x-javascript', 'text/javascript', 'application/json')), + 'JavascriptLexer': ('pygments.lexers.web', 'JavaScript', ('js', 'javascript'), ('*.js',), ('application/javascript', 'application/x-javascript', 'text/x-javascript', 'text/javascript')), 'JavascriptPhpLexer': ('pygments.lexers.templates', 'JavaScript+PHP', ('js+php', 'javascript+php'), (), ('application/x-javascript+php', 'text/x-javascript+php', 'text/javascript+php')), 'JavascriptSmartyLexer': ('pygments.lexers.templates', 'JavaScript+Smarty', ('js+smarty', 'javascript+smarty'), (), ('application/x-javascript+smarty', 'text/x-javascript+smarty', 'text/javascript+smarty')), 'JspLexer': ('pygments.lexers.templates', 'Java Server Page', ('jsp',), ('*.jsp',), ('application/x-jsp',)), @@ -143,6 +145,7 @@ LEXERS = { 'ModelicaLexer': ('pygments.lexers.other', 'Modelica', ('modelica',), ('*.mo',), ('text/x-modelica',)), 'Modula2Lexer': ('pygments.lexers.compiled', 'Modula-2', ('modula2', 'm2'), ('*.def', '*.mod'), ('text/x-modula2',)), 'MoinWikiLexer': ('pygments.lexers.text', 'MoinMoin/Trac Wiki markup', ('trac-wiki', 'moin'), (), ('text/x-trac-wiki',)), + 'MoonScriptLexer': ('pygments.lexers.agile', 'MoonScript', ('moon', 'moonscript'), ('*.moon',), ('text/x-moonscript', 'application/x-moonscript')), 'MuPADLexer': ('pygments.lexers.math', 'MuPAD', ('mupad',), ('*.mu',), ()), 'MxmlLexer': ('pygments.lexers.web', 'MXML', ('mxml',), ('*.mxml',), ()), 'MySqlLexer': ('pygments.lexers.other', 'MySQL', ('mysql',), (), ('text/x-mysql',)), @@ -164,6 +167,7 @@ LEXERS = { 'OcamlLexer': ('pygments.lexers.functional', 'OCaml', ('ocaml',), ('*.ml', '*.mli', '*.mll', '*.mly'), ('text/x-ocaml',)), 'OctaveLexer': ('pygments.lexers.math', 'Octave', ('octave',), ('*.m',), ('text/octave',)), 'OocLexer': ('pygments.lexers.compiled', 'Ooc', ('ooc',), ('*.ooc',), ('text/x-ooc',)), + 'OpaLexer': ('pygments.lexers.functional', 'Opa', ('opa',), ('*.opa',), ('text/x-opa',)), 'PerlLexer': ('pygments.lexers.agile', 'Perl', ('perl', 'pl'), ('*.pl', '*.pm'), ('text/x-perl', 'application/x-perl')), 'PhpLexer': ('pygments.lexers.web', 'PHP', ('php', 'php3', 'php4', 'php5'), ('*.php', '*.php[345]'), ('text/x-php',)), 'PlPgsqlLexer': ('pygments.lexers.postgres', 'PL/pgSQL', ('plpgsql',), (), ('text/x-plpgsql',)), @@ -205,6 +209,7 @@ LEXERS = { 'ScssLexer': ('pygments.lexers.web', 'SCSS', ('scss',), ('*.scss',), ('text/x-scss',)), 'SmalltalkLexer': ('pygments.lexers.other', 'Smalltalk', ('smalltalk', 'squeak'), ('*.st',), ('text/x-smalltalk',)), 'SmartyLexer': ('pygments.lexers.templates', 'Smarty', ('smarty',), ('*.tpl',), ('application/x-smarty',)), + 'SnobolLexer': ('pygments.lexers.other', 'Snobol', ('snobol',), ('*.snobol',), ('text/x-snobol',)), 'SourcesListLexer': ('pygments.lexers.text', 'Debian Sourcelist', ('sourceslist', 'sources.list'), ('sources.list',), ()), 'SqlLexer': ('pygments.lexers.other', 'SQL', ('sql',), ('*.sql',), ('text/x-sql',)), 'SqliteConsoleLexer': ('pygments.lexers.other', 'sqlite3con', ('sqlite3',), ('*.sqlite3-console',), ('text/x-sqlite3-console',)), diff --git a/pygments/lexers/agile.py b/pygments/lexers/agile.py index 0c0de54e..6fcb7183 100644 --- a/pygments/lexers/agile.py +++ b/pygments/lexers/agile.py @@ -21,9 +21,9 @@ from pygments import unistring as uni __all__ = ['PythonLexer', 'PythonConsoleLexer', 'PythonTracebackLexer', 'RubyLexer', 'RubyConsoleLexer', 'PerlLexer', 'LuaLexer', - 'MiniDLexer', 'IoLexer', 'TclLexer', 'ClojureLexer', - 'Python3Lexer', 'Python3TracebackLexer', 'FactorLexer', - 'IokeLexer', 'FancyLexer', 'GroovyLexer'] + 'MoonScriptLexer', 'MiniDLexer', 'IoLexer', 'TclLexer', + 'ClojureLexer', 'Python3Lexer', 'Python3TracebackLexer', + 'FactorLexer', 'IokeLexer', 'FancyLexer', 'GroovyLexer'] # b/w compatibility from pygments.lexers.functional import SchemeLexer @@ -105,12 +105,12 @@ class PythonLexer(RegexLexer): r'WindowsError|ZeroDivisionError)\b', Name.Exception), ], 'numbers': [ - (r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?', Number.Float), - (r'\d+[eE][+-]?[0-9]+', Number.Float), - (r'0[0-7]+', Number.Oct), + (r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?j?', Number.Float), + (r'\d+[eE][+-]?[0-9]+j?', Number.Float), + (r'0[0-7]+j?', Number.Oct), (r'0[xX][a-fA-F0-9]+', Number.Hex), (r'\d+L', Number.Integer.Long), - (r'\d+', Number.Integer) + (r'\d+j?', Number.Integer) ], 'backtick': [ ('`.*?`', String.Backtick), @@ -199,7 +199,7 @@ class Python3Lexer(RegexLexer): tokens = PythonLexer.tokens.copy() tokens['keywords'] = [ (r'(assert|break|continue|del|elif|else|except|' - r'finally|for|global|if|lambda|pass|raise|' + r'finally|for|global|if|lambda|pass|raise|nonlocal|' r'return|try|while|yield|as|with|True|False|None)\b', Keyword), ] tokens['builtins'] = [ @@ -824,19 +824,19 @@ class PerlLexer(RegexLexer): mimetypes = ['text/x-perl', 'application/x-perl'] flags = re.DOTALL | re.MULTILINE - # TODO: give this a perl guy who knows how to parse perl... + # TODO: give this to a perl guy who knows how to parse perl... tokens = { 'balanced-regex': [ - (r'/(\\\\|\\/|[^/])*/[egimosx]*', String.Regex, '#pop'), - (r'!(\\\\|\\!|[^!])*![egimosx]*', String.Regex, '#pop'), + (r'/(\\\\|\\[^\\]|[^\\/])*/[egimosx]*', String.Regex, '#pop'), + (r'!(\\\\|\\[^\\]|[^\\!])*![egimosx]*', String.Regex, '#pop'), (r'\\(\\\\|[^\\])*\\[egimosx]*', String.Regex, '#pop'), - (r'{(\\\\|\\}|[^}])*}[egimosx]*', String.Regex, '#pop'), - (r'<(\\\\|\\>|[^>])*>[egimosx]*', String.Regex, '#pop'), - (r'\[(\\\\|\\\]|[^\]])*\][egimosx]*', String.Regex, '#pop'), - (r'\((\\\\|\\\)|[^\)])*\)[egimosx]*', String.Regex, '#pop'), - (r'@(\\\\|\\\@|[^\@])*@[egimosx]*', String.Regex, '#pop'), - (r'%(\\\\|\\\%|[^\%])*%[egimosx]*', String.Regex, '#pop'), - (r'\$(\\\\|\\\$|[^\$])*\$[egimosx]*', String.Regex, '#pop'), + (r'{(\\\\|\\[^\\]|[^\\}])*}[egimosx]*', String.Regex, '#pop'), + (r'<(\\\\|\\[^\\]|[^\\>])*>[egimosx]*', String.Regex, '#pop'), + (r'\[(\\\\|\\[^\\]|[^\\\]])*\][egimosx]*', String.Regex, '#pop'), + (r'\((\\\\|\\[^\\]|[^\\\)])*\)[egimosx]*', String.Regex, '#pop'), + (r'@(\\\\|\\[^\\]|[^\\\@])*@[egimosx]*', String.Regex, '#pop'), + (r'%(\\\\|\\[^\\]|[^\\\%])*%[egimosx]*', String.Regex, '#pop'), + (r'\$(\\\\|\\[^\\]|[^\\\$])*\$[egimosx]*', String.Regex, '#pop'), ], 'root': [ (r'\#.*?$', Comment.Single), @@ -848,20 +848,26 @@ class PerlLexer(RegexLexer): bygroups(Keyword, Text, Name, Text, Punctuation, Text), 'format'), (r'(eq|lt|gt|le|ge|ne|not|and|or|cmp)\b', Operator.Word), # common delimiters - (r's/(\\\\|\\/|[^/])*/(\\\\|\\/|[^/])*/[egimosx]*', String.Regex), + (r's/(\\\\|\\[^\\]|[^\\/])*/(\\\\|\\[^\\]|[^\\/])*/[egimosx]*', + String.Regex), (r's!(\\\\|\\!|[^!])*!(\\\\|\\!|[^!])*![egimosx]*', String.Regex), (r's\\(\\\\|[^\\])*\\(\\\\|[^\\])*\\[egimosx]*', String.Regex), - (r's@(\\\\|\\@|[^@])*@(\\\\|\\@|[^@])*@[egimosx]*', String.Regex), - (r's%(\\\\|\\%|[^%])*%(\\\\|\\%|[^%])*%[egimosx]*', String.Regex), + (r's@(\\\\|\\[^\\]|[^\\@])*@(\\\\|\\[^\\]|[^\\@])*@[egimosx]*', + String.Regex), + (r's%(\\\\|\\[^\\]|[^\\%])*%(\\\\|\\[^\\]|[^\\%])*%[egimosx]*', + String.Regex), # balanced delimiters - (r's{(\\\\|\\}|[^}])*}\s*', String.Regex, 'balanced-regex'), - (r's<(\\\\|\\>|[^>])*>\s*', String.Regex, 'balanced-regex'), - (r's\[(\\\\|\\\]|[^\]])*\]\s*', String.Regex, 'balanced-regex'), - (r's\((\\\\|\\\)|[^\)])*\)\s*', String.Regex, 'balanced-regex'), - - (r'm?/(\\\\|\\/|[^/\n])*/[gcimosx]*', String.Regex), + (r's{(\\\\|\\[^\\]|[^\\}])*}\s*', String.Regex, 'balanced-regex'), + (r's<(\\\\|\\[^\\]|[^\\>])*>\s*', String.Regex, 'balanced-regex'), + (r's\[(\\\\|\\[^\\]|[^\\\]])*\]\s*', String.Regex, + 'balanced-regex'), + (r's\((\\\\|\\[^\\]|[^\\\)])*\)\s*', String.Regex, + 'balanced-regex'), + + (r'm?/(\\\\|\\[^\\]|[^\\/\n])*/[gcimosx]*', String.Regex), (r'm(?=[/!\\{<\[\(@%\$])', String.Regex, 'balanced-regex'), - (r'((?<==~)|(?<=\())\s*/(\\\\|\\/|[^/])*/[gcimosx]*', String.Regex), + (r'((?<==~)|(?<=\())\s*/(\\\\|\\[^\\]|[^\\/])*/[gcimosx]*', + String.Regex), (r'\s+', Text), (r'(abs|accept|alarm|atan2|bind|binmode|bless|caller|chdir|' r'chmod|chomp|chop|chown|chr|chroot|close|closedir|connect|' @@ -903,9 +909,9 @@ class PerlLexer(RegexLexer): Number.Float), (r'(?i)\d+(_\d*)*e[+-]?\d+(_\d*)*', Number.Float), (r'\d+(_\d+)*', Number.Integer), - (r"'(\\\\|\\'|[^'])*'", String), - (r'"(\\\\|\\"|[^"])*"', String), - (r'`(\\\\|\\`|[^`])*`', String.Backtick), + (r"'(\\\\|\\[^\\]|[^'\\])*'", String), + (r'"(\\\\|\\[^\\]|[^"\\])*"', String), + (r'`(\\\\|\\[^\\]|[^`\\])*`', String.Backtick), (r'<([^\s>]+)>', String.Regex), (r'(q|qq|qw|qr|qx)\{', String.Other, 'cb-string'), (r'(q|qq|qw|qr|qx)\(', String.Other, 'rb-string'), @@ -1112,6 +1118,73 @@ class LuaLexer(RegexLexer): yield index, token, value +class MoonScriptLexer(LuaLexer): + """ + For `MoonScript <http://moonscript.org.org>`_ source code. + + *New in Pygments 1.5.* + """ + + name = "MoonScript" + aliases = ["moon", "moonscript"] + filenames = ["*.moon"] + mimetypes = ['text/x-moonscript', 'application/x-moonscript'] + + tokens = { + 'root': [ + (r'#!(.*?)$', Comment.Preproc), + (r'', Text, 'base'), + ], + 'base': [ + ('--.*$', Comment.Single), + (r'(?i)(\d*\.\d+|\d+\.\d*)(e[+-]?\d+)?', Number.Float), + (r'(?i)\d+e[+-]?\d+', Number.Float), + (r'(?i)0x[0-9a-f]*', Number.Hex), + (r'\d+', Number.Integer), + (r'\n', Text), + (r'[^\S\n]+', Text), + (r'(?s)\[(=*)\[.*?\]\1\]', String), + (r'(->|=>)', Name.Function), + (r':[a-zA-Z_][a-zA-Z0-9_]*', Name.Variable), + (r'(==|!=|~=|<=|>=|\.\.|\.\.\.|[=+\-*/%^<>#!.\\:])', Operator), + (r'[;,]', Punctuation), + (r'[\[\]\{\}\(\)]', Keyword.Type), + (r"(class|extends|if|then|super|do|with|import|export|" + r"while|elseif|return|for|in|from|when|using|else|" + r"and|or|not|switch|break)\b", Keyword), + (r'(true|false|nil)\b', Keyword.Constant), + (r'(and|or|not)\b', Operator.Word), + (r'(self)\b', Name.Builtin.Pseudo), + (r'[a-zA-Z_][a-zA-Z0-9_]*:', Name.Variable), + (r'@[a-zA-Z_][a-zA-Z0-9_]*', Name.Variable.Class), + (r'[A-Z]\w+', Name.Class), # proper name + (r'[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)?', Name), + ("'", String.Single, combined('stringescape', 'sqs')), + ('"', String.Double, combined('stringescape', 'dqs')) + ], + 'stringescape': [ + (r'''\\([abfnrtv\\"']|\d{1,3})''', String.Escape) + ], + 'sqs': [ + ("'", String.Single, '#pop'), + (".", String) + ], + 'dqs': [ + ('"', String.Double, '#pop'), + (".", String) + ] + } + + def get_tokens_unprocessed(self, text): + # set . as Operator instead of Punctuation + for index, token, value in \ + LuaLexer.get_tokens_unprocessed(self, text): + if token == Punctuation and value == ".": + token = Operator + yield index, token, value + + + class MiniDLexer(RegexLexer): """ For `MiniD <http://www.dsource.org/projects/minid>`_ (a D-like scripting @@ -1430,8 +1503,8 @@ class ClojureLexer(RegexLexer): valid_name = r'[\w!$%*+,<=>?/.-]+' def _multi_escape(entries): - return '|'.join([re.escape(entry) + '(?![\\w-!$%*+,<=>?/.-])' - for entry in entries]) + return '(?:' + '|'.join(map(re.escape, entries)) + \ + ')?![\\w!$%*+,<=>?/.-]' tokens = { 'root' : [ diff --git a/pygments/lexers/compiled.py b/pygments/lexers/compiled.py index 5da2b875..af153a54 100644 --- a/pygments/lexers/compiled.py +++ b/pygments/lexers/compiled.py @@ -1232,8 +1232,10 @@ class DylanLexer(RegexLexer): (r'\$[a-zA-Z0-9-]+', Name.Constant), (r'[!$%&*/:<>=?~^.+\[\]{}-]+', Operator), (r'\s+', Text), + (r'#"[a-zA-Z0-9-]+"', Keyword), (r'#[a-zA-Z0-9-]+', Keyword), - (r'[a-zA-Z0-9-]+', Name.Variable), + (r'#(\(|\[)', Punctuation), + (r'[a-zA-Z0-9-_]+', Name.Variable), ], 'string': [ (r'"', String, '#pop'), @@ -2296,7 +2298,7 @@ class AdaLexer(RegexLexer): include('numbers'), (r"'[^']'", String.Character), (r'([a-z0-9_]+)(\s*|[(,])', bygroups(Name, using(this))), - (r"(<>|=>|:=|[\(\)\|:;,.'])", Punctuation), + (r"(<>|=>|:=|[()|:;,.'])", Punctuation), (r'[*<>+=/&-]', Operator), (r'\n+', Text), ], @@ -2317,7 +2319,7 @@ class AdaLexer(RegexLexer): ], 'end' : [ ('(if|case|record|loop|select)', Keyword.Reserved), - ('"[^"]+"|[a-zA-Z0-9_]+', Name.Function), + ('"[^"]+"|[a-zA-Z0-9_.]+', Name.Function), ('[\n\s]+', Text), (';', Punctuation, '#pop'), ], @@ -2337,11 +2339,12 @@ class AdaLexer(RegexLexer): ], 'import': [ (r'[a-z0-9_.]+', Name.Namespace, '#pop'), + (r'', Text, '#pop'), ], 'formal_part' : [ (r'\)', Punctuation, '#pop'), - (r'([a-z0-9_]+)(\s*)(,|:[^=])', bygroups(Name.Variable, - Text, Punctuation)), + (r'[a-z0-9_]+', Name.Variable), + (r',|:[^=]', Punctuation), (r'(in|not|null|out|access)\b', Keyword.Reserved), include('root'), ], diff --git a/pygments/lexers/dotnet.py b/pygments/lexers/dotnet.py index f78df352..20f645e1 100644 --- a/pygments/lexers/dotnet.py +++ b/pygments/lexers/dotnet.py @@ -92,7 +92,7 @@ class CSharpLexer(RegexLexer): (r'\n', Text), (r'[~!%^&*()+=|\[\]:;,.<>/?-]', Punctuation), (r'[{}]', Punctuation), - (r'@"(\\\\|\\"|[^"])*"', String), + (r'@"(""|[^"])*"', String), (r'"(\\\\|\\"|[^"\n])*["\n]', String), (r"'\\.'|'[^\\]'", String.Char), (r"[0-9](\.[0-9]*)?([eE][+-][0-9]+)?" @@ -111,10 +111,12 @@ class CSharpLexer(RegexLexer): r'ref|return|sealed|sizeof|stackalloc|static|' r'switch|this|throw|true|try|typeof|' r'unchecked|unsafe|virtual|void|while|' - r'get|set|new|partial|yield|add|remove|value)\b', Keyword), + r'get|set|new|partial|yield|add|remove|value|alias|ascending|' + r'descending|from|group|into|orderby|select|where|' + r'join|equals)\b', Keyword), (r'(global)(::)', bygroups(Keyword, Punctuation)), - (r'(bool|byte|char|decimal|double|float|int|long|object|sbyte|' - r'short|string|uint|ulong|ushort)\b\??', Keyword.Type), + (r'(bool|byte|char|decimal|double|dynamic|float|int|long|object|' + r'sbyte|short|string|uint|ulong|ushort|var)\b\??', Keyword.Type), (r'(class|struct)(\s+)', bygroups(Keyword, Text), 'class'), (r'(namespace|using)(\s+)', bygroups(Keyword, Text), 'namespace'), (cs_ident, Name), @@ -214,7 +216,7 @@ class NemerleLexer(RegexLexer): (r'[~!%^&*()+=|\[\]:;,.<>/?-]', Punctuation), (r'[{}]', Punctuation), - (r'@"(\\\\|\\"|[^"])*"', String), + (r'@"(""|[^"])*"', String), (r'"(\\\\|\\"|[^"\n])*["\n]', String), (r"'\\.'|'[^\\]'", String.Char), (r"0[xX][0-9a-fA-F]+[Ll]?", Number), @@ -278,8 +280,9 @@ class NemerleLexer(RegexLexer): ], 'splice-string-content': [ (r'if|match', Keyword), - (r'[~!%^&*+=|\[\]:;,.<>/?-]', Punctuation), + (r'[~!%^&*+=|\[\]:;,.<>/?-\\"$ ]', Punctuation), (cs_ident, Name), + (r'\d+', Number), (r'\(', Punctuation, '#push'), (r'\)', Punctuation, '#pop') ] @@ -581,7 +584,7 @@ class FSharpLexer(RegexLexer): Name.Namespace, 'dotted'), (r'\b([A-Z][A-Za-z0-9_\']*)', Name.Class), (r'//.*?\n', Comment.Single), - (r'\(\*', Comment, 'comment'), + (r'\(\*(?!\))', Comment, 'comment'), (r'\b(%s)\b' % '|'.join(keywords), Keyword), (r'(%s)' % '|'.join(keyopts), Operator), (r'(%s|%s)?%s' % (infix_syms, prefix_syms, operators), Operator), diff --git a/pygments/lexers/functional.py b/pygments/lexers/functional.py index a8102cc8..c6f5421f 100644 --- a/pygments/lexers/functional.py +++ b/pygments/lexers/functional.py @@ -14,11 +14,11 @@ import re from pygments.lexer import Lexer, RegexLexer, bygroups, include, do_insertions from pygments.token import Text, Comment, Operator, Keyword, Name, \ String, Number, Punctuation, Literal, Generic, Error - +import pygments.formatters __all__ = ['SchemeLexer', 'CommonLispLexer', 'HaskellLexer', 'LiterateHaskellLexer', 'SMLLexer', 'OcamlLexer', 'ErlangLexer', - 'ErlangShellLexer'] + 'ErlangShellLexer', 'OpaLexer'] class SchemeLexer(RegexLexer): @@ -1079,3 +1079,323 @@ class ErlangShellLexer(Lexer): erlexer.get_tokens_unprocessed(curcode)): yield item + +class OpaLexer(RegexLexer): + """ + Lexer for the Opa language (http://opalang.org). + + *New in Pygments 1.5.* + """ + + name = 'Opa' + aliases = ['opa'] + filenames = ['*.opa'] + mimetypes = ['text/x-opa'] + + # most of these aren't strictly keywords + # but if you color only real keywords, you might just + # as well not color anything + keywords = [ + 'and', 'as', 'begin', 'css', 'database', 'db', 'do', 'else', 'end', + 'external', 'forall', 'if', 'import', 'match', 'package', 'parser', + 'rec', 'server', 'then', 'type', 'val', 'with', 'xml_parser' + ] + + # matches both stuff and `stuff` + ident_re = r'(([a-zA-Z_]\w*)|(`[^`]*`))' + + op_re = r'[.=\-<>,@~%/+?*&^!]' + punc_re = r'[()\[\],;|]' # '{' and '}' are treated elsewhere + # because they are also used for inserts + + tokens = { + # copied from the caml lexer, should be adapted + 'escape-sequence': [ + (r'\\[\\\"\'ntr}]', String.Escape), + (r'\\[0-9]{3}', String.Escape), + (r'\\x[0-9a-fA-F]{2}', String.Escape), + ], + + # factorizing these rules, because they are inserted many times + 'comments': [ + (r'/\*', Comment, 'nested-comment'), + (r'//.*?$', Comment), + ], + 'comments-and-spaces': [ + include('comments'), + (r'\s+', Text), + ], + + 'root': [ + include('comments-and-spaces'), + # keywords + (r'\b(%s)\b' % '|'.join(keywords), Keyword), + # directives + # we could parse the actual set of directives instead of anything + # starting with @, but this is troublesome + # because it needs to be adjusted all the time + # and assuming we parse only sources that compile, it is useless + (r'@'+ident_re+r'\b', Name.Builtin.Pseudo), + + # number literals + (r'-?.[\d]+([eE][+\-]?\d+)', Number.Float), + (r'-?\d+.\d*([eE][+\-]?\d+)', Number.Float), + (r'-?\d+[eE][+\-]?\d+', Number.Float), + (r'0[xX][\da-fA-F]+', Number.Hex), + (r'0[oO][0-7]+', Number.Oct), + (r'0[bB][01]+', Number.Binary), + (r'\d+', Number.Integer), + # color literals + (r'#[\da-fA-F]{3,6}', Number.Integer), + + # string literals + (r'"', String.Double, 'string'), + # char literal, should be checked because this is the regexp from + # the caml lexer + (r"'(?:(\\[\\\"'ntbr ])|(\\[0-9]{3})|(\\x[0-9a-fA-F]{2})|.)'", + String.Char), + + # this is meant to deal with embedded exprs in strings + # every time we find a '}' we pop a state so that if we were + # inside a string, we are back in the string state + # as a consequence, we must also push a state every time we find a + # '{' or else we will have errors when parsing {} for instance + (r'{', Operator, '#push'), + (r'}', Operator, '#pop'), + + # html literals + # this is a much more strict that the actual parser, + # since a<b would not be parsed as html + # but then again, the parser is way too lax, and we can't hope + # to have something as tolerant + (r'<(?=[a-zA-Z>])', String.Single, 'html-open-tag'), + + # db path + # matching the '[_]' in '/a[_]' because it is a part + # of the syntax of the db path definition + # unfortunately, i don't know how to match the ']' in + # /a[1], so this is somewhat inconsistent + (r'[@?!]?(/\w+)+(\[_\])?', Name.Variable), + # putting the same color on <- as on db path, since + # it can be used only to mean Db.write + (r'<-(?!'+op_re+r')', Name.Variable), + + # 'modules' + # although modules are not distinguished by their names as in caml + # the standard library seems to follow the convention that modules + # only area capitalized + (r'\b([A-Z]\w*)(?=\.)', Name.Namespace), + + # operators + # = has a special role because this is the only + # way to syntactic distinguish binding constructions + # unfortunately, this colors the equal in {x=2} too + (r'=(?!'+op_re+r')', Keyword), + (r'(%s)+' % op_re, Operator), + (r'(%s)+' % punc_re, Operator), + + # coercions + (r':', Operator, 'type'), + # type variables + # we need this rule because we don't parse specially type + # definitions so in "type t('a) = ...", "'a" is parsed by 'root' + ("'"+ident_re, Keyword.Type), + + # id literal, #something, or #{expr} + (r'#'+ident_re, String.Single), + (r'#(?={)', String.Single), + + # identifiers + # this avoids to color '2' in 'a2' as an integer + (ident_re, Text), + + # default, not sure if that is needed or not + # (r'.', Text), + ], + + # it is quite painful to have to parse types to know where they end + # this is the general rule for a type + # a type is either: + # * -> ty + # * type-with-slash + # * type-with-slash -> ty + # * type-with-slash (, type-with-slash)+ -> ty + # + # the code is pretty funky in here, but this code would roughly + # translate in caml to: + # let rec type stream = + # match stream with + # | [< "->"; stream >] -> type stream + # | [< ""; stream >] -> + # type_with_slash stream + # type_lhs_1 stream; + # and type_1 stream = ... + 'type': [ + include('comments-and-spaces'), + (r'->', Keyword.Type), + (r'', Keyword.Type, ('#pop', 'type-lhs-1', 'type-with-slash')), + ], + + # parses all the atomic or closed constructions in the syntax of type + # expressions: record types, tuple types, type constructors, basic type + # and type variables + 'type-1': [ + include('comments-and-spaces'), + (r'\(', Keyword.Type, ('#pop', 'type-tuple')), + (r'~?{', Keyword.Type, ('#pop', 'type-record')), + (ident_re+r'\(', Keyword.Type, ('#pop', 'type-tuple')), + (ident_re, Keyword.Type, '#pop'), + ("'"+ident_re, Keyword.Type), + # this case is not in the syntax but sometimes + # we think we are parsing types when in fact we are parsing + # some css, so we just pop the states until we get back into + # the root state + (r'', Keyword.Type, '#pop'), + ], + + # type-with-slash is either: + # * type-1 + # * type-1 (/ type-1)+ + 'type-with-slash': [ + include('comments-and-spaces'), + (r'', Keyword.Type, ('#pop', 'slash-type-1', 'type-1')), + ], + 'slash-type-1': [ + include('comments-and-spaces'), + ('/', Keyword.Type, ('#pop', 'type-1')), + # same remark as above + (r'', Keyword.Type, '#pop'), + ], + + # we go in this state after having parsed a type-with-slash + # while trying to parse a type + # and at this point we must determine if we are parsing an arrow + # type (in which case we must continue parsing) or not (in which + # case we stop) + 'type-lhs-1': [ + include('comments-and-spaces'), + (r'->', Keyword.Type, ('#pop', 'type')), + (r'(?=,)', Keyword.Type, ('#pop', 'type-arrow')), + (r'', Keyword.Type, '#pop'), + ], + 'type-arrow': [ + include('comments-and-spaces'), + # the look ahead here allows to parse f(x : int, y : float -> truc) + # correctly + (r',(?=[^:]*?->)', Keyword.Type, 'type-with-slash'), + (r'->', Keyword.Type, ('#pop', 'type')), + # same remark as above + (r'', Keyword.Type, '#pop'), + ], + + # no need to do precise parsing for tuples and records + # because they are closed constructions, so we can simply + # find the closing delimiter + # note that this function would be not work if the source + # contained identifiers like `{)` (although it could be patched + # to support it) + 'type-tuple': [ + include('comments-and-spaces'), + (r'[^\(\)/*]+', Keyword.Type), + (r'[/*]', Keyword.Type), + (r'\(', Keyword.Type, '#push'), + (r'\)', Keyword.Type, '#pop'), + ], + 'type-record': [ + include('comments-and-spaces'), + (r'[^{}/*]+', Keyword.Type), + (r'[/*]', Keyword.Type), + (r'{', Keyword.Type, '#push'), + (r'}', Keyword.Type, '#pop'), + ], + +# 'type-tuple': [ +# include('comments-and-spaces'), +# (r'\)', Keyword.Type, '#pop'), +# (r'', Keyword.Type, ('#pop', 'type-tuple-1', 'type-1')), +# ], +# 'type-tuple-1': [ +# include('comments-and-spaces'), +# (r',?\s*\)', Keyword.Type, '#pop'), # ,) is a valid end of tuple, in (1,) +# (r',', Keyword.Type, 'type-1'), +# ], +# 'type-record':[ +# include('comments-and-spaces'), +# (r'}', Keyword.Type, '#pop'), +# (r'~?(?:\w+|`[^`]*`)', Keyword.Type, 'type-record-field-expr'), +# ], +# 'type-record-field-expr': [ +# +# ], + + 'nested-comment': [ + (r'[^/*]+', Comment), + (r'/\*', Comment, '#push'), + (r'\*/', Comment, '#pop'), + (r'[/*]', Comment), + ], + + # the coy pasting between string and single-string + # is kinda sad. Is there a way to avoid that?? + 'string': [ + (r'[^\\"{]+', String.Double), + (r'"', String.Double, '#pop'), + (r'{', Operator, 'root'), + include('escape-sequence'), + ], + 'single-string': [ + (r'[^\\\'{]+', String.Double), + (r'\'', String.Double, '#pop'), + (r'{', Operator, 'root'), + include('escape-sequence'), + ], + + # all the html stuff + # can't really reuse some existing html parser + # because we must be able to parse embedded expressions + + # we are in this state after someone parsed the '<' that + # started the html literal + 'html-open-tag': [ + (r'[\w\-:]+', String.Single, ('#pop', 'html-attr')), + (r'>', String.Single, ('#pop', 'html-content')), + ], + + # we are in this state after someone parsed the '</' that + # started the end of the closing tag + 'html-end-tag': [ + # this is a star, because </> is allowed + (r'[\w\-:]*>', String.Single, '#pop'), + ], + + # we are in this state after having parsed '<ident(:ident)?' + # we thus parse a possibly empty list of attributes + 'html-attr': [ + (r'\s+', Text), + (r'[\w\-:]+=', String.Single, 'html-attr-value'), + (r'/>', String.Single, '#pop'), + (r'>', String.Single, ('#pop', 'html-content')), + ], + + 'html-attr-value': [ + (r"'", String.Single, ('#pop', 'single-string')), + (r'"', String.Single, ('#pop', 'string')), + (r'#'+ident_re, String.Single, '#pop'), + (r'#(?={)', String.Single, ('#pop', 'root')), + (r'{', Operator, ('#pop', 'root')), # this is a tail call! + ], + + # we should probably deal with '\' escapes here + 'html-content': [ + (r'<!--', Comment, 'html-comment'), + (r'</', String.Single, ('#pop', 'html-end-tag')), + (r'<', String.Single, 'html-open-tag'), + (r'{', Operator, 'root'), + (r'.|\s+', String.Single), + ], + + 'html-comment': [ + (r'-->', Comment, '#pop'), + (r'[^\-]+|-', Comment), + ], + } diff --git a/pygments/lexers/other.py b/pygments/lexers/other.py index c5172922..44cf4f47 100644 --- a/pygments/lexers/other.py +++ b/pygments/lexers/other.py @@ -15,7 +15,7 @@ from pygments.lexer import Lexer, RegexLexer, include, bygroups, using, \ this, do_insertions, combined from pygments.token import Error, Punctuation, Literal, Token, \ Text, Comment, Operator, Keyword, Name, String, Number, Generic -from pygments.util import shebang_matches +from pygments.util import ClassNotFound, shebang_matches from pygments.lexers.web import HtmlLexer @@ -27,7 +27,7 @@ __all__ = ['SqlLexer', 'MySqlLexer', 'SqliteConsoleLexer', 'BrainfuckLexer', 'NewspeakLexer', 'GherkinLexer', 'AsymptoteLexer', 'PostScriptLexer', 'AutohotkeyLexer', 'GoodDataCLLexer', 'MaqlLexer', 'ProtoBufLexer', 'HybrisLexer', 'AwkLexer', - 'Cfengine3Lexer', 'ECLLexer'] + 'Cfengine3Lexer', 'HttpLexer', 'SnobolLexer', 'ECLLexer'] line_re = re.compile('.*?\n') @@ -221,17 +221,19 @@ class MySqlLexer(RegexLexer): class ECLLexer(RegexLexer): """ - Lexer for the declarative big-data `ECL <http://hpccsystems.com/community/docs/ecl-language-reference/html>`_ + Lexer for the declarative big-data `ECL + <http://hpccsystems.com/community/docs/ecl-language-reference/html>`_ language. - *New in Pygments 0.7.* + *New in Pygments 1.5.* """ + name = 'ECL' aliases = ['ecl'] filenames = ['*.ecl'] mimetypes = ['application/x-ecl'] - flags = re.IGNORECASE + flags = re.IGNORECASE | re.MULTILINE tokens = { 'root': [ @@ -265,16 +267,17 @@ class ECLLexer(RegexLexer): ], 'types': [ (r'(RECORD|END)[^\d]', Keyword.Declaration), - (r'(ASCII|BIG_ENDIAN|BOOLEAN|DATA|DECIMAL|EBCDIC|INTEGER|PATTERN|' + (r'((?:ASCII|BIG_ENDIAN|BOOLEAN|DATA|DECIMAL|EBCDIC|INTEGER|PATTERN|' r'QSTRING|REAL|RECORD|RULE|SET OF|STRING|TOKEN|UDECIMAL|UNICODE|' - r'UNSIGNED|VARSTRING|VARUNICODE)\d*\s+', Keyword.Type), + r'UNSIGNED|VARSTRING|VARUNICODE)\d*)(\s+)', + bygroups(Keyword.Type, Text)), ], 'keywords': [ (r'(APPLY|ASSERT|BUILD|BUILDINDEX|EVALUATE|FAIL|KEYDIFF|KEYPATCH|' r'LOADXML|NOTHOR|NOTIFY|OUTPUT|PARALLEL|SEQUENTIAL|SOAPCALL|WAIT' r'CHECKPOINT|DEPRECATED|FAILCODE|FAILMESSAGE|FAILURE|GLOBAL|' r'INDEPENDENT|ONWARNING|PERSIST|PRIORITY|RECOVERY|STORED|SUCCESS|' - r'WAIT|WHEN)\W', Keyword.Reserved), + r'WAIT|WHEN)\b', Keyword.Reserved), # These are classed differently, check later (r'(ALL|AND|ANY|AS|ATMOST|BEFORE|BEGINC\+\+|BEST|BETWEEN|CASE|CONST|' r'COUNTER|CSV|DESCEND|ENCRYPT|ENDC\+\+|ENDMACRO|EXCEPT|EXCLUSIVE|' @@ -287,7 +290,7 @@ class ECLLexer(RegexLexer): r'RIGHT|SCAN|SELF|SEPARATOR|SERVICE|SHARED|SKEW|SKIP|SQL|STORE|' r'TERMINATOR|THOR|THRESHOLD|TOKEN|TRANSFORM|TRIM|TRUE|TYPE|' r'UNICODEORDER|UNSORTED|VALIDATE|VIRTUAL|WHOLE|WILD|WITHIN|XML|' - r'XPATH|__COMPRESSED__)\W', Keyword.Reserved), + r'XPATH|__COMPRESSED__)\b', Keyword.Reserved), ], 'functions': [ (r'(ABS|ACOS|ALLNODES|ASCII|ASIN|ASSTRING|ATAN|ATAN2|AVE|CASE|' @@ -306,11 +309,12 @@ class ECLLexer(RegexLexer): r'SOAPCALL|SORT|SORTED|SQRT|STEPPED|STORED|SUM|TABLE|TAN|TANH|' r'THISNODE|TOPN|TOUNICODE|TRANSFER|TRIM|TRUNCATE|TYPEOF|UNGROUP|' r'UNICODEORDER|VARIANCE|WHICH|WORKUNIT|XMLDECODE|XMLENCODE|' - r'XMLTEXT|XMLUNICODE)\W', Name.Function), + r'XMLTEXT|XMLUNICODE)\b', Name.Function), ], 'string': [ (r'"', String, '#pop'), (r'\'', String, '#pop'), + (r'[^"\']+', String), ], } @@ -3012,11 +3016,12 @@ class Cfengine3Lexer(RegexLexer): bygroups(Keyword.Reserved,Text,Operator,Text)), (r'"', String, 'string'), (r'(\w+)(\()', bygroups(Name.Function, Punctuation)), - (r'([\w.!&|]+)(::)', bygroups(Name.Class, Punctuation)), + (r'([\w.!&|\(\)]+)(::)', bygroups(Name.Class, Punctuation)), (r'(\w+)(:)', bygroups(Keyword.Declaration,Punctuation)), (r'@[\{\(][^\)\}]+[\}\)]', Name.Variable), (r'[(){},;]', Punctuation), (r'=>', Operator), + (r'->', Operator), (r'\d+\.\d+', Number.Float), (r'\d+', Number.Integer), (r'\w+', Name.Function), @@ -3041,3 +3046,133 @@ class Cfengine3Lexer(RegexLexer): (r'\s+', Text), ], } + + +class HttpLexer(RegexLexer): + """ + Lexer for HTTP sessions. + + *New in Pygments 1.5.* + """ + + name = 'HTTP' + aliases = ['http'] + + flags = re.DOTALL + + def header_callback(self, match): + if match.group(1).lower() == 'content-type': + content_type = match.group(5).strip() + if ';' in content_type: + content_type = content_type[:content_type.find(';')].strip() + self.content_type = content_type + yield match.start(1), Name.Attribute, match.group(1) + yield match.start(2), Text, match.group(2) + yield match.start(3), Operator, match.group(3) + yield match.start(4), Text, match.group(4) + yield match.start(5), Literal, match.group(5) + yield match.start(6), Text, match.group(6) + + def content_callback(self, match): + content_type = getattr(self, 'content_type', None) + content = match.group() + offset = match.start() + if content_type: + from pygments.lexers import get_lexer_for_mimetype + try: + lexer = get_lexer_for_mimetype(content_type) + except ClassNotFound: + pass + else: + for idx, token, value in lexer.get_tokens_unprocessed(content): + yield offset + idx, token, value + return + yield offset, Text, content + + tokens = { + 'root': [ + (r'(GET|POST|PUT|DELETE|HEAD|OPTIONS|TRACE)( +)([^ ]+)( +)' + r'(HTTPS?)(/)(1\.[01])(\r?\n|$)', + bygroups(Name.Function, Text, Name.Namespace, Text, + Keyword.Reserved, Operator, Number, Text), + 'headers'), + (r'(HTTPS?)(/)(1\.[01])( +)(\d{3})( +)([^\r\n]+)(\r?\n|$)', + bygroups(Keyword.Reserved, Operator, Number, Text, Number, + Text, Name.Exception, Text), + 'headers'), + ], + 'headers': [ + (r'([^\s:]+)( *)(:)( *)([^\r\n]+)(\r?\n|$)', header_callback), + (r'\r?\n', Text, 'content') + ], + 'content': [ + (r'.+', content_callback) + ] + } + + +class SnobolLexer(RegexLexer): + """ + Lexer for the SNOBOL4 programming language. + + Recognizes the common ASCII equivalents of the original SNOBOL4 operators. + Does not require spaces around binary operators. + + *New in Pygments 1.5.* + """ + + name = "Snobol" + aliases = ["snobol"] + filenames = ['*.snobol'] + mimetypes = ['text/x-snobol'] + + tokens = { + # root state, start of line + # comments, continuation lines, and directives start in column 1 + # as do labels + 'root': [ + (r'\*.*\n', Comment), + (r'[\+\.] ', Punctuation, 'statement'), + (r'-.*\n', Comment), + (r'END\s*\n', Name.Label, 'heredoc'), + (r'[A-Za-z\$][\w$]*', Name.Label, 'statement'), + (r'\s+', Text, 'statement'), + ], + # statement state, line after continuation or label + 'statement': [ + (r'\s*\n', Text, '#pop'), + (r'\s+', Text), + (r'(?<=[^\w.])(LT|LE|EQ|NE|GE|GT|INTEGER|IDENT|DIFFER|LGT|SIZE|' + r'REPLACE|TRIM|DUPL|REMDR|DATE|TIME|EVAL|APPLY|OPSYN|LOAD|UNLOAD|' + r'LEN|SPAN|BREAK|ANY|NOTANY|TAB|RTAB|REM|POS|RPOS|FAIL|FENCE|' + r'ABORT|ARB|ARBNO|BAL|SUCCEED|INPUT|OUTPUT|TERMINAL)(?=[^\w.])', + Name.Builtin), + (r'[A-Za-z][\w\.]*', Name), + # ASCII equivalents of original operators + # | for the EBCDIC equivalent, ! likewise + # \ for EBCDIC negation + (r'\*\*|[\?\$\.!%\*/#+\-@\|&\\!=]', Operator), + (r'"[^"]*"', String), + (r"'[^']*'", String), + # Accept SPITBOL syntax for real numbers + # as well as Macro SNOBOL4 + (r'[0-9]+(?=[^\.EeDd])', Number.Integer), + (r'[0-9]+(\.[0-9]*)?([EDed][-+]?[0-9]+)?', Number.Float), + # Goto + (r':', Punctuation, 'goto'), + (r'[\(\)<>,;]', Punctuation), + ], + # Goto block + 'goto': [ + (r'\s*\n', Text, "#pop:2"), + (r'\s+', Text), + (r'F|S', Keyword), + (r'(\()([A-Za-z][\w.]*)(\))', + bygroups(Punctuation, Name.Label, Punctuation)) + ], + # everything after the END statement is basically one + # big heredoc. + 'heredoc': [ + (r'.*\n', String.Heredoc) + ] + } diff --git a/pygments/lexers/pypylog.py b/pygments/lexers/pypylog.py index c3af3387..48f0dbb9 100644 --- a/pygments/lexers/pypylog.py +++ b/pygments/lexers/pypylog.py @@ -37,6 +37,7 @@ class PyPyLogLexer(RegexLexer): "jit-log": [ (r"\[\w+\] jit-log-.*?}$", Keyword, "#pop"), + (r"^\+\d+: ", Comment), (r"[ifp]\d+", Name), (r"ptr\d+", Name), (r"(\()([\w_]+(?:\.[\w_]+)?)(\))", @@ -45,9 +46,9 @@ class PyPyLogLexer(RegexLexer): (r"(\d+\.\d+|inf|-inf)", Number.Float), (r"-?\d+", Number.Integer), (r"'.*'", String), - (r"(None|descr|ConstClass|ConstPtr)", Name), + (r"(None|descr|ConstClass|ConstPtr|TargetToken)", Name), (r"<.*?>", Name.Builtin), - (r"(debug_merge_point|jump|finish)", Name.Class), + (r"(label|debug_merge_point|jump|finish)", Name.Class), (r"(int_add_ovf|int_add|int_sub_ovf|int_sub|int_mul_ovf|int_mul|" r"int_floordiv|int_mod|int_lshift|int_rshift|int_and|int_or|" r"int_xor|int_eq|int_ne|int_ge|int_gt|int_le|int_lt|int_is_zero|" @@ -55,14 +56,15 @@ class PyPyLogLexer(RegexLexer): r"uint_floordiv|uint_ge|uint_lt|" r"float_add|float_sub|float_mul|float_truediv|" r"float_eq|float_ne|float_ge|float_gt|float_le|float_lt|float_abs|" - r"ptr_eq|" - r"cast_int_to_float|cast_float_to_int|cast_opaque_ptr|" - r"force_token|quasiimmut_field|same_as|virtual_ref_finish|virtual_ref|" + r"ptr_eq|ptr_ne|instance_ptr_eq|instance_ptr_ne|" + r"cast_int_to_float|cast_float_to_int|" + r"force_token|quasiimmut_field|same_as|virtual_ref_finish|virtual_ref|mark_opaque_ptr|" r"call_may_force|call_assembler|call_loopinvariant|call_release_gil|call_pure|call|" r"new_with_vtable|new_array|newstr|newunicode|new|" r"arraylen_gc|" r"getarrayitem_gc_pure|getarrayitem_gc|setarrayitem_gc|" - r"getarrayitem_raw|setarrayitem_raw|getfield_gc_pure|getfield_gc|" + r"getarrayitem_raw|setarrayitem_raw|getfield_gc_pure|getfield_gc|getinteriorfield_gc|" + r"getinteriorfield_gc|setinteriorfield_gc|" r"getfield_raw|setfield_gc|setfield_raw|" r"strgetitem|strsetitem|strlen|copystrcontent|" r"unicodegetitem|unicodesetitem|unicodelen|" diff --git a/pygments/lexers/web.py b/pygments/lexers/web.py index 148762fd..f76e3420 100644 --- a/pygments/lexers/web.py +++ b/pygments/lexers/web.py @@ -22,7 +22,7 @@ from pygments.lexers.agile import RubyLexer from pygments.lexers.compiled import ScalaLexer -__all__ = ['HtmlLexer', 'XmlLexer', 'JavascriptLexer', 'CssLexer', +__all__ = ['HtmlLexer', 'XmlLexer', 'JavascriptLexer', 'JSONLexer', 'CssLexer', 'PhpLexer', 'ActionScriptLexer', 'XsltLexer', 'ActionScript3Lexer', 'MxmlLexer', 'HaxeLexer', 'HamlLexer', 'SassLexer', 'ScssLexer', 'ObjectiveJLexer', 'CoffeeScriptLexer', 'DuelLexer', 'ScamlLexer', @@ -36,9 +36,9 @@ class JavascriptLexer(RegexLexer): name = 'JavaScript' aliases = ['js', 'javascript'] - filenames = ['*.js', '*.json'] + filenames = ['*.js', ] mimetypes = ['application/javascript', 'application/x-javascript', - 'text/x-javascript', 'text/javascript', 'application/json'] + 'text/x-javascript', 'text/javascript', ] flags = re.DOTALL tokens = { @@ -89,6 +89,74 @@ class JavascriptLexer(RegexLexer): } +class JSONLexer(RegexLexer): + """ + For JSON data structures. + + *New in Pygments 1.5.* + """ + + name = 'JSON' + aliases = ['json'] + filenames = ['*.json'] + mimetypes = [ 'application/json', ] + + flags = re.DOTALL + tokens = { + 'whitespace': [ + (r'\s+', Text), + ], + + # represents a simple terminal value + 'simplevalue':[ + (r'(true|false|null)\b', Keyword.Constant), + (r'-?[0-9]+', Number.Integer), + (r'"(\\\\|\\"|[^"])*"', String.Double), + ], + + + # the right hand side of an object, after the attribute name + 'objectattribute': [ + include('value'), + (r':', Punctuation), + # comma terminates the attribute but expects more + (r',', Punctuation, '#pop'), + # a closing bracket terminates the entire object, so pop twice + (r'}', Punctuation, ('#pop', '#pop')), + ], + + # a json object - { attr, attr, ... } + 'objectvalue': [ + include('whitespace'), + (r'"(\\\\|\\"|[^"])*"', Name.Tag, 'objectattribute'), + (r'}', Punctuation, '#pop'), + ], + + # json array - [ value, value, ... } + 'arrayvalue': [ + include('whitespace'), + include('value'), + (r',', Punctuation), + (r']', Punctuation, '#pop'), + ], + + # a json value - either a simple value or a complex value (object or array) + 'value': [ + include('whitespace'), + include('simplevalue'), + (r'{', Punctuation, 'objectvalue'), + (r'\[', Punctuation, 'arrayvalue'), + ], + + + # the root of a json document whould be a value + 'root': [ + include('value'), + ], + + } + + class ActionScriptLexer(RegexLexer): """ For ActionScript source code. @@ -1728,7 +1796,7 @@ class CoffeeScriptLexer(RegexLexer): 'slashstartsregex': [ include('commentsandwhitespace'), (r'///', String.Regex, ('#pop', 'multilineregex')), - (r'/(\\.|[^[/\\\n]|\[(\\.|[^\]\\\n])*])+/' + (r'/(?! )(\\.|[^[/\\\n]|\[(\\.|[^\]\\\n])*])+/' r'([gim]+\b|\B)', String.Regex, '#pop'), (r'', Text, '#pop'), ], @@ -1766,8 +1834,9 @@ class CoffeeScriptLexer(RegexLexer): ("'", String, 'sqs'), ], 'strings': [ - (r'[^#\\\'"]+', String) # note that all coffee script strings are multi-line. - # hashmarks, quotes and backslashes must be parsed one at a time + (r'[^#\\\'"]+', String), + # note that all coffee script strings are multi-line. + # hashmarks, quotes and backslashes must be parsed one at a time ], 'interpoling_string' : [ (r'}', String.Interpol, "#pop"), diff --git a/pygments/util.py b/pygments/util.py index 429e40a1..a400624b 100644 --- a/pygments/util.py +++ b/pygments/util.py @@ -118,7 +118,7 @@ def make_analysator(f): return 0.0 try: return min(1.0, max(0.0, float(rv))) - except ValueError: + except (ValueError, TypeError): return 0.0 text_analyse.__doc__ = f.__doc__ return staticmethod(text_analyse) |