diff options
29 files changed, 2846 insertions, 54 deletions
@@ -22,6 +22,7 @@ Other contributors, listed alphabetically, are: * Frits van Bommel -- assembler lexers * Pierre Bourdon -- bugfixes * Hiram Chirino -- Scaml and Jade lexers +* Leaf Corcoran -- MoonScript lexer * Christopher Creutzig -- MuPAD lexer * Pete Curry -- bugfixes * Owen Durni -- haXe lexer @@ -36,6 +37,7 @@ Other contributors, listed alphabetically, are: * Matt Good -- Genshi, Cheetah lexers * Patrick Gotthardt -- PHP namespaces support * Olivier Guibe -- Asymptote lexer +* Martin Harriman -- SNOBOL lexer * Matthew Harrison -- SVG formatter * Steven Hazel -- Tcl lexer * Aslak Hellesøy -- Gherkin lexer @@ -45,6 +47,7 @@ Other contributors, listed alphabetically, are: * Ben Hollis -- Mason lexer * Tim Howard -- BlitzMax lexer * Dennis Kaarsemaker -- sources.list lexer +* Adam Koprowski -- Opa lexer * Benjamin Kowarsch -- Modula-2 lexer * Marek Kubica -- Scheme lexer * Jochen Kupperschmidt -- Markdown processor @@ -57,6 +60,7 @@ Other contributors, listed alphabetically, are: * Stephen McKamey -- Duel/JBST lexer * Brian McKenna -- F# lexer * Lukas Meuser -- BBCode formatter, Lua lexer +* Hong Minhee -- HTTP lexer * Michael Mior -- Awk lexer * Paulo Moura -- Logtalk lexer * Mher Movsisyan -- DTD lexer @@ -69,7 +73,9 @@ Other contributors, listed alphabetically, are: * Ronny Pfannschmidt -- BBCode lexer * Benjamin Peterson -- Test suite refactoring * Dominik Picheta -- Nimrod lexer +* Clément Prévost -- UrbiScript lexer * Justin Reidy -- MXML lexer +* Norman Richards -- JSON lexer * Lubomir Rintel -- GoodData MAQL and CL lexers * Andre Roberge -- Tango style * Konrad Rudolph -- LaTeX formatter enhancements @@ -88,6 +94,7 @@ Other contributors, listed alphabetically, are: * Jeremy Thurgood -- Erlang, Squid config lexers * Erick Tryzelaar -- Felix lexer * Daniele Varrazzo -- PostgreSQL lexers +* Abe Voelker -- OpenEdge ABL lexer * Whitney Young -- ObjectiveC lexer * Nathan Weizenbaum -- Haml and Sass lexers * Dietmar Winkler -- Modelica lexer @@ -6,7 +6,7 @@ http://bitbucket.org/birkenfeld/pygments-main/issues. Version 1.5 ----------- -(codename not decided, released Jul xx, 2011) +(codename not decided, released 2012) - Lexers added: @@ -24,6 +24,14 @@ Version 1.5 * Octave (PR#22) * Standard ML (PR#14) * CFengine3 (#601) + * Opa (PR#37) + * HTTP sessions (PR#42) + * JSON (PR#31) + * SNOBOL (PR#30) + * MoonScript (PR#43) + * ECL (PR#29) + * Urbiscript (PR#17) + * OpenEdge ABL (PR#27) - In the LaTeX formatter, escape special &, < and > chars (#648). diff --git a/pygments/formatters/html.py b/pygments/formatters/html.py index 1f0ca680..f6c6400a 100644 --- a/pygments/formatters/html.py +++ b/pygments/formatters/html.py @@ -596,7 +596,7 @@ class HtmlFormatter(Formatter): def _wrap_lineanchors(self, inner): s = self.lineanchors - i = 0 + i = self.linenostart - 1 # subtract 1 since we have to increment i *before* yielding for t, line in inner: if t: i += 1 diff --git a/pygments/lexer.py b/pygments/lexer.py index 53ea5ac1..33427ae3 100644 --- a/pygments/lexer.py +++ b/pygments/lexer.py @@ -439,7 +439,7 @@ class RegexLexerMeta(LexerMeta): def __call__(cls, *args, **kwds): """Instantiate cls after preprocessing its token definitions.""" - if not hasattr(cls, '_tokens'): + if '_tokens' not in cls.__dict__: cls._all_tokens = {} cls._tmpname = 0 if hasattr(cls, 'token_variants') and cls.token_variants: diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py index 2a848737..73b26967 100644 --- a/pygments/lexers/_mapping.py +++ b/pygments/lexers/_mapping.py @@ -75,6 +75,7 @@ LEXERS = { 'DtdLexer': ('pygments.lexers.web', 'DTD', ('dtd',), ('*.dtd',), ('application/xml-dtd',)), 'DuelLexer': ('pygments.lexers.web', 'Duel', ('duel', 'Duel Engine', 'Duel View', 'JBST', 'jbst', 'JsonML+BST'), ('*.duel', '*.jbst'), ('text/x-duel', 'text/x-jbst')), 'DylanLexer': ('pygments.lexers.compiled', 'Dylan', ('dylan',), ('*.dylan', '*.dyl'), ('text/x-dylan',)), + 'ECLLexer': ('pygments.lexers.other', 'ECL', ('ecl',), ('*.ecl',), ('application/x-ecl',)), 'ECLexer': ('pygments.lexers.compiled', 'eC', ('ec',), ('*.ec', '*.eh'), ('text/x-echdr', 'text/x-ecsrc')), 'ErbLexer': ('pygments.lexers.templates', 'ERB', ('erb',), (), ('application/x-ruby-templating',)), 'ErlangLexer': ('pygments.lexers.functional', 'Erlang', ('erlang',), ('*.erl', '*.hrl'), ('text/x-erlang',)), @@ -108,17 +109,19 @@ LEXERS = { 'HtmlLexer': ('pygments.lexers.web', 'HTML', ('html',), ('*.html', '*.htm', '*.xhtml', '*.xslt'), ('text/html', 'application/xhtml+xml')), 'HtmlPhpLexer': ('pygments.lexers.templates', 'HTML+PHP', ('html+php',), ('*.phtml',), ('application/x-php', 'application/x-httpd-php', 'application/x-httpd-php3', 'application/x-httpd-php4', 'application/x-httpd-php5')), 'HtmlSmartyLexer': ('pygments.lexers.templates', 'HTML+Smarty', ('html+smarty',), (), ('text/html+smarty',)), + 'HttpLexer': ('pygments.lexers.other', 'HTTP', ('http',), (), ()), 'HybrisLexer': ('pygments.lexers.other', 'Hybris', ('hybris', 'hy'), ('*.hy', '*.hyb'), ('text/x-hybris', 'application/x-hybris')), 'IniLexer': ('pygments.lexers.text', 'INI', ('ini', 'cfg'), ('*.ini', '*.cfg'), ('text/x-ini',)), 'IoLexer': ('pygments.lexers.agile', 'Io', ('io',), ('*.io',), ('text/x-iosrc',)), 'IokeLexer': ('pygments.lexers.agile', 'Ioke', ('ioke', 'ik'), ('*.ik',), ('text/x-iokesrc',)), 'IrcLogsLexer': ('pygments.lexers.text', 'IRC logs', ('irc',), ('*.weechatlog',), ('text/x-irclog',)), + 'JSONLexer': ('pygments.lexers.web', 'JSON', ('json',), ('*.json',), ('application/json',)), 'JadeLexer': ('pygments.lexers.web', 'Jade', ('jade', 'JADE'), ('*.jade',), ('text/x-jade',)), 'JavaLexer': ('pygments.lexers.compiled', 'Java', ('java',), ('*.java',), ('text/x-java',)), 'JavascriptDjangoLexer': ('pygments.lexers.templates', 'JavaScript+Django/Jinja', ('js+django', 'javascript+django', 'js+jinja', 'javascript+jinja'), (), ('application/x-javascript+django', 'application/x-javascript+jinja', 'text/x-javascript+django', 'text/x-javascript+jinja', 'text/javascript+django', 'text/javascript+jinja')), 'JavascriptErbLexer': ('pygments.lexers.templates', 'JavaScript+Ruby', ('js+erb', 'javascript+erb', 'js+ruby', 'javascript+ruby'), (), ('application/x-javascript+ruby', 'text/x-javascript+ruby', 'text/javascript+ruby')), 'JavascriptGenshiLexer': ('pygments.lexers.templates', 'JavaScript+Genshi Text', ('js+genshitext', 'js+genshi', 'javascript+genshitext', 'javascript+genshi'), (), ('application/x-javascript+genshi', 'text/x-javascript+genshi', 'text/javascript+genshi')), - 'JavascriptLexer': ('pygments.lexers.web', 'JavaScript', ('js', 'javascript'), ('*.js', '*.json'), ('application/javascript', 'application/x-javascript', 'text/x-javascript', 'text/javascript', 'application/json')), + 'JavascriptLexer': ('pygments.lexers.web', 'JavaScript', ('js', 'javascript'), ('*.js',), ('application/javascript', 'application/x-javascript', 'text/x-javascript', 'text/javascript')), 'JavascriptPhpLexer': ('pygments.lexers.templates', 'JavaScript+PHP', ('js+php', 'javascript+php'), (), ('application/x-javascript+php', 'text/x-javascript+php', 'text/javascript+php')), 'JavascriptSmartyLexer': ('pygments.lexers.templates', 'JavaScript+Smarty', ('js+smarty', 'javascript+smarty'), (), ('application/x-javascript+smarty', 'text/x-javascript+smarty', 'text/javascript+smarty')), 'JspLexer': ('pygments.lexers.templates', 'Java Server Page', ('jsp',), ('*.jsp',), ('application/x-jsp',)), @@ -142,6 +145,7 @@ LEXERS = { 'ModelicaLexer': ('pygments.lexers.other', 'Modelica', ('modelica',), ('*.mo',), ('text/x-modelica',)), 'Modula2Lexer': ('pygments.lexers.compiled', 'Modula-2', ('modula2', 'm2'), ('*.def', '*.mod'), ('text/x-modula2',)), 'MoinWikiLexer': ('pygments.lexers.text', 'MoinMoin/Trac Wiki markup', ('trac-wiki', 'moin'), (), ('text/x-trac-wiki',)), + 'MoonScriptLexer': ('pygments.lexers.agile', 'MoonScript', ('moon', 'moonscript'), ('*.moon',), ('text/x-moonscript', 'application/x-moonscript')), 'MuPADLexer': ('pygments.lexers.math', 'MuPAD', ('mupad',), ('*.mu',), ()), 'MxmlLexer': ('pygments.lexers.web', 'MXML', ('mxml',), ('*.mxml',), ()), 'MySqlLexer': ('pygments.lexers.other', 'MySQL', ('mysql',), (), ('text/x-mysql',)), @@ -163,6 +167,8 @@ LEXERS = { 'OcamlLexer': ('pygments.lexers.functional', 'OCaml', ('ocaml',), ('*.ml', '*.mli', '*.mll', '*.mly'), ('text/x-ocaml',)), 'OctaveLexer': ('pygments.lexers.math', 'Octave', ('octave',), ('*.m',), ('text/octave',)), 'OocLexer': ('pygments.lexers.compiled', 'Ooc', ('ooc',), ('*.ooc',), ('text/x-ooc',)), + 'OpaLexer': ('pygments.lexers.functional', 'Opa', ('opa',), ('*.opa',), ('text/x-opa',)), + 'OpenEdgeLexer': ('pygments.lexers.other', 'OpenEdge ABL', ('openedge', 'abl', 'progress'), ('*.p', '*.cls'), ('text/x-openedge', 'application/x-openedge')), 'PerlLexer': ('pygments.lexers.agile', 'Perl', ('perl', 'pl'), ('*.pl', '*.pm'), ('text/x-perl', 'application/x-perl')), 'PhpLexer': ('pygments.lexers.web', 'PHP', ('php', 'php3', 'php4', 'php5'), ('*.php', '*.php[345]'), ('text/x-php',)), 'PlPgsqlLexer': ('pygments.lexers.postgres', 'PL/pgSQL', ('plpgsql',), (), ('text/x-plpgsql',)), @@ -204,6 +210,7 @@ LEXERS = { 'ScssLexer': ('pygments.lexers.web', 'SCSS', ('scss',), ('*.scss',), ('text/x-scss',)), 'SmalltalkLexer': ('pygments.lexers.other', 'Smalltalk', ('smalltalk', 'squeak'), ('*.st',), ('text/x-smalltalk',)), 'SmartyLexer': ('pygments.lexers.templates', 'Smarty', ('smarty',), ('*.tpl',), ('application/x-smarty',)), + 'SnobolLexer': ('pygments.lexers.other', 'Snobol', ('snobol',), ('*.snobol',), ('text/x-snobol',)), 'SourcesListLexer': ('pygments.lexers.text', 'Debian Sourcelist', ('sourceslist', 'sources.list'), ('sources.list',), ()), 'SqlLexer': ('pygments.lexers.other', 'SQL', ('sql',), ('*.sql',), ('text/x-sql',)), 'SqliteConsoleLexer': ('pygments.lexers.other', 'sqlite3con', ('sqlite3',), ('*.sqlite3-console',), ('text/x-sqlite3-console',)), @@ -214,6 +221,7 @@ LEXERS = { 'TcshLexer': ('pygments.lexers.other', 'Tcsh', ('tcsh', 'csh'), ('*.tcsh', '*.csh'), ('application/x-csh',)), 'TexLexer': ('pygments.lexers.text', 'TeX', ('tex', 'latex'), ('*.tex', '*.aux', '*.toc'), ('text/x-tex', 'text/x-latex')), 'TextLexer': ('pygments.lexers.special', 'Text only', ('text',), ('*.txt',), ('text/plain',)), + 'UrbiscriptLexer': ('pygments.lexers.other', 'UrbiScript', ('urbiscript',), ('*.u',), ('application/x-urbiscript',)), 'ValaLexer': ('pygments.lexers.compiled', 'Vala', ('vala', 'vapi'), ('*.vala', '*.vapi'), ('text/x-vala',)), 'VbNetAspxLexer': ('pygments.lexers.dotnet', 'aspx-vb', ('aspx-vb',), ('*.aspx', '*.asax', '*.ascx', '*.ashx', '*.asmx', '*.axd'), ()), 'VbNetLexer': ('pygments.lexers.dotnet', 'VB.net', ('vb.net', 'vbnet'), ('*.vb', '*.bas'), ('text/x-vbnet', 'text/x-vba')), diff --git a/pygments/lexers/agile.py b/pygments/lexers/agile.py index a2a54d6c..80c1bde7 100644 --- a/pygments/lexers/agile.py +++ b/pygments/lexers/agile.py @@ -21,9 +21,9 @@ from pygments import unistring as uni __all__ = ['PythonLexer', 'PythonConsoleLexer', 'PythonTracebackLexer', 'RubyLexer', 'RubyConsoleLexer', 'PerlLexer', 'LuaLexer', - 'MiniDLexer', 'IoLexer', 'TclLexer', 'ClojureLexer', - 'Python3Lexer', 'Python3TracebackLexer', 'FactorLexer', - 'IokeLexer', 'FancyLexer', 'GroovyLexer'] + 'MoonScriptLexer', 'MiniDLexer', 'IoLexer', 'TclLexer', + 'ClojureLexer', 'Python3Lexer', 'Python3TracebackLexer', + 'FactorLexer', 'IokeLexer', 'FancyLexer', 'GroovyLexer'] # b/w compatibility from pygments.lexers.functional import SchemeLexer @@ -105,12 +105,12 @@ class PythonLexer(RegexLexer): r'WindowsError|ZeroDivisionError)\b', Name.Exception), ], 'numbers': [ - (r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?', Number.Float), - (r'\d+[eE][+-]?[0-9]+', Number.Float), - (r'0[0-7]+', Number.Oct), + (r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?j?', Number.Float), + (r'\d+[eE][+-]?[0-9]+j?', Number.Float), + (r'0[0-7]+j?', Number.Oct), (r'0[xX][a-fA-F0-9]+', Number.Hex), (r'\d+L', Number.Integer.Long), - (r'\d+', Number.Integer) + (r'\d+j?', Number.Integer) ], 'backtick': [ ('`.*?`', String.Backtick), @@ -199,7 +199,7 @@ class Python3Lexer(RegexLexer): tokens = PythonLexer.tokens.copy() tokens['keywords'] = [ (r'(assert|break|continue|del|elif|else|except|' - r'finally|for|global|if|lambda|pass|raise|' + r'finally|for|global|if|lambda|pass|raise|nonlocal|' r'return|try|while|yield|as|with|True|False|None)\b', Keyword), ] tokens['builtins'] = [ @@ -1053,7 +1053,7 @@ class LuaLexer(RegexLexer): (r'(local)\b', Keyword.Declaration), (r'(true|false|nil)\b', Keyword.Constant), - (r'(function)(\s+)', bygroups(Keyword, Text), 'funcname'), + (r'(function)\b', Keyword, 'funcname'), (r'[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)?', Name), @@ -1062,6 +1062,7 @@ class LuaLexer(RegexLexer): ], 'funcname': [ + (r'\s+', Text), ('(?:([A-Za-z_][A-Za-z0-9_]*)(\.))?([A-Za-z_][A-Za-z0-9_]*)', bygroups(Name.Class, Punctuation, Name.Function), '#pop'), # inline function @@ -1118,6 +1119,73 @@ class LuaLexer(RegexLexer): yield index, token, value +class MoonScriptLexer(LuaLexer): + """ + For `MoonScript <http://moonscript.org.org>`_ source code. + + *New in Pygments 1.5.* + """ + + name = "MoonScript" + aliases = ["moon", "moonscript"] + filenames = ["*.moon"] + mimetypes = ['text/x-moonscript', 'application/x-moonscript'] + + tokens = { + 'root': [ + (r'#!(.*?)$', Comment.Preproc), + (r'', Text, 'base'), + ], + 'base': [ + ('--.*$', Comment.Single), + (r'(?i)(\d*\.\d+|\d+\.\d*)(e[+-]?\d+)?', Number.Float), + (r'(?i)\d+e[+-]?\d+', Number.Float), + (r'(?i)0x[0-9a-f]*', Number.Hex), + (r'\d+', Number.Integer), + (r'\n', Text), + (r'[^\S\n]+', Text), + (r'(?s)\[(=*)\[.*?\]\1\]', String), + (r'(->|=>)', Name.Function), + (r':[a-zA-Z_][a-zA-Z0-9_]*', Name.Variable), + (r'(==|!=|~=|<=|>=|\.\.|\.\.\.|[=+\-*/%^<>#!.\\:])', Operator), + (r'[;,]', Punctuation), + (r'[\[\]\{\}\(\)]', Keyword.Type), + (r"(class|extends|if|then|super|do|with|import|export|" + r"while|elseif|return|for|in|from|when|using|else|" + r"and|or|not|switch|break)\b", Keyword), + (r'(true|false|nil)\b', Keyword.Constant), + (r'(and|or|not)\b', Operator.Word), + (r'(self)\b', Name.Builtin.Pseudo), + (r'[a-zA-Z_][a-zA-Z0-9_]*:', Name.Variable), + (r'@[a-zA-Z_][a-zA-Z0-9_]*', Name.Variable.Class), + (r'[A-Z]\w+', Name.Class), # proper name + (r'[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)?', Name), + ("'", String.Single, combined('stringescape', 'sqs')), + ('"', String.Double, combined('stringescape', 'dqs')) + ], + 'stringescape': [ + (r'''\\([abfnrtv\\"']|\d{1,3})''', String.Escape) + ], + 'sqs': [ + ("'", String.Single, '#pop'), + (".", String) + ], + 'dqs': [ + ('"', String.Double, '#pop'), + (".", String) + ] + } + + def get_tokens_unprocessed(self, text): + # set . as Operator instead of Punctuation + for index, token, value in \ + LuaLexer.get_tokens_unprocessed(self, text): + if token == Punctuation and value == ".": + token = Operator + yield index, token, value + + + class MiniDLexer(RegexLexer): """ For `MiniD <http://www.dsource.org/projects/minid>`_ (a D-like scripting diff --git a/pygments/lexers/compiled.py b/pygments/lexers/compiled.py index 5da2b875..af153a54 100644 --- a/pygments/lexers/compiled.py +++ b/pygments/lexers/compiled.py @@ -1232,8 +1232,10 @@ class DylanLexer(RegexLexer): (r'\$[a-zA-Z0-9-]+', Name.Constant), (r'[!$%&*/:<>=?~^.+\[\]{}-]+', Operator), (r'\s+', Text), + (r'#"[a-zA-Z0-9-]+"', Keyword), (r'#[a-zA-Z0-9-]+', Keyword), - (r'[a-zA-Z0-9-]+', Name.Variable), + (r'#(\(|\[)', Punctuation), + (r'[a-zA-Z0-9-_]+', Name.Variable), ], 'string': [ (r'"', String, '#pop'), @@ -2296,7 +2298,7 @@ class AdaLexer(RegexLexer): include('numbers'), (r"'[^']'", String.Character), (r'([a-z0-9_]+)(\s*|[(,])', bygroups(Name, using(this))), - (r"(<>|=>|:=|[\(\)\|:;,.'])", Punctuation), + (r"(<>|=>|:=|[()|:;,.'])", Punctuation), (r'[*<>+=/&-]', Operator), (r'\n+', Text), ], @@ -2317,7 +2319,7 @@ class AdaLexer(RegexLexer): ], 'end' : [ ('(if|case|record|loop|select)', Keyword.Reserved), - ('"[^"]+"|[a-zA-Z0-9_]+', Name.Function), + ('"[^"]+"|[a-zA-Z0-9_.]+', Name.Function), ('[\n\s]+', Text), (';', Punctuation, '#pop'), ], @@ -2337,11 +2339,12 @@ class AdaLexer(RegexLexer): ], 'import': [ (r'[a-z0-9_.]+', Name.Namespace, '#pop'), + (r'', Text, '#pop'), ], 'formal_part' : [ (r'\)', Punctuation, '#pop'), - (r'([a-z0-9_]+)(\s*)(,|:[^=])', bygroups(Name.Variable, - Text, Punctuation)), + (r'[a-z0-9_]+', Name.Variable), + (r',|:[^=]', Punctuation), (r'(in|not|null|out|access)\b', Keyword.Reserved), include('root'), ], diff --git a/pygments/lexers/dotnet.py b/pygments/lexers/dotnet.py index f78df352..20f645e1 100644 --- a/pygments/lexers/dotnet.py +++ b/pygments/lexers/dotnet.py @@ -92,7 +92,7 @@ class CSharpLexer(RegexLexer): (r'\n', Text), (r'[~!%^&*()+=|\[\]:;,.<>/?-]', Punctuation), (r'[{}]', Punctuation), - (r'@"(\\\\|\\"|[^"])*"', String), + (r'@"(""|[^"])*"', String), (r'"(\\\\|\\"|[^"\n])*["\n]', String), (r"'\\.'|'[^\\]'", String.Char), (r"[0-9](\.[0-9]*)?([eE][+-][0-9]+)?" @@ -111,10 +111,12 @@ class CSharpLexer(RegexLexer): r'ref|return|sealed|sizeof|stackalloc|static|' r'switch|this|throw|true|try|typeof|' r'unchecked|unsafe|virtual|void|while|' - r'get|set|new|partial|yield|add|remove|value)\b', Keyword), + r'get|set|new|partial|yield|add|remove|value|alias|ascending|' + r'descending|from|group|into|orderby|select|where|' + r'join|equals)\b', Keyword), (r'(global)(::)', bygroups(Keyword, Punctuation)), - (r'(bool|byte|char|decimal|double|float|int|long|object|sbyte|' - r'short|string|uint|ulong|ushort)\b\??', Keyword.Type), + (r'(bool|byte|char|decimal|double|dynamic|float|int|long|object|' + r'sbyte|short|string|uint|ulong|ushort|var)\b\??', Keyword.Type), (r'(class|struct)(\s+)', bygroups(Keyword, Text), 'class'), (r'(namespace|using)(\s+)', bygroups(Keyword, Text), 'namespace'), (cs_ident, Name), @@ -214,7 +216,7 @@ class NemerleLexer(RegexLexer): (r'[~!%^&*()+=|\[\]:;,.<>/?-]', Punctuation), (r'[{}]', Punctuation), - (r'@"(\\\\|\\"|[^"])*"', String), + (r'@"(""|[^"])*"', String), (r'"(\\\\|\\"|[^"\n])*["\n]', String), (r"'\\.'|'[^\\]'", String.Char), (r"0[xX][0-9a-fA-F]+[Ll]?", Number), @@ -278,8 +280,9 @@ class NemerleLexer(RegexLexer): ], 'splice-string-content': [ (r'if|match', Keyword), - (r'[~!%^&*+=|\[\]:;,.<>/?-]', Punctuation), + (r'[~!%^&*+=|\[\]:;,.<>/?-\\"$ ]', Punctuation), (cs_ident, Name), + (r'\d+', Number), (r'\(', Punctuation, '#push'), (r'\)', Punctuation, '#pop') ] @@ -581,7 +584,7 @@ class FSharpLexer(RegexLexer): Name.Namespace, 'dotted'), (r'\b([A-Z][A-Za-z0-9_\']*)', Name.Class), (r'//.*?\n', Comment.Single), - (r'\(\*', Comment, 'comment'), + (r'\(\*(?!\))', Comment, 'comment'), (r'\b(%s)\b' % '|'.join(keywords), Keyword), (r'(%s)' % '|'.join(keyopts), Operator), (r'(%s|%s)?%s' % (infix_syms, prefix_syms, operators), Operator), diff --git a/pygments/lexers/functional.py b/pygments/lexers/functional.py index a8102cc8..c6f5421f 100644 --- a/pygments/lexers/functional.py +++ b/pygments/lexers/functional.py @@ -14,11 +14,11 @@ import re from pygments.lexer import Lexer, RegexLexer, bygroups, include, do_insertions from pygments.token import Text, Comment, Operator, Keyword, Name, \ String, Number, Punctuation, Literal, Generic, Error - +import pygments.formatters __all__ = ['SchemeLexer', 'CommonLispLexer', 'HaskellLexer', 'LiterateHaskellLexer', 'SMLLexer', 'OcamlLexer', 'ErlangLexer', - 'ErlangShellLexer'] + 'ErlangShellLexer', 'OpaLexer'] class SchemeLexer(RegexLexer): @@ -1079,3 +1079,323 @@ class ErlangShellLexer(Lexer): erlexer.get_tokens_unprocessed(curcode)): yield item + +class OpaLexer(RegexLexer): + """ + Lexer for the Opa language (http://opalang.org). + + *New in Pygments 1.5.* + """ + + name = 'Opa' + aliases = ['opa'] + filenames = ['*.opa'] + mimetypes = ['text/x-opa'] + + # most of these aren't strictly keywords + # but if you color only real keywords, you might just + # as well not color anything + keywords = [ + 'and', 'as', 'begin', 'css', 'database', 'db', 'do', 'else', 'end', + 'external', 'forall', 'if', 'import', 'match', 'package', 'parser', + 'rec', 'server', 'then', 'type', 'val', 'with', 'xml_parser' + ] + + # matches both stuff and `stuff` + ident_re = r'(([a-zA-Z_]\w*)|(`[^`]*`))' + + op_re = r'[.=\-<>,@~%/+?*&^!]' + punc_re = r'[()\[\],;|]' # '{' and '}' are treated elsewhere + # because they are also used for inserts + + tokens = { + # copied from the caml lexer, should be adapted + 'escape-sequence': [ + (r'\\[\\\"\'ntr}]', String.Escape), + (r'\\[0-9]{3}', String.Escape), + (r'\\x[0-9a-fA-F]{2}', String.Escape), + ], + + # factorizing these rules, because they are inserted many times + 'comments': [ + (r'/\*', Comment, 'nested-comment'), + (r'//.*?$', Comment), + ], + 'comments-and-spaces': [ + include('comments'), + (r'\s+', Text), + ], + + 'root': [ + include('comments-and-spaces'), + # keywords + (r'\b(%s)\b' % '|'.join(keywords), Keyword), + # directives + # we could parse the actual set of directives instead of anything + # starting with @, but this is troublesome + # because it needs to be adjusted all the time + # and assuming we parse only sources that compile, it is useless + (r'@'+ident_re+r'\b', Name.Builtin.Pseudo), + + # number literals + (r'-?.[\d]+([eE][+\-]?\d+)', Number.Float), + (r'-?\d+.\d*([eE][+\-]?\d+)', Number.Float), + (r'-?\d+[eE][+\-]?\d+', Number.Float), + (r'0[xX][\da-fA-F]+', Number.Hex), + (r'0[oO][0-7]+', Number.Oct), + (r'0[bB][01]+', Number.Binary), + (r'\d+', Number.Integer), + # color literals + (r'#[\da-fA-F]{3,6}', Number.Integer), + + # string literals + (r'"', String.Double, 'string'), + # char literal, should be checked because this is the regexp from + # the caml lexer + (r"'(?:(\\[\\\"'ntbr ])|(\\[0-9]{3})|(\\x[0-9a-fA-F]{2})|.)'", + String.Char), + + # this is meant to deal with embedded exprs in strings + # every time we find a '}' we pop a state so that if we were + # inside a string, we are back in the string state + # as a consequence, we must also push a state every time we find a + # '{' or else we will have errors when parsing {} for instance + (r'{', Operator, '#push'), + (r'}', Operator, '#pop'), + + # html literals + # this is a much more strict that the actual parser, + # since a<b would not be parsed as html + # but then again, the parser is way too lax, and we can't hope + # to have something as tolerant + (r'<(?=[a-zA-Z>])', String.Single, 'html-open-tag'), + + # db path + # matching the '[_]' in '/a[_]' because it is a part + # of the syntax of the db path definition + # unfortunately, i don't know how to match the ']' in + # /a[1], so this is somewhat inconsistent + (r'[@?!]?(/\w+)+(\[_\])?', Name.Variable), + # putting the same color on <- as on db path, since + # it can be used only to mean Db.write + (r'<-(?!'+op_re+r')', Name.Variable), + + # 'modules' + # although modules are not distinguished by their names as in caml + # the standard library seems to follow the convention that modules + # only area capitalized + (r'\b([A-Z]\w*)(?=\.)', Name.Namespace), + + # operators + # = has a special role because this is the only + # way to syntactic distinguish binding constructions + # unfortunately, this colors the equal in {x=2} too + (r'=(?!'+op_re+r')', Keyword), + (r'(%s)+' % op_re, Operator), + (r'(%s)+' % punc_re, Operator), + + # coercions + (r':', Operator, 'type'), + # type variables + # we need this rule because we don't parse specially type + # definitions so in "type t('a) = ...", "'a" is parsed by 'root' + ("'"+ident_re, Keyword.Type), + + # id literal, #something, or #{expr} + (r'#'+ident_re, String.Single), + (r'#(?={)', String.Single), + + # identifiers + # this avoids to color '2' in 'a2' as an integer + (ident_re, Text), + + # default, not sure if that is needed or not + # (r'.', Text), + ], + + # it is quite painful to have to parse types to know where they end + # this is the general rule for a type + # a type is either: + # * -> ty + # * type-with-slash + # * type-with-slash -> ty + # * type-with-slash (, type-with-slash)+ -> ty + # + # the code is pretty funky in here, but this code would roughly + # translate in caml to: + # let rec type stream = + # match stream with + # | [< "->"; stream >] -> type stream + # | [< ""; stream >] -> + # type_with_slash stream + # type_lhs_1 stream; + # and type_1 stream = ... + 'type': [ + include('comments-and-spaces'), + (r'->', Keyword.Type), + (r'', Keyword.Type, ('#pop', 'type-lhs-1', 'type-with-slash')), + ], + + # parses all the atomic or closed constructions in the syntax of type + # expressions: record types, tuple types, type constructors, basic type + # and type variables + 'type-1': [ + include('comments-and-spaces'), + (r'\(', Keyword.Type, ('#pop', 'type-tuple')), + (r'~?{', Keyword.Type, ('#pop', 'type-record')), + (ident_re+r'\(', Keyword.Type, ('#pop', 'type-tuple')), + (ident_re, Keyword.Type, '#pop'), + ("'"+ident_re, Keyword.Type), + # this case is not in the syntax but sometimes + # we think we are parsing types when in fact we are parsing + # some css, so we just pop the states until we get back into + # the root state + (r'', Keyword.Type, '#pop'), + ], + + # type-with-slash is either: + # * type-1 + # * type-1 (/ type-1)+ + 'type-with-slash': [ + include('comments-and-spaces'), + (r'', Keyword.Type, ('#pop', 'slash-type-1', 'type-1')), + ], + 'slash-type-1': [ + include('comments-and-spaces'), + ('/', Keyword.Type, ('#pop', 'type-1')), + # same remark as above + (r'', Keyword.Type, '#pop'), + ], + + # we go in this state after having parsed a type-with-slash + # while trying to parse a type + # and at this point we must determine if we are parsing an arrow + # type (in which case we must continue parsing) or not (in which + # case we stop) + 'type-lhs-1': [ + include('comments-and-spaces'), + (r'->', Keyword.Type, ('#pop', 'type')), + (r'(?=,)', Keyword.Type, ('#pop', 'type-arrow')), + (r'', Keyword.Type, '#pop'), + ], + 'type-arrow': [ + include('comments-and-spaces'), + # the look ahead here allows to parse f(x : int, y : float -> truc) + # correctly + (r',(?=[^:]*?->)', Keyword.Type, 'type-with-slash'), + (r'->', Keyword.Type, ('#pop', 'type')), + # same remark as above + (r'', Keyword.Type, '#pop'), + ], + + # no need to do precise parsing for tuples and records + # because they are closed constructions, so we can simply + # find the closing delimiter + # note that this function would be not work if the source + # contained identifiers like `{)` (although it could be patched + # to support it) + 'type-tuple': [ + include('comments-and-spaces'), + (r'[^\(\)/*]+', Keyword.Type), + (r'[/*]', Keyword.Type), + (r'\(', Keyword.Type, '#push'), + (r'\)', Keyword.Type, '#pop'), + ], + 'type-record': [ + include('comments-and-spaces'), + (r'[^{}/*]+', Keyword.Type), + (r'[/*]', Keyword.Type), + (r'{', Keyword.Type, '#push'), + (r'}', Keyword.Type, '#pop'), + ], + +# 'type-tuple': [ +# include('comments-and-spaces'), +# (r'\)', Keyword.Type, '#pop'), +# (r'', Keyword.Type, ('#pop', 'type-tuple-1', 'type-1')), +# ], +# 'type-tuple-1': [ +# include('comments-and-spaces'), +# (r',?\s*\)', Keyword.Type, '#pop'), # ,) is a valid end of tuple, in (1,) +# (r',', Keyword.Type, 'type-1'), +# ], +# 'type-record':[ +# include('comments-and-spaces'), +# (r'}', Keyword.Type, '#pop'), +# (r'~?(?:\w+|`[^`]*`)', Keyword.Type, 'type-record-field-expr'), +# ], +# 'type-record-field-expr': [ +# +# ], + + 'nested-comment': [ + (r'[^/*]+', Comment), + (r'/\*', Comment, '#push'), + (r'\*/', Comment, '#pop'), + (r'[/*]', Comment), + ], + + # the coy pasting between string and single-string + # is kinda sad. Is there a way to avoid that?? + 'string': [ + (r'[^\\"{]+', String.Double), + (r'"', String.Double, '#pop'), + (r'{', Operator, 'root'), + include('escape-sequence'), + ], + 'single-string': [ + (r'[^\\\'{]+', String.Double), + (r'\'', String.Double, '#pop'), + (r'{', Operator, 'root'), + include('escape-sequence'), + ], + + # all the html stuff + # can't really reuse some existing html parser + # because we must be able to parse embedded expressions + + # we are in this state after someone parsed the '<' that + # started the html literal + 'html-open-tag': [ + (r'[\w\-:]+', String.Single, ('#pop', 'html-attr')), + (r'>', String.Single, ('#pop', 'html-content')), + ], + + # we are in this state after someone parsed the '</' that + # started the end of the closing tag + 'html-end-tag': [ + # this is a star, because </> is allowed + (r'[\w\-:]*>', String.Single, '#pop'), + ], + + # we are in this state after having parsed '<ident(:ident)?' + # we thus parse a possibly empty list of attributes + 'html-attr': [ + (r'\s+', Text), + (r'[\w\-:]+=', String.Single, 'html-attr-value'), + (r'/>', String.Single, '#pop'), + (r'>', String.Single, ('#pop', 'html-content')), + ], + + 'html-attr-value': [ + (r"'", String.Single, ('#pop', 'single-string')), + (r'"', String.Single, ('#pop', 'string')), + (r'#'+ident_re, String.Single, '#pop'), + (r'#(?={)', String.Single, ('#pop', 'root')), + (r'{', Operator, ('#pop', 'root')), # this is a tail call! + ], + + # we should probably deal with '\' escapes here + 'html-content': [ + (r'<!--', Comment, 'html-comment'), + (r'</', String.Single, ('#pop', 'html-end-tag')), + (r'<', String.Single, 'html-open-tag'), + (r'{', Operator, 'root'), + (r'.|\s+', String.Single), + ], + + 'html-comment': [ + (r'-->', Comment, '#pop'), + (r'[^\-]+|-', Comment), + ], + } diff --git a/pygments/lexers/other.py b/pygments/lexers/other.py index e73154aa..f85c6427 100644 --- a/pygments/lexers/other.py +++ b/pygments/lexers/other.py @@ -12,10 +12,10 @@ import re from pygments.lexer import Lexer, RegexLexer, include, bygroups, using, \ - this, do_insertions, combined + this, do_insertions, combined, ExtendedRegexLexer from pygments.token import Error, Punctuation, Literal, Token, \ Text, Comment, Operator, Keyword, Name, String, Number, Generic -from pygments.util import shebang_matches +from pygments.util import ClassNotFound, shebang_matches from pygments.lexers.web import HtmlLexer @@ -27,7 +27,8 @@ __all__ = ['SqlLexer', 'MySqlLexer', 'SqliteConsoleLexer', 'BrainfuckLexer', 'NewspeakLexer', 'GherkinLexer', 'AsymptoteLexer', 'PostScriptLexer', 'AutohotkeyLexer', 'GoodDataCLLexer', 'MaqlLexer', 'ProtoBufLexer', 'HybrisLexer', 'AwkLexer', - 'Cfengine3Lexer'] + 'Cfengine3Lexer', 'HttpLexer', 'SnobolLexer', 'ECLLexer', + 'UrbiscriptLexer', 'OpenEdgeLexer'] line_re = re.compile('.*?\n') @@ -219,6 +220,107 @@ class MySqlLexer(RegexLexer): } +class ECLLexer(RegexLexer): + """ + Lexer for the declarative big-data `ECL + <http://hpccsystems.com/community/docs/ecl-language-reference/html>`_ + language. + + *New in Pygments 1.5.* + """ + + name = 'ECL' + aliases = ['ecl'] + filenames = ['*.ecl'] + mimetypes = ['application/x-ecl'] + + flags = re.IGNORECASE | re.MULTILINE + + tokens = { + 'root': [ + include('whitespace'), + include('statements'), + ], + 'whitespace': [ + (r'\s+', Text), + (r'\/\/.*', Comment.Single), + (r'/(\\\n)?[*](.|\n)*?[*](\\\n)?/', Comment.Multiline), + ], + 'statements': [ + include('types'), + include('keywords'), + include('functions'), + include('hash'), + (r'"', String, 'string'), + (r'\'', String, 'string'), + (r'(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+[LlUu]*', Number.Float), + (r'(\d+\.\d*|\.\d+|\d+[fF])[fF]?', Number.Float), + (r'0x[0-9a-fA-F]+[LlUu]*', Number.Hex), + (r'0[0-7]+[LlUu]*', Number.Oct), + (r'\d+[LlUu]*', Number.Integer), + (r'\*/', Error), + (r'[~!%^&*+=|?:<>/-]+', Operator), + (r'[{}()\[\],.;]', Punctuation), + (r'[a-zA-Z_][a-zA-Z0-9_]*', Name), + ], + 'hash': [ + (r'^#.*$', Comment.Preproc), + ], + 'types': [ + (r'(RECORD|END)[^\d]', Keyword.Declaration), + (r'((?:ASCII|BIG_ENDIAN|BOOLEAN|DATA|DECIMAL|EBCDIC|INTEGER|PATTERN|' + r'QSTRING|REAL|RECORD|RULE|SET OF|STRING|TOKEN|UDECIMAL|UNICODE|' + r'UNSIGNED|VARSTRING|VARUNICODE)\d*)(\s+)', + bygroups(Keyword.Type, Text)), + ], + 'keywords': [ + (r'(APPLY|ASSERT|BUILD|BUILDINDEX|EVALUATE|FAIL|KEYDIFF|KEYPATCH|' + r'LOADXML|NOTHOR|NOTIFY|OUTPUT|PARALLEL|SEQUENTIAL|SOAPCALL|WAIT' + r'CHECKPOINT|DEPRECATED|FAILCODE|FAILMESSAGE|FAILURE|GLOBAL|' + r'INDEPENDENT|ONWARNING|PERSIST|PRIORITY|RECOVERY|STORED|SUCCESS|' + r'WAIT|WHEN)\b', Keyword.Reserved), + # These are classed differently, check later + (r'(ALL|AND|ANY|AS|ATMOST|BEFORE|BEGINC\+\+|BEST|BETWEEN|CASE|CONST|' + r'COUNTER|CSV|DESCEND|ENCRYPT|ENDC\+\+|ENDMACRO|EXCEPT|EXCLUSIVE|' + r'EXPIRE|EXPORT|EXTEND|FALSE|FEW|FIRST|FLAT|FULL|FUNCTION|GROUP|' + r'HEADER|HEADING|HOLE|IFBLOCK|IMPORT|IN|JOINED|KEEP|KEYED|LAST|' + r'LEFT|LIMIT|LOAD|LOCAL|LOCALE|LOOKUP|MACRO|MANY|MAXCOUNT|' + r'MAXLENGTH|MIN SKEW|MODULE|INTERFACE|NAMED|NOCASE|NOROOT|NOSCAN|' + r'NOSORT|NOT|OF|ONLY|OPT|OR|OUTER|OVERWRITE|PACKED|PARTITION|' + r'PENALTY|PHYSICALLENGTH|PIPE|QUOTE|RELATIONSHIP|REPEAT|RETURN|' + r'RIGHT|SCAN|SELF|SEPARATOR|SERVICE|SHARED|SKEW|SKIP|SQL|STORE|' + r'TERMINATOR|THOR|THRESHOLD|TOKEN|TRANSFORM|TRIM|TRUE|TYPE|' + r'UNICODEORDER|UNSORTED|VALIDATE|VIRTUAL|WHOLE|WILD|WITHIN|XML|' + r'XPATH|__COMPRESSED__)\b', Keyword.Reserved), + ], + 'functions': [ + (r'(ABS|ACOS|ALLNODES|ASCII|ASIN|ASSTRING|ATAN|ATAN2|AVE|CASE|' + r'CHOOSE|CHOOSEN|CHOOSESETS|CLUSTERSIZE|COMBINE|CORRELATION|COS|' + r'COSH|COUNT|COVARIANCE|CRON|DATASET|DEDUP|DEFINE|DENORMALIZE|' + r'DISTRIBUTE|DISTRIBUTED|DISTRIBUTION|EBCDIC|ENTH|ERROR|EVALUATE|' + r'EVENT|EVENTEXTRA|EVENTNAME|EXISTS|EXP|FAILCODE|FAILMESSAGE|' + r'FETCH|FROMUNICODE|GETISVALID|GLOBAL|GRAPH|GROUP|HASH|HASH32|' + r'HASH64|HASHCRC|HASHMD5|HAVING|IF|INDEX|INTFORMAT|ISVALID|' + r'ITERATE|JOIN|KEYUNICODE|LENGTH|LIBRARY|LIMIT|LN|LOCAL|LOG|LOOP|' + r'MAP|MATCHED|MATCHLENGTH|MATCHPOSITION|MATCHTEXT|MATCHUNICODE|' + r'MAX|MERGE|MERGEJOIN|MIN|NOLOCAL|NONEMPTY|NORMALIZE|PARSE|PIPE|' + r'POWER|PRELOAD|PROCESS|PROJECT|PULL|RANDOM|RANGE|RANK|RANKED|' + r'REALFORMAT|RECORDOF|REGEXFIND|REGEXREPLACE|REGROUP|REJECTED|' + r'ROLLUP|ROUND|ROUNDUP|ROW|ROWDIFF|SAMPLE|SET|SIN|SINH|SIZEOF|' + r'SOAPCALL|SORT|SORTED|SQRT|STEPPED|STORED|SUM|TABLE|TAN|TANH|' + r'THISNODE|TOPN|TOUNICODE|TRANSFER|TRIM|TRUNCATE|TYPEOF|UNGROUP|' + r'UNICODEORDER|VARIANCE|WHICH|WORKUNIT|XMLDECODE|XMLENCODE|' + r'XMLTEXT|XMLUNICODE)\b', Name.Function), + ], + 'string': [ + (r'"', String, '#pop'), + (r'\'', String, '#pop'), + (r'[^"\']+', String), + ], + } + + + class SqliteConsoleLexer(Lexer): """ Lexer for example sessions using sqlite3. @@ -2915,11 +3017,12 @@ class Cfengine3Lexer(RegexLexer): bygroups(Keyword.Reserved,Text,Operator,Text)), (r'"', String, 'string'), (r'(\w+)(\()', bygroups(Name.Function, Punctuation)), - (r'([\w.!&|]+)(::)', bygroups(Name.Class, Punctuation)), + (r'([\w.!&|\(\)]+)(::)', bygroups(Name.Class, Punctuation)), (r'(\w+)(:)', bygroups(Keyword.Declaration,Punctuation)), (r'@[\{\(][^\)\}]+[\}\)]', Name.Variable), (r'[(){},;]', Punctuation), (r'=>', Operator), + (r'->', Operator), (r'\d+\.\d+', Number.Float), (r'\d+', Number.Integer), (r'\w+', Name.Function), @@ -2944,3 +3047,760 @@ class Cfengine3Lexer(RegexLexer): (r'\s+', Text), ], } + + +class HttpLexer(RegexLexer): + """ + Lexer for HTTP sessions. + + *New in Pygments 1.5.* + """ + + name = 'HTTP' + aliases = ['http'] + + flags = re.DOTALL + + def header_callback(self, match): + if match.group(1).lower() == 'content-type': + content_type = match.group(5).strip() + if ';' in content_type: + content_type = content_type[:content_type.find(';')].strip() + self.content_type = content_type + yield match.start(1), Name.Attribute, match.group(1) + yield match.start(2), Text, match.group(2) + yield match.start(3), Operator, match.group(3) + yield match.start(4), Text, match.group(4) + yield match.start(5), Literal, match.group(5) + yield match.start(6), Text, match.group(6) + + def content_callback(self, match): + content_type = getattr(self, 'content_type', None) + content = match.group() + offset = match.start() + if content_type: + from pygments.lexers import get_lexer_for_mimetype + try: + lexer = get_lexer_for_mimetype(content_type) + except ClassNotFound: + pass + else: + for idx, token, value in lexer.get_tokens_unprocessed(content): + yield offset + idx, token, value + return + yield offset, Text, content + + tokens = { + 'root': [ + (r'(GET|POST|PUT|DELETE|HEAD|OPTIONS|TRACE)( +)([^ ]+)( +)' + r'(HTTPS?)(/)(1\.[01])(\r?\n|$)', + bygroups(Name.Function, Text, Name.Namespace, Text, + Keyword.Reserved, Operator, Number, Text), + 'headers'), + (r'(HTTPS?)(/)(1\.[01])( +)(\d{3})( +)([^\r\n]+)(\r?\n|$)', + bygroups(Keyword.Reserved, Operator, Number, Text, Number, + Text, Name.Exception, Text), + 'headers'), + ], + 'headers': [ + (r'([^\s:]+)( *)(:)( *)([^\r\n]+)(\r?\n|$)', header_callback), + (r'\r?\n', Text, 'content') + ], + 'content': [ + (r'.+', content_callback) + ] + } + + +class SnobolLexer(RegexLexer): + """ + Lexer for the SNOBOL4 programming language. + + Recognizes the common ASCII equivalents of the original SNOBOL4 operators. + Does not require spaces around binary operators. + + *New in Pygments 1.5.* + """ + + name = "Snobol" + aliases = ["snobol"] + filenames = ['*.snobol'] + mimetypes = ['text/x-snobol'] + + tokens = { + # root state, start of line + # comments, continuation lines, and directives start in column 1 + # as do labels + 'root': [ + (r'\*.*\n', Comment), + (r'[\+\.] ', Punctuation, 'statement'), + (r'-.*\n', Comment), + (r'END\s*\n', Name.Label, 'heredoc'), + (r'[A-Za-z\$][\w$]*', Name.Label, 'statement'), + (r'\s+', Text, 'statement'), + ], + # statement state, line after continuation or label + 'statement': [ + (r'\s*\n', Text, '#pop'), + (r'\s+', Text), + (r'(?<=[^\w.])(LT|LE|EQ|NE|GE|GT|INTEGER|IDENT|DIFFER|LGT|SIZE|' + r'REPLACE|TRIM|DUPL|REMDR|DATE|TIME|EVAL|APPLY|OPSYN|LOAD|UNLOAD|' + r'LEN|SPAN|BREAK|ANY|NOTANY|TAB|RTAB|REM|POS|RPOS|FAIL|FENCE|' + r'ABORT|ARB|ARBNO|BAL|SUCCEED|INPUT|OUTPUT|TERMINAL)(?=[^\w.])', + Name.Builtin), + (r'[A-Za-z][\w\.]*', Name), + # ASCII equivalents of original operators + # | for the EBCDIC equivalent, ! likewise + # \ for EBCDIC negation + (r'\*\*|[\?\$\.!%\*/#+\-@\|&\\!=]', Operator), + (r'"[^"]*"', String), + (r"'[^']*'", String), + # Accept SPITBOL syntax for real numbers + # as well as Macro SNOBOL4 + (r'[0-9]+(?=[^\.EeDd])', Number.Integer), + (r'[0-9]+(\.[0-9]*)?([EDed][-+]?[0-9]+)?', Number.Float), + # Goto + (r':', Punctuation, 'goto'), + (r'[\(\)<>,;]', Punctuation), + ], + # Goto block + 'goto': [ + (r'\s*\n', Text, "#pop:2"), + (r'\s+', Text), + (r'F|S', Keyword), + (r'(\()([A-Za-z][\w.]*)(\))', + bygroups(Punctuation, Name.Label, Punctuation)) + ], + # everything after the END statement is basically one + # big heredoc. + 'heredoc': [ + (r'.*\n', String.Heredoc) + ] + } + + +class UrbiscriptLexer(ExtendedRegexLexer): + """ + For UrbiScript source code. + + *New in Pygments 1.5.* + """ + + name = 'UrbiScript' + aliases = ['urbiscript'] + filenames = ['*.u'] + mimetypes = ['application/x-urbiscript'] + + flags = re.DOTALL + + ## TODO + # - handle Experimental and deprecated tags with specific tokens + # - handle Angles and Durations with specific tokens + + def blob_callback(lexer, match, ctx): + text_before_blob = match.group(1) + blob_start = match.group(2) + blob_size_str = match.group(3) + blob_size = int(blob_size_str) + yield match.start(), String, text_before_blob + ctx.pos += len(text_before_blob) + + # if blob size doesn't match blob format (example : "\B(2)(aaa)") + # yield blob as a string + if ctx.text[match.end() + blob_size] != ")": + result = "\\B(" + blob_size_str + ")(" + yield match.start(), String, result + ctx.pos += len(result) + return + + # if blob is well formated, yield as Escape + blob_text = blob_start + ctx.text[match.end():match.end()+blob_size] + ")" + yield match.start(), String.Escape, blob_text + ctx.pos = match.end() + blob_size + 1 # +1 is the ending ")" + + tokens = { + 'root': [ + (r'\s+', Text), + # comments + (r'//.*?\n', Comment), + (r'/\*', Comment.Multiline, 'comment'), + (r'(?:every|for|loop|while)(?:;|&|\||,)',Keyword), + (r'(?:assert|at|break|case|catch|closure|compl|continue|' + r'default|else|enum|every|external|finally|for|freezeif|if|new|' + r'onleave|return|stopif|switch|this|throw|timeout|try|' + r'waituntil|whenever|while)\b', Keyword), + (r'(?:asm|auto|bool|char|const_cast|delete|double|dynamic_cast|' + r'explicit|export|extern|float|friend|goto|inline|int|' + r'long|mutable|namespace|register|reinterpret_cast|short|' + r'signed|sizeof|static_cast|struct|template|typedef|typeid|' + r'typename|union|unsigned|using|virtual|volatile|' + r'wchar_t)\b', Keyword.Reserved), + # deprecated keywords, use a meaningfull token when available + (r'(?:emit|foreach|internal|loopn|static)\b', Keyword), + # ignored keywords, use a meaningfull token when available + (r'(?:private|protected|public)\b', Keyword), + (r'(?:var|do|const|function|class)\b', Keyword.Declaration), + (r'(?:true|false|nil|void)\b', Keyword.Constant), + (r'(?:Barrier|Binary|Boolean|CallMessage|Channel|Code|' + r'Comparable|Container|Control|Date|Dictionary|Directory|' + r'Duration|Enumeration|Event|Exception|Executable|File|Finalizable|' + r'Float|FormatInfo|Formatter|Global|Group|Hash|InputStream|' + r'IoService|Job|Kernel|Lazy|List|Loadable|Lobby|Location|Logger|Math|' + r'Mutex|nil|Object|Orderable|OutputStream|Pair|Path|Pattern|Position|' + r'Primitive|Process|Profile|PseudoLazy|PubSub|RangeIterable|Regexp|' + r'Semaphore|Server|Singleton|Socket|StackFrame|Stream|String|System|' + r'Tag|Timeout|Traceable|TrajectoryGenerator|Triplet|Tuple' + r'|UObject|UValue|UVar)\b', Name.Builtin), + (r'(?:this)\b', Name.Builtin.Pseudo), + # don't match single | and & + (r'(?:[-=+*%/<>~^:]+|\.&?|\|\||&&)', Operator), + (r'(?:and_eq|and|bitand|bitor|in|not|not_eq|or_eq|or|xor_eq|xor)\b', + Operator.Word), + (r'[{}\[\]()]+', Punctuation), + (r'(?:;|\||,|&|\?|!)+', Punctuation), + (r'[$a-zA-Z_][a-zA-Z0-9_]*', Name.Other), + (r'0x[0-9a-fA-F]+', Number.Hex), + # Float, Integer, Angle and Duration + (r'(?:[0-9]+(?:(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?)?' + r'((?:rad|deg|grad)|(?:ms|s|min|h|d))?)\b', Number.Float), + # handle binary blob in strings + (r'"', String.Double, "string.double"), + (r"'", String.Single, "string.single"), + ], + 'string.double': [ + (r'((?:\\\\|\\"|[^"])*?)(\\B\((\d+)\)\()', blob_callback), + (r'(\\\\|\\"|[^"])*?"', String.Double, '#pop'), + ], + 'string.single': [ + (r"((?:\\\\|\\'|[^'])*?)(\\B\((\d+)\)\()", blob_callback), + (r"(\\\\|\\'|[^'])*?'", String.Single, '#pop'), + ], + # from http://pygments.org/docs/lexerdevelopment/#changing-states + 'comment': [ + (r'[^*/]', Comment.Multiline), + (r'/\*', Comment.Multiline, '#push'), + (r'\*/', Comment.Multiline, '#pop'), + (r'[*/]', Comment.Multiline), + ] + } + + +class OpenEdgeLexer(RegexLexer): + """ + Lexer for `OpenEdge ABL (formerly Progress) + <http://web.progress.com/en/openedge/abl.html>`_ source code. + + *New in Pygments 1.5.* + """ + name = 'OpenEdge ABL' + aliases = ['openedge', 'abl', 'progress'] + filenames = ['*.p', '*.cls'] + mimetypes = ['text/x-openedge', 'application/x-openedge'] + + types = (r'(?i)(^|(?<=[^0-9a-z_\-]))(CHARACTER|CHAR|CHARA|CHARAC|CHARACT|CHARACTE|' + r'COM-HANDLE|DATE|DATETIME|DATETIME-TZ|' + r'DECIMAL|DEC|DECI|DECIM|DECIMA|HANDLE|' + r'INT64|INTEGER|INT|INTE|INTEG|INTEGE|' + r'LOGICAL|LONGCHAR|MEMPTR|RAW|RECID|ROWID)\s*($|(?=[^0-9a-z_\-]))') + + keywords = (r'(?i)(^|(?<=[^0-9a-z_\-]))(ABSOLUTE|ABS|ABSO|ABSOL|ABSOLU|ABSOLUT|ACCELERATOR|' + r'ACCUM|ACCUMULATE|ACCUM|ACCUMU|ACCUMUL|ACCUMULA|ACCUMULAT|' + r'ACTIVE-FORM|ACTIVE-WINDOW|ADD|ADD-BUFFER|' + r'ADD-CALC-COLUMN|ADD-COLUMNS-FROM|ADD-EVENTS-PROCEDURE|' + r'ADD-FIELDS-FROM|ADD-FIRST|ADD-INDEX-FIELD|ADD-LAST|' + r'ADD-LIKE-COLUMN|ADD-LIKE-FIELD|ADD-LIKE-INDEX|' + r'ADD-NEW-FIELD|ADD-NEW-INDEX|ADD-SCHEMA-LOCATION|ADD-SUPER-PROCEDURE|' + r'ADM-DATA|ADVISE|ALERT-BOX|ALIAS|ALL|ALLOW-COLUMN-SEARCHING|' + r'ALLOW-REPLICATION|ALTER|ALWAYS-ON-TOP|AMBIGUOUS|AMBIG|AMBIGU|AMBIGUO|AMBIGUOU|' + r'ANALYZE|ANALYZ|AND|ANSI-ONLY|ANY|ANYWHERE|APPEND|APPL-ALERT-BOXES|' + r'APPL-ALERT|APPL-ALERT-|APPL-ALERT-B|APPL-ALERT-BO|APPL-ALERT-BOX|APPL-ALERT-BOXE|' + r'APPL-CONTEXT-ID|APPLICATION|APPLY|APPSERVER-INFO|APPSERVER-PASSWORD|' + r'APPSERVER-USERID|ARRAY-MESSAGE|AS|ASC|ASCENDING|ASCE|ASCEN|' + r'ASCEND|ASCENDI|ASCENDIN|ASK-OVERWRITE|ASSEMBLY|ASSIGN|' + r'ASYNCHRONOUS|ASYNC-REQUEST-COUNT|ASYNC-REQUEST-HANDLE|AT|' + r'ATTACHED-PAIRLIST|ATTR-SPACE|ATTR|ATTRI|ATTRIB|ATTRIBU|ATTRIBUT|' + r'AUDIT-CONTROL|AUDIT-ENABLED|AUDIT-EVENT-CONTEXT|AUDIT-POLICY|' + r'AUTHENTICATION-FAILED|AUTHORIZATION|AUTO-COMPLETION|AUTO-COMP|' + r'AUTO-COMPL|AUTO-COMPLE|AUTO-COMPLET|AUTO-COMPLETI|AUTO-COMPLETIO|' + r'AUTO-ENDKEY|AUTO-END-KEY|AUTO-GO|AUTO-INDENT|AUTO-IND|' + r'AUTO-INDE|AUTO-INDEN|AUTOMATIC|AUTO-RESIZE|AUTO-RETURN|AUTO-RET|' + r'AUTO-RETU|AUTO-RETUR|AUTO-SYNCHRONIZE|AUTO-ZAP|AUTO-Z|AUTO-ZA|' + r'AVAILABLE|AVAIL|AVAILA|AVAILAB|AVAILABL|AVAILABLE-FORMATS|' + r'AVERAGE|AVE|AVER|AVERA|AVERAG|AVG|BACKGROUND|BACK|BACKG|' + r'BACKGR|BACKGRO|BACKGROU|BACKGROUN|BACKWARDS|BACKWARD|' + r'BASE64-DECODE|BASE64-ENCODE|BASE-ADE|BASE-KEY|BATCH-MODE|BATCH|' + r'BATCH-|BATCH-M|BATCH-MO|BATCH-MOD|BATCH-SIZE|BEFORE-HIDE|BEFORE-H|' + r'BEFORE-HI|BEFORE-HID|BEGIN-EVENT-GROUP|BEGINS|BELL|BETWEEN|' + r'BGCOLOR|BGC|BGCO|BGCOL|BGCOLO|BIG-ENDIAN|BINARY|BIND|BIND-WHERE|' + r'BLANK|BLOCK-ITERATION-DISPLAY|BORDER-BOTTOM-CHARS|BORDER-B|' + r'BORDER-BO|BORDER-BOT|BORDER-BOTT|BORDER-BOTTO|BORDER-BOTTOM-PIXELS|' + r'BORDER-BOTTOM-P|BORDER-BOTTOM-PI|BORDER-BOTTOM-PIX|' + r'BORDER-BOTTOM-PIXE|BORDER-BOTTOM-PIXEL|BORDER-LEFT-CHARS|BORDER-L|' + r'BORDER-LE|BORDER-LEF|BORDER-LEFT|BORDER-LEFT-|BORDER-LEFT-C|' + r'BORDER-LEFT-CH|BORDER-LEFT-CHA|BORDER-LEFT-CHAR|BORDER-LEFT-PIXELS|' + r'BORDER-LEFT-P|BORDER-LEFT-PI|BORDER-LEFT-PIX|BORDER-LEFT-PIXE|' + r'BORDER-LEFT-PIXEL|BORDER-RIGHT-CHARS|BORDER-R|BORDER-RI|BORDER-RIG|' + r'BORDER-RIGH|BORDER-RIGHT|BORDER-RIGHT-|BORDER-RIGHT-C|BORDER-RIGHT-CH|' + r'BORDER-RIGHT-CHA|BORDER-RIGHT-CHAR|BORDER-RIGHT-PIXELS|BORDER-RIGHT-P|' + r'BORDER-RIGHT-PI|BORDER-RIGHT-PIX|BORDER-RIGHT-PIXE|BORDER-RIGHT-PIXEL|' + r'BORDER-TOP-CHARS|BORDER-T|BORDER-TO|BORDER-TOP|BORDER-TOP-|BORDER-TOP-C|' + r'BORDER-TOP-CH|BORDER-TOP-CHA|BORDER-TOP-CHAR|BORDER-TOP-PIXELS|' + r'BORDER-TOP-P|BORDER-TOP-PI|BORDER-TOP-PIX|BORDER-TOP-PIXE|BORDER-TOP-PIXEL|' + r'BOX|BOX-SELECTABLE|BOX-SELECT|BOX-SELECTA|BOX-SELECTAB|BOX-SELECTABL|' + r'BREAK|BROWSE|BUFFER|BUFFER-CHARS|BUFFER-COMPARE|BUFFER-COPY|BUFFER-CREATE|' + r'BUFFER-DELETE|BUFFER-FIELD|BUFFER-HANDLE|BUFFER-LINES|BUFFER-NAME|' + r'BUFFER-RELEASE|BUFFER-VALUE|BUTTON|BUTTONS|BUTTON|BY|BY-POINTER|' + r'BY-VARIANT-POINTER|CACHE|CACHE-SIZE|CALL|CALL-NAME|CALL-TYPE|CANCEL-BREAK|' + r'CANCEL-BUTTON|CAN-CREATE|CAN-DELETE|CAN-DO|CAN-FIND|CAN-QUERY|CAN-READ|' + r'CAN-SET|CAN-WRITE|CAPS|CAREFUL-PAINT|CASE|CASE-SENSITIVE|CASE-SEN|' + r'CASE-SENS|CASE-SENSI|CASE-SENSIT|CASE-SENSITI|CASE-SENSITIV|' + r'CAST|CATCH|CDECL|CENTERED|CENTER|CENTERE|CHAINED|CHARACTER_LENGTH|' + r'CHARSET|CHECK|CHECKED|CHOOSE|CHR|CLASS|CLASS-TYPE|CLEAR|' + r'CLEAR-APPL-CONTEXT|CLEAR-LOG|CLEAR-SELECTION|CLEAR-SELECT|' + r'CLEAR-SELECTI|CLEAR-SELECTIO|CLEAR-SORT-ARROWS|CLEAR-SORT-ARROW|' + r'CLIENT-CONNECTION-ID|CLIENT-PRINCIPAL|CLIENT-TTY|CLIENT-TYPE|' + r'CLIENT-WORKSTATION|CLIPBOARD|CLOSE|CLOSE-LOG|CODE|CODEBASE-LOCATOR|' + r'CODEPAGE|CODEPAGE-CONVERT|COLLATE|COL-OF|COLON|COLON-ALIGNED|' + r'COLON-ALIGN|COLON-ALIGNE|COLOR|COLOR-TABLE|COLUMN|COL|COLU|COLUM|' + r'COLUMN-BGCOLOR|COLUMN-DCOLOR|COLUMN-FGCOLOR|COLUMN-FONT|COLUMN-LABEL|' + r'COLUMN-LAB|COLUMN-LABE|COLUMN-MOVABLE|COLUMN-OF|COLUMN-PFCOLOR|' + r'COLUMN-READ-ONLY|COLUMN-RESIZABLE|COLUMNS|COLUMN-SCROLLING|' + r'COMBO-BOX|COMMAND|COMPARES|COMPILE|COMPILER|COMPLETE|COM-SELF|' + r'CONFIG-NAME|CONNECT|CONNECTED|CONSTRUCTOR|CONTAINS|CONTENTS|CONTEXT|' + r'CONTEXT-HELP|CONTEXT-HELP-FILE|CONTEXT-HELP-ID|CONTEXT-POPUP|' + r'CONTROL|CONTROL-BOX|CONTROL-FRAME|CONVERT|CONVERT-3D-COLORS|' + r'CONVERT-TO-OFFSET|CONVERT-TO-OFFS|CONVERT-TO-OFFSE|COPY-DATASET|' + r'COPY-LOB|COPY-SAX-ATTRIBUTES|COPY-TEMP-TABLE|COUNT|COUNT-OF|' + r'CPCASE|CPCOLL|CPINTERNAL|CPLOG|CPPRINT|CPRCODEIN|CPRCODEOUT|' + r'CPSTREAM|CPTERM|CRC-VALUE|CREATE|CREATE-LIKE|CREATE-LIKE-SEQUENTIAL|' + r'CREATE-NODE-NAMESPACE|CREATE-RESULT-LIST-ENTRY|CREATE-TEST-FILE|' + r'CURRENT|CURRENT_DATE|CURRENT_DATE|CURRENT-CHANGED|CURRENT-COLUMN|' + r'CURRENT-ENVIRONMENT|CURRENT-ENV|CURRENT-ENVI|CURRENT-ENVIR|' + r'CURRENT-ENVIRO|CURRENT-ENVIRON|CURRENT-ENVIRONM|CURRENT-ENVIRONME|' + r'CURRENT-ENVIRONMEN|CURRENT-ITERATION|CURRENT-LANGUAGE|CURRENT-LANG|' + r'CURRENT-LANGU|CURRENT-LANGUA|CURRENT-LANGUAG|CURRENT-QUERY|' + r'CURRENT-RESULT-ROW|CURRENT-ROW-MODIFIED|CURRENT-VALUE|CURRENT-WINDOW|' + r'CURSOR|CURS|CURSO|CURSOR-CHAR|CURSOR-LINE|CURSOR-OFFSET|DATABASE|' + r'DATA-BIND|DATA-ENTRY-RETURN|DATA-ENTRY-RET|DATA-ENTRY-RETU|' + r'DATA-ENTRY-RETUR|DATA-RELATION|DATA-REL|DATA-RELA|DATA-RELAT|' + r'DATA-RELATI|DATA-RELATIO|DATASERVERS|DATASET|DATASET-HANDLE|DATA-SOURCE|' + r'DATA-SOURCE-COMPLETE-MAP|DATA-SOURCE-MODIFIED|DATA-SOURCE-ROWID|' + r'DATA-TYPE|DATA-T|DATA-TY|DATA-TYP|DATE-FORMAT|DATE-F|DATE-FO|' + r'DATE-FOR|DATE-FORM|DATE-FORMA|DAY|DBCODEPAGE|DBCOLLATION|DBNAME|' + r'DBPARAM|DB-REFERENCES|DBRESTRICTIONS|DBREST|DBRESTR|DBRESTRI|' + r'DBRESTRIC|DBRESTRICT|DBRESTRICTI|DBRESTRICTIO|DBRESTRICTION|' + r'DBTASKID|DBTYPE|DBVERSION|DBVERS|DBVERSI|DBVERSIO|DCOLOR|' + r'DDE|DDE-ERROR|DDE-ID|DDE-I|DDE-ITEM|DDE-NAME|DDE-TOPIC|DEBLANK|' + r'DEBUG|DEBU|DEBUG-ALERT|DEBUGGER|DEBUG-LIST|DECIMALS|DECLARE|' + r'DECLARE-NAMESPACE|DECRYPT|DEFAULT|DEFAULT-BUFFER-HANDLE|' + r'DEFAULT-BUTTON|DEFAUT-B|DEFAUT-BU|DEFAUT-BUT|DEFAUT-BUTT|DEFAUT-BUTTO|' + r'DEFAULT-COMMIT|DEFAULT-EXTENSION|DEFAULT-EX|DEFAULT-EXT|DEFAULT-EXTE|' + r'DEFAULT-EXTEN|DEFAULT-EXTENS|DEFAULT-EXTENSI|DEFAULT-EXTENSIO|' + r'DEFAULT-NOXLATE|DEFAULT-NOXL|DEFAULT-NOXLA|DEFAULT-NOXLAT|' + r'DEFAULT-VALUE|DEFAULT-WINDOW|DEFINED|' + r'DEFINE-USER-EVENT-MANAGER|DELETE|DEL|DELE|DELET|DELETE-CHARACTER|' + r'DELETE-CHAR|DELETE-CHARA|DELETE-CHARAC|DELETE-CHARACT|DELETE-CHARACTE|' + r'DELETE-CURRENT-ROW|DELETE-LINE|DELETE-RESULT-LIST-ENTRY|DELETE-SELECTED-ROW|' + r'DELETE-SELECTED-ROWS|DELIMITER|DESC|DESCENDING|DESC|DESCE|DESCEN|' + r'DESCEND|DESCENDI|DESCENDIN|DESELECT-FOCUSED-ROW|DESELECTION|' + r'DESELECT-ROWS|DESELECT-SELECTED-ROW|DESTRUCTOR|DIALOG-BOX|' + r'DICTIONARY|DICT|DICTI|DICTIO|DICTION|DICTIONA|DICTIONAR|' + r'DIR|DISABLE|DISABLE-AUTO-ZAP|DISABLED|DISABLE-DUMP-TRIGGERS|' + r'DISABLE-LOAD-TRIGGERS|DISCONNECT|DISCON|DISCONN|DISCONNE|DISCONNEC|' + r'DISP|DISPLAY|DISP|DISPL|DISPLA|DISPLAY-MESSAGE|DISPLAY-TYPE|' + r'DISPLAY-T|DISPLAY-TY|DISPLAY-TYP|DISTINCT|DO|DOMAIN-DESCRIPTION|' + r'DOMAIN-NAME|DOMAIN-TYPE|DOS|DOUBLE|DOWN|DRAG-ENABLED|DROP|DROP-DOWN|' + r'DROP-DOWN-LIST|DROP-FILE-NOTIFY|DROP-TARGET|DUMP|DYNAMIC|' + r'DYNAMIC-FUNCTION|EACH|ECHO|EDGE-CHARS|EDGE|EDGE-|EDGE-C|' + r'EDGE-CH|EDGE-CHA|EDGE-CHAR|EDGE-PIXELS|EDGE-P|EDGE-PI|EDGE-PIX|' + r'EDGE-PIXE|EDGE-PIXEL|EDIT-CAN-PASTE|EDIT-CAN-UNDO|EDIT-CLEAR|' + r'EDIT-COPY|EDIT-CUT|EDITING|EDITOR|EDIT-PASTE|EDIT-UNDO|ELSE|' + r'EMPTY|EMPTY-TEMP-TABLE|ENABLE|ENABLED-FIELDS|ENCODE|ENCRYPT|' + r'ENCRYPT-AUDIT-MAC-KEY|ENCRYPTION-SALT|END|END-DOCUMENT|' + r'END-ELEMENT|END-EVENT-GROUP|END-FILE-DROP|ENDKEY|END-KEY|' + r'END-MOVE|END-RESIZE|END-ROW-RESIZE|END-USER-PROMPT|ENTERED|' + r'ENTRY|EQ|ERROR|ERROR-COLUMN|ERROR-COL|ERROR-COLU|ERROR-COLUM|' + r'ERROR-ROW|ERROR-STACK-TRACE|ERROR-STATUS|ERROR-STAT|ERROR-STATU|' + r'ESCAPE|ETIME|EVENT-GROUP-ID|EVENT-PROCEDURE|EVENT-PROCEDURE-CONTEXT|' + r'EVENTS|EVENT|EVENT-TYPE|EVENT-T|EVENT-TY|EVENT-TYP|EXCEPT|' + r'EXCLUSIVE-ID|EXCLUSIVE-LOCK|EXCLUSIVE|EXCLUSIVE-|EXCLUSIVE-L|' + r'EXCLUSIVE-LO|EXCLUSIVE-LOC|EXCLUSIVE-WEB-USER|EXECUTE|EXISTS|' + r'EXP|EXPAND|EXPANDABLE|EXPLICIT|EXPORT|EXPORT-PRINCIPAL|EXTENDED|' + r'EXTENT|EXTERNAL|FALSE|FETCH|FETCH-SELECTED-ROW|FGCOLOR|FGC|FGCO|' + r'FGCOL|FGCOLO|FIELD|FIELDS|FIELD|FILE|FILE-CREATE-DATE|' + r'FILE-CREATE-TIME|FILE-INFORMATION|FILE-INFO|FILE-INFOR|FILE-INFORM|' + r'FILE-INFORMA|FILE-INFORMAT|FILE-INFORMATI|FILE-INFORMATIO|FILE-MOD-DATE|' + r'FILE-MOD-TIME|FILENAME|FILE-NAME|FILE-OFFSET|FILE-OFF|FILE-OFFS|FILE-OFFSE|' + r'FILE-SIZE|FILE-TYPE|FILL|FILLED|FILL-IN|FILTERS|FINAL|FINALLY|FIND|' + r'FIND-BY-ROWID|FIND-CASE-SENSITIVE|FIND-CURRENT|FINDER|FIND-FIRST|' + r'FIND-GLOBAL|FIND-LAST|FIND-NEXT-OCCURRENCE|FIND-PREV-OCCURRENCE|' + r'FIND-SELECT|FIND-UNIQUE|FIND-WRAP-AROUND|FIRST|FIRST-ASYNCH-REQUEST|' + r'FIRST-CHILD|FIRST-COLUMN|FIRST-FORM|FIRST-OBJECT|FIRST-OF|' + r'FIRST-PROCEDURE|FIRST-PROC|FIRST-PROCE|FIRST-PROCED|FIRST-PROCEDU|FIRST-PROCEDUR|' + r'FIRST-SERVER|FIRST-TAB-ITEM|FIRST-TAB-I|FIRST-TAB-IT|FIRST-TAB-ITE|' + r'FIT-LAST-COLUMN|FIXED-ONLY|FLAT-BUTTON|FLOAT|FOCUS|FOCUSED-ROW|' + r'FOCUSED-ROW-SELECTED|FONT|FONT-TABLE|FOR|FORCE-FILE|' + r'FOREGROUND|FORE|FOREG|FOREGR|FOREGRO|FOREGROU|FOREGROUN|' + r'FORM|FORMAT|FORM|FORMA|FORMATTED|FORMATTE|FORM-LONG-INPUT|' + r'FORWARD|FORWARDS|FORWARD|FRAGMENT|FRAGMEN|FRAME|FRAM|' + r'FRAME-COL|FRAME-DB|FRAME-DOWN|FRAME-FIELD|FRAME-FILE|' + r'FRAME-INDEX|FRAME-INDE|FRAME-LINE|FRAME-NAME|FRAME-ROW|' + r'FRAME-SPACING|FRAME-SPA|FRAME-SPAC|FRAME-SPACI|FRAME-SPACIN|' + r'FRAME-VALUE|FRAME-VAL|FRAME-VALU|FRAME-X|FRAME-Y|FREQUENCY|FROM|' + r'FROM-CHARS|FROM-C|FROM-CH|FROM-CHA|FROM-CHAR|' + r'FROM-CURRENT|FROM-CUR|FROM-CURR|FROM-CURRE|FROM-CURREN|' + r'FROM-PIXELS|FROM-P|FROM-PI|FROM-PIX|FROM-PIXE|FROM-PIXEL|' + r'FULL-HEIGHT-CHARS|FULL-HEIGHT|FULL-HEIGHT-|FULL-HEIGHT-C|FULL-HEIGHT-CH|FULL-HEIGHT-CHA|FULL-HEIGHT-CHAR|' + r'FULL-HEIGHT-PIXELS|FULL-HEIGHT-P|FULL-HEIGHT-PI|FULL-HEIGHT-PIX|FULL-HEIGHT-PIXE|FULL-HEIGHT-PIXEL|' + r'FULL-PATHNAME|FULL-PATHN|FULL-PATHNA|FULL-PATHNAM|' + r'FULL-WIDTH-CHARS|FULL-WIDTH|FULL-WIDTH-|FULL-WIDTH-C|FULL-WIDTH-CH|FULL-WIDTH-CHA|FULL-WIDTH-CHAR|' + r'FULL-WIDTH-PIXELS|FULL-WIDTH-P|FULL-WIDTH-PI|FULL-WIDTH-PIX|FULL-WIDTH-PIXE|FULL-WIDTH-PIXEL|' + r'FUNCTION|FUNCTION-CALL-TYPE|GATEWAYS|GATEWAY|GE|GENERATE-MD5|' + r'GENERATE-PBE-KEY|GENERATE-PBE-SALT|GENERATE-RANDOM-KEY|GENERATE-UUID|GET|' + r'GET-ATTR-CALL-TYPE|GET-ATTRIBUTE-NODE|GET-BINARY-DATA|' + r'GET-BLUE-VALUE|GET-BLUE|GET-BLUE-|GET-BLUE-V|GET-BLUE-VA|GET-BLUE-VAL|GET-BLUE-VALU|' + r'GET-BROWSE-COLUMN|GET-BUFFER-HANDLEGETBYTE|GET-BYTE|GET-CALLBACK-PROC-CONTEXT|' + r'GET-CALLBACK-PROC-NAME|GET-CGI-LIST|GET-CGI-LONG-VALUE|GET-CGI-VALUE|' + r'GET-CODEPAGES|GET-COLLATIONS|GET-CONFIG-VALUE|GET-CURRENT|GET-DOUBLE|' + r'GET-DROPPED-FILE|GET-DYNAMIC|GET-ERROR-COLUMN|GET-ERROR-ROW|GET-FILE|' + r'GET-FILE-NAME|GET-FILE-OFFSET|GET-FILE-OFFSE|GET-FIRST|GET-FLOAT|' + r'GET-GREEN-VALUE|GET-GREEN|GET-GREEN-|GET-GREEN-V|GET-GREEN-VA|GET-GREEN-VAL|GET-GREEN-VALU|' + r'GET-INDEX-BY-NAMESPACE-NAME|GET-INDEX-BY-QNAME|GET-INT64|GET-ITERATION|' + r'GET-KEY-VALUE|GET-KEY-VAL|GET-KEY-VALU|GET-LAST|GET-LOCALNAME-BY-INDEX|' + r'GET-LONG|GET-MESSAGE|GET-NEXT|GET-NUMBER|GET-POINTER-VALUE|' + r'GET-PREV|GET-PRINTERS|GET-PROPERTY|GET-QNAME-BY-INDEX|' + r'GET-RED-VALUE|GET-RED|GET-RED-|GET-RED-V|GET-RED-VA|GET-RED-VAL|GET-RED-VALU|' + r'GET-REPOSITIONED-ROW|GET-RGB-VALUE|' + r'GET-SELECTED-WIDGET|GET-SELECTED|GET-SELECTED-|GET-SELECTED-W|GET-SELECTED-WI|GET-SELECTED-WID|GET-SELECTED-WIDG|GET-SELECTED-WIDGE|' + r'GET-SHORT|GET-SIGNATURE|GET-SIZE|GET-STRING|GET-TAB-ITEM|' + r'GET-TEXT-HEIGHT-CHARS|GET-TEXT-HEIGHT|GET-TEXT-HEIGHT-|GET-TEXT-HEIGHT-C|GET-TEXT-HEIGHT-CH|GET-TEXT-HEIGHT-CHA|GET-TEXT-HEIGHT-CHAR|' + r'GET-TEXT-HEIGHT-PIXELS|GET-TEXT-HEIGHT-P|GET-TEXT-HEIGHT-PI|GET-TEXT-HEIGHT-PIX|GET-TEXT-HEIGHT-PIXE|GET-TEXT-HEIGHT-PIXEL|' + r'GET-TEXT-WIDTH-CHARS|GET-TEXT-WIDTH|GET-TEXT-WIDTH-|GET-TEXT-WIDTH-C|GET-TEXT-WIDTH-CH|GET-TEXT-WIDTH-CHA|GET-TEXT-WIDTH-CHAR|' + r'GET-TEXT-WIDTH-PIXELS|GET-TEXT-WIDTH-P|GET-TEXT-WIDTH-PI|GET-TEXT-WIDTH-PIX|GET-TEXT-WIDTH-PIXE|GET-TEXT-WIDTH-PIXEL|' + r'GET-TYPE-BY-INDEX|GET-TYPE-BY-NAMESPACE-NAME|GET-TYPE-BY-QNAME|GET-UNSIGNED-LONG|' + r'GET-UNSIGNED-SHORT|GET-URI-BY-INDEX|GET-VALUE-BY-INDEX|GET-VALUE-BY-NAMESPACE-NAME|' + r'GET-VALUE-BY-QNAME|GET-WAIT-STATE|GLOBAL|GO-ON|' + r'GO-PENDING|GO-PEND|GO-PENDI|GO-PENDIN|GRANT|' + r'GRAPHIC-EDGE|GRAPHIC-E|GRAPHIC-ED|GRAPHIC-EDG|' + r'GRID-FACTOR-HORIZONTAL|GRID-FACTOR-H|GRID-FACTOR-HO|GRID-FACTOR-HOR|GRID-FACTOR-HORI|GRID-FACTOR-HORIZ|GRID-FACTOR-HORIZO|GRID-FACTOR-HORIZON|GRID-FACTOR-HORIZONT|GRID-FACTOR-HORIZONTA|' + r'GRID-FACTOR-VERTICAL|GRID-FACTOR-V|GRID-FACTOR-VE|GRID-FACTOR-VER|GRID-FACTOR-VERT|GRID-FACTOR-VERT|GRID-FACTOR-VERTI|GRID-FACTOR-VERTIC|GRID-FACTOR-VERTICA|' + r'GRID-SNAP|' + r'GRID-UNIT-HEIGHT-CHARS|GRID-UNIT-HEIGHT|GRID-UNIT-HEIGHT-|GRID-UNIT-HEIGHT-C|GRID-UNIT-HEIGHT-CH|GRID-UNIT-HEIGHT-CHA|' + r'GRID-UNIT-HEIGHT-PIXELS|GRID-UNIT-HEIGHT-P|GRID-UNIT-HEIGHT-PI|GRID-UNIT-HEIGHT-PIX|GRID-UNIT-HEIGHT-PIXE|GRID-UNIT-HEIGHT-PIXEL|' + r'GRID-UNIT-WIDTH-CHARS|GRID-UNIT-WIDTH|GRID-UNIT-WIDTH-|GRID-UNIT-WIDTH-C|GRID-UNIT-WIDTH-CH|GRID-UNIT-WIDTH-CHA|GRID-UNIT-WIDTH-CHAR|' + r'GRID-UNIT-WIDTH-PIXELS|GRID-UNIT-WIDTH-P|GRID-UNIT-WIDTH-PI|GRID-UNIT-WIDTH-PIX|GRID-UNIT-WIDTH-PIXE|GRID-UNIT-WIDTH-PIXEL|' + r'GRID-VISIBLE|GROUP|GT|GUID|HANDLER|HAS-RECORDS|HAVING|HEADER|' + r'HEIGHT-CHARS|HEIGHT|HEIGHT-|HEIGHT-C|HEIGHT-CH|HEIGHT-CHA|HEIGHT-CHAR|' + r'HEIGHT-PIXELS|HEIGHT-P|HEIGHT-PI|HEIGHT-PIX|HEIGHT-PIXE|HEIGHT-PIXEL|' + r'HELP|HEX-DECODE|HEX-ENCODE|HIDDEN|HIDE|' + r'HORIZONTAL|HORI|HORIZ|HORIZO|HORIZON|HORIZONT|HORIZONTA|' + r'HOST-BYTE-ORDER|HTML-CHARSET|HTML-END-OF-LINE|HTML-END-OF-PAGE|' + r'HTML-FRAME-BEGIN|HTML-FRAME-END|HTML-HEADER-BEGIN|HTML-HEADER-END|' + r'HTML-TITLE-BEGIN|HTML-TITLE-END|HWND|ICON|IF|' + r'IMAGE|IMAGE-DOWN|IMAGE-INSENSITIVE|IMAGE-SIZE|' + r'IMAGE-SIZE-CHARS|IMAGE-SIZE-C|IMAGE-SIZE-CH|IMAGE-SIZE-CHA|IMAGE-SIZE-CHAR|' + r'IMAGE-SIZE-PIXELS|IMAGE-SIZE-P|IMAGE-SIZE-PI|IMAGE-SIZE-PIX|IMAGE-SIZE-PIXE|IMAGE-SIZE-PIXEL|' + r'IMAGE-UP|IMMEDIATE-DISPLAY|IMPLEMENTS|IMPORT|IMPORT-PRINCIPAL|' + r'IN|INCREMENT-EXCLUSIVE-ID|INDEX|INDEXED-REPOSITION|INDEX-HINT|' + r'INDEX-INFORMATION|INDICATOR|' + r'INFORMATION|INFO|INFOR|INFORM|INFORMA|INFORMAT|INFORMATI|INFORMATIO|' + r'IN-HANDLE|' + r'INHERIT-BGCOLOR|INHERIT-BGC|INHERIT-BGCO|INHERIT-BGCOL|INHERIT-BGCOLO|' + r'INHERIT-FGCOLOR|INHERIT-FGC|INHERIT-FGCO|INHERIT-FGCOL|INHERIT-FGCOLO|' + r'INHERITS|INITIAL|INIT|INITI|INITIA|INITIAL-DIR|INITIAL-FILTER|' + r'INITIALIZE-DOCUMENT-TYPE|INITIATE|INNER-CHARS|INNER-LINES|INPUT|' + r'INPUT-OUTPUT|INPUT-O|INPUT-OU|INPUT-OUT|INPUT-OUTP|INPUT-OUTPU|' + r'INPUT-VALUE|INSERT|INSERT-ATTRIBUTE|' + r'INSERT-BACKTAB|INSERT-B|INSERT-BA|INSERT-BAC|INSERT-BACK|INSERT-BACKT|INSERT-BACKTA|' + r'INSERT-FILE|INSERT-ROW|INSERT-STRING|INSERT-TAB|INSERT-T|INSERT-TA|' + r'INTERFACE|INTERNAL-ENTRIES|INTO|INVOKE|IS|' + r'IS-ATTR-SPACE|IS-ATTR|IS-ATTR-|IS-ATTR-S|IS-ATTR-SP|IS-ATTR-SPA|IS-ATTR-SPAC|' + r'IS-CLASS|IS-CLAS|IS-LEAD-BYTE|IS-ATTR|IS-OPEN|IS-PARAMETER-SET|IS-ROW-SELECTED|' + r'IS-SELECTED|ITEM|ITEMS-PER-ROW|JOIN|JOIN-BY-SQLDB|KBLABEL|KEEP-CONNECTION-OPEN|' + r'KEEP-FRAME-Z-ORDER|KEEP-FRAME-Z|KEEP-FRAME-Z-|KEEP-FRAME-Z-O|KEEP-FRAME-Z-OR|KEEP-FRAME-Z-ORD|KEEP-FRAME-Z-ORDE|' + r'KEEP-MESSAGES|KEEP-SECURITY-CACHE|KEEP-TAB-ORDER|KEY|KEYCODE|KEY-CODE|' + r'KEYFUNCTION|KEYFUNC|KEYFUNCT|KEYFUNCTI|KEYFUNCTIO|' + r'KEY-FUNCTION|KEY-FUNC|KEY-FUNCT|KEY-FUNCTI|KEY-FUNCTIO|' + r'KEYLABEL|KEY-LABEL|KEYS|KEYWORD|KEYWORD-ALL|LABEL|' + r'LABEL-BGCOLOR|LABEL-BGC|LABEL-BGCO|LABEL-BGCOL|LABEL-BGCOLO|' + r'LABEL-DCOLOR|LABEL-DC|LABEL-DCO|LABEL-DCOL|LABEL-DCOLO|' + r'LABEL-FGCOLOR|LABEL-FGC|LABEL-FGCO|LABEL-FGCOL|LABEL-FGCOLO|' + r'LABEL-FONT|' + r'LABEL-PFCOLOR|LABEL-PFC|LABEL-PFCO|LABEL-PFCOL|LABEL-PFCOLO|' + r'LABELS|LANDSCAPE|LANGUAGES|LANGUAGE|LARGE|LARGE-TO-SMALL|LAST|' + r'LAST-ASYNCH-REQUEST|LAST-BATCH|LAST-CHILD|LAST-EVENT|LAST-EVEN|LAST-FORM|' + r'LASTKEY|LAST-KEY|LAST-OBJECT|LAST-OF|' + r'LAST-PROCEDURE|LAST-PROCE|LAST-PROCED|LAST-PROCEDU|LAST-PROCEDUR|' + r'LAST-SERVER|LAST-TAB-ITEM|LAST-TAB-I|LAST-TAB-IT|LAST-TAB-ITE|' + r'LC|LDBNAME|LE|LEAVE|LEFT-ALIGNED|LEFT-ALIGN|LEFT-ALIGNE|LEFT-TRIM|' + r'LENGTH|LIBRARY|LIKE|LIKE-SEQUENTIAL|LINE|LINE-COUNTER|LINE-COUNT|LINE-COUNTE|' + r'LIST-EVENTS|LISTING|LISTI|LISTIN|LIST-ITEM-PAIRS|LIST-ITEMS|' + r'LIST-PROPERTY-NAMES|LIST-QUERY-ATTRS|LIST-SET-ATTRS|LIST-WIDGETS|' + r'LITERAL-QUESTION|LITTLE-ENDIAN|LOAD|LOAD-DOMAINS|LOAD-ICON|' + r'LOAD-IMAGE|LOAD-IMAGE-DOWN|LOAD-IMAGE-INSENSITIVE|LOAD-IMAGE-UP|' + r'LOAD-MOUSE-POINTER|LOAD-MOUSE-P|LOAD-MOUSE-PO|LOAD-MOUSE-POI|LOAD-MOUSE-POIN|LOAD-MOUSE-POINT|LOAD-MOUSE-POINTE|' + r'LOAD-PICTURE|LOAD-SMALL-ICON|LOCAL-NAME|LOCATOR-COLUMN-NUMBER|' + r'LOCATOR-LINE-NUMBER|LOCATOR-PUBLIC-ID|LOCATOR-SYSTEM-ID|LOCATOR-TYPE|' + r'LOCKED|LOCK-REGISTRATION|LOG|LOG-AUDIT-EVENT|LOGIN-EXPIRATION-TIMESTAMP|' + r'LOGIN-HOST|LOGIN-STATE|LOG-MANAGER|LOGOUT|LOOKAHEAD|LOOKUP|LT|' + r'MACHINE-CLASS|MANDATORY|MANUAL-HIGHLIGHT|MAP|MARGIN-EXTRA|' + r'MARGIN-HEIGHT-CHARS|MARGIN-HEIGHT|MARGIN-HEIGHT-|MARGIN-HEIGHT-C|MARGIN-HEIGHT-CH|MARGIN-HEIGHT-CHA|MARGIN-HEIGHT-CHAR|' + r'MARGIN-HEIGHT-PIXELS|MARGIN-HEIGHT-P|MARGIN-HEIGHT-PI|MARGIN-HEIGHT-PIX|MARGIN-HEIGHT-PIXE|MARGIN-HEIGHT-PIXEL|' + r'MARGIN-WIDTH-CHARS|MARGIN-WIDTH|MARGIN-WIDTH-|MARGIN-WIDTH-C|MARGIN-WIDTH-CH|MARGIN-WIDTH-CHA|MARGIN-WIDTH-CHAR|' + r'MARGIN-WIDTH-PIXELS|MARGIN-WIDTH-P|MARGIN-WIDTH-PI|MARGIN-WIDTH-PIX|MARGIN-WIDTH-PIXE|MARGIN-WIDTH-PIXEL|' + r'MARK-NEW|MARK-ROW-STATE|MATCHES|MAX|MAX-BUTTON|' + r'MAX-CHARS|MAX-DATA-GUESS|MAX-HEIGHT|' + r'MAX-HEIGHT-CHARS|MAX-HEIGHT-C|MAX-HEIGHT-CH|MAX-HEIGHT-CHA|MAX-HEIGHT-CHAR|' + r'MAX-HEIGHT-PIXELS|MAX-HEIGHT-P|MAX-HEIGHT-PI|MAX-HEIGHT-PIX|MAX-HEIGHT-PIXE|MAX-HEIGHT-PIXEL|' + r'MAXIMIZE|MAXIMUM|MAX|MAXI|MAXIM|MAXIMU|MAXIMUM-LEVEL|MAX-ROWS|' + r'MAX-SIZE|MAX-VALUE|MAX-VAL|MAX-VALU|MAX-WIDTH|' + r'MAX-WIDTH-CHARS|MAX-WIDTH|MAX-WIDTH-|MAX-WIDTH-C|MAX-WIDTH-CH|MAX-WIDTH-CHA|MAX-WIDTH-CHAR|' + r'MAX-WIDTH-PIXELS|MAX-WIDTH-P|MAX-WIDTH-PI|MAX-WIDTH-PIX|MAX-WIDTH-PIXE|MAX-WIDTH-PIXEL|' + r'MD5-DIGEST|MEMBER|MEMPTR-TO-NODE-VALUE|MENU|MENUBAR|MENU-BAR|MENU-ITEM|' + r'MENU-KEY|MENU-K|MENU-KE|MENU-MOUSE|MENU-M|MENU-MO|MENU-MOU|MENU-MOUS|' + r'MERGE-BY-FIELD|MESSAGE|MESSAGE-AREA|MESSAGE-AREA-FONT|MESSAGE-LINES|' + r'METHOD|MIN|MIN-BUTTON|' + r'MIN-COLUMN-WIDTH-CHARS|MIN-COLUMN-WIDTH-C|MIN-COLUMN-WIDTH-CH|MIN-COLUMN-WIDTH-CHA|MIN-COLUMN-WIDTH-CHAR|' + r'MIN-COLUMN-WIDTH-PIXELS|MIN-COLUMN-WIDTH-P|MIN-COLUMN-WIDTH-PI|MIN-COLUMN-WIDTH-PIX|MIN-COLUMN-WIDTH-PIXE|MIN-COLUMN-WIDTH-PIXEL|' + r'MIN-HEIGHT-CHARS|MIN-HEIGHT|MIN-HEIGHT-|MIN-HEIGHT-C|MIN-HEIGHT-CH|MIN-HEIGHT-CHA|MIN-HEIGHT-CHAR|' + r'MIN-HEIGHT-PIXELS|MIN-HEIGHT-P|MIN-HEIGHT-PI|MIN-HEIGHT-PIX|MIN-HEIGHT-PIXE|MIN-HEIGHT-PIXEL|' + r'MINIMUM|MIN|MINI|MINIM|MINIMU|MIN-SIZE|' + r'MIN-VALUE|MIN-VAL|MIN-VALU|' + r'MIN-WIDTH-CHARS|MIN-WIDTH|MIN-WIDTH-|MIN-WIDTH-C|MIN-WIDTH-CH|MIN-WIDTH-CHA|MIN-WIDTH-CHAR|' + r'MIN-WIDTH-PIXELS|MIN-WIDTH-P|MIN-WIDTH-PI|MIN-WIDTH-PIX|MIN-WIDTH-PIXE|MIN-WIDTH-PIXEL|' + r'MODIFIED|MODULO|MOD|MODU|MODUL|MONTH|MOUSE|' + r'MOUSE-POINTER|MOUSE-P|MOUSE-PO|MOUSE-POI|MOUSE-POIN|MOUSE-POINT|MOUSE-POINTE|' + r'MOVABLE|' + r'MOVE-AFTER-TAB-ITEM|MOVE-AFTER|MOVE-AFTER-|MOVE-AFTER-T|MOVE-AFTER-TA|MOVE-AFTER-TAB|MOVE-AFTER-TAB-|MOVE-AFTER-TAB-I|MOVE-AFTER-TAB-IT|MOVE-AFTER-TAB-ITE|' + r'MOVE-BEFORE-TAB-ITEM|MOVE-BEFOR|MOVE-BEFORE|MOVE-BEFORE-|MOVE-BEFORE-T|MOVE-BEFORE-TA|MOVE-BEFORE-TAB|MOVE-BEFORE-TAB-|MOVE-BEFORE-TAB-I|MOVE-BEFORE-TAB-IT|MOVE-BEFORE-TAB-ITE|' + r'MOVE-COLUMN|MOVE-COL|MOVE-COLU|MOVE-COLUM|' + r'MOVE-TO-BOTTOM|MOVE-TO-B|MOVE-TO-BO|MOVE-TO-BOT|MOVE-TO-BOTT|MOVE-TO-BOTTO|' + r'MOVE-TO-EOF|MOVE-TO-TOP|MOVE-TO-T|MOVE-TO-TO|MPE|MULTI-COMPILE|MULTIPLE|' + r'MULTIPLE-KEY|MULTITASKING-INTERVAL|MUST-EXIST|NAME|NAMESPACE-PREFIX|' + r'NAMESPACE-URI|NATIVE|NE|NEEDS-APPSERVER-PROMPT|NEEDS-PROMPT|NEW|' + r'NEW-INSTANCE|NEW-ROW|NEXT|NEXT-COLUMN|NEXT-PROMPT|NEXT-ROWID|' + r'NEXT-SIBLING|NEXT-TAB-ITEM|NEXT-TAB-I|NEXT-TAB-IT|NEXT-TAB-ITE|' + r'NEXT-VALUE|NO|NO-APPLY|NO-ARRAY-MESSAGE|NO-ASSIGN|' + r'NO-ATTR-LIST|NO-ATTR|NO-ATTR-|NO-ATTR-L|NO-ATTR-LI|NO-ATTR-LIS|' + r'NO-ATTR-SPACE|NO-ATTR|NO-ATTR-|NO-ATTR-S|NO-ATTR-SP|NO-ATTR-SPA|NO-ATTR-SPAC|' + r'NO-AUTO-VALIDATE|NO-BIND-WHERE|NO-BOX|NO-CONSOLE|NO-CONVERT|' + r'NO-CONVERT-3D-COLORS|NO-CURRENT-VALUE|NO-DEBUG|NODE-VALUE-TO-MEMPTR|' + r'NO-DRAG|NO-ECHO|NO-EMPTY-SPACE|NO-ERROR|NO-FILL|NO-F|NO-FI|' + r'NO-FIL|NO-FOCUS|NO-HELP|NO-HIDE|NO-INDEX-HINT|' + r'NO-INHERIT-BGCOLOR|NO-INHERIT-BGC|NO-INHERIT-BGCO|LABEL-BGCOL|LABEL-BGCOLO|' + r'NO-INHERIT-FGCOLOR|NO-INHERIT-FGC|NO-INHERIT-FGCO|NO-INHERIT-FGCOL|NO-INHERIT-FGCOLO|' + r'NO-JOIN-BY-SQLDB|NO-LABELS|NO-LABE|NO-LOBS|NO-LOCK|' + r'NO-LOOKAHEAD|NO-MAP|' + r'NO-MESSAGE|NO-MES|NO-MESS|NO-MESSA|NO-MESSAG|' + r'NONAMESPACE-SCHEMA-LOCATION|NONE|NO-PAUSE|' + r'NO-PREFETCH|NO-PREFE|NO-PREFET|NO-PREFETC|NORMALIZE|' + r'NO-ROW-MARKERS|NO-SCROLLBAR-VERTICAL|NO-SEPARATE-CONNECTION|' + r'NO-SEPARATORS|NOT|NO-TAB-STOP|NOT-ACTIVE|' + r'NO-UNDERLINE|NO-UND|NO-UNDE|NO-UNDER|NO-UNDERL|NO-UNDERLI|NO-UNDERLIN|' + r'NO-UNDO|' + r'NO-VALIDATE|NO-VAL|NO-VALI|NO-VALID|NO-VALIDA|NO-VALIDAT|NOW|' + r'NO-WAIT|NO-WORD-WRAP|NULL|NUM-ALIASES|NUM-ALI|NUM-ALIA|NUM-ALIAS|NUM-ALIASE|' + r'NUM-BUFFERS|NUM-BUTTONS|NUM-BUT|NUM-BUTT|NUM-BUTTO|NUM-BUTTON|' + r'NUM-COLUMNS|NUM-COL|NUM-COLU|NUM-COLUM|NUM-COLUMN|NUM-COPIES|' + r'NUM-DBS|NUM-DROPPED-FILES|NUM-ENTRIES|NUMERIC|' + r'NUMERIC-FORMAT|NUMERIC-F|NUMERIC-FO|NUMERIC-FOR|NUMERIC-FORM|NUMERIC-FORMA|' + r'NUM-FIELDS|NUM-FORMATS|NUM-ITEMS|NUM-ITERATIONS|NUM-LINES|' + r'NUM-LOCKED-COLUMNS|NUM-LOCKED-COL|NUM-LOCKED-COLU|NUM-LOCKED-COLUM|NUM-LOCKED-COLUMN|' + r'NUM-MESSAGES|NUM-PARAMETERS|NUM-REFERENCES|NUM-REPLACED|NUM-RESULTS|NUM-SELECTED-ROWS|' + r'NUM-SELECTED-WIDGETS|NUM-SELECTED|NUM-SELECTED-|NUM-SELECTED-W|NUM-SELECTED-WI|NUM-SELECTED-WID|NUM-SELECTED-WIDG|NUM-SELECTED-WIDGE|NUM-SELECTED-WIDGET|' + r'NUM-TABS|NUM-TO-RETAIN|NUM-VISIBLE-COLUMNS|OCTET-LENGTH|OF|' + r'OFF|OK|OK-CANCEL|OLD|ON|' + r'ON-FRAME-BORDER|ON-FRAME|ON-FRAME-|ON-FRAME-B|ON-FRAME-BO|ON-FRAME-BOR|ON-FRAME-BORD|ON-FRAME-BORDE|' + r'OPEN|OPSYS|OPTION|OR|ORDERED-JOIN|ORDINAL|' + r'OS-APPEND|OS-COMMAND|OS-COPY|OS-CREATE-DIR|OS-DELETE|OS-DIR|' + r'OS-DRIVES|OS-DRIVE|OS-ERROR|OS-GETENV|OS-RENAME|OTHERWISE|' + r'OUTPUT|OVERLAY|OVERRIDE|OWNER|PAGE|' + r'PAGE-BOTTOM|PAGE-BOT|PAGE-BOTT|PAGE-BOTTO|PAGED|' + r'PAGE-NUMBER|PAGE-NUM|PAGE-NUMB|PAGE-NUMBE|PAGE-SIZE|' + r'PAGE-TOP|PAGE-WIDTH|PAGE-WID|PAGE-WIDT|' + r'PARAMETER|PARAM|PARAME|PARAMET|PARAMETE|' + r'PARENT|PARSE-STATUS|PARTIAL-KEY|PASCAL|PASSWORD-FIELD|PATHNAME|PAUSE|' + r'PBE-HASH-ALGORITHM|PBE-HASH-ALG|PBE-HASH-ALGO|PBE-HASH-ALGOR|PBE-HASH-ALGORI|PBE-HASH-ALGORIT|PBE-HASH-ALGORITH|' + r'PBE-KEY-ROUNDS|PDBNAME|PERSISTENT|PERSIST|PERSISTE|PERSISTEN|' + r'PERSISTENT-CACHE-DISABLED|PFCOLOR|PFC|PFCO|PFCOL|PFCOLO|PIXELS|' + r'PIXELS-PER-COLUMN|PIXELS-PER-COL|PIXELS-PER-COLU|PIXELS-PER-COLUM|' + r'PIXELS-PER-ROW|POPUP-MENU|POPUP-M|POPUP-ME|POPUP-MEN|' + r'POPUP-ONLY|POPUP-O|POPUP-ON|POPUP-ONL|PORTRAIT|POSITION|' + r'PRECISION|PREFER-DATASET|PREPARED|PREPARE-STRING|' + r'PREPROCESS|PREPROC|PREPROCE|PREPROCES|' + r'PRESELECT|PRESEL|PRESELE|PRESELEC|PREV|PREV-COLUMN|' + r'PREV-SIBLING|' + r'PREV-TAB-ITEM|PREV-TAB-I|PREV-TAB-IT|PREV-TAB-ITE|' + r'PRIMARY|PRINTER|PRINTER-CONTROL-HANDLE|PRINTER-HDC|' + r'PRINTER-NAME|PRINTER-PORT|PRINTER-SETUP|PRIVATE|' + r'PRIVATE-DATA|PRIVATE-D|PRIVATE-DA|PRIVATE-DAT|' + r'PRIVILEGES|' + r'PROCEDURE|PROCE|PROCED|PROCEDU|PROCEDUR|' + r'PROCEDURE-CALL-TYPE|' + r'PROCESS|' + r'PROC-HANDLE|PROC-HA|PROC-HAN|PROC-HAND|PROC-HANDL|' + r'PROC-STATUS|PROC-ST|PROC-STA|PROC-STAT|PROC-STATU|' + r'proc-text|proc-text-buffer|' + r'PROFILER|PROGRAM-NAME|PROGRESS|' + r'PROGRESS-SOURCE|PROGRESS-S|PROGRESS-SO|PROGRESS-SOU|PROGRESS-SOUR|PROGRESS-SOURC|' + r'PROMPT|PROMPT-FOR|PROMPT-F|PROMPT-FO|PROMSGS|PROPATH|' + r'PROPERTY|PROTECTED|PROVERSION|PROVERS|PROVERSI|PROVERSIO|' + r'PROXY|PROXY-PASSWORD|PROXY-USERID|PUBLIC|PUBLIC-ID|' + r'PUBLISH|PUBLISHED-EVENTS|PUT|PUTBYTE|PUT-BYTE|PUT-DOUBLE|' + r'PUT-FLOAT|PUT-INT64|PUT-KEY-VALUE|PUT-KEY-VAL|PUT-KEY-VALU|PUT-LONG|' + r'PUT-SHORT|PUT-STRING|PUT-UNSIGNED-LONG|QUERY|QUERY-CLOSE|QUERY-OFF-END|' + r'QUERY-OPEN|QUERY-PREPARE|QUERY-TUNING|QUESTION|QUIT|QUOTER|' + r'RADIO-BUTTONS|RADIO-SET|RANDOM|RAW-TRANSFER|' + r'RCODE-INFORMATION|RCODE-INFO|RCODE-INFOR|RCODE-INFORM|RCODE-INFORMA|RCODE-INFORMAT|RCODE-INFORMATI|RCODE-INFORMATIO|' + r'READ-AVAILABLE|READ-EXACT-NUM|READ-FILE|READKEY|READ-ONLY|READ-XML|READ-XMLSCHEMA|' + r'REAL|RECORD-LENGTH|RECTANGLE|RECT|RECTA|RECTAN|RECTANG|RECTANGL|' + r'RECURSIVE|REFERENCE-ONLY|REFRESH|REFRESHABLE|REFRESH-AUDIT-POLICY|' + r'REGISTER-DOMAIN|RELEASE|REMOTE|REMOVE-EVENTS-PROCEDURE|REMOVE-SUPER-PROCEDURE|' + r'REPEAT|REPLACE|REPLACE-SELECTION-TEXT|REPOSITION|REPOSITION-BACKWARD|' + r'REPOSITION-FORWARD|REPOSITION-MODE|REPOSITION-TO-ROW|REPOSITION-TO-ROWID|' + r'REQUEST|RESET|RESIZABLE|RESIZA|RESIZAB|RESIZABL|RESIZE|RESTART-ROW|' + r'RESTART-ROWID|RETAIN|RETAIN-SHAPE|RETRY|RETRY-CANCEL|RETURN|' + r'RETURN-INSERTED|RETURN-INS|RETURN-INSE|RETURN-INSER|RETURN-INSERT|RETURN-INSERTE|' + r'RETURNS|RETURN-TO-START-DIR|RETURN-TO-START-DI|' + r'RETURN-VALUE|RETURN-VAL|RETURN-VALU|' + r'RETURN-VALUE-DATA-TYPE|REVERSE-FROM|REVERT|' + r'REVOKE|RGB-VALUE|RIGHT-ALIGNED|RETURN-ALIGN|RETURN-ALIGNE|' + r'RIGHT-TRIM|R-INDEX|ROLES|ROUND|ROUTINE-LEVEL|ROW|' + r'ROW-HEIGHT-CHARS|HEIGHT|ROW-HEIGHT-PIXELS|HEIGHT-P|ROW-MARKERS|' + r'ROW-OF|ROW-RESIZABLE|RULE|RUN|RUN-PROCEDURE|SAVE|SAVE-AS|' + r'SAVE-FILE|SAX-COMPLETE|SAX-COMPLE|SAX-COMPLET|SAX-PARSE|SAX-PARSE-FIRST|' + r'SAX-PARSE-NEXT|SAX-PARSER-ERROR|SAX-RUNNING|SAX-UNINITIALIZED|' + r'SAX-WRITE-BEGIN|SAX-WRITE-COMPLETE|SAX-WRITE-CONTENT|SAX-WRITE-ELEMENT|' + r'SAX-WRITE-ERROR|SAX-WRITE-IDLE|SAX-WRITER|SAX-WRITE-TAG|SCHEMA|' + r'SCHEMA-LOCATION|SCHEMA-MARSHAL|SCHEMA-PATH|SCREEN|SCREEN-IO|' + r'SCREEN-LINES|SCREEN-VALUE|SCREEN-VAL|SCREEN-VALU|SCROLL|SCROLLABLE|' + r'SCROLLBAR-HORIZONTAL|SCROLLBAR-H|SCROLLBAR-HO|SCROLLBAR-HOR|SCROLLBAR-HORI|SCROLLBAR-HORIZ|SCROLLBAR-HORIZO|SCROLLBAR-HORIZON|SCROLLBAR-HORIZONT|SCROLLBAR-HORIZONTA|' + r'SCROLL-BARS|' + r'SCROLLBAR-VERTICAL|SCROLLBAR-V|SCROLLBAR-VE|SCROLLBAR-VER|SCROLLBAR-VERT|SCROLLBAR-VERTI|SCROLLBAR-VERTIC|SCROLLBAR-VERTICA|' + r'SCROLL-DELTA|' + r'SCROLLED-ROW-POSITION|SCROLLED-ROW-POS|SCROLLED-ROW-POSI|SCROLLED-ROW-POSIT|SCROLLED-ROW-POSITI|SCROLLED-ROW-POSITIO|' + r'SCROLLING|SCROLL-OFFSET|SCROLL-TO-CURRENT-ROW|SCROLL-TO-ITEM|SCROLL-TO-I|SCROLL-TO-IT|SCROLL-TO-ITE|' + r'SCROLL-TO-SELECTED-ROW|SDBNAME|SEAL|SEAL-TIMESTAMP|SEARCH|SEARCH-SELF|SEARCH-TARGET|' + r'SECTION|SECURITY-POLICY|SEEK|SELECT|SELECTABLE|SELECT-ALL|' + r'SELECTED|SELECT-FOCUSED-ROW|SELECTION|SELECTION-END|SELECTION-LIST|' + r'SELECTION-START|SELECTION-TEXT|SELECT-NEXT-ROW|SELECT-PREV-ROW|' + r'SELECT-ROW|SELF|SEND|send-sql-statement|send-sql|SENSITIVE|' + r'SEPARATE-CONNECTION|SEPARATOR-FGCOLOR|SEPARATORS|SERVER|' + r'SERVER-CONNECTION-BOUND|SERVER-CONNECTION-BOUND-REQUEST|' + r'SERVER-CONNECTION-CONTEXT|SERVER-CONNECTION-ID|SERVER-OPERATING-MODE|' + r'SESSION|SESSION-ID|SET|SET-APPL-CONTEXT|SET-ATTR-CALL-TYPE|SET-ATTRIBUTE-NODE|' + r'SET-BLUE-VALUE|SET-BLUE|SET-BLUE-|SET-BLUE-V|SET-BLUE-VA|SET-BLUE-VAL|SET-BLUE-VALU|' + r'SET-BREAK|SET-BUFFERS|SET-CALLBACK|SET-CLIENT|SET-COMMIT|SET-CONTENTS|' + r'SET-CURRENT-VALUE|SET-DB-CLIENT|SET-DYNAMIC|SET-EVENT-MANAGER-OPTION|' + r'SET-GREEN-VALUE|SET-GREEN|SET-GREEN-|SET-GREEN-V|SET-GREEN-VA|SET-GREEN-VAL|SET-GREEN-VALU|' + r'SET-INPUT-SOURCE|SET-OPTION|SET-OUTPUT-DESTINATION|SET-PARAMETER|SET-POINTER-VALUE|' + r'SET-PROPERTY|SET-RED-VALUE|SET-RED|SET-RED-|SET-RED-V|SET-RED-VA|SET-RED-VAL|SET-RED-VALU|' + r'SET-REPOSITIONED-ROW|SET-RGB-VALUE|SET-ROLLBACK|SET-SELECTION|SET-SIZE|' + r'SET-SORT-ARROW|SETUSERID|SETUSER|SETUSERI|SET-WAIT-STATE|SHA1-DIGEST|SHARED|' + r'SHARE-LOCK|SHARE|SHARE-|SHARE-L|SHARE-LO|SHARE-LOC|SHOW-IN-TASKBAR|SHOW-STATS|SHOW-STAT|' + r'SIDE-LABEL-HANDLE|SIDE-LABEL-H|SIDE-LABEL-HA|SIDE-LABEL-HAN|SIDE-LABEL-HAND|SIDE-LABEL-HANDL|' + r'SIDE-LABELS|SIDE-LAB|SIDE-LABE|SIDE-LABEL|' + r'SILENT|SIMPLE|SINGLE|SIZE|' + r'SIZE-CHARS|SIZE-C|SIZE-CH|SIZE-CHA|SIZE-CHAR|' + r'SIZE-PIXELS|SIZE-P|SIZE-PI|SIZE-PIX|SIZE-PIXE|SIZE-PIXEL|SKIP|' + r'SKIP-DELETED-RECORD|SLIDER|SMALL-ICON|SMALLINT|SMALL-TITLE|SOME|SORT|' + r'SORT-ASCENDING|SORT-NUMBER|SOURCE|SOURCE-PROCEDURE|SPACE|SQL|SQRT|' + r'SSL-SERVER-NAME|STANDALONE|START|START-DOCUMENT|START-ELEMENT|START-MOVE|' + r'START-RESIZE|START-ROW-RESIZE|STATE-DETAIL|STATIC|STATUS|STATUS-AREA|STATUS-AREA-FONT|' + r'STDCALL|STOP|STOP-PARSING|STOPPED|STOPPE|' + r'STORED-PROCEDURE|STORED-PROC|STORED-PROCE|STORED-PROCED|STORED-PROCEDU|STORED-PROCEDUR|' + r'STREAM|STREAM-HANDLE|STREAM-IO|STRETCH-TO-FIT|STRICT|STRING|STRING-VALUE|STRING-XREF|' + r'SUB-AVERAGE|SUB-AVE|SUB-AVER|SUB-AVERA|SUB-AVERAG|' + r'SUB-COUNT|SUB-MAXIMUM|SUM-MAX|SUM-MAXI|SUM-MAXIM|SUM-MAXIMU|SUB-MENU|SUBSUB-|' + r'MINIMUM|SUB-MIN|SUBSCRIBE|SUBSTITUTE|SUBST|SUBSTI|SUBSTIT|SUBSTITU|SUBSTITUT|' + r'SUBSTRING|SUBSTR|SUBSTRI|SUBSTRIN|SUB-TOTAL|SUBTYPE|SUM|SUPER|SUPER-PROCEDURES|' + r'SUPPRESS-NAMESPACE-PROCESSING|' + r'SUPPRESS-WARNINGS|SUPPRESS-W|SUPPRESS-WA|SUPPRESS-WAR|SUPPRESS-WARN|SUPPRESS-WARNI|SUPPRESS-WARNIN|SUPPRESS-WARNING|' + r'SYMMETRIC-ENCRYPTION-ALGORITHM|SYMMETRIC-ENCRYPTION-IV|SYMMETRIC-ENCRYPTION-KEY|SYMMETRIC-SUPPORT|' + r'SYSTEM-ALERT-BOXES|SYSTEM-ALERT|SYSTEM-ALERT-|SYSTEM-ALERT-B|SYSTEM-ALERT-BO|SYSTEM-ALERT-BOX|SYSTEM-ALERT-BOXE|' + r'SYSTEM-DIALOG|SYSTEM-HELP|SYSTEM-ID|TABLE|TABLE-HANDLE|TABLE-NUMBER|TAB-POSITION|' + r'TAB-STOP|TARGET|TARGET-PROCEDURE|' + r'TEMP-DIRECTORY|TEMP-DIR|TEMP-DIRE|TEMP-DIREC|TEMP-DIRECT|TEMP-DIRECTO|TEMP-DIRECTOR|' + r'TEMP-TABLE|TEMP-TABLE-PREPARE|TERM|TERMINAL|TERM|TERMI|TERMIN|TERMINA|' + r'TERMINATE|TEXT|TEXT-CURSOR|TEXT-SEG-GROW|TEXT-SELECTED|THEN|' + r'THIS-OBJECT|THIS-PROCEDURE|THREE-D|THROW|THROUGH|THRU|TIC-MARKS|TIME|' + r'TIME-SOURCE|TITLE|' + r'TITLE-BGCOLOR|TITLE-BGC|TITLE-BGCO|TITLE-BGCOL|TITLE-BGCOLO|' + r'TITLE-DCOLOR|TITLE-DC|TITLE-DCO|TITLE-DCOL|TITLE-DCOLO|' + r'TITLE-FGCOLOR|TITLE-FGC|TITLE-FGCO|TITLE-FGCOL|TITLE-FGCOLO|' + r'TITLE-FONT|TITLE-FO|TITLE-FON|' + r'TO|TODAY|TOGGLE-BOX|TOOLTIP|TOOLTIPS|TOPIC|TOP-NAV-QUERY|TOP-ONLY|' + r'TO-ROWID|TOTAL|TRAILING|TRANS|TRANSACTION|TRANSACTION-MODE|' + r'TRANS-INIT-PROCEDURE|TRANSPARENT|TRIGGER|TRIGGERS|TRIM|' + r'TRUE|TRUNCATE|TRUNC|TRUNCA|TRUNCAT|TYPE|TYPE-OF|' + r'UNBOX|UNBUFFERED|UNBUFF|UNBUFFE|UNBUFFER|UNBUFFERE|' + r'UNDERLINE|UNDERL|UNDERLI|UNDERLIN|UNDO|' + r'UNFORMATTED|UNFORM|UNFORMA|UNFORMAT|UNFORMATT|UNFORMATTE|UNION|' + r'UNIQUE|UNIQUE-ID|UNIQUE-MATCH|UNIX|UNLESS-HIDDEN|UNLOAD|' + r'UNSIGNED-LONG|UNSUBSCRIBE|UP|UPDATE|UPDATE-ATTRIBUTE|' + r'URL|URL-DECODE|URL-ENCODE|URL-PASSWORD|URL-USERID|USE|' + r'USE-DICT-EXPS|USE-FILENAME|USE-INDEX|USER|USE-REVVIDEO|' + r'USERID|USER-ID|USE-TEXT|USE-UNDERLINE|USE-WIDGET-POOL|' + r'USING|V6DISPLAY|V6FRAME|VALIDATE|VALIDATE-EXPRESSION|' + r'VALIDATE-MESSAGE|VALIDATE-SEAL|VALIDATION-ENABLED|VALID-EVENT|' + r'VALID-HANDLE|VALID-OBJECT|VALUE|VALUE-CHANGED|VALUES|' + r'VARIABLE|VAR|VARI|VARIA|VARIAB|VARIABL|VERBOSE|' + r'VERSION|VERTICAL|VERT|VERTI|VERTIC|VERTICA|' + r'VIEW|VIEW-AS|VIEW-FIRST-COLUMN-ON-REOPEN|' + r'VIRTUAL-HEIGHT-CHARS|VIRTUAL-HEIGHT|VIRTUAL-HEIGHT-|VIRTUAL-HEIGHT-C|VIRTUAL-HEIGHT-CH|VIRTUAL-HEIGHT-CHA|VIRTUAL-HEIGHT-CHAR|' + r'VIRTUAL-HEIGHT-PIXELS|VIRTUAL-HEIGHT-P|VIRTUAL-HEIGHT-PI|VIRTUAL-HEIGHT-PIX|VIRTUAL-HEIGHT-PIXE|VIRTUAL-HEIGHT-PIXEL|' + r'VIRTUAL-WIDTH-CHARS|VIRTUAL-WIDTH|VIRTUAL-WIDTH-|VIRTUAL-WIDTH-C|VIRTUAL-WIDTH-CH|VIRTUAL-WIDTH-CHA|VIRTUAL-WIDTH-CHAR|' + r'VIRTUAL-WIDTH-PIXELS|VIRTUAL-WIDTH-P|VIRTUAL-WIDTH-PI|VIRTUAL-WIDTH-PIX|VIRTUAL-WIDTH-PIXE|VIRTUAL-WIDTH-PIXEL|' + r'VISIBLE|VOID|WAIT|WAIT-FOR|WARNING|WEB-CONTEXT|WEEKDAY|WHEN|' + r'WHERE|WHILE|WIDGET|' + r'WIDGET-ENTER|WIDGET-E|WIDGET-EN|WIDGET-ENT|WIDGET-ENTE|' + r'WIDGET-ID|' + r'WIDGET-LEAVE|WIDGET-L|WIDGET-LE|WIDGET-LEA|WIDGET-LEAV|' + r'WIDGET-POOL|WIDTH|' + r'WIDTH-CHARS|WIDTH|WIDTH-|WIDTH-C|WIDTH-CH|WIDTH-CHA|WIDTH-CHAR|' + r'WIDTH-PIXELS|WIDTH-P|WIDTH-PI|WIDTH-PIX|WIDTH-PIXE|WIDTH-PIXEL|' + r'WINDOW|' + r'WINDOW-MAXIMIZED|WINDOW-MAXIM|WINDOW-MAXIMI|WINDOW-MAXIMIZ|WINDOW-MAXIMIZE|' + r'WINDOW-MINIMIZED|WINDOW-MINIM|WINDOW-MINIMI|WINDOW-MINIMIZ|WINDOW-MINIMIZE|' + r'WINDOW-NAME|WINDOW-NORMAL|WINDOW-STATE|WINDOW-STA|WINDOW-STAT|' + r'WINDOW-SYSTEM|WITH|WORD-INDEX|WORD-WRAP|WORK-AREA-HEIGHT-PIXELS|' + r'WORK-AREA-WIDTH-PIXELS|WORK-AREA-X|WORK-AREA-Y|WORKFILE|' + r'WORK-TABLE|WORK-TAB|WORK-TABL|WRITE|WRITE-CDATA|WRITE-CHARACTERS|' + r'WRITE-COMMENT|WRITE-DATA-ELEMENT|WRITE-EMPTY-ELEMENT|WRITE-ENTITY-REF|' + r'WRITE-EXTERNAL-DTD|WRITE-FRAGMENT|WRITE-MESSAGE|' + r'WRITE-PROCESSING-INSTRUCTION|WRITE-STATUS|WRITE-XML|WRITE-XMLSCHEMA|' + r'X|XCODE|XML-DATA-TYPE|XML-NODE-TYPE|XML-SCHEMA-PATH|' + r'XML-SUPPRESS-NAMESPACE-PROCESSING|X-OF|XREF|' + r'XREF-XML|Y|YEAR|YEAR-OFFSET|YES|YES-NO|' + r'YES-NO-CANCEL|Y-OF)\s*($|(?=[^0-9a-z_\-]))') + + tokens = { + 'root': [ + (r'/\*', Comment.Multiline, 'comment'), + (r'\{', Comment.Preproc, 'preprocessor'), + (r'\s*&.*', Comment.Preproc), + (r'0[xX][0-9a-fA-F]+[LlUu]*', Number.Hex), + (r'(?i)(DEFINE|DEF|DEFI|DEFIN)\b', Keyword.Declaration), + (types, Keyword.Type), + (keywords, Name.Builtin), + (r'"(\\\\|\\"|[^"])*"', String.Double), + (r"'(\\\\|\\'|[^'])*'", String.Single), + (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float), + (r'[0-9]+', Number.Integer), + (r'\s+', Text), + (r'[\\+\\-\\*\\/\\=]', Operator), + (r'[\\.\\:\\(\\)]', Punctuation), + (r'.', Name.Variable), # Lazy catch-all + ], + 'comment': [ + (r'[^*/]', Comment.Multiline), + (r'/\*', Comment.Multiline, '#push'), + (r'\*/', Comment.Multiline, '#pop'), + (r'[*/]', Comment.Multiline) + ], + 'preprocessor': [ + (r'[^{}]', Comment.Preproc), + (r'{', Comment.Preproc, '#push'), + (r'}', Comment.Preproc, '#pop'), + ], + } diff --git a/pygments/lexers/pypylog.py b/pygments/lexers/pypylog.py index c3af3387..48f0dbb9 100644 --- a/pygments/lexers/pypylog.py +++ b/pygments/lexers/pypylog.py @@ -37,6 +37,7 @@ class PyPyLogLexer(RegexLexer): "jit-log": [ (r"\[\w+\] jit-log-.*?}$", Keyword, "#pop"), + (r"^\+\d+: ", Comment), (r"[ifp]\d+", Name), (r"ptr\d+", Name), (r"(\()([\w_]+(?:\.[\w_]+)?)(\))", @@ -45,9 +46,9 @@ class PyPyLogLexer(RegexLexer): (r"(\d+\.\d+|inf|-inf)", Number.Float), (r"-?\d+", Number.Integer), (r"'.*'", String), - (r"(None|descr|ConstClass|ConstPtr)", Name), + (r"(None|descr|ConstClass|ConstPtr|TargetToken)", Name), (r"<.*?>", Name.Builtin), - (r"(debug_merge_point|jump|finish)", Name.Class), + (r"(label|debug_merge_point|jump|finish)", Name.Class), (r"(int_add_ovf|int_add|int_sub_ovf|int_sub|int_mul_ovf|int_mul|" r"int_floordiv|int_mod|int_lshift|int_rshift|int_and|int_or|" r"int_xor|int_eq|int_ne|int_ge|int_gt|int_le|int_lt|int_is_zero|" @@ -55,14 +56,15 @@ class PyPyLogLexer(RegexLexer): r"uint_floordiv|uint_ge|uint_lt|" r"float_add|float_sub|float_mul|float_truediv|" r"float_eq|float_ne|float_ge|float_gt|float_le|float_lt|float_abs|" - r"ptr_eq|" - r"cast_int_to_float|cast_float_to_int|cast_opaque_ptr|" - r"force_token|quasiimmut_field|same_as|virtual_ref_finish|virtual_ref|" + r"ptr_eq|ptr_ne|instance_ptr_eq|instance_ptr_ne|" + r"cast_int_to_float|cast_float_to_int|" + r"force_token|quasiimmut_field|same_as|virtual_ref_finish|virtual_ref|mark_opaque_ptr|" r"call_may_force|call_assembler|call_loopinvariant|call_release_gil|call_pure|call|" r"new_with_vtable|new_array|newstr|newunicode|new|" r"arraylen_gc|" r"getarrayitem_gc_pure|getarrayitem_gc|setarrayitem_gc|" - r"getarrayitem_raw|setarrayitem_raw|getfield_gc_pure|getfield_gc|" + r"getarrayitem_raw|setarrayitem_raw|getfield_gc_pure|getfield_gc|getinteriorfield_gc|" + r"getinteriorfield_gc|setinteriorfield_gc|" r"getfield_raw|setfield_gc|setfield_raw|" r"strgetitem|strsetitem|strlen|copystrcontent|" r"unicodegetitem|unicodesetitem|unicodelen|" diff --git a/pygments/lexers/web.py b/pygments/lexers/web.py index 148762fd..f76e3420 100644 --- a/pygments/lexers/web.py +++ b/pygments/lexers/web.py @@ -22,7 +22,7 @@ from pygments.lexers.agile import RubyLexer from pygments.lexers.compiled import ScalaLexer -__all__ = ['HtmlLexer', 'XmlLexer', 'JavascriptLexer', 'CssLexer', +__all__ = ['HtmlLexer', 'XmlLexer', 'JavascriptLexer', 'JSONLexer', 'CssLexer', 'PhpLexer', 'ActionScriptLexer', 'XsltLexer', 'ActionScript3Lexer', 'MxmlLexer', 'HaxeLexer', 'HamlLexer', 'SassLexer', 'ScssLexer', 'ObjectiveJLexer', 'CoffeeScriptLexer', 'DuelLexer', 'ScamlLexer', @@ -36,9 +36,9 @@ class JavascriptLexer(RegexLexer): name = 'JavaScript' aliases = ['js', 'javascript'] - filenames = ['*.js', '*.json'] + filenames = ['*.js', ] mimetypes = ['application/javascript', 'application/x-javascript', - 'text/x-javascript', 'text/javascript', 'application/json'] + 'text/x-javascript', 'text/javascript', ] flags = re.DOTALL tokens = { @@ -89,6 +89,74 @@ class JavascriptLexer(RegexLexer): } +class JSONLexer(RegexLexer): + """ + For JSON data structures. + + *New in Pygments 1.5.* + """ + + name = 'JSON' + aliases = ['json'] + filenames = ['*.json'] + mimetypes = [ 'application/json', ] + + flags = re.DOTALL + tokens = { + 'whitespace': [ + (r'\s+', Text), + ], + + # represents a simple terminal value + 'simplevalue':[ + (r'(true|false|null)\b', Keyword.Constant), + (r'-?[0-9]+', Number.Integer), + (r'"(\\\\|\\"|[^"])*"', String.Double), + ], + + + # the right hand side of an object, after the attribute name + 'objectattribute': [ + include('value'), + (r':', Punctuation), + # comma terminates the attribute but expects more + (r',', Punctuation, '#pop'), + # a closing bracket terminates the entire object, so pop twice + (r'}', Punctuation, ('#pop', '#pop')), + ], + + # a json object - { attr, attr, ... } + 'objectvalue': [ + include('whitespace'), + (r'"(\\\\|\\"|[^"])*"', Name.Tag, 'objectattribute'), + (r'}', Punctuation, '#pop'), + ], + + # json array - [ value, value, ... } + 'arrayvalue': [ + include('whitespace'), + include('value'), + (r',', Punctuation), + (r']', Punctuation, '#pop'), + ], + + # a json value - either a simple value or a complex value (object or array) + 'value': [ + include('whitespace'), + include('simplevalue'), + (r'{', Punctuation, 'objectvalue'), + (r'\[', Punctuation, 'arrayvalue'), + ], + + + # the root of a json document whould be a value + 'root': [ + include('value'), + ], + + } + + class ActionScriptLexer(RegexLexer): """ For ActionScript source code. @@ -1728,7 +1796,7 @@ class CoffeeScriptLexer(RegexLexer): 'slashstartsregex': [ include('commentsandwhitespace'), (r'///', String.Regex, ('#pop', 'multilineregex')), - (r'/(\\.|[^[/\\\n]|\[(\\.|[^\]\\\n])*])+/' + (r'/(?! )(\\.|[^[/\\\n]|\[(\\.|[^\]\\\n])*])+/' r'([gim]+\b|\B)', String.Regex, '#pop'), (r'', Text, '#pop'), ], @@ -1766,8 +1834,9 @@ class CoffeeScriptLexer(RegexLexer): ("'", String, 'sqs'), ], 'strings': [ - (r'[^#\\\'"]+', String) # note that all coffee script strings are multi-line. - # hashmarks, quotes and backslashes must be parsed one at a time + (r'[^#\\\'"]+', String), + # note that all coffee script strings are multi-line. + # hashmarks, quotes and backslashes must be parsed one at a time ], 'interpoling_string' : [ (r'}', String.Interpol, "#pop"), diff --git a/pygments/util.py b/pygments/util.py index 429e40a1..a400624b 100644 --- a/pygments/util.py +++ b/pygments/util.py @@ -118,7 +118,7 @@ def make_analysator(f): return 0.0 try: return min(1.0, max(0.0, float(rv))) - except ValueError: + except (ValueError, TypeError): return 0.0 text_analyse.__doc__ = f.__doc__ return staticmethod(text_analyse) diff --git a/scripts/find_error.py b/scripts/find_error.py index 199e9c5c..1b793790 100755 --- a/scripts/find_error.py +++ b/scripts/find_error.py @@ -123,7 +123,7 @@ def main(fn, lexer=None, options={}): for tok, state in map(None, tokens, states): show_token(tok, state) else: - for i in range(len(tokens) - num, len(tokens)): + for i in range(max(len(tokens) - num, 0), len(tokens)): show_token(tokens[i], states[i]) print 'Error token:' l = len(repr(val)) diff --git a/tests/examplefiles/classes.dylan b/tests/examplefiles/classes.dylan index ff435b77..6dd55ff2 100644 --- a/tests/examplefiles/classes.dylan +++ b/tests/examplefiles/classes.dylan @@ -22,3 +22,19 @@ end function; define constant $blue-car = make(<car>, model: "Viper"); define constant $black-car = make(<car>, model: "Town Car", sunroof?: #t); define constant $red-car = make(<car>, model: "F40", sunroof?: #f); + +define method foo() => _ :: <boolean> + #t +end method; + +define method foo() => _ :: <boolean>; + #t +end method; + +define method \+() +end; + +define constant $symbol = #"hello"; +define variable *vector* = #[3.5, 5] +define constant $list = #(1, 2); +define constant $pair = #(1 . "foo") diff --git a/tests/examplefiles/example.cls b/tests/examplefiles/example.cls new file mode 100644 index 00000000..d36ad6f0 --- /dev/null +++ b/tests/examplefiles/example.cls @@ -0,0 +1,15 @@ +USING Progress.Lang.*. + +CLASS Test INHERITS Progress.Sucks: + + DEFINE PRIVATE VARIABLE cTest AS CHAR NO-UNDO. + + CONSTRUCTOR PUBLIC Test(): + SUPER(). + MESSAGE "Why are you punishing yourself by coding in this language?". + END CONSTRUCTOR. + + METHOD PUBLIC LOGICAL Blowup(INPUT iTime AS INT): + END. + +END CLASS. diff --git a/tests/examplefiles/example.moon b/tests/examplefiles/example.moon new file mode 100644 index 00000000..d4415e32 --- /dev/null +++ b/tests/examplefiles/example.moon @@ -0,0 +1,629 @@ +-- transform.moon +-- Leaf Corcoran (leafot@gmail.com) 2011 +-- +-- This is part of the MoonScript compiler. See <http://moonscript.org> +-- MoonScript is licensed under the MIT License +-- + +module "moonscript.transform", package.seeall + +types = require "moonscript.types" +util = require "moonscript.util" +data = require "moonscript.data" + +import reversed from util +import ntype, build, smart_node, is_slice from types +import insert from table + +export Statement, Value, NameProxy, LocalName, Run + +-- always declares as local +class LocalName + new: (@name) => self[1] = "temp_name" + get_name: => @name + +class NameProxy + new: (@prefix) => + self[1] = "temp_name" + + get_name: (scope) => + if not @name + @name = scope\free_name @prefix, true + @name + + chain: (...) => + items = {...} -- todo: fix ... propagation + items = for i in *items + if type(i) == "string" + {"dot", i} + else + i + + build.chain { + base: self + unpack items + } + + index: (key) => + build.chain { + base: self, {"index", key} + } + + __tostring: => + if @name + ("name<%s>")\format @name + else + ("name<prefix(%s)>")\format @prefix + +class Run + new: (@fn) => + self[1] = "run" + + call: (state) => + self.fn state + +-- transform the last stm is a list of stms +-- will puke on group +apply_to_last = (stms, fn) -> + -- find last (real) exp + last_exp_id = 0 + for i = #stms, 1, -1 + stm = stms[i] + if stm and util.moon.type(stm) != Run + last_exp_id = i + break + + return for i, stm in ipairs stms + if i == last_exp_id + fn stm + else + stm + +-- is a body a sindle expression/statement +is_singular = (body) -> + return false if #body != 1 + if "group" == ntype body + is_singular body[2] + else + true + +constructor_name = "new" + +class Transformer + new: (@transformers, @scope) => + @seen_nodes = {} + + transform: (scope, node, ...) => + -- print scope, node, ... + return node if @seen_nodes[node] + @seen_nodes[node] = true + while true + transformer = @transformers[ntype node] + res = if transformer + transformer(scope, node, ...) or node + else + node + return node if res == node + node = res + + __call: (node, ...) => + @transform @scope, node, ... + + instance: (scope) => + Transformer @transformers, scope + + can_transform: (node) => + @transformers[ntype node] != nil + +construct_comprehension = (inner, clauses) -> + current_stms = inner + for _, clause in reversed clauses + t = clause[1] + current_stms = if t == "for" + _, names, iter = unpack clause + {"foreach", names, iter, current_stms} + elseif t == "when" + _, cond = unpack clause + {"if", cond, current_stms} + else + error "Unknown comprehension clause: "..t + current_stms = {current_stms} + + current_stms[1] + +Statement = Transformer { + assign: (node) => + _, names, values = unpack node + -- bubble cascading assigns + if #values == 1 and types.cascading[ntype values[1]] + values[1] = @transform.statement values[1], (stm) -> + t = ntype stm + if types.is_value stm + {"assign", names, {stm}} + else + stm + + build.group { + {"declare", names} + values[1] + } + else + node + + export: (node) => + -- assign values if they are included + if #node > 2 + if node[2] == "class" + cls = smart_node node[3] + build.group { + {"export", {cls.name}} + cls + } + else + build.group { + node + build.assign { + names: node[2] + values: node[3] + } + } + else + nil + + update: (node) => + _, name, op, exp = unpack node + op_final = op\match "^(.+)=$" + error "Unknown op: "..op if not op_final + build.assign_one name, {"exp", name, op_final, exp} + + import: (node) => + _, names, source = unpack node + + stubs = for name in *names + if type(name) == "table" + name + else + {"dot", name} + + real_names = for name in *names + type(name) == "table" and name[2] or name + + if type(source) == "string" + build.assign { + names: real_names + values: [build.chain { base: source, stub} for stub in *stubs] + } + else + source_name = NameProxy "table" + build.group { + {"declare", real_names} + build["do"] { + build.assign_one source_name, source + build.assign { + names: real_names + values: [build.chain { base: source_name, stub} for stub in *stubs] + } + } + } + + comprehension: (node, action) => + _, exp, clauses = unpack node + + action = action or (exp) -> {exp} + construct_comprehension action(exp), clauses + + -- handle cascading return decorator + if: (node, ret) => + if ret + smart_node node + -- mutate all the bodies + node['then'] = apply_to_last node['then'], ret + for i = 4, #node + case = node[i] + body_idx = #node[i] + case[body_idx] = apply_to_last case[body_idx], ret + node + + with: (node, ret) => + _, exp, block = unpack node + scope_name = NameProxy "with" + build["do"] { + build.assign_one scope_name, exp + Run => @set "scope_var", scope_name + build.group block + if ret + ret scope_name + } + + foreach: (node) => + smart_node node + if ntype(node.iter) == "unpack" + list = node.iter[2] + + index_name = NameProxy "index" + list_name = NameProxy "list" + + slice_var = nil + bounds = if is_slice list + slice = list[#list] + table.remove list + table.remove slice, 1 + + slice[2] = if slice[2] and slice[2] != "" + max_tmp_name = NameProxy "max" + slice_var = build.assign_one max_tmp_name, slice[2] + {"exp", max_tmp_name, "<", 0 + "and", {"length", list_name}, "+", max_tmp_name + "or", max_tmp_name } + else + {"length", list_name} + + slice + else + {1, {"length", list_name}} + + build.group { + build.assign_one list_name, list + slice_var + build["for"] { + name: index_name + bounds: bounds + body: { + {"assign", node.names, {list_name\index index_name}} + build.group node.body + } + } + } + + switch: (node, ret) => + _, exp, conds = unpack node + exp_name = NameProxy "exp" + + -- convert switch conds into if statment conds + convert_cond = (cond) -> + t, case_exp, body = unpack cond + out = {} + insert out, t == "case" and "elseif" or "else" + if t != "else" + insert out, {"exp", case_exp, "==", exp_name} if t != "else" + else + body = case_exp + + if ret + body = apply_to_last body, ret + + insert out, body + + out + + first = true + if_stm = {"if"} + for cond in *conds + if_cond = convert_cond cond + if first + first = false + insert if_stm, if_cond[2] + insert if_stm, if_cond[3] + else + insert if_stm, if_cond + + build.group { + build.assign_one exp_name, exp + if_stm + } + + class: (node) => + _, name, parent_val, body = unpack node + + -- split apart properties and statements + statements = {} + properties = {} + for item in *body + switch item[1] + when "stm" + insert statements, item[2] + when "props" + for tuple in *item[2,] + insert properties, tuple + + -- find constructor + constructor = nil + properties = for tuple in *properties + if tuple[1] == constructor_name + constructor = tuple[2] + nil + else + tuple + + parent_cls_name = NameProxy "parent" + base_name = NameProxy "base" + self_name = NameProxy "self" + cls_name = NameProxy "class" + + if not constructor + constructor = build.fndef { + args: {{"..."}} + arrow: "fat" + body: { + build["if"] { + cond: parent_cls_name + then: { + build.chain { base: "super", {"call", {"..."}} } + } + } + } + } + else + smart_node constructor + constructor.arrow = "fat" + + cls = build.table { + {"__init", constructor} + {"__base", base_name} + {"__name", {"string", '"', name}} -- "quote the string" + {"__parent", parent_cls_name} + } + + -- look up a name in the class object + class_lookup = build["if"] { + cond: {"exp", "val", "==", "nil", "and", parent_cls_name} + then: { + parent_cls_name\index"name" + } + } + insert class_lookup, {"else", {"val"}} + + cls_mt = build.table { + {"__index", build.fndef { + args: {{"cls"}, {"name"}} + body: { + build.assign_one LocalName"val", build.chain { + base: "rawget", {"call", {base_name, "name"}} + } + class_lookup + } + }} + {"__call", build.fndef { + args: {{"cls"}, {"..."}} + body: { + build.assign_one self_name, build.chain { + base: "setmetatable" + {"call", {"{}", base_name}} + } + build.chain { + base: "cls.__init" + {"call", {self_name, "..."}} + } + self_name + } + }} + } + + cls = build.chain { + base: "setmetatable" + {"call", {cls, cls_mt}} + } + + value = nil + with build + value = .block_exp { + Run => + @set "super", (block, chain) -> + if chain + slice = [item for item in *chain[3,]] + new_chain = {"chain", parent_cls_name} + + head = slice[1] + + if head == nil + return parent_cls_name + + switch head[1] + -- calling super, inject calling name and self into chain + when "call" + calling_name = block\get"current_block" + slice[1] = {"call", {"self", unpack head[2]}} + act = if ntype(calling_name) != "value" then "index" else "dot" + insert new_chain, {act, calling_name} + + -- colon call on super, replace class with self as first arg + when "colon" + call = head[3] + insert new_chain, {"dot", head[2]} + slice[1] = { "call", { "self", unpack call[2] } } + + insert new_chain, item for item in *slice + + new_chain + else + parent_cls_name + + .assign_one parent_cls_name, parent_val == "" and "nil" or parent_val + .assign_one base_name, {"table", properties} + .assign_one base_name\chain"__index", base_name + + build["if"] { + cond: parent_cls_name + then: { + .chain { + base: "setmetatable" + {"call", { + base_name, + .chain { base: parent_cls_name, {"dot", "__base"}} + }} + } + } + } + + .assign_one cls_name, cls + .assign_one base_name\chain"__class", cls_name + + .group if #statements > 0 { + .assign_one LocalName"self", cls_name + .group statements + } else {} + + cls_name + } + + value = .group { + .declare names: {name} + .assign { + names: {name} + values: {value} + } + } + + value +} + +class Accumulator + body_idx: { for: 4, while: 3, foreach: 4 } + + new: => + @accum_name = NameProxy "accum" + @value_name = NameProxy "value" + @len_name = NameProxy "len" + + -- wraps node and mutates body + convert: (node) => + index = @body_idx[ntype node] + node[index] = @mutate_body node[index] + @wrap node + + -- wrap the node into a block_exp + wrap: (node) => + build.block_exp { + build.assign_one @accum_name, build.table! + build.assign_one @len_name, 0 + node + @accum_name + } + + -- mutates the body of a loop construct to save last value into accumulator + -- can optionally skip nil results + mutate_body: (body, skip_nil=true) => + val = if not skip_nil and is_singular body + with body[1] + body = {} + else + body = apply_to_last body, (n) -> + build.assign_one @value_name, n + @value_name + + update = { + {"update", @len_name, "+=", 1} + build.assign_one @accum_name\index(@len_name), val + } + + if skip_nil + table.insert body, build["if"] { + cond: {"exp", @value_name, "!=", "nil"} + then: update + } + else + table.insert body, build.group update + + body + +default_accumulator = (node) => + Accumulator!\convert node + + +implicitly_return = (scope) -> + fn = (stm) -> + t = ntype stm + if types.manual_return[t] or not types.is_value stm + stm + elseif types.cascading[t] + scope.transform.statement stm, fn + else + if t == "comprehension" and not types.comprehension_has_value stm + stm + else + {"return", stm} + + fn + +Value = Transformer { + for: default_accumulator + while: default_accumulator + foreach: default_accumulator + + comprehension: (node) => + a = Accumulator! + node = @transform.statement node, (exp) -> + a\mutate_body {exp}, false + a\wrap node + + tblcomprehension: (node) => + _, key_exp, value_exp, clauses = unpack node + + accum = NameProxy "tbl" + dest = build.chain { base: accum, {"index", key_exp} } + inner = build.assign_one dest, value_exp + + build.block_exp { + build.assign_one accum, build.table! + construct_comprehension {inner}, clauses + accum + } + + fndef: (node) => + smart_node node + node.body = apply_to_last node.body, implicitly_return self + node + + if: (node) => build.block_exp { node } + with: (node) => build.block_exp { node } + switch: (node) => + build.block_exp { node } + + -- pull out colon chain + chain: (node) => + stub = node[#node] + if type(stub) == "table" and stub[1] == "colon_stub" + table.remove node, #node + + base_name = NameProxy "base" + fn_name = NameProxy "fn" + + is_super = node[2] == "super" + @transform.value build.block_exp { + build.assign { + names: {base_name} + values: {node} + } + + build.assign { + names: {fn_name} + values: { + build.chain { base: base_name, {"dot", stub[2]} } + } + } + + build.fndef { + args: {{"..."}} + body: { + build.chain { + base: fn_name, {"call", {is_super and "self" or base_name, "..."}} + } + } + } + } + + block_exp: (node) => + _, body = unpack node + + fn = nil + arg_list = {} + + insert body, Run => + if @has_varargs + insert arg_list, "..." + insert fn.args, {"..."} + + fn = smart_node build.fndef body: body + build.chain { base: {"parens", fn}, {"call", arg_list} } +} + diff --git a/tests/examplefiles/example.p b/tests/examplefiles/example.p new file mode 100644 index 00000000..e8c17e33 --- /dev/null +++ b/tests/examplefiles/example.p @@ -0,0 +1,34 @@ +{include.i} +{nested.i {include.i}} + +&SCOPED-DEFINE MY_NAME "Abe" + +DEF VAR i AS INT NO-UNDO. +i = 0xABE + 1337 / (1 * 1.00) + +def var clowercasetest as char no-undo. +DEF VAR vardashtest AS DATETIME-TZ NO-UNDO. + +DEFINE TEMP-TABLE ttNames NO-UNDO + FIELD cName AS CHAR + INDEX IXPK_ttNames IS PRIMARY UNIQUE cName. + +/* One-line comment */ +/* Two-line + Comment */ +/* + Nested + /* + Multiline + /* + Comment + */ + */ +*/ + +CREATE ttNames. +ASSIGN ttNames.cName = {&MY_NAME}. + +FOR EACH ttNames: + MESSAGE "Hello, " + ttNames.cName + '!' VIEW-AS ALERT-BOX. +END. diff --git a/tests/examplefiles/example.snobol b/tests/examplefiles/example.snobol new file mode 100644 index 00000000..26ca5cf4 --- /dev/null +++ b/tests/examplefiles/example.snobol @@ -0,0 +1,15 @@ +-SOME RANDOM DIRECTIVE WOULD GO HERE +* +* SNOBOL4 example file for lexer +* + SOME.THING_OR_OTHER32 = 1 + 1.0 - 1E3 * 1E-3 ** 2.718284590E0 ++ :F(END)S(IN_LOOP) + PATTERN = LEN(3) ("GAR" | "BAR") +IN_LOOP THING = INPUT :F(END) + THING LEN(3) ("GAR" | "BAR") :S(OK) + OUTPUT = THING " : Failure!" :(IN_LOOP) +OK OUTPUT = THING ' : "Success"!' :(IN_LOOP) +END +FOOBAR +FOOGAR +THiNIg diff --git a/tests/examplefiles/example.u b/tests/examplefiles/example.u new file mode 100644 index 00000000..42c85902 --- /dev/null +++ b/tests/examplefiles/example.u @@ -0,0 +1,548 @@ + // This is a one line comment. + /* an inner comment */ + + /* nested /* comments */ */ + + /* + /* + Multi-line. + */ + */ + +// Binary blob escape. +//"some text \B(3)("\") ouhyeah" == "\"\\\""; +"some text \B(3)("\") ouhyeah" == "\"\\\""; +'some text \B(3)('\') ouhyeah' == '\'\\\''; + +//"\B(4)()"'()"; +"\B(4)()"'()"; +'\B(4)()'"()'; + +//blob size limits +"hey ! \B(0)() oh !" + +//blob format is wrong +"hey ! \B(2)(aaa) oh !" +"hey ! \B(100)(aaa) oh !" + +//multiple blob in a string +"hey ! \B(3)(aaa) hey ! \B(3)(aaa) oh !" + +// multiple digits blob size +"hey ! \B(10)(aaaaaaaaaa) !" +"hey ! \B(10)(aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa) !" +"hey ! \B(100)(a) !" + +// multiple digits blob size +"hey ! \B(007)(aaaaaaa) !" +"hey ! \B(007)(aa) !" +"hey ! \B(007)(aaaaaaaaaaaaaaaaaa) !" + +// deprecated and restricted keyworks +emit Event.new; +static int main(); + +loopn (2) {echo("a");}; + +foreach (var i : [1,2,3,4]) { + echo(i); +}; + +function() {}; + +var 'if'; +var this.'else'; + +var '%x'; +var '1 2 3'; +var this.'[]'; + +// angles +pi == 180deg; +pi == 200grad; + +// Dictionary +[ => ]; // The empty dictionary + +// duration +1d == 24h; +0.5d == 12h; +1h == 60min; +1min == 60s; +1s == 1000ms; + +1s == 1; +1s 2s 3s == 6; +1s 1ms == 1.001; +1ms 1s == 1.001; + + + 1 == 1; + 1 == 1.0; + 1.2 == 1.2000; + 1.234e6 == 1234000; + 1e+11 == 1E+11; + 1e10 == 10000000000; + 1e30 == 1e10 * 1e10 * 1e10; + + +0.000001; + +0.0000001; + +0.00000000001; + +1e+3; + +1E-5; + + +1.; +// [00004701:error] !!! syntax error: unexpected ; + + 0x2a == 42; + 0x2A == 42; + 0xabcdef == 11259375; + 0xABCDEF == 11259375; +0xFFFFFFFF == 4294967295; + + +//123foo; +//[00005658:error] !!! syntax error: invalid token: '123foo' +//12.3foo; +//[00018827:error] !!! syntax error: invalid token: '12.3foo' +0xabcdef; +//[00060432] 11259375 +//0xabcdefg; +//[00061848:error] !!! syntax error: invalid token: '0xabcdefg' + + +[]; // The empty list +[1, 2, 3]; + +// Special characters. +"\"" == "\""; +"\\" == "\\"; + +// ASCII characters. +"\a" == "\007"; "\a" == "\x07"; +"\b" == "\010"; "\b" == "\x08"; +"\f" == "\014"; "\f" == "\x0c"; +"\n" == "\012"; "\n" == "\x0a"; +"\r" == "\015"; "\r" == "\x0d"; +"\t" == "\011"; "\t" == "\x09"; +"\v" == "\013"; "\v" == "\x0b"; + +// Octal escapes. +"\0" == "\00"; "\0" == "\000"; +"\0000" == "\0""0"; +"\062\063" == "23"; + +// Hexadecimal escapes. +"\x00" == "\0"; +"\x32\x33" == "23"; + + + +"foo" "bar" "baz" == "foobarbaz"; + +// Tuples +(); +[00000000] () +(1,); +[00000000] (1,) +(1, 2); +[00000000] (1, 2) +(1, 2, 3, 4,); +[00000000] (1, 2, 3, 4) + +function Global.verboseId(var x) +{ + echo(x) | x +}|; +class verboseId(Global).math : verboseId(Math) +{ +}; + +{ + for (3) + { + sleep(1s); + echo("ping"); + }, + sleep(0.5s); + for (3) + { + sleep(1s); + echo("pong"); + }, +}; + + 1 + 1 == 2; + 1 - 2 == -1; + 2 * 3 == 6; + 10 / 2 == 5; + 2 ** 10 == 1024; + -(1 + 2) == -3; + 1 + 2 * 3 == 7; + (1 + 2) * 3 == 9; + -2 ** 2 == -4; + - - - - 1 == 1; + +a = b +a += b +a -= b +a *= b +a /= b +a %= b +a ^= b + + +var value = 0|; +var valueAlias = value|; +value += 10; +valueAlias; +var myList = []|; +var myList.specialFeature = 42|; +myList += [1, 2, 3]; +myList.specialFeature; +var myOtherList = myList + [4, 5]; +myOtherList.specialFeature; +var something = []|; +var somethingElse = something|; +something += [1, 2]; +somethingElse += [3, 4]; +something; + + +class Counter +{ + var count = 0; + function init (n) { var this.count = n }; + // Display the value, and the identity. + function asString() { "%s @ %s" % [count, uid ] }; + function '+'(var n) { new(count + n) }; + function '-'(var n) { new(count - n) }; +}|; + + +class ImmutableCounter : Counter +{ + function '+='(var n) { this + n }; + function '-='(var n) { this - n }; +}|; + +var ic1 = ImmutableCounter.new(0); +var ic2 = ic1; + +ic1 += 1; +ic1; +ic2; + + +a << b +a >> b +a ^ b + +4 << 2 == 16; +4 >> 2 == 1; + +!a +a && b +a || b + +true && true; +true || false; +!true == false; +true || (1 / 0); +(false && (1 / 0)) == false; + +a == b +a != b +a === b +a !== b +a ~= b +a =~= b +a < b +a <= b +a > b +a >= b + +assert{ + ! (0 < 0); + 0 <= 0; + 0 == 0; + 0 !== 0; +}; + +a in b +a not in b +a[args] +a[args] = v + +1 in [0, 1, 2]; +3 not in [0, 1, 2]; + +"one" in ["zero" => 0, "one" => 1, "two" => 2]; +"three" not in ["zero" => 0, "one" => 1, "two" => 2]; + +a.b +a.b(args) +a->b +a->b = v +a.&b + +var obj = Object.new|; +function obj.f() { 24 }|; + + +var f = function(a, b) { + echo(b + a); +}| +f(1, 0); + + +function g3() +{ + return; // Stop execution at this point and return void + echo(0); // This is not executed +}| + +Object.setProperty, to define/set a property. +Object.getProperty, to get a property. +Object.removeProperty, to delete a property. +Object.hasProperty, to test for the existence of a property. +Object.properties, to get all the properties of a slot. + +enum Suit +{ + hearts, + diamonds, + clubs, + spades, // Last comma is optional +}; + +for (var suit in Suit) + echo("%s the ace of %s." % [find_ace(suit), suit]); + +switch ( ("foo", [1, 2]) ) +{ + // The pattern does not match the values of the list. + case ("foo", [2, 1]): + echo("fail"); + + // The pattern does not match the tuple. + case ["foo", [1, 2]]: + echo("fail"); + + // The pattern matches and binds the variable "l" + // but the condition is not verified. + case ("foo", var l) if l.size == 0: + echo("fail"); + + // The pattern matches. + case ("foo", [var a, var b]): + echo("foo(%s, %s)" % [a, b]); +}; +//[00000000] *** foo(1, 2) + +{ + ["b" => var b, "a" => var a] = ["a" => 1, "b" => 2, "c" => 3]; + echo("a = %d, b = %d" % [a, b]); +}; +//[00000000] *** a = 1, b = 2 + + +switch (["speed" => 2, "time" => 6s]) +{ + case ["speed" => var s] if s > 3: + echo("Too fast"); + case ["speed" => var s, "time" => var t] if s * t > 10: + echo("Too far"); +}; +//[00000000] *** Too far + + +try +{ + throw ("message", 0) +} +catch (var e if e.isA(Exception)) +{ + echo(e.message) +} +catch ((var msg, var value) if value.isA(Float)) +{ + echo("%s: %d" % [msg, value]) +}; +//[00000000] *** message: 0 + + +{ + var e = Event.new; + at (e?(var msg, var value) if value % 2 == 0) + echo("%s: %d" % [msg, value]); + + // Does not trigger the "at" because the guard is not verified. + e!("message", 1); + + // Trigger the "at". + e!("message", 2); +}; +//[00000000] *** message: 2 + +for (var i = 0; i < 8; i++) +{ + if (i % 2 != 0) + continue; + echo(i); +}; + +do (1024) +{ + assert(this == 1024); + assert(sqrt == 32); + setSlot("y", 23); +}.y; + +{ + var n = 10|; + var res = []|; + loop;{ + n--; + res << n; + if (n == 0) + break + }; + res +} + + +{ + var n = 10|; + var res = []|; + loop|{ + n--; + res << n; + if (n == 0) + break + }; + res +} + + +var j = 3| +while (0 < j) +{ + echo(j); + j--; +}; + + +{ + var i = 4| + while| (true) + { + i -= 1; + echo ("in: " + i); + if (i == 1) + break + else if (i == 2) + continue; + echo ("out: " + i); + }; +}; + + + +function test(e) +{ + try + { throw e; } + catch (0) + { echo("zero") } + catch ([var x, var y]) + { echo(x + y) } +} | {}; + +try { echo("try") } +catch { echo("catch")} +else { echo("else")}; + + +try +{ + echo("inside"); +} +finally +{ + echo("finally"); +}; +//[00000001] *** inside +//[00000002] *** finally + +at (e?(var start) ~ 1s) + echo("in : %s" % (time - start).round) +onleave + echo("out: %s" % (time - start).round); + +// This emission is too short to trigger the at. +e!(time); + +// This one is long enough. +// The body triggers 1s after the emission started. +e!(time) ~ 2s; +//[00001000] *** in : 1 +//[00002000] *** out: 2 + + +timeout (2.1s) + every (1s) + echo("Are you still there?"); +//[00000000] *** Are you still there? +//[00001000] *** Are you still there? +//[00002000] *** Are you still there? + + every| (1s) + { + echo("aba"); + }; + +for, (var i = 3; 0 < i; i -= 1) +{ + echo (i); +}; + + +for& (var i: [0, 1, 2]) +{ + echo (i * i); +}; + +loop,{ +}; + + +waituntil (e?(1, var b)); + +whenever (e?("arg", var arg) if arg % 2) + echo("e (%s) on" % arg) +else + echo("e off"); + + + while, (i) + { + var j = i -= 1; + }| + + +var y = 0; +{ + sleep(0.5s); + y = 100 smooth:3s, +}, + + + + diff --git a/tests/examplefiles/http_request_example b/tests/examplefiles/http_request_example new file mode 100644 index 00000000..5d2a1d52 --- /dev/null +++ b/tests/examplefiles/http_request_example @@ -0,0 +1,14 @@ +POST /demo/submit/ HTTP/1.1
+Host: pygments.org
+Connection: keep-alivk
+Cache-Control: max-age=0
+Origin: http://pygments.org
+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.63 Safari/535.7
+Content-Type: application/x-www-form-urlencoded
+Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8
+Referer: http://pygments.org/
+Accept-Encoding: gzip,deflate,sdch
+Accept-Language: en-US,en;q=0.8
+Accept-Charset: windows-949,utf-8;q=0.7,*;q=0.3
+
+name=test&lang=text&code=asdf&user= diff --git a/tests/examplefiles/http_response_example b/tests/examplefiles/http_response_example new file mode 100644 index 00000000..bf53d61d --- /dev/null +++ b/tests/examplefiles/http_response_example @@ -0,0 +1,27 @@ +HTTP/1.1 200 OK
+Date: Tue, 13 Dec 2011 00:11:44 GMT
+Status: 200 OK
+X-Transaction: 50b85fff78dab4a3
+X-RateLimit-Limit: 150
+ETag: "b31143be48ebfe7512b65fe64fe092f3"
+X-Frame-Options: SAMEORIGIN
+Last-Modified: Tue, 13 Dec 2011 00:11:44 GMT
+X-RateLimit-Remaining: 145
+X-Runtime: 0.01190
+X-Transaction-Mask: a6183ffa5f8ca943ff1b53b5644ef1145f6f285d
+Content-Type: application/json; charset=utf-8
+Content-Length: 2389
+Pragma: no-cache
+X-RateLimit-Class: api
+X-Revision: DEV
+Expires: Tue, 31 Mar 1981 05:00:00 GMT
+Cache-Control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0
+X-MID: a55f21733bc52bb11d1fc58f9b51b4974fbb8f83
+X-RateLimit-Reset: 1323738416
+Set-Cookie: k=10.34.234.116.1323735104238974; path=/; expires=Tue, 20-Dec-11 00:11:44 GMT; domain=.twitter.com
+Set-Cookie: guest_id=v1%3A13237351042425496; domain=.twitter.com; path=/; expires=Thu, 12-Dec-2013 12:11:44 GMT
+Set-Cookie: _twitter_sess=BAh7CDoPY3JlYXRlZF9hdGwrCPS6wjQ0AToHaWQiJTFiMTlhY2E1ZjczYThk%250ANDUwMWQxNjMwZGU2YTQ1ODBhIgpmbGFzaElDOidBY3Rpb25Db250cm9sbGVy%250AOjpGbGFzaDo6Rmxhc2hIYXNoewAGOgpAdXNlZHsA--6b502f30a083e8a41a64f10930e142ea362b1561; domain=.twitter.com; path=/; HttpOnly
+Vary: Accept-Encoding
+Server: tfe
+
+[{"contributors_enabled":false,"profile_background_tile":true,"followers_count":644,"protected":false,"profile_image_url":"http:\/\/a0.twimg.com\/profile_images\/69064242\/gb_normal.jpg","screen_name":"birkenfeld","default_profile_image":false,"following":null,"friends_count":88,"profile_sidebar_fill_color":"7AC3EE","url":"http:\/\/pythonic.pocoo.org\/","name":"Georg Brandl","default_profile":false,"is_translator":false,"utc_offset":3600,"profile_sidebar_border_color":"65B0DA","description":"","profile_background_image_url_https":"https:\/\/si0.twimg.com\/images\/themes\/theme10\/bg.gif","favourites_count":0,"profile_use_background_image":true,"created_at":"Tue Dec 30 22:25:11 +0000 2008","status":{"retweet_count":10,"favorited":false,"geo":null,"possibly_sensitive":false,"coordinates":null,"in_reply_to_screen_name":null,"in_reply_to_status_id_str":null,"retweeted":false,"in_reply_to_status_id":null,"in_reply_to_user_id_str":null,"created_at":"Sat Jul 09 13:42:35 +0000 2011","truncated":false,"id_str":"89690914515206144","contributors":null,"place":null,"source":"web","in_reply_to_user_id":null,"id":89690914515206144,"retweeted_status":{"retweet_count":10,"favorited":false,"geo":null,"possibly_sensitive":false,"coordinates":null,"in_reply_to_screen_name":null,"in_reply_to_status_id_str":null,"retweeted":false,"in_reply_to_status_id":null,"in_reply_to_user_id_str":null,"created_at":"Sat Jul 09 13:07:04 +0000 2011","truncated":false,"id_str":"89681976755372032","contributors":null,"place":null,"source":"web","in_reply_to_user_id":null,"id":89681976755372032,"text":"Excellent Python posts from @mitsuhiko - http:\/\/t.co\/k1wt6e4 and @ncoghlan_dev - http:\/\/t.co\/eTxacgZ (links fixed)"},"text":"RT @jessenoller: Excellent Python posts from @mitsuhiko - http:\/\/t.co\/k1wt6e4 and @ncoghlan_dev - http:\/\/t.co\/eTxacgZ (links fixed)"},"follow_request_sent":null,"statuses_count":553,"geo_enabled":false,"notifications":null,"profile_text_color":"3D1957","id_str":"18490730","lang":"en","profile_background_image_url":"http:\/\/a1.twimg.com\/images\/themes\/theme10\/bg.gif","profile_image_url_https":"https:\/\/si0.twimg.com\/profile_images\/69064242\/gb_normal.jpg","show_all_inline_media":true,"listed_count":65,"profile_link_color":"FF0000","verified":false,"id":18490730,"time_zone":"Berlin","profile_background_color":"642D8B","location":"Bavaria, Germany"}] diff --git a/tests/examplefiles/nemerle_sample.n b/tests/examplefiles/nemerle_sample.n index 2c05033a..5236857d 100644 --- a/tests/examplefiles/nemerle_sample.n +++ b/tests/examplefiles/nemerle_sample.n @@ -13,13 +13,15 @@ namespace Demo.Ns public virtual someMethod(str : string) : list[double] { def x = "simple string"; - def x = $"simple $splice string $(spliceMethod())"; + def x = $"simple $splice string $(spliceMethod() + 1)"; def x = <# recursive <# string #> sample #>; def x = $<# recursive $splice <# string #> sample + ..$(lst; "; "; x => $"x * 2 = $(x * 2)") str #>; + def x = @"somestring \"; def localFunc(arg) { @@ -80,6 +82,6 @@ namespace Demo.Ns macro sampleMacro(expr) syntax ("write", expr) { - <[ WriteLine($expr) ]> + <[ WriteLine($(expr : dyn)) ]> } } diff --git a/tests/examplefiles/test.cs b/tests/examplefiles/test.cs index ffa9bfea..faab7e42 100644 --- a/tests/examplefiles/test.cs +++ b/tests/examplefiles/test.cs @@ -153,6 +153,29 @@ namespace Diva.Core { public OpenerTask (string fileName) { this.fileName = fileName; + var verbatimString = @"c:\test\"; + + var verbatimStringWithNewline = @"test \\ \n \t \r +a +b +c"; + var verbatimStringWithEscapedQuotes = @"He said +""she says \"" is not an escaped character in verbatimstrings"" +"; + + int[] numbers = { 5,6,4,2,4,6,8,9,7,0 }; + var linqExample = from n in numbers + where n > 5 + select n; + + var anotherlinqExample = from n in numbers + orderby n descending + select n; + + int[] someMoreNumbers = { 8,2,17,34,8,9,9,5,3,4,2,1,5 }; + var moreLinq = from n in numbers + join mn in moreNumbers on n equals mn + 2 + select new {n, mn}; } public override void Reset () diff --git a/tests/examplefiles/test.ecl b/tests/examplefiles/test.ecl new file mode 100644 index 00000000..b686492a --- /dev/null +++ b/tests/examplefiles/test.ecl @@ -0,0 +1,58 @@ +/*############################################################################## + + Copyright (C) 2011 HPCC Systems. + + All rights reserved. This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as + published by the Free Software Foundation, either version 3 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. +############################################################################## */ + +#option ('slidingJoins', true); + +namesRecord := + RECORD +string20 surname; +string10 forename; +integer2 age; +integer2 dadAge; +integer2 mumAge; + END; + +namesRecord2 := + record +string10 extra; +namesRecord; + end; + +namesTable := dataset('x',namesRecord,FLAT); +namesTable2 := dataset('y',namesRecord2,FLAT); + +integer2 aveAgeL(namesRecord l) := (l.dadAge+l.mumAge)/2; +integer2 aveAgeR(namesRecord2 r) := (r.dadAge+r.mumAge)/2; + +// Standard join on a function of left and right +output(join(namesTable, namesTable2, aveAgeL(left) = aveAgeR(right))); + +//Several simple examples of sliding join syntax +output(join(namesTable, namesTable2, left.age >= right.age - 10 and left.age <= right.age +10)); +output(join(namesTable, namesTable2, left.age between right.age - 10 and right.age +10)); +output(join(namesTable, namesTable2, left.age between right.age + 10 and right.age +30)); +output(join(namesTable, namesTable2, left.age between (right.age + 20) - 10 and (right.age +20) + 10)); +output(join(namesTable, namesTable2, aveAgeL(left) between aveAgeR(right)+10 and aveAgeR(right)+40)); + +//Same, but on strings. Also includes age to ensure sort is done by non-sliding before sliding. +output(join(namesTable, namesTable2, left.surname between right.surname[1..10]+'AAAAAAAAAA' and right.surname[1..10]+'ZZZZZZZZZZ' and left.age=right.age)); +output(join(namesTable, namesTable2, left.surname between right.surname[1..10]+'AAAAAAAAAA' and right.surname[1..10]+'ZZZZZZZZZZ' and left.age=right.age,all)); + +//This should not generate a self join +output(join(namesTable, namesTable, left.age between right.age - 10 and right.age +10)); + diff --git a/tests/test_basic_api.py b/tests/test_basic_api.py index 02261d24..680a6085 100644 --- a/tests/test_basic_api.py +++ b/tests/test_basic_api.py @@ -57,7 +57,7 @@ def test_lexer_classes(): assert 'root' in cls._tokens, \ '%s has no root state' % cls - if cls.name == 'XQuery': # XXX temporary + if cls.name in ['XQuery', 'Opa']: # XXX temporary return tokens = list(inst.get_tokens(test_content)) diff --git a/tests/test_examplefiles.py b/tests/test_examplefiles.py index e5dbcf4c..9326660c 100644 --- a/tests/test_examplefiles.py +++ b/tests/test_examplefiles.py @@ -60,8 +60,9 @@ def check_lexer(lx, absfn, outfn): tokens = [] for type, val in lx.get_tokens(text): ntext.append(val) - assert type != Error, 'lexer %s generated error token for %s' % \ - (lx, absfn) + assert type != Error, \ + 'lexer %s generated error token for %s: %r at position %d' % \ + (lx, absfn, val, len(u''.join(ntext))) tokens.append((type, val)) if u''.join(ntext) != text: print '\n'.join(difflib.unified_diff(u''.join(ntext).splitlines(), diff --git a/tests/test_html_formatter.py b/tests/test_html_formatter.py index 5a506755..58a50c74 100644 --- a/tests/test_html_formatter.py +++ b/tests/test_html_formatter.py @@ -75,6 +75,38 @@ class HtmlFormatterTest(unittest.TestCase): fmt = HtmlFormatter(**optdict) fmt.format(tokensource, outfile) + def test_linenos(self): + optdict = dict(linenos=True) + outfile = StringIO.StringIO() + fmt = HtmlFormatter(**optdict) + fmt.format(tokensource, outfile) + html = outfile.getvalue() + self.assert_(re.search("<pre>\s+1\s+2\s+3", html)) + + def test_linenos_with_startnum(self): + optdict = dict(linenos=True, linenostart=5) + outfile = StringIO.StringIO() + fmt = HtmlFormatter(**optdict) + fmt.format(tokensource, outfile) + html = outfile.getvalue() + self.assert_(re.search("<pre>\s+5\s+6\s+7", html)) + + def test_lineanchors(self): + optdict = dict(lineanchors="foo") + outfile = StringIO.StringIO() + fmt = HtmlFormatter(**optdict) + fmt.format(tokensource, outfile) + html = outfile.getvalue() + self.assert_(re.search("<pre><a name=\"foo-1\">", html)) + + def test_lineanchors_with_startnum(self): + optdict = dict(lineanchors="foo", linenostart=5) + outfile = StringIO.StringIO() + fmt = HtmlFormatter(**optdict) + fmt.format(tokensource, outfile) + html = outfile.getvalue() + self.assert_(re.search("<pre><a name=\"foo-5\">", html)) + def test_valid_output(self): # test all available wrappers fmt = HtmlFormatter(full=True, linenos=True, noclasses=True, diff --git a/tests/test_util.py b/tests/test_util.py index 0876cf70..af1f4e44 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -13,6 +13,12 @@ import os from pygments import util +class FakeLexer(object): + def analyse(text): + return float(text) + analyse = util.make_analysator(analyse) + + class UtilTest(unittest.TestCase): def test_getoptions(self): @@ -53,12 +59,36 @@ class UtilTest(unittest.TestCase): self.assertEquals(util.docstring_headline(f1), "docstring headline") self.assertEquals(util.docstring_headline(f2), "docstring headline") - def test_analysator(self): - class X(object): + def test_analysator_returns_float(self): + # If an analysator wrapped by make_analysator returns a floating point + # number, then that number will be returned by the wrapper. + self.assertEquals(FakeLexer.analyse('0.5'), 0.5) + + def test_analysator_returns_boolean(self): + # If an analysator wrapped by make_analysator returns a boolean value, + # then the wrapper will return 1.0 if the boolean was True or 0.0 if + # it was False. + self.assertEquals(FakeLexer.analyse(True), 1.0) + self.assertEquals(FakeLexer.analyse(False), 0.0) + + def test_analysator_raises_exception(self): + # If an analysator wrapped by make_analysator raises an exception, + # then the wrapper will return 0.0. + class ErrorLexer(object): def analyse(text): - return 0.5 + raise RuntimeError('something bad happened') analyse = util.make_analysator(analyse) - self.assertEquals(X.analyse(''), 0.5) + self.assertEquals(ErrorLexer.analyse(''), 0.0) + + def test_analysator_value_error(self): + # When converting the analysator's return value to a float a + # ValueError may occur. If that happens 0.0 is returned instead. + self.assertEquals(FakeLexer.analyse('bad input'), 0.0) + + def test_analysator_type_error(self): + # When converting the analysator's return value to a float a + # TypeError may occur. If that happens 0.0 is returned instead. + self.assertEquals(FakeLexer.analyse(None), 0.0) def test_shebang_matches(self): self.assert_(util.shebang_matches('#!/usr/bin/env python', r'python(2\.\d)?')) |