diff options
author | mitsuhiko <devnull@localhost> | 2008-03-01 19:04:13 +0100 |
---|---|---|
committer | mitsuhiko <devnull@localhost> | 2008-03-01 19:04:13 +0100 |
commit | b5df43edb9bc89308e8b0258c164dc15a3ec013f (patch) | |
tree | a0ec597754b5f409f421e90c5b48ea9792c2e0e1 | |
parent | 94ee873d3f3cce954475cb69e43cfb44d831d8ae (diff) | |
parent | c49d1e47d170cfce8f2985692582cb4d1eadddda (diff) | |
download | pygments-b5df43edb9bc89308e8b0258c164dc15a3ec013f.tar.gz |
Automated merge with ssh://team@pocoo.org/pygments-main
-rw-r--r-- | CHANGES | 16 | ||||
-rw-r--r-- | docs/src/quickstart.txt | 13 | ||||
-rw-r--r-- | external/rst-directive.py | 4 | ||||
-rwxr-xr-x | pygmentize | 5 | ||||
-rw-r--r-- | pygments/__init__.py | 4 | ||||
-rw-r--r-- | pygments/cmdline.py | 4 | ||||
-rwxr-xr-x | pygments/formatters/_mapping.py | 2 | ||||
-rw-r--r-- | pygments/formatters/latex.py | 2 | ||||
-rw-r--r-- | pygments/lexers/_mapping.py | 9 | ||||
-rw-r--r-- | pygments/lexers/agile.py | 56 | ||||
-rw-r--r-- | pygments/lexers/compiled.py | 2 | ||||
-rw-r--r-- | pygments/lexers/other.py | 74 | ||||
-rw-r--r-- | pygments/lexers/text.py | 50 | ||||
-rw-r--r-- | pygments/lexers/web.py | 6 | ||||
-rw-r--r-- | scripts/find_error.py | 7 | ||||
-rw-r--r-- | setup.cfg | 2 | ||||
-rw-r--r-- | setup.py | 1 | ||||
-rw-r--r-- | tests/examplefiles/Squeak.st | 505 |
18 files changed, 734 insertions, 28 deletions
@@ -3,7 +3,21 @@ Pygments changelog Version 1.0 ----------- -(codename not selected, released XXX YY, 2007) +(codename not selected, released XXX YY, 2008) + +- Lexers added: + + * Io + * Smalltalk + * Darcs patches + +- Handle format specifications in the Perl lexer. + +- Add more file name extensions for the C++, INI and XML lexers. + +- Fixes in the IRC and MuPad lexers. + +- Handle KeyboardInterrupts gracefully in pygmentize. - Lexers added: diff --git a/docs/src/quickstart.txt b/docs/src/quickstart.txt index 93f32c0f..b3f73ae5 100644 --- a/docs/src/quickstart.txt +++ b/docs/src/quickstart.txt @@ -183,9 +183,20 @@ To output HTML, use the ``-f`` option:: $ pygmentize -f html -o test.html test.py to write an HTML-highlighted version of test.py to the file test.html. +Note that it will only be a snippet of HTML, if you want a full HTML document, +use the "full" option:: -The stylesheet can be created with:: + $ pygmentize -f html -O full -o test.html test.py + +This will produce a full HTML document with included stylesheet. + +A style can be selected with ``-O style=<name>``. + +If you need a stylesheet for an existing HTML file using Pygments CSS classes, +it can be created with:: $ pygmentize -S default -f html > style.css +where ``default`` is the style name. + More options and tricks and be found in the `command line referene <cmdline.txt>`_. diff --git a/external/rst-directive.py b/external/rst-directive.py index 9c3d1b78..9ed14a67 100644 --- a/external/rst-directive.py +++ b/external/rst-directive.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """ - The Pygments MoinMoin Parser - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + The Pygments reStructuredText directive + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ This fragment is a Docutils_ 0.4 directive that renders source code (to HTML only, currently) via Pygments. @@ -1,4 +1,7 @@ #!/usr/bin/env python import sys, pygments.cmdline -sys.exit(pygments.cmdline.main(sys.argv)) +try: + sys.exit(pygments.cmdline.main(sys.argv)) +except KeyboardInterrupt: + sys.exit(1) diff --git a/pygments/__init__.py b/pygments/__init__.py index d9788de4..5482b0b6 100644 --- a/pygments/__init__.py +++ b/pygments/__init__.py @@ -16,6 +16,10 @@ * it is usable as a command-line tool and as a library * ... and it highlights even Brainfuck! + The `Pygments tip`_ is installable with ``easy_install Pygments==dev``. + + .. _Pygments tip: http://dev.pocoo.org/hg/pygments-main/archive/tip.tar.gz#egg=Pygments-dev + :copyright: 2006-2007 by Georg Brandl, Armin Ronacher and others. :license: BSD, see LICENSE for more details. """ diff --git a/pygments/cmdline.py b/pygments/cmdline.py index d886bdfc..996207aa 100644 --- a/pygments/cmdline.py +++ b/pygments/cmdline.py @@ -122,7 +122,7 @@ def _print_help(what, name): cls = find_filter_class(name) print "Help on the %s filter:" % name print dedent(cls.__doc__) - except ClassNotFound: + except AttributeError: print >>sys.stderr, "%s not found!" % what @@ -253,7 +253,7 @@ def main(args=sys.argv): for p_opt in P_opts: try: name, value = p_opt.split('=', 1) - except: + except ValueError: parsed_opts[p_opt] = True else: parsed_opts[name] = value diff --git a/pygments/formatters/_mapping.py b/pygments/formatters/_mapping.py index e2ed1a83..776bc4d3 100755 --- a/pygments/formatters/_mapping.py +++ b/pygments/formatters/_mapping.py @@ -33,7 +33,7 @@ FORMATTERS = { NullFormatter: ('Text only', ('text', 'null'), ('*.txt',), 'Output the text unchanged without any formatting.'), RawTokenFormatter: ('Raw tokens', ('raw', 'tokens'), ('*.raw',), 'Format tokens as a raw representation for storing token streams.'), RtfFormatter: ('RTF', ('rtf',), ('*.rtf',), 'Format tokens as RTF markup. This formatter automatically outputs full RTF documents with color information and other useful stuff. Perfect for Copy and Paste into Microsoft\xc2\xae Word\xc2\xae documents.'), - SvgFormatter: ('SVG', ('svg',), ('*.svg',), 'Format tokens as an SVG graphics file. This formatter is still experimental.'), + SvgFormatter: ('SVG', ('svg',), ('*.svg',), 'Format tokens as an SVG graphics file. This formatter is still experimental. Each line of code is a ``<text>`` element with explicit ``x`` and ``y`` coordinates containing ``<tspan>`` elements with the individual token styles.'), Terminal256Formatter: ('Terminal256', ('terminal256', 'console256', '256'), (), 'Format tokens with ANSI color sequences, for output in a 256-color terminal or console. Like in `TerminalFormatter` color sequences are terminated at newlines, so that paging the output works correctly.'), TerminalFormatter: ('Terminal', ('terminal', 'console'), (), 'Format tokens with ANSI color sequences, for output in a text console. Color sequences are terminated at newlines, so that paging the output works correctly.') } diff --git a/pygments/formatters/latex.py b/pygments/formatters/latex.py index 7bd8e8b9..1d37d11b 100644 --- a/pygments/formatters/latex.py +++ b/pygments/formatters/latex.py @@ -231,7 +231,7 @@ class LatexFormatter(Formatter): else: outfile.write(value) - outfile.write('\n\\end{Verbatim}\n') + outfile.write('\\end{Verbatim}\n') if self.full: realoutfile.write(DOC_TEMPLATE % diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py index ec5df71a..5ee6d0f2 100644 --- a/pygments/lexers/_mapping.py +++ b/pygments/lexers/_mapping.py @@ -26,7 +26,7 @@ LEXERS = { 'CObjdumpLexer': ('pygments.lexers.asm', 'c-objdump', ('c-objdump',), ('*.c-objdump',), ('text/x-c-objdump',)), 'CSharpLexer': ('pygments.lexers.dotnet', 'C#', ('csharp', 'c#'), ('*.cs',), ('text/x-csharp',)), 'CommonLispLexer': ('pygments.lexers.functional', 'Common Lisp', ('common-lisp', 'cl'), ('*.cl', '*.lisp', '*.el'), ('text/x-common-lisp',)), - 'CppLexer': ('pygments.lexers.compiled', 'C++', ('cpp', 'c++'), ('*.cpp', '*.hpp', '*.c++', '*.h++'), ('text/x-c++hdr', 'text/x-c++src')), + 'CppLexer': ('pygments.lexers.compiled', 'C++', ('cpp', 'c++'), ('*.cpp', '*.hpp', '*.c++', '*.h++', '*.cc', '*.hh', '*.cxx', '*.hxx'), ('text/x-c++hdr', 'text/x-c++src')), 'CppObjdumpLexer': ('pygments.lexers.asm', 'cpp-objdump', ('cpp-objdump', 'c++-objdumb', 'cxx-objdump'), ('*.cpp-objdump', '*.c++-objdump', '*.cxx-objdump'), ('text/x-cpp-objdump',)), 'CssDjangoLexer': ('pygments.lexers.templates', 'CSS+Django/Jinja', ('css+django', 'css+jinja'), (), ('text/css+django', 'text/css+jinja')), 'CssErbLexer': ('pygments.lexers.templates', 'CSS+Ruby', ('css+erb', 'css+ruby'), (), ('text/css+ruby',)), @@ -36,6 +36,7 @@ LEXERS = { 'CssSmartyLexer': ('pygments.lexers.templates', 'CSS+Smarty', ('css+smarty',), (), ('text/css+smarty',)), 'DLexer': ('pygments.lexers.compiled', 'D', ('d',), ('*.d', '*.di'), ('text/x-dsrc',)), 'DObjdumpLexer': ('pygments.lexers.asm', 'd-objdump', ('d-objdump',), ('*.d-objdump',), ('text/x-d-objdump',)), + 'DarcsPatchLexer': ('pygments.lexers.text', 'Darcs Patch', ('dpatch',), ('*.dpatch', '*.darcspatch'), ()), 'DebianControlLexer': ('pygments.lexers.text', 'Debian Control file', ('control',), ('control',), ()), 'DelphiLexer': ('pygments.lexers.compiled', 'Delphi', ('delphi', 'pas', 'pascal', 'objectpascal'), ('*.pas',), ('text/x-pascal',)), 'DiffLexer': ('pygments.lexers.text', 'Diff', ('diff',), ('*.diff', '*.patch'), ('text/x-diff', 'text/x-patch')), @@ -54,7 +55,8 @@ LEXERS = { 'HtmlLexer': ('pygments.lexers.web', 'HTML', ('html',), ('*.html', '*.htm', '*.xhtml', '*.xslt'), ('text/html', 'application/xhtml+xml')), 'HtmlPhpLexer': ('pygments.lexers.templates', 'HTML+PHP', ('html+php',), ('*.phtml',), ('application/x-php', 'application/x-httpd-php', 'application/x-httpd-php3', 'application/x-httpd-php4', 'application/x-httpd-php5')), 'HtmlSmartyLexer': ('pygments.lexers.templates', 'HTML+Smarty', ('html+smarty',), (), ('text/html+smarty',)), - 'IniLexer': ('pygments.lexers.text', 'INI', ('ini', 'cfg'), ('*.ini', '*.cfg'), ('text/x-ini',)), + 'IniLexer': ('pygments.lexers.text', 'INI', ('ini', 'cfg'), ('*.ini', '*.cfg', '*.properties'), ('text/x-ini',)), + 'IoLexer': ('pygments.lexers.agile', 'Io', ('io',), ('*.io',), ('text/x-iosrc',)), 'IrcLogsLexer': ('pygments.lexers.text', 'IRC logs', ('irc',), ('*.weechatlog',), ('text/x-irclog',)), 'JavaLexer': ('pygments.lexers.compiled', 'Java', ('java',), ('*.java',), ('text/x-java',)), 'JavascriptDjangoLexer': ('pygments.lexers.templates', 'JavaScript+Django/Jinja', ('js+django', 'javascript+django', 'js+jinja', 'javascript+jinja'), (), ('application/x-javascript+django', 'application/x-javascript+jinja', 'text/x-javascript+django', 'text/x-javascript+jinja', 'text/javascript+django', 'text/javascript+jinja')), @@ -99,6 +101,7 @@ LEXERS = { 'RubyConsoleLexer': ('pygments.lexers.agile', 'Ruby irb session', ('rbcon', 'irb'), (), ('text/x-ruby-shellsession',)), 'RubyLexer': ('pygments.lexers.agile', 'Ruby', ('rb', 'ruby'), ('*.rb', '*.rbw', 'Rakefile', '*.rake', '*.gemspec', '*.rbx'), ('text/x-ruby', 'application/x-ruby')), 'SchemeLexer': ('pygments.lexers.functional', 'Scheme', ('scheme', 'scm'), ('*.scm',), ('text/x-scheme', 'application/x-scheme')), + 'SmalltalkLexer': ('pygments.lexers.other', 'Smalltalk', ('smalltalk', 'squeak'), ('*.st',), ('text/x-smalltalk',)), 'SmartyLexer': ('pygments.lexers.templates', 'Smarty', ('smarty',), ('*.tpl',), ('application/x-smarty',)), 'SourcesListLexer': ('pygments.lexers.text', 'Debian Sourcelist', ('sourceslist', 'sources.list'), ('sources.list',), ()), 'SqlLexer': ('pygments.lexers.other', 'SQL', ('sql',), ('*.sql',), ('text/x-sql',)), @@ -110,7 +113,7 @@ LEXERS = { 'VimLexer': ('pygments.lexers.text', 'VimL', ('vim',), ('*.vim', '.vimrc'), ('text/x-vim',)), 'XmlDjangoLexer': ('pygments.lexers.templates', 'XML+Django/Jinja', ('xml+django', 'xml+jinja'), (), ('application/xml+django', 'application/xml+jinja')), 'XmlErbLexer': ('pygments.lexers.templates', 'XML+Ruby', ('xml+erb', 'xml+ruby'), (), ('application/xml+ruby',)), - 'XmlLexer': ('pygments.lexers.web', 'XML', ('xml',), ('*.xml', '*.xsl', '*.rss', '*.xslt'), ('text/xml', 'application/xml', 'image/svg+xml', 'application/rss+xml', 'application/atom+xml', 'application/xsl+xml', 'application/xslt+xml')), + 'XmlLexer': ('pygments.lexers.web', 'XML', ('xml',), ('*.xml', '*.xsl', '*.rss', '*.xslt', '*.xsd', '*.wsdl'), ('text/xml', 'application/xml', 'image/svg+xml', 'application/rss+xml', 'application/atom+xml', 'application/xsl+xml', 'application/xslt+xml')), 'XmlPhpLexer': ('pygments.lexers.templates', 'XML+PHP', ('xml+php',), (), ('application/xml+php',)), 'XmlSmartyLexer': ('pygments.lexers.templates', 'XML+Smarty', ('xml+smarty',), (), ('application/xml+smarty',)) } diff --git a/pygments/lexers/agile.py b/pygments/lexers/agile.py index ecee4295..60bc097a 100644 --- a/pygments/lexers/agile.py +++ b/pygments/lexers/agile.py @@ -26,7 +26,7 @@ from pygments.util import get_bool_opt, get_list_opt, shebang_matches __all__ = ['PythonLexer', 'PythonConsoleLexer', 'PythonTracebackLexer', 'RubyLexer', 'RubyConsoleLexer', 'PerlLexer', 'LuaLexer', - 'MiniDLexer', 'TclLexer'] + 'MiniDLexer', 'IoLexer'] # b/w compatibility from pygments.lexers.functional import SchemeLexer @@ -653,10 +653,12 @@ class PerlLexer(RegexLexer): tokens = { 'root': [ (r'\#.*?$', Comment.Single), - (r'=[a-zA-Z0-9]+\s+.*\n[.\n]*?\n\s*=cut', Comment.Multiline), + (r'=[a-zA-Z0-9]+\s+.*?\n[.\n]*?\n\s*=cut', Comment.Multiline), (r'(case|continue|do|else|elsif|for|foreach|if|last|my|' r'next|our|redo|reset|then|unless|until|while|use|' r'print|new|BEGIN|END|return)\b', Keyword), + (r'(format)(\s+)([a-zA-Z0-9_]+)(\s*)(=)(\s*\n)', + bygroups(Keyword, Text, Name, Text, Punctuation, Text), 'format'), (r'(eq|lt|gt|le|ge|ne|not|and|or|cmp)\b', Operator.Word), (r's/(\\\\|\\/|[^/])*/(\\\\|\\/|[^/])*/[egimosx]*', String.Regex), (r'm?/(\\\\|\\/|[^/\n])*/[gcimosx]*', String.Regex), @@ -716,6 +718,10 @@ class PerlLexer(RegexLexer): # of punctuation in Perl! (r'(?=\w)', Name, 'name'), ], + 'format': [ + (r'\.\n', String.Interpol, '#pop'), + (r'[^\n]*\n', String.Interpol), + ], 'varname': [ (r'\s+', Text), (r'\{', Punctuation, '#pop'), # hash syntax? @@ -952,7 +958,7 @@ class MiniDLexer(RegexLexer): r'|[-/.&|\+<>!()\[\]{}?,;:=*%^~#]', Punctuation ), # Identifier - (r'[a-zA-Z_]\w*', Name), + (r'[a-zA-Z_](\w|::)*', Name), ], 'nestedcomment': [ (r'[^+/]+', Comment), @@ -963,6 +969,50 @@ class MiniDLexer(RegexLexer): } +class IoLexer(RegexLexer): + """ + For `Io <http://iolanguage.com/>`_ (a small, prototype-based + programming language) source. + + *New in Pygments 1.0.* + """ + name = 'Io' + filenames = ['*.io'] + aliases = ['io'] + mimetypes = ['text/x-iosrc'] + tokens = { + 'root': [ + (r'\n', Text), + (r'\s+', Text), + # Comments + (r'//(.*?)\n', Comment), + (r'#(.*?)\n', Comment), + (r'/(\\\n)?[*](.|\n)*?[*](\\\n)?/', Comment), + (r'/\+', Comment, 'nestedcomment'), + # DoubleQuotedString + (r'"(\\\\|\\"|[^"])*"', String), + # Operators + (r':=|=|\(|\)|;|,|\*|-|\+|>|<|@|!|/|\||\^|\.|%|&|\[|\]|\{|\}', Operator), + # keywords + (r'(clone|do|doFile|doString|method|for|if|else|elseif|then)', Keyword), + # constants + (r'nil|false|true', Name.Constant), + # names + ('Object|list|List|Map|args|Sequence|Coroutine|File', Name.Builtin), + ('[a-zA-Z_][a-zA-Z0-9_]*', Name), + # numbers + (r'(\d+\.?\d*|\d*\.\d+)([eE][+-]?[0-9]+)?', Number.Float), + (r'\d+', Number.Integer) + ], + 'nestedcomment': [ + (r'[^+/]+', Comment), + (r'/\+', Comment, '#push'), + (r'\+/', Comment, '#pop'), + (r'[+/]', Comment), + ] + } + + class TclLexer(RegexLexer): """ For Tcl source code. diff --git a/pygments/lexers/compiled.py b/pygments/lexers/compiled.py index a247e7e3..d37e0a57 100644 --- a/pygments/lexers/compiled.py +++ b/pygments/lexers/compiled.py @@ -134,7 +134,7 @@ class CppLexer(RegexLexer): """ name = 'C++' aliases = ['cpp', 'c++'] - filenames = ['*.cpp', '*.hpp', '*.c++', '*.h++'] + filenames = ['*.cpp', '*.hpp', '*.c++', '*.h++', '*.cc', '*.hh', '*.cxx', '*.hxx'] mimetypes = ['text/x-c++hdr', 'text/x-c++src'] tokens = { diff --git a/pygments/lexers/other.py b/pygments/lexers/other.py index 6d2ecf20..3598b890 100644 --- a/pygments/lexers/other.py +++ b/pygments/lexers/other.py @@ -18,7 +18,8 @@ from pygments.util import shebang_matches __all__ = ['SqlLexer', 'MySqlLexer', 'BrainfuckLexer', 'BashLexer', - 'BatchLexer', 'BefungeLexer', 'RedcodeLexer', 'MOOCodeLexer'] + 'BatchLexer', 'BefungeLexer', 'RedcodeLexer', 'MOOCodeLexer', + 'SmalltalkLexer'] class SqlLexer(RegexLexer): @@ -469,3 +470,74 @@ class MOOCodeLexer(RegexLexer): (r'([a-zA-Z_0-9]+)', Text), ] } + +class SmalltalkLexer(RegexLexer): + """ + For `Smalltalk <http://www.smalltalk.org/>`_ syntax. + Contributed by Stefan Matthias Aust. + + *New in Pygments 1.0.* + """ + name = 'Smalltalk' + filenames = ['*.st'] + aliases = ['smalltalk', 'squeak'] + mimetypes = ['text/x-smalltalk'] + + tokens = { + 'root' : [ + # Squeak fileout format (optional) + (r'^"[^"]*"!', Keyword), + (r"^'[^']*'!", Keyword), + (r'^(!)(\w+)( commentStamp: )(.*?)( prior: .*?!\n)(.*?)(!)', + bygroups(Keyword, Name.Class, Keyword, String, Keyword, Text, Keyword)), + (r'^(!)(\w+(?: class)?)( methodsFor: )(\'[^\']*\')(.*?!)', + bygroups(Keyword, Name.Class, Keyword, String, Keyword)), + (r'^(\w+)( subclass: )(#\w+)' + r'(\s+instanceVariableNames: )(.*?)' + r'(\s+classVariableNames: )(.*?)' + r'(\s+poolDictionaries: )(.*?)' + r'(\s+category: )(.*?)(!)', + bygroups(Name.Class, Keyword, String.Symbol, Keyword, String, Keyword, + String, Keyword, String, Keyword, String, Keyword)), + (r'^(\w+(?: class)?)(\s+instanceVariableNames: )(.*?)(!)', + bygroups(Name.Class, Keyword, String, Keyword)), + (r'(!\n)(\].*)(! !)$', bygroups(Keyword, Text, Keyword)), + (r'! !$', Keyword), + # skip whitespace and comments + (r'\s+', Text), + (r'"[^"]*"', Comment), + # method patterns + (r'^(\w+)(\s*:\s*)(\w+\s*)', bygroups(Name.Function, Punctuation, + Name.Variable), 'pattern'), + (r'^([-+*/\\~<>=|&!?,@%]+\s*)(\w+)', bygroups(Name.Function, Name.Variable)), + (r'^(\w+)', Name.Function), + # literals + (r'\'[^\']*\'', String), + (r'\$.', String.Char), + (r'#\(', String.Symbol, 'parenth'), + (r'(\d+r)?-?\d+(\.\d+)?(e-?\d+)?', Number), + (r'#("[^"]*"|[-+*/\\~<>=|&!?,@%]+|[\w:]+)', String.Symbol), + # blocks variables + (r'(\[\s*)((?::\w+\s*)+)(\|)', bygroups(Text, Name.Variable, Text)), + # temporaries + (r'(\|)([\w\s]*)(\|)', bygroups(Operator, Name.Variable, Operator)), + # names + (r'\b(ifTrue:|ifFalse:|whileTrue:|whileFalse:|timesRepeat:)', Name.Builtin), + (r'\b(self|super)\b', Name.Builtin.Pseudo), + (r'\b[A-Z]\w*:', Name), + (r'\b[A-Z]\w*\b', Name), #Name.Class), + (r'\w+:?|[-+*/\\~<>=|&!?,@%]+', Name), #Name.Function), + # syntax + (r'\^|:=', Operator), + (r'[\[\](){}.;]', Text), + ], + 'parenth' : [ + (r'\)', String.Symbol, '#pop'), + include('root'), + ], + 'pattern' : [ + (r'(\w+)(\s*:\s*)(\w+\s*)', bygroups(Name.Function, Punctuation, + Name.Variable)), + (r'', Text, '#pop'), + ], + } diff --git a/pygments/lexers/text.py b/pygments/lexers/text.py index d0a806d9..f4c5a1ba 100644 --- a/pygments/lexers/text.py +++ b/pygments/lexers/text.py @@ -11,7 +11,8 @@ Dennis Kaarsemaker, Kumar Appaiah <akumar@ee.iitm.ac.in>, Varun Hiremath <varunhiremath@gmail.com>, - Jeremy Thurgood. + Jeremy Thurgood, + Max Battcher. :license: BSD, see LICENSE for more details. """ @@ -26,14 +27,15 @@ from pygments.lexer import RegexLexer, bygroups, include, using, this, \ do_insertions from pygments.token import Punctuation, \ Text, Comment, Keyword, Name, String, Generic, Operator, Number, \ - Whitespace + Whitespace, Literal from pygments.util import get_bool_opt __all__ = ['IniLexer', 'SourcesListLexer', 'MakefileLexer', 'DiffLexer', 'IrcLogsLexer', 'TexLexer', 'GroffLexer', 'ApacheConfLexer', 'BBCodeLexer', 'MoinWikiLexer', 'RstLexer', 'VimLexer', - 'GettextLexer', 'SquidConfLexer', 'DebianControlLexer'] + 'GettextLexer', 'SquidConfLexer', 'DebianControlLexer', + 'DarcsPatchLexer'] class IniLexer(RegexLexer): @@ -43,7 +45,7 @@ class IniLexer(RegexLexer): name = 'INI' aliases = ['ini', 'cfg'] - filenames = ['*.ini', '*.cfg'] + filenames = ['*.ini', '*.cfg', '*.properties'] mimetypes = ['text/x-ini'] tokens = { @@ -198,6 +200,44 @@ class DiffLexer(RegexLexer): return 0.9 +class DarcsPatchLexer(RegexLexer): + """ + DarcsPatchLexer is a lexer for the various versions of the darcs patch + format. Examples of this format are derived by commands such as + ``darcs annotate --patch`` and ``darcs send``. + + *New in Pygments 1.0.* + """ + name = 'Darcs Patch' + aliases = ['dpatch'] + filenames = ['*.dpatch', '*.darcspatch'] + + tokens = { + 'root': [ + (r'<', Operator), + (r'>', Operator), + (r'{', Operator, 'patch'), + (r'(\[)((?:TAG )?)(.*)(\n)(.*)(\*\*)(\d+)(\s?)', bygroups(Operator, Keyword, Name, Text, + Name, Operator, Literal.Date, Text), 'comment'), + (r'New patches:', Generic.Heading), + (r'Context:', Generic.Heading), + (r'Patch bundle hash:', Generic.Heading), + (r'\s+|\w+', Text), + ], + 'comment': [ + (r' .*\n', Comment), + (r'\]', Operator, "#pop"), + ], + 'patch': [ + (r'}', Operator, "#pop"), + (r'(\w+)(.*\n)', bygroups(Keyword, Text)), + (r'\+.*\n', Generic.Inserted), + (r'-.*\n', Generic.Deleted), + (r'.*\n', Text), + ], + } + + class IrcLogsLexer(RegexLexer): """ Lexer for IRC logs in *irssi*, *xchat* or *weechat* style. @@ -233,7 +273,7 @@ class IrcLogsLexer(RegexLexer): ("^" + timestamp + r'(\s*<.*>\s*)$', bygroups(Comment.Preproc, Name.Tag)), # normal msgs ("^" + timestamp + r""" - (\s*<.*>\s*) # Nick """, + (\s*<.*?>\s*) # Nick """, bygroups(Comment.Preproc, Name.Tag), 'msg'), # /me msgs ("^" + timestamp + r""" diff --git a/pygments/lexers/web.py b/pygments/lexers/web.py index d4bfa765..8166e266 100644 --- a/pygments/lexers/web.py +++ b/pygments/lexers/web.py @@ -58,7 +58,7 @@ class JavascriptLexer(RegexLexer): r'window)\b', Name.Builtin), (r'[$a-zA-Z_][a-zA-Z0-9_]*', Name.Other), (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float), - (r'0x[0-9a-f]+', Number.Hex), + (r'0x[0-9a-fA-F]+', Number.Hex), (r'[0-9]+', Number.Integer), (r'"(\\\\|\\"|[^"])*"', String.Double), (r"'(\\\\|\\'|[^'])*'", String.Single), @@ -438,7 +438,7 @@ class PhpLexer(RegexLexer): RegexLexer.get_tokens_unprocessed(self, text, stack): if token is Name.Other: if value in self._functions: - yield index, Name.Function, value + yield index, Name.Builtin, value continue yield index, token, value @@ -459,7 +459,7 @@ class XmlLexer(RegexLexer): name = 'XML' aliases = ['xml'] - filenames = ['*.xml', '*.xsl', '*.rss', '*.xslt'] + filenames = ['*.xml', '*.xsl', '*.rss', '*.xslt', '*.xsd', '*.wsdl'] mimetypes = ['text/xml', 'application/xml', 'image/svg+xml', 'application/rss+xml', 'application/atom+xml', 'application/xsl+xml', 'application/xslt+xml'] diff --git a/scripts/find_error.py b/scripts/find_error.py index e210028a..a7719b3f 100644 --- a/scripts/find_error.py +++ b/scripts/find_error.py @@ -40,12 +40,17 @@ def main(fn): for type, val in lx.get_tokens(text): if type == Error: print "Error parsing", fn - print "\n".join([' ' + repr(x) for x in ntext[-5:]]) + print "\n".join([' ' + repr(x) for x in ntext[-num:]]) print `val` + "<<<" return ntext.append((type,val)) +num = 10 + if __name__ == "__main__": + if sys.argv[1][:2] == '-n': + num = int(sys.argv[1][2:]) + del sys.argv[1] for f in sys.argv[1:]: main(f) @@ -1,6 +1,6 @@ [egg_info] tag_build = dev -tag_svn_revision = true +tag_date = true [aliases] release = egg_info -RDb '' @@ -19,7 +19,6 @@ setup( long_description = pygments.__doc__, keywords = 'syntax highlighting', packages = find_packages(), - scripts = ['pygmentize'], entry_points = { 'console_scripts': [ 'pygmentize = pygments.cmdline:main', diff --git a/tests/examplefiles/Squeak.st b/tests/examplefiles/Squeak.st new file mode 100644 index 00000000..cf36ce85 --- /dev/null +++ b/tests/examplefiles/Squeak.st @@ -0,0 +1,505 @@ +{ import: st80 } + +Object assert: condition +[ + condition ifFalse: [self error: 'assertion failed'] +] + +{ import: SqueakImage } +{ import: SqueakObject } +{ import: SqueakIO } +{ import: SqueakBitBlt } + +{ import: String-paths } + +NilObject := [ 0 ] +FalseObject := [ 1 ] +TrueObject := [ 2 ] +SchedulerAssociation := [ 3 ] +ClassBitmap := [ 4 ] +ClassInteger := [ 5 ] +ClassString := [ 6 ] +ClassArray := [ 7 ] +ClassFloat := [ 9 ] +ClassMethodContext := [ 10 ] +ClassBlockContext := [ 11 ] +ClassPoint := [ 12 ] +ClassLargePositiveInteger := [ 13 ] +TheDisplay := [ 14 ] +ClassMessage := [ 15 ] +ClassCompiledMethod := [ 16 ] +TheLowSpaceSemaphore := [ 17 ] +ClassSemaphore := [ 18 ] +ClassCharacter := [ 19 ] +SelectorDoesNotUnderstand := [ 20 ] +SelectorCannotReturn := [ 21 ] +TheInputSemaphore := [ 22 ] +SpecialSelectors := [ 23 ] +SqueakCharacterTable := [ 24 ] +SelectorMustBeBoolean := [ 25 ] +ClassByteArray := [ 26 ] +ClassProcess := [ 27 ] +CompactClasses := [ 28 ] +TheTimerSemaphore := [ 29 ] +TheInterruptSemaphore := [ 30 ] +FloatProto := [ 31 ] +SelectorCannotInterpret := [ 34 ] +MethodContextProto := [ 35 ] +BlockContextProto := [ 37 ] +ExternalObjectsArray := [ 38 ] +ClassPseudoContext := [ 39 ] +ClassTranslatedMethod := [ 40 ] +TheFinalizationSemaphore := [ 41 ] +ClassLargeNegativeInteger := [ 42 ] +ClassExternalAddress := [ 43 ] +ClassExternalStructure := [ 44 ] +ClassExternalData := [ 45 ] +ClassExternalFunction := [ 46 ] +ClassExternalLibrary := [ 47 ] +SelectorAboutToReturn := [ 48 ] + +Class_superclass := [ 0 ] +Class_mdict := [ 1 ] +Class_format := [ 2 ] +Class_name := [ 6 ] + +Association_key := [ 0 ] +Association_value := [ 1 ] + +ProcSched_processLists := [ 0 ] +ProcSched_activeProcess := [ 1 ] + +Process_suspendedContext := [ 1 ] +Process_priority := [ 2 ] +Process_myList := [ 3 ] + +Context_sender := [ 0 ] +Context_instructionPointer := [ 1 ] +Context_stackPointer := [ 2 ] +BlockContext_argumentCount := [ 3 ] MethodContext_method := [ 3 ] +BlockContext_initialPC := [ 4 ] MethodContext_receiverMap := [ 4 ] +BlockContext_home := [ 5 ] MethodContext_receiver := [ 5 ] + +Context_stackFrameStart := [ 5 ] +Context_tempFrameStart := [ 6 ] +Context_smallFrameSize := [ 16 ] "indexable fields" +Context_largeFrameSize := [ 56 ] "indexable fields" + +MethodDictionary_array := [ 1 ] +MethodDictionary_selectorStart := [ 2 ] + +Point_x := [ 0 ] +Point_y := [ 1 ] + +Character_value := [ 0 ] + +Link_nextLink := [ 0 ] + +LinkedList_firstLink := [ 0 ] +LinkedList_lastLink := [ 1 ] + +Semaphore_excessSignals := [ 2 ] + +Message_selector := [ 0 ] +Message_arguments := [ 1 ] +Message_lookupClass := [ 2 ] + +MethodCacheSize := [ 4096 ] +MethodCacheMask := [ MethodCacheSize - 1 ] + +MaxPrimitiveIndex := [ 700 ] + +Squeak : Object ( bytecodeTable primitiveTable viPath vmPath lastHash specialObjectsArray nilObj trueObj falseObj + currentBytecode activeContext homeContext receiver method instructionPointer stackPointer + messageSelector argumentCount receiverClass lkupClass methodClass newMethod primitiveIndex + successFlag lastTick nextPollTick nextWakeupTick interruptPending interruptKeycode + interruptCheckCounter interruptCheckCounterFeedbackReset interruptChecksEveryNms + methodCacheSelector methodCacheClass methodCacheMethod methodCachePrimitive + thisSessionID io bitBlt cacheHits cacheMisses ) + +" + INITIALISATION +" + +FormatTable := [ + (Array new: 16) + at: 1 put: FormatEmpty; " 0 no fields" + at: 2 put: FormatFixed; " 1 fixed fields only (all containing pointers)" + at: 3 put: FormatPointers; " 2 indexable fields only (all containing pointers)" + at: 4 put: FormatPointers; " 3 both fixed and indexable fields (all containing pointers)" + at: 5 put: FormatWeak; " 4 both fixed and indexable weak fields (all containing pointers)" + at: 6 put: nil; " 5 unused" + at: 7 put: FormatWords; " 6 indexable word fields only (no pointers)" + at: 8 put: nil; " 7 unused" + at: 9 put: FormatBytes; " 8-11 indexable byte fields only (no pointers)" + at: 10 put: FormatBytes; + at: 11 put: FormatBytes; + at: 12 put: FormatBytes; + at: 13 put: FormatMethod; "12-15 compiled methods" + at: 14 put: FormatMethod; + at: 15 put: FormatMethod; + at: 16 put: FormatMethod; + yourself +] + +Initialisers := [ + #( initEmpty:fromOop:inImage:withMap: + initPointers:fromOop:inImage:withMap: + initPointers:fromOop:inImage:withMap: + initWeak:fromOop:inImage:withMap: + initWords:fromOop:inImage:withMap: + initBytes:fromOop:inImage:withMap: + initMethod:fromOop:inImage:withMap: ) collect: [:s | s asSelector] +] + +Squeak interpret: imageName +[ + | image | + image := SqueakImage open: imageName. + self := self on: image. + viPath := imageName realpath. + vmPath := (Smalltalk argumentAt: 1) realpath dirname, '/'. + self interpret. +] + +Squeak on: image +[ + | map | + Squeak := self := self new. + + map := Array new: 1 + image maxObjectIndex. + SqueakObject initialiseHeap: map size. + + 'instantiate objects...' putln. + + image oopsDo: [:imobj | + | format | + format := image formatOf: imobj. + map at: 1 + (image indexOf: imobj) + put: (format >= 12 ifTrue: [SqueakMethod] ifFalse: [SqueakObject]) new]. + + lastHash := image lastHash. + specialObjectsArray := map at: 1 + (image indexOf: image specialObjectsOop). + + 'initialise objects...' putln. + + image oopsDo: [:imobj | + | format object size | + format := FormatTable at: 1 + (image formatOf: imobj). + object := map at: 1 + (image indexOf: imobj). + size := image sizeBitsOf: imobj. + object initClass: (map at: 1 + (image indexOf: (image fetchClassOf: imobj))) + hash: (image hashBitsOf: imobj) + format: format. + self _perform: (Initialisers at: 1 + format) w: object w: imobj w: image w: map]. + + bitBlt := SqueakBitBlt withInterpreter: self getProxy. + io := SqueakIO new. + + thisSessionID := io mSecs. + + interruptCheckCounter := 0. + interruptCheckCounterFeedbackReset := 1000. + interruptChecksEveryNms := 3. + nextPollTick := 0. + nextWakeupTick := 0. + lastTick := 0. + interruptKeycode := 2094. "cmd-." + interruptPending := false. + + nilObj := self splObj: NilObject. + trueObj := self splObj: TrueObject. + falseObj := self splObj: FalseObject. + + self initialiseBytecodeTable. + self initialisePrimitiveTable. + self initialiseDebugging. + + methodCacheSelector := Array new: MethodCacheSize. + methodCacheClass := Array new: MethodCacheSize. + methodCacheMethod := Array new: MethodCacheSize. + methodCachePrimitive := Array new: MethodCacheSize. + + cacheHits := 0. + cacheMisses := 0. + + self loadInitialContext. + self printCallStack: activeContext. +] + +Squeak initEmpty: obj fromOop: imobj inImage: image withMap: map [] + +Squeak initPointers: obj fromOop: imobj inImage: image withMap: map +[ + | size field | + size := image sizeBitsOf: imobj. + obj initFixed: (image fixedFieldsOf: imobj). + size := size // 4 - 1. + obj initPointers: size. + field := imobj + 4. + 0 to: size - 1 do: [:i | + | ptr | + ptr := image longAt: field. + ptr := (image isIntegerObject: ptr) ifTrue: [image intAt: field] ifFalse: [map at: 1 + (image indexOf: ptr)]. + self assert: (ptr isSmallInteger or: [ptr isSqueakObject]). + obj storePointer: i with: ptr. + field := field + 4]. +] + +Squeak initWords: obj fromOop: imobj inImage: image withMap: map +[ + | size | + size := image sizeBitsOf: imobj. + size := size / 4 - 1. + obj initWords: size. + 0 to: size - 1 do: [:i | obj storeWord: i with: (image longAt: imobj + 4 + (4 * i))]. +] + +Squeak initBytes: obj fromOop: imobj inImage: image withMap: map +[ + | size | + size := image sizeBitsOf: imobj. + size := size - 4 - ((image formatOf: imobj) bitAnd: 3). + obj initBytes: size. + 0 to: size - 1 do: [:i | obj storeByte: i with: (image byteAt: imobj + 4 + i)]. +] + +Squeak initMethod: obj fromOop: imobj inImage: image withMap: map +[ + | header size field | + self initBytes: obj fromOop: imobj inImage: image withMap: map. + header := image longAt: imobj + 4. + size := 1 "header" + (image literalCountOfHeader: header). + obj initLiterals: size. + field := imobj + 4. + 0 to: size - 1 do: [:i | + | ptr | + ptr := image longAt: field. + ptr := (image isIntegerObject: ptr) ifTrue: [image intAt: field] ifFalse: [map at: 1 + (image indexOf: ptr)]. + self assert: (ptr isSmallInteger or: [ptr isSqueakObject]). + obj storeLiteral: i with: ptr. + field := field + 4]. +] + +Squeak initialiseBytecodeTable +[ + bytecodeTable := Array new: 256. + self table: bytecodeTable from: self bytecodeTable +] + +Squeak initialisePrimitiveTable +[ + primitiveTable := Array new: 1 + MaxPrimitiveIndex. + self table: primitiveTable from: self primitiveTable +] + +Squeak table: anArray from: specArray +[ + "SpecArray is an array of either (index selector) or (index1 index2 selector)." + | contiguous | + contiguous := 0. + specArray do: [ :spec | + (spec at: 1) == contiguous ifFalse: [self error: 'Non-contiguous table entry']. + spec size = 2 + ifTrue: + [anArray at: ((spec at: 1) + 1) put: (SelectorTable at: (spec at: 2)). + contiguous := contiguous + 1] + ifFalse: + [(spec at: 1) to: (spec at: 2) do: [:i | anArray at: (i + 1) put: (SelectorTable at: (spec at: 3))]. + contiguous := contiguous + ((spec at: 2) - (spec at: 1)) + 1]]. +] + +Squeak loadInitialContext +[ + | proc | + proc := self processScheduler fetchPointer: ProcSched_activeProcess. + activeContext := proc fetchPointer: Process_suspendedContext. + self fetchContextRegisters: activeContext. +] + +" + ACCESSING +" + +Squeak argumentCount [ ^argumentCount ] +Squeak failed [ ^successFlag not ] +Squeak nilObject [ ^nilObj ] +Squeak trueObject [ ^trueObj ] +Squeak falseObject [ ^falseObj ] + +Squeak splObj: offset +[ + ^specialObjectsArray fetchPointer: offset +] + +Squeak processScheduler +[ + ^(self splObj: SchedulerAssociation) fetchPointer: Association_value. +] + +" + CONTEXTS +" + +Squeak fetchContextRegisters: activeCntx +[ + | home | + home := (activeCntx instVarAt: MethodContext_method) isSmallInteger + ifTrue: [activeCntx instVarAt: BlockContext_home] + ifFalse: [activeCntx]. + homeContext := home. + receiver := home instVarAt: MethodContext_receiver. + method := home instVarAt: MethodContext_method. + instructionPointer := activeCntx instVarAt: Context_instructionPointer. + stackPointer := activeCntx instVarAt: Context_stackPointer. +] + +Squeak storeContextRegisters: aContext +[ + aContext instVarAt: Context_instructionPointer put: instructionPointer. + aContext instVarAt: Context_stackPointer put: stackPointer. +] + +" + ... +" + +SmallInteger class [ ^Squeak splObj: ClassInteger ] + +Squeak makePointX: x Y: y +[ + | point | + (point := SqueakObject newFixed: (self splObj: ClassPoint) hash: self newObjectHash size: 2) + instVarAt: Point_x put: x; + instVarAt: Point_y put: y. + ^point +] + +Squeak makeString: aString +[ + | string | + string := SqueakObject newBytes: (self splObj: ClassString) hash: self newObjectHash indexableSize: aString size. + 1 to: string size do: [:i | string storeByte: i - 1 with: (aString at: i) asciiValue]. + ^string +] + +Squeak makeLargeInteger: aLargeInteger +[ + | largeInt | + largeInt := SqueakObject newBytes: (self splObj: ClassLargePositiveInteger) hash: self newObjectHash indexableSize: 4. + 1 to: 4 do: [:i | largeInt storeByte: i - 1 with: (aLargeInteger digitAt: i)]. + ^largeInt +] + +Squeak makeFloat: floatValue +[ + | float | + float := SqueakObject newWords: (self splObj: ClassFloat) hash: self newObjectHash indexableSize: 2. + float setFloatValue: floatValue. + ^float +] + +Instantiators := [ + #( instantiateEmpty:hash:indexableSize: " 0 no fields" + instantiateFixed:hash:indexableSize: " 1 fixed fields only (all containing pointers)" + instantiatePointers:hash:indexableSize: " 2 indexable fields only (all containing pointers)" + instantiateFixedPointers:hash:indexableSize: " 3 both fixed and indexable fields (all containing pointers)" + instantiateWeakPointers:hash:indexableSize: " 4 both fixed and indexable weak fields (all containing pointers)" + instantiateIllegal:hash:indexableSize: " 5 unused" + instantiateWords:hash:indexableSize: " 6 indexable word fields only (no pointers)" + instantiateIllegal:hash:indexableSize: " 7 unused" + instantiateBytes:hash:indexableSize: " 8-11 indexable byte fields only (no pointers)" + instantiateBytes:hash:indexableSize: + instantiateBytes:hash:indexableSize: + instantiateBytes:hash:indexableSize: + instantiateMethod:hash:indexableSize: "12-15 compiled methods" + instantiateMethod:hash:indexableSize: + instantiateMethod:hash:indexableSize: + instantiateMethod:hash:indexableSize: + ) collect: [:s | s asSelector] +] + +Squeak instantiateClass: class indexableSize: indexableSize +[ + ^self _perform: (Instantiators at: 1 + class instSpecOfClass) + w: class + w: self newObjectHash + w: indexableSize +] + +Squeak instantiateEmpty: type hash: hash indexableSize: size +[ + self error: 'instantiateEmpty: type hash: hash indexableSize: size'. + ^SqueakObject new: type hash: hash format: FormatEmpty fixedSize: 0 fields: nil +] + +Squeak instantiateFixed: type hash: hash indexableSize: size +[ + size := type instSizeOfClass. + ^SqueakObject new: type hash: hash format: FormatFixed fixedSize: size fields: (Array new: size withAll: nilObj) +] + +Squeak instantiatePointers: type hash: hash indexableSize: size +[ + ^SqueakObject new: type hash: hash format: FormatPointers fixedSize: 0 fields: (Array new: size withAll: nilObj) +] + +Squeak instantiateFixedPointers: type hash: hash indexableSize: size +[ + | fixedSize | + self error: 'instantiateFixedPointers: type hash: hash indexableSize: size'. + fixedSize := type instSizeOfClass. + size := fixedSize + size. + ^SqueakObject new: type hash: hash format: FormatPointers fixedSize: fixedSize fields: (Array new: size withAll: nilObj) +] + +WeakArray : Array () + +Squeak instantiateWeak: type hash: hash indexableSize: size +[ + | fixedSize | + self error: 'instantiateWeak: type hash: hash indexableSize: size'. + fixedSize := type instSizeOfClass. "always 0?" + size += fixedSize + size. + ^SqueakObject new: type hash: hash format: FormatWeak fixedSize: fixedSize fields: (WeakArray new: size defaultElement: nilObj) +] + +Squeak instantiateWords: type hash: hash indexableSize: size +[ + ^SqueakObject new: type hash: hash format: FormatWords fixedSize: 0 fields: (WordArray new: size withAll: 0) +] + +Squeak instantiateBytes: type hash: hash indexableSize: size +[ + ^SqueakObject new: type hash: hash format: FormatBytes fixedSize: 0 fields: (ByteArray new: size withAll: 0) +] + +Squeak instantiateMethod: type hash: hash indexableSize: size +[ + self error: 'instantiateMethod: type hash: hash indexableSize: size'. + ^SqueakMethod new: type hash: hash format: FormatMethod fixedSize: 0 fields: (ByteArray new: size withAll: 0) +] + +{ import: Squeak-machine } +{ import: Squeak-bytecodes } +{ import: Squeak-primitives } +{ import: Squeak-debug } + +{ import: SqueakProxy } + +Squeak error: excuse +[ + $\n put. + self printCallStack: activeContext. + $\n put. + super error: excuse +] + +Object error: reason +[ + $\n put. + Squeak printCallStack. + reason putln. + { abort(); }. +] + |