diff options
| -rw-r--r-- | CHANGES | 3 | ||||
| -rw-r--r-- | Makefile | 6 | ||||
| -rw-r--r-- | docs/src/lexerdevelopment.txt | 8 | ||||
| -rw-r--r-- | pygments/__init__.py | 2 | ||||
| -rw-r--r-- | pygments/formatters/__init__.py | 2 | ||||
| -rw-r--r-- | pygments/lexer.py | 22 | ||||
| -rw-r--r-- | pygments/lexers/_mapping.py | 91 | ||||
| -rw-r--r-- | pygments/lexers/text.py | 71 | ||||
| -rw-r--r-- | pygments/token.py | 2 | ||||
| -rw-r--r-- | tests/test_using_api.py | 31 |
10 files changed, 182 insertions, 56 deletions
@@ -5,6 +5,8 @@ Version 0.7 ----------- (codename to be selected, released Feb XX, 2007) +- Added a "state" keyword argument to the `using` helper. + - Added a `commandprefix` option to the `LatexFormatter` which allows to control how the command names are constructed. @@ -14,6 +16,7 @@ Version 0.7 * Windows batch files * Trac Wiki markup * Python tracebacks + * ReStructuredText * and the Befunge esoteric programming language (yay!) - Added Mako lexers by Ben Bangert. @@ -10,11 +10,13 @@ PYTHON ?= python -export PYTHONPATH ?= $(shell python -c 'print ":".join(line.strip() for line in file("PYTHONPATH"))' 2>/dev/null) +export PYTHONPATH = $(shell echo "$$PYTHONPATH"):$(shell python -c 'print ":".join(line.strip() for line in file("PYTHONPATH"))' 2>/dev/null) -.PHONY: apidocs check clean clean-pyc codetags docs epydoc lexermap \ +.PHONY: all apidocs check clean clean-pyc codetags docs epydoc lexermap \ pylint reindent test +all: test + apidocs: epydoc check: diff --git a/docs/src/lexerdevelopment.txt b/docs/src/lexerdevelopment.txt index a004df5c..2af2683f 100644 --- a/docs/src/lexerdevelopment.txt +++ b/docs/src/lexerdevelopment.txt @@ -360,8 +360,12 @@ Here, two states are pushed onto the state stack, ``'script-content'`` and attributes and the closing ``>``, then the ``'tag'`` state is popped and the next state on top of the stack will be ``'script-content'``. -Any keywords arguments passed to ``using()`` are added to the keyword arguments -used to create the lexer. +The `using()` helper has a special keyword argument, `state`, which works as +follows: if given, the lexer to use initially is not in the ``"root"`` state, +but in the state given by this argument. This *only* works with a `RegexLexer`. + +Any other keywords arguments passed to `using()` are added to the keyword +arguments used to create the lexer. Delegating Lexer diff --git a/pygments/__init__.py b/pygments/__init__.py index 15413509..1797ad90 100644 --- a/pygments/__init__.py +++ b/pygments/__init__.py @@ -73,7 +73,7 @@ def format(tokens, formatter, outfile=None): raise TypeError('format() argument must be a formatter instance, ' 'not a class') raise - + def highlight(code, lexer, formatter, outfile=None): """ diff --git a/pygments/formatters/__init__.py b/pygments/formatters/__init__.py index 8aefe103..d3a66366 100644 --- a/pygments/formatters/__init__.py +++ b/pygments/formatters/__init__.py @@ -24,7 +24,7 @@ def _doc_desc(obj): res = [] for line in obj.__doc__.strip().splitlines(): if line.strip(): - res.append(line.strip()) + res.append(" " + line.strip()) else: break return ''.join(res) diff --git a/pygments/lexer.py b/pygments/lexer.py index 3cfe4e62..f984416a 100644 --- a/pygments/lexer.py +++ b/pygments/lexer.py @@ -295,8 +295,23 @@ def using(_other, **kwargs): """ Callback that processes the match with a different lexer. - The keyword arguments are forwarded to the lexer. + The keyword arguments are forwarded to the lexer, except `state` which + is handled separately. + + `state` specifies the state that the new lexer will start in, and can + be an enumerable such as ('root', 'inline', 'string') or a simple + string which is assumed to be on top of the root state. + + Note: For that to work, `_other` must not be an `ExtendedRegexLexer`. """ + gt_kwargs = {} + if 'state' in kwargs: + s = kwargs.pop('state') + if isinstance(s, (list, tuple)): + gt_kwargs['stack'] = s + else: + gt_kwargs['stack'] = ('root', s) + if _other is this: def callback(lexer, match, ctx=None): # if keyword arguments are given the callback @@ -308,7 +323,7 @@ def using(_other, **kwargs): else: lx = lexer s = match.start() - for i, t, v in lx.get_tokens_unprocessed(match.group()): + for i, t, v in lx.get_tokens_unprocessed(match.group(), **gt_kwargs): yield i + s, t, v if ctx: ctx.pos = match.end() @@ -319,7 +334,7 @@ def using(_other, **kwargs): lx = _other(**kwargs) s = match.start() - for i, t, v in lx.get_tokens_unprocessed(match.group()): + for i, t, v in lx.get_tokens_unprocessed(match.group(), **gt_kwargs): yield i + s, t, v if ctx: ctx.pos = match.end() @@ -444,6 +459,7 @@ class RegexLexer(Lexer): for rex, action, new_state in statetokens: m = rex.match(text, pos) if m: + # print rex.pattern if type(action) is _TokenType: yield pos, action, m.group() else: diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py index fbaaf1a5..a482e5b0 100644 --- a/pygments/lexers/_mapping.py +++ b/pygments/lexers/_mapping.py @@ -14,78 +14,79 @@ """ LEXERS = { - 'ApacheConfLexer': ('pygments.lexers.text', 'ApacheConf', ('apacheconf', 'aconf', 'apache'), ('.htaccess', 'apache.conf', 'apache2.conf'), ()), - 'BBCodeLexer': ('pygments.lexers.text', 'BBCode', ('bbcode',), (), ()), + 'ApacheConfLexer': ('pygments.lexers.text', 'ApacheConf', ('apacheconf', 'aconf', 'apache'), ('.htaccess', 'apache.conf', 'apache2.conf'), ('text/x-apacheconf',)), + 'BBCodeLexer': ('pygments.lexers.text', 'BBCode', ('bbcode',), (), ('text/x-bbcode',)), 'BashLexer': ('pygments.lexers.other', 'Bash', ('bash', 'sh'), ('*.sh',), ('application/x-sh', 'application/x-shellscript')), 'BatchLexer': ('pygments.lexers.other', 'Batchfile', ('bat',), ('*.bat', '*.cmd'), ('application/x-dos-batch',)), - 'BefungeLexer': ('pygments.lexers.other', 'Befunge', ('befunge',), ('*.befunge',), ()), + 'BefungeLexer': ('pygments.lexers.other', 'Befunge', ('befunge',), ('*.befunge',), ('application/x-befunge',)), 'BooLexer': ('pygments.lexers.dotnet', 'Boo', ('boo',), ('*.boo',), ('text/x-boo',)), - 'BrainfuckLexer': ('pygments.lexers.other', 'Brainfuck', ('brainfuck', 'bf'), ('*.bf', '*.b'), ()), + 'BrainfuckLexer': ('pygments.lexers.other', 'Brainfuck', ('brainfuck', 'bf'), ('*.bf', '*.b'), ('application/x-brainfuck',)), 'CLexer': ('pygments.lexers.compiled', 'C', ('c',), ('*.c', '*.h'), ('text/x-chdr', 'text/x-csrc')), 'CSharpLexer': ('pygments.lexers.dotnet', 'C#', ('csharp', 'c#'), ('*.cs',), ('text/x-csharp',)), 'CppLexer': ('pygments.lexers.compiled', 'C++', ('cpp', 'c++'), ('*.cpp', '*.hpp', '*.c++', '*.h++'), ('text/x-c++hdr', 'text/x-c++src')), - 'CssDjangoLexer': ('pygments.lexers.templates', 'CSS+Django/Jinja', ('css+django', 'css+jinja'), (), ()), - 'CssErbLexer': ('pygments.lexers.templates', 'CSS+Ruby', ('css+erb', 'css+ruby'), (), ()), - 'CssGenshiLexer': ('pygments.lexers.templates', 'CSS+Genshi Text', ('css+genshitext', 'css+genshi'), (), ()), + 'CssDjangoLexer': ('pygments.lexers.templates', 'CSS+Django/Jinja', ('css+django', 'css+jinja'), (), ('text/css+django', 'text/css+jinja')), + 'CssErbLexer': ('pygments.lexers.templates', 'CSS+Ruby', ('css+erb', 'css+ruby'), (), ('text/css+ruby',)), + 'CssGenshiLexer': ('pygments.lexers.templates', 'CSS+Genshi Text', ('css+genshitext', 'css+genshi'), (), ('text/css+genshi',)), 'CssLexer': ('pygments.lexers.web', 'CSS', ('css',), ('*.css',), ('text/css',)), - 'CssPhpLexer': ('pygments.lexers.templates', 'CSS+PHP', ('css+php',), (), ()), - 'CssSmartyLexer': ('pygments.lexers.templates', 'CSS+Smarty', ('css+smarty',), (), ()), + 'CssPhpLexer': ('pygments.lexers.templates', 'CSS+PHP', ('css+php',), (), ('text/css+php',)), + 'CssSmartyLexer': ('pygments.lexers.templates', 'CSS+Smarty', ('css+smarty',), (), ('text/css+smarty',)), 'DelphiLexer': ('pygments.lexers.compiled', 'Delphi', ('delphi', 'pas', 'pascal', 'objectpascal'), ('*.pas',), ('text/x-pascal',)), 'DiffLexer': ('pygments.lexers.text', 'Diff', ('diff',), ('*.diff', '*.patch'), ('text/x-diff', 'text/x-patch')), - 'DjangoLexer': ('pygments.lexers.templates', 'Django/Jinja', ('django', 'jinja'), (), ()), - 'ErbLexer': ('pygments.lexers.templates', 'ERB', ('erb',), (), ()), - 'GenshiLexer': ('pygments.lexers.templates', 'Genshi', ('genshi', 'kid', 'xml+genshi', 'xml+kid'), ('*.kid',), ()), - 'GenshiTextLexer': ('pygments.lexers.templates', 'Genshi Text', ('genshitext',), (), ()), + 'DjangoLexer': ('pygments.lexers.templates', 'Django/Jinja', ('django', 'jinja'), (), ('application/x-django-templating', 'application/x-jinja')), + 'ErbLexer': ('pygments.lexers.templates', 'ERB', ('erb',), (), ('application/x-ruby-templating',)), + 'GenshiLexer': ('pygments.lexers.templates', 'Genshi', ('genshi', 'kid', 'xml+genshi', 'xml+kid'), ('*.kid',), ('application/x-genshi', 'application/x-kid')), + 'GenshiTextLexer': ('pygments.lexers.templates', 'Genshi Text', ('genshitext',), (), ('application/x-genshi-text', 'text/x-genshi')), 'GroffLexer': ('pygments.lexers.text', 'Groff', ('groff', 'nroff', 'man'), ('*.[1234567]', '*.man'), ('application/x-troff', 'text/troff')), - 'HtmlDjangoLexer': ('pygments.lexers.templates', 'HTML+Django/Jinja', ('html+django', 'html+jinja'), (), ()), - 'HtmlGenshiLexer': ('pygments.lexers.templates', 'HTML+Genshi', ('html+genshi', 'html+kid'), (), ()), + 'HtmlDjangoLexer': ('pygments.lexers.templates', 'HTML+Django/Jinja', ('html+django', 'html+jinja'), (), ('text/html+django', 'text/html+jinja')), + 'HtmlGenshiLexer': ('pygments.lexers.templates', 'HTML+Genshi', ('html+genshi', 'html+kid'), (), ('text/html+genshi',)), 'HtmlLexer': ('pygments.lexers.web', 'HTML', ('html',), ('*.html', '*.htm', '*.xhtml'), ('text/html', 'application/xhtml+xml')), - 'HtmlPhpLexer': ('pygments.lexers.templates', 'HTML+PHP', ('html+php',), ('*.phtml',), ('text/x-php', 'application/x-php', 'application/x-httpd-php', 'application/x-httpd-php3', 'application/x-httpd-php4', 'application/x-httpd-php5')), - 'HtmlSmartyLexer': ('pygments.lexers.templates', 'HTML+Smarty', ('html+smarty',), (), ()), - 'IniLexer': ('pygments.lexers.text', 'INI', ('ini', 'cfg'), ('*.ini', '*.cfg'), ()), - 'IrcLogsLexer': ('pygments.lexers.text', 'IRC logs', ('irc',), (), ()), + 'HtmlPhpLexer': ('pygments.lexers.templates', 'HTML+PHP', ('html+php',), ('*.phtml',), ('application/x-php', 'application/x-httpd-php', 'application/x-httpd-php3', 'application/x-httpd-php4', 'application/x-httpd-php5')), + 'HtmlSmartyLexer': ('pygments.lexers.templates', 'HTML+Smarty', ('html+smarty',), (), ('text/html+smarty',)), + 'IniLexer': ('pygments.lexers.text', 'INI', ('ini', 'cfg'), ('*.ini', '*.cfg'), ('text/x-ini',)), + 'IrcLogsLexer': ('pygments.lexers.text', 'IRC logs', ('irc',), (), ('text/x-irclog',)), 'JavaLexer': ('pygments.lexers.compiled', 'Java', ('java',), ('*.java',), ('text/x-java',)), - 'JavascriptDjangoLexer': ('pygments.lexers.templates', 'JavaScript+Django/Jinja', ('js+django', 'javascript+django', 'js+jinja', 'javascript+jinja'), (), ()), - 'JavascriptErbLexer': ('pygments.lexers.templates', 'JavaScript+Ruby', ('js+erb', 'javascript+erb', 'js+ruby', 'javascript+ruby'), (), ()), - 'JavascriptGenshiLexer': ('pygments.lexers.templates', 'JavaScript+Genshi Text', ('js+genshitext', 'js+genshi', 'javascript+genshitext', 'javascript+genshi'), (), ()), + 'JavascriptDjangoLexer': ('pygments.lexers.templates', 'JavaScript+Django/Jinja', ('js+django', 'javascript+django', 'js+jinja', 'javascript+jinja'), (), ('application/x-javascript+django', 'application/x-javascript+jinja', 'text/x-javascript+django', 'text/x-javascript+jinja', 'text/javascript+django', 'text/javascript+jinja')), + 'JavascriptErbLexer': ('pygments.lexers.templates', 'JavaScript+Ruby', ('js+erb', 'javascript+erb', 'js+ruby', 'javascript+ruby'), (), ('application/x-javascript+ruby', 'text/x-javascript+ruby', 'text/javascript+ruby')), + 'JavascriptGenshiLexer': ('pygments.lexers.templates', 'JavaScript+Genshi Text', ('js+genshitext', 'js+genshi', 'javascript+genshitext', 'javascript+genshi'), (), ('application/x-javascript+genshi', 'text/x-javascript+genshi', 'text/javascript+genshi')), 'JavascriptLexer': ('pygments.lexers.web', 'JavaScript', ('js', 'javascript'), ('*.js',), ('application/x-javascript', 'text/x-javascript', 'text/javascript')), - 'JavascriptPhpLexer': ('pygments.lexers.templates', 'JavaScript+PHP', ('js+php', 'javascript+php'), (), ()), - 'JavascriptSmartyLexer': ('pygments.lexers.templates', 'JavaScript+Smarty', ('js+smarty', 'javascript+smarty'), (), ()), - 'JspLexer': ('pygments.lexers.templates', 'Java Server Page', ('jsp',), ('*.jsp',), ()), + 'JavascriptPhpLexer': ('pygments.lexers.templates', 'JavaScript+PHP', ('js+php', 'javascript+php'), (), ('application/x-javascript+php', 'text/x-javascript+php', 'text/javascript+php')), + 'JavascriptSmartyLexer': ('pygments.lexers.templates', 'JavaScript+Smarty', ('js+smarty', 'javascript+smarty'), (), ('application/x-javascript+smarty', 'text/x-javascript+smarty', 'text/javascript+smarty')), + 'JspLexer': ('pygments.lexers.templates', 'Java Server Page', ('jsp',), ('*.jsp',), ('application/x-jsp',)), 'LuaLexer': ('pygments.lexers.agile', 'Lua', ('lua',), ('*.lua',), ('text/x-lua', 'application/x-lua')), 'MakefileLexer': ('pygments.lexers.text', 'Makefile', ('make', 'makefile', 'mf'), ('*.mak', 'Makefile', 'makefile'), ('text/x-makefile',)), - 'MakoCssLexer': ('pygments.lexers.templates', 'CSS+Mako', ('css+mako',), (), ()), - 'MakoHtmlLexer': ('pygments.lexers.templates', 'HTML+Mako', ('html+mako',), (), ()), - 'MakoJavascriptLexer': ('pygments.lexers.templates', 'JavaScript+Mako', ('js+mako', 'javascript+mako'), (), ()), - 'MakoLexer': ('pygments.lexers.templates', 'Mako', ('mako',), ('*.mao',), ()), - 'MakoXmlLexer': ('pygments.lexers.templates', 'XML+Mako', ('xml+mako',), (), ()), + 'MakoCssLexer': ('pygments.lexers.templates', 'CSS+Mako', ('css+mako',), (), ('text/css+mako',)), + 'MakoHtmlLexer': ('pygments.lexers.templates', 'HTML+Mako', ('html+mako',), (), ('text/html+mako',)), + 'MakoJavascriptLexer': ('pygments.lexers.templates', 'JavaScript+Mako', ('js+mako', 'javascript+mako'), (), ('application/x-javascript+mako', 'text/x-javascript+mako', 'text/javascript+mako')), + 'MakoLexer': ('pygments.lexers.templates', 'Mako', ('mako',), ('*.mao',), ('application/x-mako',)), + 'MakoXmlLexer': ('pygments.lexers.templates', 'XML+Mako', ('xml+mako',), (), ('application/xml+mako',)), 'MoinWikiLexer': ('pygments.lexers.text', 'MoinMoin/Trac Wiki markup', ('trac-wiki', 'moin'), (), ('text/x-trac-wiki',)), - 'MyghtyCssLexer': ('pygments.lexers.templates', 'CSS+Myghty', ('css+myghty',), (), ()), - 'MyghtyHtmlLexer': ('pygments.lexers.templates', 'HTML+Myghty', ('html+myghty',), (), ()), - 'MyghtyJavascriptLexer': ('pygments.lexers.templates', 'JavaScript+Myghty', ('js+myghty', 'javascript+myghty'), (), ()), - 'MyghtyLexer': ('pygments.lexers.templates', 'Myghty', ('myghty',), ('*.myt', 'autodelegate'), ()), - 'MyghtyXmlLexer': ('pygments.lexers.templates', 'XML+Myghty', ('xml+myghty',), (), ()), + 'MyghtyCssLexer': ('pygments.lexers.templates', 'CSS+Myghty', ('css+myghty',), (), ('text/css+myghty',)), + 'MyghtyHtmlLexer': ('pygments.lexers.templates', 'HTML+Myghty', ('html+myghty',), (), ('text/html+myghty',)), + 'MyghtyJavascriptLexer': ('pygments.lexers.templates', 'JavaScript+Myghty', ('js+myghty', 'javascript+myghty'), (), ('application/x-javascript+myghty', 'text/x-javascript+myghty', 'text/javascript+mygthy')), + 'MyghtyLexer': ('pygments.lexers.templates', 'Myghty', ('myghty',), ('*.myt', 'autodelegate'), ('application/x-myghty',)), + 'MyghtyXmlLexer': ('pygments.lexers.templates', 'XML+Myghty', ('xml+myghty',), (), ('application/xml+myghty',)), 'PerlLexer': ('pygments.lexers.agile', 'Perl', ('perl', 'pl'), ('*.pl', '*.pm'), ('text/x-perl', 'application/x-perl')), - 'PhpLexer': ('pygments.lexers.web', 'PHP', ('php', 'php3', 'php4', 'php5'), ('*.php', '*.php[345]'), ()), - 'PythonConsoleLexer': ('pygments.lexers.agile', 'Python console session', ('pycon',), (), ()), + 'PhpLexer': ('pygments.lexers.web', 'PHP', ('php', 'php3', 'php4', 'php5'), ('*.php', '*.php[345]'), ('text/x-php',)), + 'PythonConsoleLexer': ('pygments.lexers.agile', 'Python console session', ('pycon',), (), ('text/x-python-doctest',)), 'PythonLexer': ('pygments.lexers.agile', 'Python', ('python', 'py'), ('*.py', '*.pyw'), ('text/x-python', 'application/x-python')), - 'PythonTracebackLexer': ('pygments.lexers.agile', 'PythonTraceback', ('pytb',), ('*.pytb',), ('text/x-python-traceback',)), + 'PythonTracebackLexer': ('pygments.lexers.agile', 'Python Traceback', ('pytb',), ('*.pytb',), ('text/x-python-traceback',)), 'RawTokenLexer': ('pygments.lexers.special', 'Raw token data', ('raw',), ('*.raw',), ('application/x-pygments-tokens',)), - 'RhtmlLexer': ('pygments.lexers.templates', 'RHTML', ('rhtml', 'html+erb', 'html+ruby'), ('*.rhtml',), ()), - 'RubyConsoleLexer': ('pygments.lexers.agile', 'Ruby irb session', ('rbcon', 'irb'), (), ()), + 'RhtmlLexer': ('pygments.lexers.templates', 'RHTML', ('rhtml', 'html+erb', 'html+ruby'), ('*.rhtml',), ('text/html+ruby',)), + 'RstLexer': ('pygments.lexers.text', 'reStructuredText', ('rst', 'restructuredtext'), ('*.rst', '*.rest'), ('text/x-rst',)), + 'RubyConsoleLexer': ('pygments.lexers.agile', 'Ruby irb session', ('rbcon', 'irb'), (), ('text/x-ruby-shellsession',)), 'RubyLexer': ('pygments.lexers.agile', 'Ruby', ('rb', 'ruby'), ('*.rb', '*.rbw', 'Rakefile', '*.rake', '*.gemspec', '*.rbx'), ('text/x-ruby', 'application/x-ruby')), 'SchemeLexer': ('pygments.lexers.agile', 'Scheme', ('scheme',), ('*.scm',), ('text/x-scheme', 'application/x-scheme')), - 'SmartyLexer': ('pygments.lexers.templates', 'Smarty', ('smarty',), ('*.tpl',), ()), + 'SmartyLexer': ('pygments.lexers.templates', 'Smarty', ('smarty',), ('*.tpl',), ('application/x-smarty',)), 'SourcesListLexer': ('pygments.lexers.text', 'Debian Sourcelist', ('sourceslist', 'sources.list'), ('sources.list',), ()), 'SqlLexer': ('pygments.lexers.other', 'SQL', ('sql',), ('*.sql',), ('text/x-sql',)), 'TexLexer': ('pygments.lexers.text', 'TeX', ('tex', 'latex'), ('*.tex', '*.aux', '*.toc'), ('text/x-tex', 'text/x-latex')), 'TextLexer': ('pygments.lexers.special', 'Text only', ('text',), ('*.txt',), ('text/plain',)), 'VbNetLexer': ('pygments.lexers.dotnet', 'VB.net', ('vb.net', 'vbnet'), ('*.vb', '*.bas'), ('text/x-vbnet', 'text/x-vba')), - 'XmlDjangoLexer': ('pygments.lexers.templates', 'XML+Django/Jinja', ('xml+django', 'xml+jinja'), (), ()), - 'XmlErbLexer': ('pygments.lexers.templates', 'XML+Ruby', ('xml+erb', 'xml+ruby'), (), ()), + 'XmlDjangoLexer': ('pygments.lexers.templates', 'XML+Django/Jinja', ('xml+django', 'xml+jinja'), (), ('application/xml+django', 'application/xml+jinja')), + 'XmlErbLexer': ('pygments.lexers.templates', 'XML+Ruby', ('xml+erb', 'xml+ruby'), (), ('application/xml+ruby',)), 'XmlLexer': ('pygments.lexers.web', 'XML', ('xml',), ('*.xml', '*.xsl', '*.rss'), ('text/xml', 'application/xml', 'image/svg+xml', 'application/rss+xml', 'application/atom+xml', 'application/xsl+xml', 'application/xslt+xml')), - 'XmlPhpLexer': ('pygments.lexers.templates', 'XML+PHP', ('xml+php',), (), ()), - 'XmlSmartyLexer': ('pygments.lexers.templates', 'XML+Smarty', ('xml+smarty',), (), ()) + 'XmlPhpLexer': ('pygments.lexers.templates', 'XML+PHP', ('xml+php',), (), ('application/xml+php',)), + 'XmlSmartyLexer': ('pygments.lexers.templates', 'XML+Smarty', ('xml+smarty',), (), ('application/xml+smarty',)) } if __name__ == '__main__': diff --git a/pygments/lexers/text.py b/pygments/lexers/text.py index 6336ecbb..89bddc65 100644 --- a/pygments/lexers/text.py +++ b/pygments/lexers/text.py @@ -25,7 +25,7 @@ from pygments.token import Punctuation, \ __all__ = ['IniLexer', 'SourcesListLexer', 'MakefileLexer', 'DiffLexer', 'IrcLogsLexer', 'TexLexer', 'GroffLexer', 'ApacheConfLexer', - 'BBCodeLexer', 'MoinWikiLexer'] + 'BBCodeLexer', 'MoinWikiLexer', 'RstLexer'] class IniLexer(RegexLexer): @@ -444,3 +444,72 @@ class MoinWikiLexer(RegexLexer): (r'.', Comment.Preproc), # allow loose { or } ], } + + +class RstLexer(RegexLexer): + """ + For MoinMoin (and Trac) Wiki markup. + + *New in Pygments 0.7.* + """ + name = 'reStructuredText' + aliases = ['rst', 'restructuredtext'] + filenames = ['*.rst', '*.rest'] + mimetypes = ["text/x-rst"] + flags = re.MULTILINE + + tokens = { + 'root': [ + # Heading with overline + (r'^(=+|-+|`+|:+|\.+|\'+|"+|~+|\^+|_+|\*+|\++|#+)(\n)(.+)(\n)(\1)(\n)', + bygroups(Generic.Heading, Text, using(this, state='inline'), Text, Generic.Heading, Text)), + # Plain heading + (r'^(\S.*)(\n)(=+|-+|`+|:+|\.+|\'+|"+|~+|\^+|_+|\*+|\++|#+)(\n)', + bygroups(Generic.Heading, Text, Generic.Heading, Text)), + + # Bulleted lists + (r'^(\s*)([-*+])( .+\n(?:\1 .+\n)*)', + bygroups(Text, Keyword, using(this, state='inline'))), + # Numbered lists + (r'^(\s*)([0-9#ivxlcmIVXLCM]+\.)( .+\n(?:\1 .+\n)*)', + bygroups(Text, Keyword, using(this, state='inline'))), + (r'^(\s*)(\(?[0-9#ivxlcmIVXLCM]+\))( .+\n(?:\1 .+\n)*)', + bygroups(Text, Keyword, using(this, state='inline'))), + # Numbered, but keep words at BOL from becoming lists + (r'^(\s*)([A-Z]+\.)( .+\n(?:\1 .+\n)+)', + bygroups(Text, Keyword, using(this, state='inline'))), + (r'^(\s*)(\(?[A-Za-z]+\))( .+\n(?:\1 .+\n)+)', + bygroups(Text, Keyword, using(this, state='inline'))), + # Introducing a section + (r'^( *\.\.)(\s*)(\w+)(::)(?:(\s*)(.+))?(\n(?:(?: +.*|)\n)+)$', + bygroups(Punctuation, Text, Number, Punctuation, Text, Number, Text)), + # A reference target + (r'^( *\.\.)(\s*)(\w+:)(.*?)$', + bygroups(Punctuation, Text, Name.Tag, using(this, state='inline'))), + # A footnote target + (r'^( *\.\.)(\s*)(\[.+\])(.*?)$', + bygroups(Punctuation, Text, Name.Tag, using(this, state='inline'))), + # Comments + (r'^ *\.\..*(\n( +.*\n|\n)+)?', Comment.Preproc), + # Definition list + (r'^([^ ].*(?<!::)\n)((?:(?: +.*)\n)+)', + bygroups(using(this, state='inline'), using(this, state='inline'))), + # Code blocks + (r'(::)(\n)((?:(?: +.*|)\n)+)', + bygroups(String.Escape, Text, String)), + include('inline'), + ], + 'inline': [ + (r'``.+?``', String), # code + # Phrase reference + (r'(``?)(.+?)(\1__?)', + bygroups(Punctuation, using(this), Punctuation)), + (r'`.+?`', Name), + (r'\*\*.+?\*\*', String), # Strong emphasis + (r'\*.+?\*', Number), # Emphasis + (r'\[.*?\]_', String), # Footnote or citation + (r'<.+?>', Name.Tag), + (r'[^\n\[*`:]+', Text), + (r'.', Text), + ], + } diff --git a/pygments/token.py b/pygments/token.py index 146045df..53a59ba6 100644 --- a/pygments/token.py +++ b/pygments/token.py @@ -1,4 +1,4 @@ - # -*- coding: utf-8 -*- +# -*- coding: utf-8 -*- """ pygments.token ~~~~~~~~~~~~~~ diff --git a/tests/test_using_api.py b/tests/test_using_api.py new file mode 100644 index 00000000..0af12441 --- /dev/null +++ b/tests/test_using_api.py @@ -0,0 +1,31 @@ +import unittest +from pygments.lexer import using, bygroups, this, RegexLexer +from pygments.token import String, Text, Keyword + +class TestLexer(RegexLexer): + tokens = { + 'root': [ + (r'#.*', using(this, state='invalid')), + (r'(")(.+?)(")', bygroups(String, using(this, state='string'), String)), + (r'[^"]+', Text), + ], + 'string': [ + (r'.+', Keyword), + ], + } + +class UsingStateTest(unittest.TestCase): + def test_basic(self): + expected = [(Text, 'a'), (String, '"'), (Keyword, 'bcd'), + (String, '"'), (Text, 'e\n')] + t = list(TestLexer().get_tokens('a"bcd"e')) + self.assertEquals(t, expected) + def test_error(self): + def gen(): + x = list(TestLexer().get_tokens('#a')) + #XXX: should probably raise a more specific exception if the state + # doesn't exist. + self.assertRaises(Exception, gen) + +if __name__ == "__main__": + unittest.main() |