[svn] various changes in pygments (does also affect the docs)

9 files changed, 170 insertions, 18 deletions

diff --git a/CHANGES b/CHANGES
index ad979ef8..35097c1f 100644
--- a/CHANGES
+++ b/CHANGES
@@ -26,6 +26,8 @@ Version 0.7
 
 - Changed behavior of `in` Operator for tokens. 
 
+- Added mimetypes for all lexers
+
 - Fixed tickets: #167, #178, #179, #180, #185.
 
 
diff --git a/docs/src/tokens.txt b/docs/src/tokens.txt
index 853bf2cb..22ccccea 100644
--- a/docs/src/tokens.txt
+++ b/docs/src/tokens.txt
@@ -81,6 +81,21 @@ of those token aliases, a number of subtypes exists (excluding the special token
 
 The `is_token_subtype()` function in the `pygments.token` module can be used to
 test if a token type is a subtype of another (such as `Name.Tag` and `Name`).
+(This is the same as ``Name.Tag in Name``. The in operator was newly introduced
+in pygments 0.7, the function still exists for backwards compatiblity)
+
+With pygments 0.7 it's also possible to convert token from strings (for example
+if you want to supply a token from the command line):
+
+.. sourcecode:: pycon
+
+    >>> from pygments.token import String, string_to_token
+    >>> string_to_token("String")
+    Token.Literal.String
+    >>> string_to_token("Token.Literal.String")
+    Token.Literal.String
+    >>> string_to_token(String)
+    Token.Literal.String
 
 
 Keyword Tokens
diff --git a/pygments/filters/__init__.py b/pygments/filters/__init__.py
index 3bf7d79a..206cf9aa 100644
--- a/pygments/filters/__init__.py
+++ b/pygments/filters/__init__.py
@@ -9,8 +9,13 @@
     :copyright: 2006 by Armin Ronacher.
     :license: BSD, see LICENSE for more details.
 """
+try:
+    set
+except NameError:
+    from sets import Set as set
+
 import re
-from pygments.token import String, Comment
+from pygments.token import String, Comment, Keyword, Name, string_to_token
 from pygments.filter import Filter
 from pygments.util import get_list_opt
 from pygments.plugin import find_plugin_filters
@@ -86,7 +91,7 @@ class KeywordCaseFilter(Filter):
         Filter.__init__(self, **options)
         case = options.get('keywordcase', 'lower')
         if case not in ('lower', 'upper', 'capitalize'):
-            raise TypeError
+            raise TypeError('unknown conversion method %r' % case)
         self.convert = getattr(unicode, case)
 
     def filter(self, lexer, stream):
@@ -97,7 +102,37 @@ class KeywordCaseFilter(Filter):
                 yield ttype, value
 
 
+class NameHighlightFilter(Filter):
+    """
+    Highlight normal name token with a different one::
+
+        filter = NameHighlightFilter(
+            highlight=['foo', 'bar', 'baz'],
+            highlight_token=Name.Function
+        )
+
+    This would highlight the names "foo", "bar" and "baz"
+    as functions. `Name.Function` is the token default.
+    """
+
+    def __init__(self, **options):
+        self.words = set(get_list_opt(options, 'highlight', []))
+        highlight_token = options.get('highlight_token')
+        if highlight_token:
+            self.highlight_token = string_to_token(highlight_token)
+        else:
+            self.highlight_token = Name.Function
+
+    def filter(self, lexer, stream):
+        for ttype, value in stream:
+            if ttype is Name and value in self.words:
+                yield self.highlight_token, value
+            else:
+                yield ttype, value
+
+
 FILTERS = {
     'codetagify':           CodeTagFilter,
-    'keywordcase':          KeywordCaseFilter
+    'keywordcase':          KeywordCaseFilter,
+    'highlight':            NameHighlightFilter
 }
diff --git a/pygments/lexers/agile.py b/pygments/lexers/agile.py
index fc9e4e46..fa702f7a 100644
--- a/pygments/lexers/agile.py
+++ b/pygments/lexers/agile.py
@@ -174,6 +174,7 @@ class PythonConsoleLexer(Lexer):
     """
     name = 'Python console session'
     aliases = ['pycon']
+    mimetypes = ['text/x-python-doctest']
 
     def get_tokens_unprocessed(self, text):
         pylexer = PythonLexer(**self.options)
@@ -584,6 +585,7 @@ class RubyConsoleLexer(Lexer):
     """
     name = 'Ruby irb session'
     aliases = ['rbcon', 'irb']
+    mimetypes = ['text/x-ruby-shellsession']
 
     _prompt_re = re.compile('irb\([a-zA-Z_][a-zA-Z0-9_]*\):\d{3}:\d+[>*] ')
 
diff --git a/pygments/lexers/other.py b/pygments/lexers/other.py
index 62654909..d4473ec4 100644
--- a/pygments/lexers/other.py
+++ b/pygments/lexers/other.py
@@ -144,6 +144,7 @@ class BrainfuckLexer(RegexLexer):
     name = 'Brainfuck'
     aliases = ['brainfuck', 'bf']
     filenames = ['*.bf', '*.b']
+    mimetypes = ['application/x-brainfuck']
 
     tokens = {
         'common': [
@@ -176,6 +177,7 @@ class BefungeLexer(RegexLexer):
     name = 'Befunge'
     aliases = ['befunge']
     filenames = ['*.befunge']
+    mimetypes = ['application/x-befunge']
 
     tokens = {
         'root': [
diff --git a/pygments/lexers/templates.py b/pygments/lexers/templates.py
index 419b7123..141837b3 100644
--- a/pygments/lexers/templates.py
+++ b/pygments/lexers/templates.py
@@ -53,6 +53,7 @@ class ErbLexer(Lexer):
 
     name = 'ERB'
     aliases = ['erb']
+    mimetypes = ['application/x-ruby-templating']
 
     _block_re = re.compile(r'(<%%|%%>|<%=|<%#|<%-|<%|-%>|%>|^%[^%].*?$)', re.M)
 
@@ -145,6 +146,7 @@ class SmartyLexer(RegexLexer):
     name = 'Smarty'
     aliases = ['smarty']
     filenames = ['*.tpl']
+    mimetypes = ['application/x-smarty']
 
     flags = re.MULTILINE | re.DOTALL
 
@@ -199,6 +201,7 @@ class DjangoLexer(RegexLexer):
 
     name = 'Django/Jinja'
     aliases = ['django', 'jinja']
+    mimetypes = ['application/x-django-templating', 'application/x-jinja']
 
     tokens = {
         'root': [
@@ -265,6 +268,7 @@ class MyghtyLexer(RegexLexer):
     name = 'Myghty'
     aliases = ['myghty']
     filenames = ['*.myt', 'autodelegate']
+    mimetypes = ['application/x-myghty']
 
     tokens = {
         'root': [
@@ -309,6 +313,7 @@ class MyghtyHtmlLexer(DelegatingLexer):
 
     name = 'HTML+Myghty'
     aliases = ['html+myghty']
+    mimetypes = ['text/html+myghty']
 
     def __init__(self, **options):
         super(MyghtyHtmlLexer, self).__init__(HtmlLexer, MyghtyLexer,
@@ -325,6 +330,7 @@ class MyghtyXmlLexer(DelegatingLexer):
 
     name = 'XML+Myghty'
     aliases = ['xml+myghty']
+    mimetypes = ['application/xml+myghty']
 
     def __init__(self, **options):
         super(MyghtyXmlLexer, self).__init__(XmlLexer, MyghtyLexer,
@@ -341,6 +347,9 @@ class MyghtyJavascriptLexer(DelegatingLexer):
 
     name = 'JavaScript+Myghty'
     aliases = ['js+myghty', 'javascript+myghty']
+    mimetypes = ['application/x-javascript+myghty',
+                 'text/x-javascript+myghty',
+                 'text/javascript+mygthy']
 
     def __init__(self, **options):
         super(MyghtyJavascriptLexer, self).__init__(JavascriptLexer,
@@ -357,6 +366,7 @@ class MyghtyCssLexer(DelegatingLexer):
 
     name = 'CSS+Myghty'
     aliases = ['css+myghty']
+    mimetypes = ['text/css+myghty']
 
     def __init__(self, **options):
         super(MyghtyCssLexer, self).__init__(CssLexer, MyghtyLexer,
@@ -376,6 +386,7 @@ class MakoLexer(RegexLexer):
     name = 'Mako'
     aliases = ['mako']
     filenames = ['*.mao']
+    mimetypes = ['application/x-mako']
 
     tokens = {
         'root': [
@@ -438,6 +449,7 @@ class MakoHtmlLexer(DelegatingLexer):
 
     name = 'HTML+Mako'
     aliases = ['html+mako']
+    mimetypes = ['text/html+mako']
 
     def __init__(self, **options):
         super(MakoHtmlLexer, self).__init__(HtmlLexer, MakoLexer,
@@ -453,6 +465,7 @@ class MakoXmlLexer(DelegatingLexer):
 
     name = 'XML+Mako'
     aliases = ['xml+mako']
+    mimetypes = ['application/xml+mako']
 
     def __init__(self, **options):
         super(MakoXmlLexer, self).__init__(XmlLexer, MakoLexer,
@@ -468,6 +481,9 @@ class MakoJavascriptLexer(DelegatingLexer):
 
     name = 'JavaScript+Mako'
     aliases = ['js+mako', 'javascript+mako']
+    mimetypes = ['application/x-javascript+mako',
+                 'text/x-javascript+mako',
+                 'text/javascript+mako']
 
     def __init__(self, **options):
         super(MakoJavascriptLexer, self).__init__(JavascriptLexer,
@@ -483,6 +499,7 @@ class MakoCssLexer(DelegatingLexer):
 
     name = 'CSS+Mako'
     aliases = ['css+mako']
+    mimetypes = ['text/css+mako']
 
     def __init__(self, **options):
         super(MakoCssLexer, self).__init__(CssLexer, MakoLexer,
@@ -499,6 +516,7 @@ class GenshiTextLexer(RegexLexer):
 
     name = 'Genshi Text'
     aliases = ['genshitext']
+    mimetypes = ['application/x-genshi-text', 'text/x-genshi']
 
     tokens = {
         'root': [
@@ -598,6 +616,7 @@ class HtmlGenshiLexer(DelegatingLexer):
     name = 'HTML+Genshi'
     aliases = ['html+genshi', 'html+kid']
     alias_filenames = ['*.html', '*.htm', '*.xhtml']
+    mimetypes = ['text/html+genshi']
 
     def __init__(self, **options):
         super(HtmlGenshiLexer, self).__init__(HtmlLexer, GenshiMarkupLexer,
@@ -622,6 +641,7 @@ class GenshiLexer(DelegatingLexer):
     aliases = ['genshi', 'kid', 'xml+genshi', 'xml+kid']
     filenames = ['*.kid']
     alias_filenames = ['*.xml']
+    mimetypes = ['application/x-genshi', 'application/x-kid']
 
     def __init__(self, **options):
         super(GenshiLexer, self).__init__(XmlLexer, GenshiMarkupLexer,
@@ -645,6 +665,9 @@ class JavascriptGenshiLexer(DelegatingLexer):
     aliases = ['js+genshitext', 'js+genshi', 'javascript+genshitext',
                'javascript+genshi']
     alias_filenames = ['*.js']
+    mimetypes = ['application/x-javascript+genshi',
+                 'text/x-javascript+genshi',
+                 'text/javascript+genshi']
 
     def __init__(self, **options):
         super(JavascriptGenshiLexer, self).__init__(JavascriptLexer,
@@ -663,6 +686,7 @@ class CssGenshiLexer(DelegatingLexer):
     name = 'CSS+Genshi Text'
     aliases = ['css+genshitext', 'css+genshi']
     alias_filenames = ['*.css']
+    mimetypes = ['text/css+genshi']
 
     def __init__(self, **options):
         super(CssGenshiLexer, self).__init__(CssLexer, GenshiTextLexer,
@@ -684,6 +708,7 @@ class RhtmlLexer(DelegatingLexer):
     aliases = ['rhtml', 'html+erb', 'html+ruby']
     filenames = ['*.rhtml']
     alias_filenames = ['*.html', '*.htm', '*.xhtml']
+    mimetypes = ['text/html+ruby']
 
     def __init__(self, **options):
         super(RhtmlLexer, self).__init__(HtmlLexer, ErbLexer, **options)
@@ -705,6 +730,7 @@ class XmlErbLexer(DelegatingLexer):
     name = 'XML+Ruby'
     aliases = ['xml+erb', 'xml+ruby']
     alias_filenames = ['*.xml']
+    mimetypes = ['application/xml+ruby']
 
     def __init__(self, **options):
         super(XmlErbLexer, self).__init__(XmlLexer, ErbLexer, **options)
@@ -723,7 +749,8 @@ class CssErbLexer(DelegatingLexer):
 
     name = 'CSS+Ruby'
     aliases = ['css+erb', 'css+ruby']
-    alias_filenames = ['*.xml']
+    alias_filenames = ['*.css']
+    mimetypes = ['text/css+ruby']
 
     def __init__(self, **options):
         super(CssErbLexer, self).__init__(CssLexer, ErbLexer, **options)
@@ -741,6 +768,9 @@ class JavascriptErbLexer(DelegatingLexer):
     name = 'JavaScript+Ruby'
     aliases = ['js+erb', 'javascript+erb', 'js+ruby', 'javascript+ruby']
     alias_filenames = ['*.js']
+    mimetypes = ['application/x-javascript+ruby',
+                 'text/x-javascript+ruby',
+                 'text/javascript+ruby']
 
     def __init__(self, **options):
         super(JavascriptErbLexer, self).__init__(JavascriptLexer, ErbLexer,
@@ -762,7 +792,7 @@ class HtmlPhpLexer(DelegatingLexer):
     filenames = ['*.phtml']
     alias_filenames = ['*.php', '*.html', '*.htm', '*.xhtml',
                        '*.php[345]']
-    mimetypes = ['text/x-php', 'application/x-php',
+    mimetypes = ['application/x-php',
                  'application/x-httpd-php', 'application/x-httpd-php3',
                  'application/x-httpd-php4', 'application/x-httpd-php5']
 
@@ -784,6 +814,7 @@ class XmlPhpLexer(DelegatingLexer):
     name = 'XML+PHP'
     aliases = ['xml+php']
     alias_filenames = ['*.xml', '*.php', '*.php[345]']
+    mimetypes = ['application/xml+php']
 
     def __init__(self, **options):
         super(XmlPhpLexer, self).__init__(XmlLexer, PhpLexer, **options)
@@ -803,6 +834,7 @@ class CssPhpLexer(DelegatingLexer):
     name = 'CSS+PHP'
     aliases = ['css+php']
     alias_filenames = ['*.css']
+    mimetypes = ['text/css+php']
 
     def __init__(self, **options):
         super(CssPhpLexer, self).__init__(CssLexer, PhpLexer, **options)
@@ -820,6 +852,9 @@ class JavascriptPhpLexer(DelegatingLexer):
     name = 'JavaScript+PHP'
     aliases = ['js+php', 'javascript+php']
     alias_filenames = ['*.js']
+    mimetypes = ['application/x-javascript+php',
+                 'text/x-javascript+php',
+                 'text/javascript+php']
 
     def __init__(self, **options):
         super(JavascriptPhpLexer, self).__init__(JavascriptLexer, PhpLexer,
@@ -840,6 +875,7 @@ class HtmlSmartyLexer(DelegatingLexer):
     name = 'HTML+Smarty'
     aliases = ['html+smarty']
     alias_filenames = ['*.html', '*.htm', '*.xhtml', '*.tpl']
+    mimetypes = ['text/html+smarty']
 
     def __init__(self, **options):
         super(HtmlSmartyLexer, self).__init__(HtmlLexer, SmartyLexer, **options)
@@ -860,6 +896,7 @@ class XmlSmartyLexer(DelegatingLexer):
     name = 'XML+Smarty'
     aliases = ['xml+smarty']
     alias_filenames = ['*.xml', '*.tpl']
+    mimetypes = ['application/xml+smarty']
 
     def __init__(self, **options):
         super(XmlSmartyLexer, self).__init__(XmlLexer, SmartyLexer, **options)
@@ -880,6 +917,7 @@ class CssSmartyLexer(DelegatingLexer):
     name = 'CSS+Smarty'
     aliases = ['css+smarty']
     alias_filenames = ['*.css', '*.tpl']
+    mimetypes = ['text/css+smarty']
 
     def __init__(self, **options):
         super(CssSmartyLexer, self).__init__(CssLexer, SmartyLexer, **options)
@@ -897,6 +935,9 @@ class JavascriptSmartyLexer(DelegatingLexer):
     name = 'JavaScript+Smarty'
     aliases = ['js+smarty', 'javascript+smarty']
     alias_filenames = ['*.js', '*.tpl']
+    mimetypes = ['application/x-javascript+smarty',
+                 'text/x-javascript+smarty',
+                 'text/javascript+smarty']
 
     def __init__(self, **options):
         super(JavascriptSmartyLexer, self).__init__(JavascriptLexer, SmartyLexer,
@@ -917,6 +958,7 @@ class HtmlDjangoLexer(DelegatingLexer):
     name = 'HTML+Django/Jinja'
     aliases = ['html+django', 'html+jinja']
     alias_filenames = ['*.html', '*.htm', '*.xhtml']
+    mimetypes = ['text/html+django', 'text/html+jinja']
 
     def __init__(self, **options):
         super(HtmlDjangoLexer, self).__init__(HtmlLexer, DjangoLexer, **options)
@@ -937,6 +979,7 @@ class XmlDjangoLexer(DelegatingLexer):
     name = 'XML+Django/Jinja'
     aliases = ['xml+django', 'xml+jinja']
     alias_filenames = ['*.xml']
+    mimetypes = ['application/xml+django', 'application/xml+jinja']
 
     def __init__(self, **options):
         super(XmlDjangoLexer, self).__init__(XmlLexer, DjangoLexer, **options)
@@ -957,6 +1000,7 @@ class CssDjangoLexer(DelegatingLexer):
     name = 'CSS+Django/Jinja'
     aliases = ['css+django', 'css+jinja']
     alias_filenames = ['*.css']
+    mimetypes = ['text/css+django', 'text/css+jinja']
 
     def __init__(self, **options):
         super(CssDjangoLexer, self).__init__(CssLexer, DjangoLexer, **options)
@@ -975,6 +1019,12 @@ class JavascriptDjangoLexer(DelegatingLexer):
     aliases = ['js+django', 'javascript+django',
                'js+jinja', 'javascript+jinja']
     alias_filenames = ['*.js']
+    mimetypes = ['application/x-javascript+django',
+                 'application/x-javascript+jinja',
+                 'text/x-javascript+django',
+                 'text/x-javascript+jinja',
+                 'text/javascript+django',
+                 'text/javascript+jinja']
 
     def __init__(self, **options):
         super(JavascriptDjangoLexer, self).__init__(JavascriptLexer, DjangoLexer,
@@ -1018,6 +1068,7 @@ class JspLexer(DelegatingLexer):
     name = 'Java Server Page'
     aliases = ['jsp']
     filenames = ['*.jsp']
+    mimetypes = ['application/x-jsp']
 
     def __init__(self, **options):
         super(JspLexer, self).__init__(XmlLexer, JspRootLexer, **options)
diff --git a/pygments/lexers/text.py b/pygments/lexers/text.py
index 29f569aa..6336ecbb 100644
--- a/pygments/lexers/text.py
+++ b/pygments/lexers/text.py
@@ -36,6 +36,7 @@ class IniLexer(RegexLexer):
     name = 'INI'
     aliases = ['ini', 'cfg']
     filenames = ['*.ini', '*.cfg']
+    mimetypes = ['text/x-ini']
 
     tokens = {
         'root': [
@@ -64,6 +65,7 @@ class SourcesListLexer(RegexLexer):
     name= 'Debian Sourcelist'
     aliases = ['sourceslist', 'sources.list']
     filenames = ['sources.list']
+    mimetype = ['application/x-debian-sourceslist']
 
     tokens = {
         'root': [
@@ -189,6 +191,7 @@ class IrcLogsLexer(RegexLexer):
 
     name = 'IRC logs'
     aliases = ['irc']
+    mimetypes = ['text/x-irclog']
 
     flags = re.VERBOSE | re.MULTILINE
     timestamp = r"""
@@ -235,6 +238,7 @@ class BBCodeLexer(RegexLexer):
 
     name = 'BBCode'
     aliases = ['bbcode']
+    mimetypes = ['text/x-bbcode']
 
     tokens = {
         'root' : [
@@ -370,6 +374,7 @@ class ApacheConfLexer(RegexLexer):
     name = 'ApacheConf'
     aliases = ['apacheconf', 'aconf', 'apache']
     filenames = ['.htaccess', 'apache.conf', 'apache2.conf']
+    mimetypes = ['text/x-apacheconf']
     flags = re.MULTILINE | re.IGNORECASE
 
     tokens = {
@@ -439,9 +444,3 @@ class MoinWikiLexer(RegexLexer):
             (r'.', Comment.Preproc), # allow loose { or }
         ],
     }
-
-
-#class RstLexer(RegexLexer):
-#    name = 'reStructuredText'
-#    aliases = ['rst', 'restructuredtext']
-#    filenames = ['*.rst']
diff --git a/pygments/lexers/web.py b/pygments/lexers/web.py
index ef480894..05622805 100644
--- a/pygments/lexers/web.py
+++ b/pygments/lexers/web.py
@@ -288,6 +288,7 @@ class PhpLexer(RegexLexer):
     name = 'PHP'
     aliases = ['php', 'php3', 'php4', 'php5']
     filenames = ['*.php', '*.php[345]']
+    mimetypes = ['text/x-php']
 
     flags = re.IGNORECASE | re.DOTALL | re.MULTILINE
     tokens = {
@@ -310,6 +311,8 @@ class PhpLexer(RegexLexer):
             (r'(class)(\s+)', bygroups(Keyword, Text), 'classname'),
             (r'(function)(\s+)(&?)(\s*)',
               bygroups(Keyword, Text, Operator, Text), 'functionname'),
+            (r'(const)(\s+)([a-zA-Z_][a-zA-Z0-9_]*)',
+              bygroups(Keyword, Text, Name.Constant)),
             (r'(and|E_PARSE|old_function|E_ERROR|or|as|E_WARNING|parent|'
              r'eval|PHP_OS|break|exit|case|extends|PHP_VERSION|cfunction|'
              r'FALSE|print|for|require|continue|foreach|require_once|'
diff --git a/pygments/token.py b/pygments/token.py
index f0b6f8c9..b99ee9b9 100644
--- a/pygments/token.py
+++ b/pygments/token.py
@@ -8,6 +8,11 @@
     :copyright: 2006 by Georg Brandl.
     :license: BSD, see LICENSE for more details.
 """
+try:
+    set
+except NameError:
+    from sets import Set as set
+
 
 class _TokenType(tuple):
     parent = None
@@ -21,6 +26,10 @@ class _TokenType(tuple):
         buf.reverse()
         return buf
 
+    def __init__(self, *args, **kwargs):
+        super(_TokenType, self).__init__(*args, **kwargs)
+        self.subtokens = set()
+
     def __contains__(self, val):
         return self is val or (
             type(val) is self.__class__ and
@@ -29,9 +38,10 @@ class _TokenType(tuple):
 
     def __getattr__(self, val):
         if not val or not val[0].isupper():
-            return tuple.__getattr__(self, val)
+            return tuple.__getattribute__(self, val)
         new = _TokenType(self + (val,))
         setattr(self, val, new)
+        self.subtokens.add(new)
         new.parent = self
         return new
 
@@ -63,14 +73,47 @@ Comment     = Token.Comment
 # Generic types for non-source code
 Generic     = Token.Generic
 
+# String and some others are not direct childs of Token.
+# alias them:
+Token.Token = Token
+Token.String = String
+Token.Number = Number
+Token.Punctuation = Punctuation
+
 
 def is_token_subtype(ttype, other):
-    """Return True if ``ttype`` is a subtype of ``other``."""
-    while ttype is not None:
-        if ttype == other:
-            return True
-        ttype = ttype.parent
-    return False
+    """
+    Return True if ``ttype`` is a subtype of ``other``.
+
+    exists for backwards compatibility. use ``ttype in other`` now.
+    """
+    return ttype in other
+
+
+def string_to_token(s):
+    """
+    Convert a string into a token::
+
+        >>> string_to_token('String.Double')
+        Token.Literal.String.Double
+        >>> string_to_token('Token.Literal.Number')
+        Token.Literal.Number
+        >>> string_to_token('')
+        Token
+
+    Tokens that are already tokens are returned unchanged:
+
+        >>> string_to_token(String)
+        Token.Literal.String
+    """
+    if isinstance(s, _TokenType):
+        return s
+    if not s:
+        return Token
+    node = Token
+    for item in s.split('.'):
+        node = getattr(node, item)
+    return node
 
 
 # Map standard token types to short names, used in CSS class naming.