Merged in protz/pygments-main/add-envname (pull request #235)

1 files changed, 85 insertions, 73 deletions

diff --git a/pygments/lexers/dotnet.py b/pygments/lexers/dotnet.py
index a603086b..b26f6843 100644
--- a/pygments/lexers/dotnet.py
+++ b/pygments/lexers/dotnet.py
@@ -5,19 +5,19 @@
 
     Lexers for .net languages.
 
-    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 import re
 
 from pygments.lexer import RegexLexer, DelegatingLexer, bygroups, include, \
-     using, this
+    using, this, default
 from pygments.token import Punctuation, \
-     Text, Comment, Operator, Keyword, Name, String, Number, Literal, Other
-from pygments.util import get_choice_opt
+    Text, Comment, Operator, Keyword, Name, String, Number, Literal, Other
+from pygments.util import get_choice_opt, iteritems
 from pygments import unistring as uni
 
-from pygments.lexers.web import XmlLexer
+from pygments.lexers.html import XmlLexer
 
 __all__ = ['CSharpLexer', 'NemerleLexer', 'BooLexer', 'VbNetLexer',
            'CSharpAspxLexer', 'VbNetAspxLexer', 'FSharpLexer']
@@ -44,13 +44,13 @@ class CSharpLexer(RegexLexer):
 
       The default value is ``basic``.
 
-      *New in Pygments 0.8.*
+      .. versionadded:: 0.8
     """
 
     name = 'C#'
     aliases = ['csharp', 'c#']
     filenames = ['*.cs']
-    mimetypes = ['text/x-csharp'] # inferred
+    mimetypes = ['text/x-csharp']  # inferred
 
     flags = re.MULTILINE | re.DOTALL | re.UNICODE
 
@@ -58,7 +58,7 @@ class CSharpLexer(RegexLexer):
     # see http://www.ecma-international.org/publications/files/ECMA-ST/Ecma-334.pdf
 
     levels = {
-        'none': '@?[_a-zA-Z][a-zA-Z0-9_]*',
+        'none': '@?[_a-zA-Z]\w*',
         'basic': ('@?[_' + uni.Lu + uni.Ll + uni.Lt + uni.Lm + uni.Nl + ']' +
                   '[' + uni.Lu + uni.Ll + uni.Lt + uni.Lm + uni.Nl +
                   uni.Nd + uni.Pc + uni.Cf + uni.Mn + uni.Mc + ']*'),
@@ -71,17 +71,17 @@ class CSharpLexer(RegexLexer):
     tokens = {}
     token_variants = True
 
-    for levelname, cs_ident in levels.items():
+    for levelname, cs_ident in iteritems(levels):
         tokens[levelname] = {
             'root': [
                 # method names
-                (r'^([ \t]*(?:' + cs_ident + r'(?:\[\])?\s+)+?)' # return type
-                 r'(' + cs_ident + ')'                           # method name
+                (r'^([ \t]*(?:' + cs_ident + r'(?:\[\])?\s+)+?)'  # return type
+                 r'(' + cs_ident + ')'                            # method name
                  r'(\s*)(\()',                               # signature start
                  bygroups(using(this), Name.Function, Text, Punctuation)),
                 (r'^\s*\[.*?\]', Name.Attribute),
                 (r'[^\S\n]+', Text),
-                (r'\\\n', Text), # line continuation
+                (r'\\\n', Text),  # line continuation
                 (r'//.*?\n', Comment.Single),
                 (r'/[*].*?[*]/', Comment.Multiline),
                 (r'\n', Text),
@@ -120,13 +120,13 @@ class CSharpLexer(RegexLexer):
                 (cs_ident, Name.Class, '#pop')
             ],
             'namespace': [
-                (r'(?=\()', Text, '#pop'), # using (resource)
+                (r'(?=\()', Text, '#pop'),  # using (resource)
                 ('(' + cs_ident + r'|\.)+', Name.Namespace, '#pop')
             ]
         }
 
     def __init__(self, **options):
-        level = get_choice_opt(options, 'unicodelevel', self.tokens.keys(), 'basic')
+        level = get_choice_opt(options, 'unicodelevel', list(self.tokens), 'basic')
         if level not in self._all_tokens:
             # compile the regexes now
             self._tokens = self.__class__.process_tokendef(level)
@@ -156,13 +156,13 @@ class NemerleLexer(RegexLexer):
 
       The default value is ``basic``.
 
-    *New in Pygments 1.5.*
+    .. versionadded:: 1.5
     """
 
     name = 'Nemerle'
     aliases = ['nemerle']
     filenames = ['*.n']
-    mimetypes = ['text/x-nemerle'] # inferred
+    mimetypes = ['text/x-nemerle']  # inferred
 
     flags = re.MULTILINE | re.DOTALL | re.UNICODE
 
@@ -170,30 +170,30 @@ class NemerleLexer(RegexLexer):
     # http://www.ecma-international.org/publications/files/ECMA-ST/Ecma-334.pdf
 
     levels = dict(
-        none = '@?[_a-zA-Z][a-zA-Z0-9_]*',
-        basic = ('@?[_' + uni.Lu + uni.Ll + uni.Lt + uni.Lm + uni.Nl + ']' +
-                 '[' + uni.Lu + uni.Ll + uni.Lt + uni.Lm + uni.Nl +
-                 uni.Nd + uni.Pc + uni.Cf + uni.Mn + uni.Mc + ']*'),
-        full = ('@?(?:_|[^' + uni.allexcept('Lu', 'Ll', 'Lt', 'Lm', 'Lo',
-                                            'Nl') + '])'
-                + '[^' + uni.allexcept('Lu', 'Ll', 'Lt', 'Lm', 'Lo', 'Nl',
-                                       'Nd', 'Pc', 'Cf', 'Mn', 'Mc') + ']*'),
+        none='@?[_a-zA-Z]\w*',
+        basic=('@?[_' + uni.Lu + uni.Ll + uni.Lt + uni.Lm + uni.Nl + ']' +
+               '[' + uni.Lu + uni.Ll + uni.Lt + uni.Lm + uni.Nl +
+               uni.Nd + uni.Pc + uni.Cf + uni.Mn + uni.Mc + ']*'),
+        full=('@?(?:_|[^' + uni.allexcept('Lu', 'Ll', 'Lt', 'Lm', 'Lo',
+                                          'Nl') + '])'
+              + '[^' + uni.allexcept('Lu', 'Ll', 'Lt', 'Lm', 'Lo', 'Nl',
+                                     'Nd', 'Pc', 'Cf', 'Mn', 'Mc') + ']*'),
     )
 
     tokens = {}
     token_variants = True
 
-    for levelname, cs_ident in levels.items():
+    for levelname, cs_ident in iteritems(levels):
         tokens[levelname] = {
             'root': [
                 # method names
-                (r'^([ \t]*(?:' + cs_ident + r'(?:\[\])?\s+)+?)' # return type
-                 r'(' + cs_ident + ')'                           # method name
+                (r'^([ \t]*(?:' + cs_ident + r'(?:\[\])?\s+)+?)'  # return type
+                 r'(' + cs_ident + ')'                            # method name
                  r'(\s*)(\()',                               # signature start
                  bygroups(using(this), Name.Function, Text, Punctuation)),
                 (r'^\s*\[.*?\]', Name.Attribute),
                 (r'[^\S\n]+', Text),
-                (r'\\\n', Text), # line continuation
+                (r'\\\n', Text),  # line continuation
                 (r'//.*?\n', Comment.Single),
                 (r'/[*].*?[*]/', Comment.Multiline),
                 (r'\n', Text),
@@ -249,7 +249,7 @@ class NemerleLexer(RegexLexer):
                 (cs_ident, Name.Class, '#pop')
             ],
             'namespace': [
-                (r'(?=\()', Text, '#pop'), # using (resource)
+                (r'(?=\()', Text, '#pop'),  # using (resource)
                 ('(' + cs_ident + r'|\.)+', Name.Namespace, '#pop')
             ],
             'splice-string': [
@@ -284,7 +284,7 @@ class NemerleLexer(RegexLexer):
         }
 
     def __init__(self, **options):
-        level = get_choice_opt(options, 'unicodelevel', self.tokens.keys(),
+        level = get_choice_opt(options, 'unicodelevel', list(self.tokens),
                                'basic')
         if level not in self._all_tokens:
             # compile the regexes now
@@ -336,7 +336,7 @@ class BooLexer(RegexLexer):
             (r'"""(\\\\|\\"|.*?)"""', String.Double),
             (r'"(\\\\|\\"|[^"]*?)"', String.Double),
             (r"'(\\\\|\\'|[^']*?)'", String.Single),
-            (r'[a-zA-Z_][a-zA-Z0-9_]*', Name),
+            (r'[a-zA-Z_]\w*', Name),
             (r'(\d+\.\d*|\d*\.\d+)([fF][+-]?[0-9]+)?', Number.Float),
             (r'[0-9][0-9\.]*(ms?|d|h|s)', Number),
             (r'0\d+', Number.Oct),
@@ -351,13 +351,13 @@ class BooLexer(RegexLexer):
             ('[*/]', Comment.Multiline)
         ],
         'funcname': [
-            ('[a-zA-Z_][a-zA-Z0-9_]*', Name.Function, '#pop')
+            ('[a-zA-Z_]\w*', Name.Function, '#pop')
         ],
         'classname': [
-            ('[a-zA-Z_][a-zA-Z0-9_]*', Name.Class, '#pop')
+            ('[a-zA-Z_]\w*', Name.Class, '#pop')
         ],
         'namespace': [
-            ('[a-zA-Z_][a-zA-Z0-9_.]*', Name.Namespace, '#pop')
+            ('[a-zA-Z_][\w.]*', Name.Namespace, '#pop')
         ]
     }
 
@@ -372,7 +372,7 @@ class VbNetLexer(RegexLexer):
     name = 'VB.net'
     aliases = ['vb.net', 'vbnet']
     filenames = ['*.vb', '*.bas']
-    mimetypes = ['text/x-vbnet', 'text/x-vba'] # (?)
+    mimetypes = ['text/x-vbnet', 'text/x-vba']  # (?)
 
     flags = re.MULTILINE | re.IGNORECASE
     tokens = {
@@ -422,16 +422,16 @@ class VbNetLexer(RegexLexer):
             (r'(?<!\.)(AddressOf|And|AndAlso|As|GetType|In|Is|IsNot|Like|Mod|'
              r'Or|OrElse|TypeOf|Xor)\b', Operator.Word),
             (r'&=|[*]=|/=|\\=|\^=|\+=|-=|<<=|>>=|<<|>>|:=|'
-             r'<=|>=|<>|[-&*/\\^+=<>]',
+             r'<=|>=|<>|[-&*/\\^+=<>\[\]]',
              Operator),
             ('"', String, 'string'),
-            ('[a-zA-Z_][a-zA-Z0-9_]*[%&@!#$]?', Name),
+            ('[a-z_]\w*[%&@!#$]?', Name),
             ('#.*?#', Literal.Date),
             (r'(\d+\.\d*|\d*\.\d+)([fF][+-]?[0-9]+)?', Number.Float),
             (r'\d+([SILDFR]|US|UI|UL)?', Number.Integer),
             (r'&H[0-9a-f]+([SILDFR]|US|UI|UL)?', Number.Integer),
             (r'&O[0-7]+([SILDFR]|US|UI|UL)?', Number.Integer),
-            (r'_\n', Text), # Line continuation
+            (r'_\n', Text),  # Line continuation
         ],
         'string': [
             (r'""', String),
@@ -439,26 +439,31 @@ class VbNetLexer(RegexLexer):
             (r'[^"]+', String),
         ],
         'dim': [
-            (r'[a-z_][a-z0-9_]*', Name.Variable, '#pop'),
-            (r'', Text, '#pop'),  # any other syntax
+            (r'[a-z_]\w*', Name.Variable, '#pop'),
+            default('#pop'),  # any other syntax
         ],
         'funcname': [
-            (r'[a-z_][a-z0-9_]*', Name.Function, '#pop'),
+            (r'[a-z_]\w*', Name.Function, '#pop'),
         ],
         'classname': [
-            (r'[a-z_][a-z0-9_]*', Name.Class, '#pop'),
+            (r'[a-z_]\w*', Name.Class, '#pop'),
         ],
         'namespace': [
-            (r'[a-z_][a-z0-9_.]*', Name.Namespace, '#pop'),
+            (r'[a-z_][\w.]*', Name.Namespace, '#pop'),
         ],
         'end': [
             (r'\s+', Text),
             (r'(Function|Sub|Property|Class|Structure|Enum|Module|Namespace)\b',
              Keyword, '#pop'),
-            (r'', Text, '#pop'),
+            default('#pop'),
         ]
     }
 
+    def analyse_text(text):
+        if re.search(r'^\s*(#If|Module|Namespace)', text,
+                     re.IGNORECASE | re.MULTILINE):
+            return 0.5
+
 
 class GenericAspxLexer(RegexLexer):
     """
@@ -483,7 +488,7 @@ class GenericAspxLexer(RegexLexer):
     }
 
 
-#TODO support multiple languages within the same source file
+# TODO support multiple languages within the same source file
 class CSharpAspxLexer(DelegatingLexer):
     """
     Lexer for highligting C# within ASP.NET pages.
@@ -495,7 +500,7 @@ class CSharpAspxLexer(DelegatingLexer):
     mimetypes = []
 
     def __init__(self, **options):
-        super(CSharpAspxLexer, self).__init__(CSharpLexer,GenericAspxLexer,
+        super(CSharpAspxLexer, self).__init__(CSharpLexer, GenericAspxLexer,
                                               **options)
 
     def analyse_text(text):
@@ -516,8 +521,8 @@ class VbNetAspxLexer(DelegatingLexer):
     mimetypes = []
 
     def __init__(self, **options):
-        super(VbNetAspxLexer, self).__init__(VbNetLexer,GenericAspxLexer,
-                                              **options)
+        super(VbNetAspxLexer, self).__init__(VbNetLexer, GenericAspxLexer,
+                                             **options)
 
     def analyse_text(text):
         if re.search(r'Page\s*Language="Vb"', text, re.I) is not None:
@@ -531,7 +536,10 @@ class FSharpLexer(RegexLexer):
     """
     For the F# language (version 3.0).
 
-    *New in Pygments 1.5.*
+    AAAAACK Strings
+    http://research.microsoft.com/en-us/um/cambridge/projects/fsharp/manual/spec.html#_Toc335818775
+
+    .. versionadded:: 1.5
     """
 
     name = 'FSharp'
@@ -540,15 +548,15 @@ class FSharpLexer(RegexLexer):
     mimetypes = ['text/x-fsharp']
 
     keywords = [
-      'abstract', 'as', 'assert', 'base', 'begin', 'class', 'default',
-      'delegate', 'do!', 'do', 'done', 'downcast', 'downto', 'elif', 'else',
-      'end', 'exception', 'extern', 'false', 'finally', 'for', 'function',
-      'fun', 'global', 'if', 'inherit', 'inline', 'interface', 'internal',
-      'in', 'lazy', 'let!', 'let', 'match', 'member', 'module', 'mutable',
-      'namespace', 'new', 'null', 'of', 'open', 'override', 'private', 'public',
-      'rec', 'return!', 'return', 'select', 'static', 'struct', 'then', 'to',
-      'true', 'try', 'type', 'upcast', 'use!', 'use', 'val', 'void', 'when',
-      'while', 'with', 'yield!', 'yield',
+        'abstract', 'as', 'assert', 'base', 'begin', 'class', 'default',
+        'delegate', 'do!', 'do', 'done', 'downcast', 'downto', 'elif', 'else',
+        'end', 'exception', 'extern', 'false', 'finally', 'for', 'function',
+        'fun', 'global', 'if', 'inherit', 'inline', 'interface', 'internal',
+        'in', 'lazy', 'let!', 'let', 'match', 'member', 'module', 'mutable',
+        'namespace', 'new', 'null', 'of', 'open', 'override', 'private', 'public',
+        'rec', 'return!', 'return', 'select', 'static', 'struct', 'then', 'to',
+        'true', 'try', 'type', 'upcast', 'use!', 'use', 'val', 'void', 'when',
+        'while', 'with', 'yield!', 'yield',
     ]
     # Reserved words; cannot hurt to color them as keywords too.
     keywords += [
@@ -559,10 +567,10 @@ class FSharpLexer(RegexLexer):
         'virtual', 'volatile',
     ]
     keyopts = [
-      '!=', '#', '&&', '&', '\(', '\)', '\*', '\+', ',', '-\.',
-      '->', '-', '\.\.', '\.', '::', ':=', ':>', ':', ';;', ';', '<-',
-      '<\]', '<', '>\]', '>', '\?\?', '\?', '\[<', '\[\|', '\[', '\]',
-      '_', '`', '{', '\|\]', '\|', '}', '~', '<@@', '<@', '=', '@>', '@@>',
+        '!=', '#', '&&', '&', '\(', '\)', '\*', '\+', ',', '-\.',
+        '->', '-', '\.\.', '\.', '::', ':=', ':>', ':', ';;', ';', '<-',
+        '<\]', '<', '>\]', '>', '\?\?', '\?', '\[<', '\[\|', '\[', '\]',
+        '_', '`', '{', '\|\]', '\|', '}', '~', '<@@', '<@', '=', '@>', '@@>',
     ]
 
     operators = r'[!$%&*+\./:<=>?@^|~-]'
@@ -589,9 +597,9 @@ class FSharpLexer(RegexLexer):
         'root': [
             (r'\s+', Text),
             (r'\(\)|\[\]', Name.Builtin.Pseudo),
-            (r'\b(?<!\.)([A-Z][A-Za-z0-9_\']*)(?=\s*\.)',
+            (r'\b(?<!\.)([A-Z][\w\']*)(?=\s*\.)',
              Name.Namespace, 'dotted'),
-            (r'\b([A-Z][A-Za-z0-9_\']*)', Name),
+            (r'\b([A-Z][\w\']*)', Name),
             (r'///.*?\n', String.Doc),
             (r'//.*?\n', Comment.Single),
             (r'\(\*(?!\))', Comment, 'comment'),
@@ -600,13 +608,13 @@ class FSharpLexer(RegexLexer):
             (r'"""', String, 'tqs'),
             (r'"', String, 'string'),
 
-            (r'\b(open|module)(\s+)([a-zA-Z0-9_.]+)',
+            (r'\b(open|module)(\s+)([\w.]+)',
              bygroups(Keyword, Text, Name.Namespace)),
-            (r'\b(let!?)(\s+)([a-zA-Z0-9_]+)',
+            (r'\b(let!?)(\s+)(\w+)',
              bygroups(Keyword, Text, Name.Variable)),
-            (r'\b(type)(\s+)([a-zA-Z0-9_]+)',
+            (r'\b(type)(\s+)(\w+)',
              bygroups(Keyword, Text, Name.Class)),
-            (r'\b(member|override)(\s+)([a-zA-Z0-9_]+)(\.)([a-zA-Z0-9_]+)',
+            (r'\b(member|override)(\s+)(\w+)(\.)(\w+)',
              bygroups(Keyword, Text, Name, Punctuation, Name.Function)),
             (r'\b(%s)\b' % '|'.join(keywords), Keyword),
             (r'(%s)' % '|'.join(keyopts), Operator),
@@ -621,23 +629,27 @@ class FSharpLexer(RegexLexer):
             (r'\d[\d_]*[uU]?[yslLnQRZINGmM]?', Number.Integer),
             (r'0[xX][\da-fA-F][\da-fA-F_]*[uU]?[yslLn]?[fF]?', Number.Hex),
             (r'0[oO][0-7][0-7_]*[uU]?[yslLn]?', Number.Oct),
-            (r'0[bB][01][01_]*[uU]?[yslLn]?', Number.Binary),
+            (r'0[bB][01][01_]*[uU]?[yslLn]?', Number.Bin),
             (r'-?\d[\d_]*(.[\d_]*)?([eE][+\-]?\d[\d_]*)[fFmM]?',
              Number.Float),
 
             (r"'(?:(\\[\\\"'ntbr ])|(\\[0-9]{3})|(\\x[0-9a-fA-F]{2}))'B?",
              String.Char),
             (r"'.'", String.Char),
-            (r"'", Keyword), # a stray quote is another syntax element
+            (r"'", Keyword),  # a stray quote is another syntax element
+
+            (r'@?"', String.Double, 'string'),
 
             (r'[~?][a-z][\w\']*:', Name.Variable),
         ],
         'dotted': [
             (r'\s+', Text),
             (r'\.', Punctuation),
-            (r'[A-Z][A-Za-z0-9_\']*(?=\s*\.)', Name.Namespace),
-            (r'[A-Z][A-Za-z0-9_\']*', Name, '#pop'),
-            (r'[a-z_][A-Za-z0-9_\']*', Name, '#pop'),
+            (r'[A-Z][\w\']*(?=\s*\.)', Name.Namespace),
+            (r'[A-Z][\w\']*', Name, '#pop'),
+            (r'[a-z_][\w\']*', Name, '#pop'),
+            # e.g. dictionary index access
+            default('#pop'),
         ],
         'comment': [
             (r'[^(*)@"]+', Comment),