merge with bgoetzmann/pygments-main

1 files changed, 266 insertions, 3 deletions

diff --git a/pygments/lexers/dotnet.py b/pygments/lexers/dotnet.py
index 48feeb85..7c13e8a4 100644
--- a/pygments/lexers/dotnet.py
+++ b/pygments/lexers/dotnet.py
@@ -10,7 +10,8 @@
 """
 import re
 
-from pygments.lexer import RegexLexer, DelegatingLexer, bygroups, using, this
+from pygments.lexer import RegexLexer, DelegatingLexer, bygroups, include, \
+     using, this
 from pygments.token import Punctuation, \
      Text, Comment, Operator, Keyword, Name, String, Number, Literal, Other
 from pygments.util import get_choice_opt
@@ -18,8 +19,8 @@ from pygments import unistring as uni
 
 from pygments.lexers.web import XmlLexer
 
-__all__ = ['CSharpLexer', 'BooLexer', 'VbNetLexer', 'CSharpAspxLexer',
-           'VbNetAspxLexer']
+__all__ = ['CSharpLexer', 'NemerleLexer', 'BooLexer', 'VbNetLexer',
+           'CSharpAspxLexer', 'VbNetAspxLexer', 'FSharpLexer']
 
 
 def _escape(st):
@@ -138,6 +139,164 @@ class CSharpLexer(RegexLexer):
         RegexLexer.__init__(self, **options)
 
 
+class NemerleLexer(RegexLexer):
+    """
+    For `Nemerle <http://nemerle.org>`_ source code.
+
+    Additional options accepted:
+
+    `unicodelevel`
+      Determines which Unicode characters this lexer allows for identifiers.
+      The possible values are:
+
+      * ``none`` -- only the ASCII letters and numbers are allowed. This
+        is the fastest selection.
+      * ``basic`` -- all Unicode characters from the specification except
+        category ``Lo`` are allowed.
+      * ``full`` -- all Unicode characters as specified in the C# specs
+        are allowed.  Note that this means a considerable slowdown since the
+        ``Lo`` category has more than 40,000 characters in it!
+
+      The default value is ``basic``.
+
+    *New in Pygments 1.5.*
+    """
+
+    name = 'Nemerle'
+    aliases = ['nemerle']
+    filenames = ['*.n']
+    mimetypes = ['text/x-nemerle'] # inferred
+
+    flags = re.MULTILINE | re.DOTALL | re.UNICODE
+
+    # for the range of allowed unicode characters in identifiers,
+    # see http://www.ecma-international.org/publications/files/ECMA-ST/Ecma-334.pdf
+
+    levels = {
+        'none': '@?[_a-zA-Z][a-zA-Z0-9_]*',
+        'basic': ('@?[_' + uni.Lu + uni.Ll + uni.Lt + uni.Lm + uni.Nl + ']' +
+                  '[' + uni.Lu + uni.Ll + uni.Lt + uni.Lm + uni.Nl +
+                  uni.Nd + uni.Pc + uni.Cf + uni.Mn + uni.Mc + ']*'),
+        'full': ('@?(?:_|[^' +
+                 _escape(uni.allexcept('Lu', 'Ll', 'Lt', 'Lm', 'Lo', 'Nl')) + '])'
+                 + '[^' + _escape(uni.allexcept('Lu', 'Ll', 'Lt', 'Lm', 'Lo',
+                                                'Nl', 'Nd', 'Pc', 'Cf', 'Mn',
+                                                'Mc')) + ']*'),
+    }
+
+    tokens = {}
+    token_variants = True
+
+    for levelname, cs_ident in levels.items():
+        tokens[levelname] = {
+            'root': [
+                # method names
+                (r'^([ \t]*(?:' + cs_ident + r'(?:\[\])?\s+)+?)' # return type
+                 r'(' + cs_ident + ')'                           # method name
+                 r'(\s*)(\()',                               # signature start
+                 bygroups(using(this), Name.Function, Text, Punctuation)),
+                (r'^\s*\[.*?\]', Name.Attribute),
+                (r'[^\S\n]+', Text),
+                (r'\\\n', Text), # line continuation
+                (r'//.*?\n', Comment.Single),
+                (r'/[*](.|\n)*?[*]/', Comment.Multiline),
+                (r'\n', Text),
+                (r'\$\s*"', String, 'splice-string'),
+                (r'\$\s*<#', String, 'splice-string2'),
+                (r'<#', String, 'recursive-string'),
+
+                (r'(<\[)\s*(' + cs_ident + ':)?', Keyword),
+                (r'\]\>', Keyword),
+
+                # quasiquotation only
+                (r'\$' + cs_ident, Name), 
+                (r'(\$)(\()', bygroups(Name, Punctuation), 'splice-string-content'),
+
+                (r'[~!%^&*()+=|\[\]:;,.<>/?-]', Punctuation),
+                (r'[{}]', Punctuation),
+                (r'@"(\\\\|\\"|[^"])*"', String),
+                (r'"(\\\\|\\"|[^"\n])*["\n]', String),
+                (r"'\\.'|'[^\\]'", String.Char),
+		(r"0[xX][0-9a-fA-F]+[Ll]?", Number),
+                (r"[0-9](\.[0-9]*)?([eE][+-][0-9]+)?[flFLdD]?", Number),
+                (r'#[ \t]*(if|endif|else|elif|define|undef|'
+                 r'line|error|warning|region|endregion|pragma)\b.*?\n',
+                 Comment.Preproc),
+                (r'\b(extern)(\s+)(alias)\b', bygroups(Keyword, Text,
+                 Keyword)),
+                (r'(abstract|and|as|base|catch|def|delegate|'
+                 r'enum|event|extern|false|finally|'
+                 r'fun|implements|interface|internal|'
+                 r'is|macro|match|matches|module|mutable|new|'
+                 r'null|out|override|params|partial|private|'
+                 r'protected|public|ref|sealed|static|'
+                 r'syntax|this|throw|true|try|type|typeof|'
+                 r'virtual|volatile|when|where|with|'
+                 r'assert|assert2|async|break|checked|continue|do|else|'
+                 r'ensures|for|foreach|if|late|lock|new|nolate|'
+                 r'otherwise|regexp|repeat|requires|return|surroundwith|'
+                 r'unchecked|unless|using|while|yield)\b', Keyword),
+                (r'(global)(::)', bygroups(Keyword, Punctuation)),
+                (r'(bool|byte|char|decimal|double|float|int|long|object|sbyte|'
+                 r'short|string|uint|ulong|ushort|void|array|list)\b\??',
+                 Keyword.Type),
+                (r'(:>?)\s*(' + cs_ident + r'\??)',
+                 bygroups(Punctuation, Keyword.Type)),
+                (r'(class|struct|variant|module)(\s+)',
+                 bygroups(Keyword, Text), 'class'),
+                (r'(namespace|using)(\s+)', bygroups(Keyword, Text),
+                 'namespace'),
+                (cs_ident, Name),
+            ],
+            'class': [
+                (cs_ident, Name.Class, '#pop')
+            ],
+            'namespace': [
+                (r'(?=\()', Text, '#pop'), # using (resource)
+                ('(' + cs_ident + r'|\.)+', Name.Namespace, '#pop')
+            ],
+            'splice-string': [
+                (r'[^"$]',  String), 
+                (r'\$' + cs_ident, Name),
+                (r'(\$)(\()', bygroups(Name, Punctuation),
+                 'splice-string-content'),
+                (r'\\"',  String),
+                (r'"',  String, '#pop')
+            ],
+            'splice-string2': [
+                (r'[^#<>$]',  String), 
+                (r'\$' + cs_ident, Name),
+                (r'(\$)(\()', bygroups(Name, Punctuation),
+                 'splice-string-content'),
+                (r'<#',  String, '#push'),
+                (r'#>',  String, '#pop')
+            ],
+            'recursive-string': [
+                (r'[^#<>]',  String),
+                (r'<#',  String, '#push'),
+                (r'#>',  String, '#pop')
+            ],
+            'splice-string-content': [
+                (r'if|match', Keyword),
+                (r'[~!%^&*+=|\[\]:;,.<>/?-]', Punctuation),
+                (cs_ident, Name), 
+                (r'\(', Punctuation, '#push'),
+                (r'\)', Punctuation, '#pop')
+            ]
+        }
+
+    def __init__(self, **options):
+        level = get_choice_opt(options, 'unicodelevel', self.tokens.keys(),
+                               'basic')
+        if level not in self._all_tokens:
+            # compile the regexes now
+            self._tokens = self.__class__.process_tokendef(level)
+        else:
+            self._tokens = self._all_tokens[level]
+
+        RegexLexer.__init__(self, **options)
+
+
 class BooLexer(RegexLexer):
     """
     For `Boo <http://boo.codehaus.org/>`_ source code.
@@ -365,3 +524,107 @@ class VbNetAspxLexer(DelegatingLexer):
             return 0.2
         elif re.search(r'script[^>]+language=["\']vb', text, re.I) is not None:
             return 0.15
+
+
+# Very close to functional.OcamlLexer
+class FSharpLexer(RegexLexer):
+    """
+    For the F# language.
+
+    *New in Pygments 1.5.*
+    """
+
+    name = 'FSharp'
+    aliases = ['fsharp']
+    filenames = ['*.fs', '*.fsi']
+    mimetypes = ['text/x-fsharp']
+
+    keywords = [
+      'abstract', 'and', 'as', 'assert', 'base', 'begin', 'class',
+      'default', 'delegate', 'do', 'do!', 'done', 'downcast',
+      'downto', 'elif', 'else', 'end', 'exception', 'extern',
+      'false', 'finally', 'for', 'fun', 'function', 'global', 'if',
+      'in', 'inherit', 'inline', 'interface', 'internal', 'lazy',
+      'let', 'let!', 'match', 'member', 'module', 'mutable',
+      'namespace', 'new', 'null', 'of', 'open', 'or', 'override',
+      'private', 'public', 'rec', 'return', 'return!', 'sig',
+      'static', 'struct', 'then', 'to', 'true', 'try', 'type',
+      'upcast', 'use', 'use!', 'val', 'void', 'when', 'while',
+      'with', 'yield', 'yield!'
+    ]
+    keyopts = [
+      '!=','#','&','&&','\(','\)','\*','\+',',','-',
+      '-\.','->','\.','\.\.',':','::',':=',':>',';',';;','<',
+      '<-','>','>]','\?','\?\?','\[','\[<','\[>','\[\|',
+      ']','_','`','{','\|','\|]','}','~','<@','=','@>'
+    ]
+
+    operators = r'[!$%&*+\./:<=>?@^|~-]'
+    word_operators = ['and', 'asr', 'land', 'lor', 'lsl', 'lxor', 'mod', 'not', 'or']
+    prefix_syms = r'[!?~]'
+    infix_syms = r'[=<>@^|&+\*/$%-]'
+    primitives = ['unit', 'int', 'float', 'bool', 'string', 'char', 'list', 'array',
+                  'byte', 'sbyte', 'int16', 'uint16', 'uint32', 'int64', 'uint64'
+                  'nativeint', 'unativeint', 'decimal', 'void', 'float32', 'single',
+                  'double']
+
+    tokens = {
+        'escape-sequence': [
+            (r'\\[\\\"\'ntbr]', String.Escape),
+            (r'\\[0-9]{3}', String.Escape),
+            (r'\\x[0-9a-fA-F]{2}', String.Escape),
+        ],
+        'root': [
+            (r'\s+', Text),
+            (r'false|true|\(\)|\[\]', Name.Builtin.Pseudo),
+            (r'\b([A-Z][A-Za-z0-9_\']*)(?=\s*\.)',
+             Name.Namespace, 'dotted'),
+            (r'\b([A-Z][A-Za-z0-9_\']*)', Name.Class),
+            (r'//.*?\n', Comment.Single),
+            (r'\(\*', Comment, 'comment'),
+            (r'\b(%s)\b' % '|'.join(keywords), Keyword),
+            (r'(%s)' % '|'.join(keyopts), Operator),
+            (r'(%s|%s)?%s' % (infix_syms, prefix_syms, operators), Operator),
+            (r'\b(%s)\b' % '|'.join(word_operators), Operator.Word),
+            (r'\b(%s)\b' % '|'.join(primitives), Keyword.Type),
+
+            (r'#[ \t]*(if|endif|else|line|nowarn|light)\b.*?\n',
+             Comment.Preproc),
+
+            (r"[^\W\d][\w']*", Name),
+
+            (r'\d[\d_]*', Number.Integer),
+            (r'0[xX][\da-fA-F][\da-fA-F_]*', Number.Hex),
+            (r'0[oO][0-7][0-7_]*', Number.Oct),
+            (r'0[bB][01][01_]*', Number.Binary),
+            (r'-?\d[\d_]*(.[\d_]*)?([eE][+\-]?\d[\d_]*)', Number.Float),
+
+            (r"'(?:(\\[\\\"'ntbr ])|(\\[0-9]{3})|(\\x[0-9a-fA-F]{2}))'",
+             String.Char),
+            (r"'.'", String.Char),
+            (r"'", Keyword), # a stray quote is another syntax element
+
+            (r'"', String.Double, 'string'),
+
+            (r'[~?][a-z][\w\']*:', Name.Variable),
+        ],
+        'comment': [
+            (r'[^(*)]+', Comment),
+            (r'\(\*', Comment, '#push'),
+            (r'\*\)', Comment, '#pop'),
+            (r'[(*)]', Comment),
+        ],
+        'string': [
+            (r'[^\\"]+', String.Double),
+            include('escape-sequence'),
+            (r'\\\n', String.Double),
+            (r'"', String.Double, '#pop'),
+        ],
+        'dotted': [
+            (r'\s+', Text),
+            (r'\.', Punctuation),
+            (r'[A-Z][A-Za-z0-9_\']*(?=\s*\.)', Name.Namespace),
+            (r'[A-Z][A-Za-z0-9_\']*', Name.Class, '#pop'),
+            (r'[a-z_][A-Za-z0-9_\']*', Name, '#pop'),
+        ],
+    }