F# lexer: rewrite with newest spec for F# 3.0 (#842).

2 files changed, 90 insertions, 47 deletions

diff --git a/CHANGES b/CHANGES
index 9b2cafc5..9288b306 100644
--- a/CHANGES
+++ b/CHANGES
@@ -53,6 +53,8 @@ Version 1.7
 
 - Prolog lexer: add different kinds of numeric literals (#864).
 
+- F# lexer: rewrite with newest spec for F# 3.0 (#842).
+
 
 Version 1.6
 -----------
diff --git a/pygments/lexers/dotnet.py b/pygments/lexers/dotnet.py
index bdd9edc1..a603086b 100644
--- a/pygments/lexers/dotnet.py
+++ b/pygments/lexers/dotnet.py
@@ -529,7 +529,7 @@ class VbNetAspxLexer(DelegatingLexer):
 # Very close to functional.OcamlLexer
 class FSharpLexer(RegexLexer):
     """
-    For the F# language.
+    For the F# language (version 3.0).
 
     *New in Pygments 1.5.*
     """
@@ -540,91 +540,132 @@ class FSharpLexer(RegexLexer):
     mimetypes = ['text/x-fsharp']
 
     keywords = [
-      'abstract', 'and', 'as', 'assert', 'base', 'begin', 'class',
-      'default', 'delegate', 'do', 'do!', 'done', 'downcast',
-      'downto', 'elif', 'else', 'end', 'exception', 'extern',
-      'false', 'finally', 'for', 'fun', 'function', 'global', 'if',
-      'in', 'inherit', 'inline', 'interface', 'internal', 'lazy',
-      'let', 'let!', 'match', 'member', 'module', 'mutable',
-      'namespace', 'new', 'null', 'of', 'open', 'or', 'override',
-      'private', 'public', 'rec', 'return', 'return!', 'sig',
-      'static', 'struct', 'then', 'to', 'true', 'try', 'type',
-      'upcast', 'use', 'use!', 'val', 'void', 'when', 'while',
-      'with', 'yield', 'yield!'
+      'abstract', 'as', 'assert', 'base', 'begin', 'class', 'default',
+      'delegate', 'do!', 'do', 'done', 'downcast', 'downto', 'elif', 'else',
+      'end', 'exception', 'extern', 'false', 'finally', 'for', 'function',
+      'fun', 'global', 'if', 'inherit', 'inline', 'interface', 'internal',
+      'in', 'lazy', 'let!', 'let', 'match', 'member', 'module', 'mutable',
+      'namespace', 'new', 'null', 'of', 'open', 'override', 'private', 'public',
+      'rec', 'return!', 'return', 'select', 'static', 'struct', 'then', 'to',
+      'true', 'try', 'type', 'upcast', 'use!', 'use', 'val', 'void', 'when',
+      'while', 'with', 'yield!', 'yield',
+    ]
+    # Reserved words; cannot hurt to color them as keywords too.
+    keywords += [
+        'atomic', 'break', 'checked', 'component', 'const', 'constraint',
+        'constructor', 'continue', 'eager', 'event', 'external', 'fixed',
+        'functor', 'include', 'method', 'mixin', 'object', 'parallel',
+        'process', 'protected', 'pure', 'sealed', 'tailcall', 'trait',
+        'virtual', 'volatile',
     ]
     keyopts = [
-      '!=','#','&&','&','\(','\)','\*','\+',',','-\.',
-      '->','-','\.\.','\.','::',':=',':>',':',';;',';','<-',
-      '<','>]','>','\?\?','\?','\[<','\[>','\[\|','\[',
-      ']','_','`','{','\|\]','\|','}','~','<@','=','@>'
+      '!=', '#', '&&', '&', '\(', '\)', '\*', '\+', ',', '-\.',
+      '->', '-', '\.\.', '\.', '::', ':=', ':>', ':', ';;', ';', '<-',
+      '<\]', '<', '>\]', '>', '\?\?', '\?', '\[<', '\[\|', '\[', '\]',
+      '_', '`', '{', '\|\]', '\|', '}', '~', '<@@', '<@', '=', '@>', '@@>',
     ]
 
     operators = r'[!$%&*+\./:<=>?@^|~-]'
-    word_operators = ['and', 'asr', 'land', 'lor', 'lsl', 'lxor', 'mod', 'not', 'or']
+    word_operators = ['and', 'or', 'not']
     prefix_syms = r'[!?~]'
     infix_syms = r'[=<>@^|&+\*/$%-]'
-    primitives = ['unit', 'int', 'float', 'bool', 'string', 'char', 'list', 'array',
-                  'byte', 'sbyte', 'int16', 'uint16', 'uint32', 'int64', 'uint64'
-                  'nativeint', 'unativeint', 'decimal', 'void', 'float32', 'single',
-                  'double']
+    primitives = [
+        'sbyte', 'byte', 'char', 'nativeint', 'unativeint', 'float32', 'single',
+        'float', 'double', 'int8', 'uint8', 'int16', 'uint16', 'int32',
+        'uint32', 'int64', 'uint64', 'decimal', 'unit', 'bool', 'string',
+        'list', 'exn', 'obj', 'enum',
+    ]
+
+    # See http://msdn.microsoft.com/en-us/library/dd233181.aspx and/or
+    # http://fsharp.org/about/files/spec.pdf for reference.  Good luck.
 
     tokens = {
         'escape-sequence': [
-            (r'\\[\\\"\'ntbr]', String.Escape),
+            (r'\\[\\\"\'ntbrafv]', String.Escape),
             (r'\\[0-9]{3}', String.Escape),
-            (r'\\x[0-9a-fA-F]{2}', String.Escape),
+            (r'\\u[0-9a-fA-F]{4}', String.Escape),
+            (r'\\U[0-9a-fA-F]{8}', String.Escape),
         ],
         'root': [
             (r'\s+', Text),
-            (r'false|true|\(\)|\[\]', Name.Builtin.Pseudo),
-            (r'\b([A-Z][A-Za-z0-9_\']*)(?=\s*\.)',
+            (r'\(\)|\[\]', Name.Builtin.Pseudo),
+            (r'\b(?<!\.)([A-Z][A-Za-z0-9_\']*)(?=\s*\.)',
              Name.Namespace, 'dotted'),
-            (r'\b([A-Z][A-Za-z0-9_\']*)', Name.Class),
+            (r'\b([A-Z][A-Za-z0-9_\']*)', Name),
+            (r'///.*?\n', String.Doc),
             (r'//.*?\n', Comment.Single),
             (r'\(\*(?!\))', Comment, 'comment'),
+
+            (r'@"', String, 'lstring'),
+            (r'"""', String, 'tqs'),
+            (r'"', String, 'string'),
+
+            (r'\b(open|module)(\s+)([a-zA-Z0-9_.]+)',
+             bygroups(Keyword, Text, Name.Namespace)),
+            (r'\b(let!?)(\s+)([a-zA-Z0-9_]+)',
+             bygroups(Keyword, Text, Name.Variable)),
+            (r'\b(type)(\s+)([a-zA-Z0-9_]+)',
+             bygroups(Keyword, Text, Name.Class)),
+            (r'\b(member|override)(\s+)([a-zA-Z0-9_]+)(\.)([a-zA-Z0-9_]+)',
+             bygroups(Keyword, Text, Name, Punctuation, Name.Function)),
             (r'\b(%s)\b' % '|'.join(keywords), Keyword),
             (r'(%s)' % '|'.join(keyopts), Operator),
             (r'(%s|%s)?%s' % (infix_syms, prefix_syms, operators), Operator),
             (r'\b(%s)\b' % '|'.join(word_operators), Operator.Word),
             (r'\b(%s)\b' % '|'.join(primitives), Keyword.Type),
-
-            (r'#[ \t]*(if|endif|else|line|nowarn|light)\b.*?\n',
+            (r'#[ \t]*(if|endif|else|line|nowarn|light|\d+)\b.*?\n',
              Comment.Preproc),
 
             (r"[^\W\d][\w']*", Name),
 
-            (r'\d[\d_]*', Number.Integer),
-            (r'0[xX][\da-fA-F][\da-fA-F_]*', Number.Hex),
-            (r'0[oO][0-7][0-7_]*', Number.Oct),
-            (r'0[bB][01][01_]*', Number.Binary),
-            (r'-?\d[\d_]*(.[\d_]*)?([eE][+\-]?\d[\d_]*)', Number.Float),
+            (r'\d[\d_]*[uU]?[yslLnQRZINGmM]?', Number.Integer),
+            (r'0[xX][\da-fA-F][\da-fA-F_]*[uU]?[yslLn]?[fF]?', Number.Hex),
+            (r'0[oO][0-7][0-7_]*[uU]?[yslLn]?', Number.Oct),
+            (r'0[bB][01][01_]*[uU]?[yslLn]?', Number.Binary),
+            (r'-?\d[\d_]*(.[\d_]*)?([eE][+\-]?\d[\d_]*)[fFmM]?',
+             Number.Float),
 
-            (r"'(?:(\\[\\\"'ntbr ])|(\\[0-9]{3})|(\\x[0-9a-fA-F]{2}))'",
+            (r"'(?:(\\[\\\"'ntbr ])|(\\[0-9]{3})|(\\x[0-9a-fA-F]{2}))'B?",
              String.Char),
             (r"'.'", String.Char),
             (r"'", Keyword), # a stray quote is another syntax element
 
-            (r'"', String.Double, 'string'),
-
             (r'[~?][a-z][\w\']*:', Name.Variable),
         ],
+        'dotted': [
+            (r'\s+', Text),
+            (r'\.', Punctuation),
+            (r'[A-Z][A-Za-z0-9_\']*(?=\s*\.)', Name.Namespace),
+            (r'[A-Z][A-Za-z0-9_\']*', Name, '#pop'),
+            (r'[a-z_][A-Za-z0-9_\']*', Name, '#pop'),
+        ],
         'comment': [
-            (r'[^(*)]+', Comment),
+            (r'[^(*)@"]+', Comment),
             (r'\(\*', Comment, '#push'),
             (r'\*\)', Comment, '#pop'),
-            (r'[(*)]', Comment),
+            # comments cannot be closed within strings in comments
+            (r'@"', String, 'lstring'),
+            (r'"""', String, 'tqs'),
+            (r'"', String, 'string'),
+            (r'[(*)@]', Comment),
         ],
         'string': [
-            (r'[^\\"]+', String.Double),
+            (r'[^\\"]+', String),
             include('escape-sequence'),
-            (r'\\\n', String.Double),
-            (r'"', String.Double, '#pop'),
+            (r'\\\n', String),
+            (r'\n', String),  # newlines are allowed in any string
+            (r'"B?', String, '#pop'),
         ],
-        'dotted': [
-            (r'\s+', Text),
-            (r'\.', Punctuation),
-            (r'[A-Z][A-Za-z0-9_\']*(?=\s*\.)', Name.Namespace),
-            (r'[A-Z][A-Za-z0-9_\']*', Name.Class, '#pop'),
-            (r'[a-z_][A-Za-z0-9_\']*', Name, '#pop'),
+        'lstring': [
+            (r'[^"]+', String),
+            (r'\n', String),
+            (r'""', String),
+            (r'"B?', String, '#pop'),
+        ],
+        'tqs': [
+            (r'[^"]+', String),
+            (r'\n', String),
+            (r'"""B?', String, '#pop'),
+            (r'"', String),
         ],
     }