Merge branch 'lexers_de' of https://github.com/blu-base/pygments into blu-base-lexers_de

1 files changed, 70 insertions, 58 deletions

diff --git a/pygments/lexers/dotnet.py b/pygments/lexers/dotnet.py
index 299ee8bd..c04d2a0a 100644
--- a/pygments/lexers/dotnet.py
+++ b/pygments/lexers/dotnet.py
@@ -11,8 +11,8 @@ import re
 
 from pygments.lexer import RegexLexer, DelegatingLexer, bygroups, include, \
     using, this, default, words
-from pygments.token import Punctuation, \
-    Text, Comment, Operator, Keyword, Name, String, Number, Literal, Other
+from pygments.token import Punctuation, Text, Comment, Operator, Keyword, \
+        Name, String, Number, Literal, Other, Whitespace
 from pygments.util import get_choice_opt
 from pygments import unistring as uni
 
@@ -74,16 +74,17 @@ class CSharpLexer(RegexLexer):
         tokens[levelname] = {
             'root': [
                 # method names
-                (r'^([ \t]*(?:' + cs_ident + r'(?:\[\])?\s+)+?)'  # return type
+                (r'^([ \t]*)((?:' + cs_ident + r'(?:\[\])?\s+)+?)'  # return type
                  r'(' + cs_ident + ')'                            # method name
                  r'(\s*)(\()',                               # signature start
-                 bygroups(using(this), Name.Function, Text, Punctuation)),
-                (r'^\s*\[.*?\]', Name.Attribute),
-                (r'[^\S\n]+', Text),
-                (r'\\\n', Text),  # line continuation
+                 bygroups(Whitespace, using(this), Name.Function, Whitespace, 
+                     Punctuation)),
+                (r'^(\s*)(\[.*?\])', bygroups(Whitespace, Name.Attribute)),
+                (r'[^\S\n]+', Whitespace),
+                (r'(\\)(\n)', bygroups(Text, Whitespace)),  # line continuation
                 (r'//.*?\n', Comment.Single),
                 (r'/[*].*?[*]/', Comment.Multiline),
-                (r'\n', Text),
+                (r'\n', Whitespace),
                 (r'[~!%^&*()+=|\[\]:;,.<>/?-]', Punctuation),
                 (r'[{}]', Punctuation),
                 (r'@"(""|[^"])*"', String),
@@ -91,10 +92,11 @@ class CSharpLexer(RegexLexer):
                 (r"'\\.'|'[^\\]'", String.Char),
                 (r"[0-9](\.[0-9]*)?([eE][+-][0-9]+)?"
                  r"[flFLdD]?|0[xX][0-9a-fA-F]+[Ll]?", Number),
-                (r'#[ \t]*(if|endif|else|elif|define|undef|'
-                 r'line|error|warning|region|endregion|pragma)\b.*?\n',
-                 Comment.Preproc),
-                (r'\b(extern)(\s+)(alias)\b', bygroups(Keyword, Text,
+                (r'(#)([ \t]*)(if|endif|else|elif|define|undef|'
+                 r'line|error|warning|region|endregion|pragma)\b(.*?)(\n)',
+                 bygroups(Comment.Preproc, Whitespace, Comment.Preproc, 
+                     Comment.Preproc, Whitespace)),
+                (r'\b(extern)(\s+)(alias)\b', bygroups(Keyword, Whitespace,
                  Keyword)),
                 (r'(abstract|as|async|await|base|break|by|case|catch|'
                  r'checked|const|continue|default|delegate|'
@@ -111,8 +113,8 @@ class CSharpLexer(RegexLexer):
                 (r'(global)(::)', bygroups(Keyword, Punctuation)),
                 (r'(bool|byte|char|decimal|double|dynamic|float|int|long|object|'
                  r'sbyte|short|string|uint|ulong|ushort|var)\b\??', Keyword.Type),
-                (r'(class|struct)(\s+)', bygroups(Keyword, Text), 'class'),
-                (r'(namespace|using)(\s+)', bygroups(Keyword, Text), 'namespace'),
+                (r'(class|struct)(\s+)', bygroups(Keyword, Whitespace), 'class'),
+                (r'(namespace|using)(\s+)', bygroups(Keyword, Whitespace), 'namespace'),
                 (cs_ident, Name),
             ],
             'class': [
@@ -187,21 +189,25 @@ class NemerleLexer(RegexLexer):
         tokens[levelname] = {
             'root': [
                 # method names
-                (r'^([ \t]*(?:' + cs_ident + r'(?:\[\])?\s+)+?)'  # return type
+                (r'^([ \t]*)((?:' + cs_ident + r'(?:\[\])?\s+)+?)'  # return type
                  r'(' + cs_ident + ')'                            # method name
                  r'(\s*)(\()',                               # signature start
-                 bygroups(using(this), Name.Function, Text, Punctuation)),
-                (r'^\s*\[.*?\]', Name.Attribute),
-                (r'[^\S\n]+', Text),
-                (r'\\\n', Text),  # line continuation
+                 bygroups(Whitespace, using(this), Name.Function, Whitespace, \
+                     Punctuation)),
+                (r'^(\s*)(\[.*?\])', bygroups(Whitespace, Name.Attribute)),
+                (r'[^\S\n]+', Whitespace),
+                (r'(\\)(\n)', bygroups(Text, Whitespace)),  # line continuation
                 (r'//.*?\n', Comment.Single),
                 (r'/[*].*?[*]/', Comment.Multiline),
-                (r'\n', Text),
-                (r'\$\s*"', String, 'splice-string'),
-                (r'\$\s*<#', String, 'splice-string2'),
+                (r'\n', Whitespace),
+                (r'(\$)(\s*)(")', bygroups(String, Whitespace, String),
+                    'splice-string'),
+                (r'(\$)(\s*)(<#)', bygroups(String, Whitespace, String),
+                    'splice-string2'),
                 (r'<#', String, 'recursive-string'),
 
-                (r'(<\[)\s*(' + cs_ident + ':)?', Keyword),
+                (r'(<\[)(\s*)(' + cs_ident + ':)?', bygroups(Keyword, 
+                    Whitespace, Keyword)),
                 (r'\]\>', Keyword),
 
                 # quasiquotation only
@@ -216,11 +222,10 @@ class NemerleLexer(RegexLexer):
                 (r"'\\.'|'[^\\]'", String.Char),
                 (r"0[xX][0-9a-fA-F]+[Ll]?", Number),
                 (r"[0-9](\.[0-9]*)?([eE][+-][0-9]+)?[flFLdD]?", Number),
-                (r'#[ \t]*(if|endif|else|elif|define|undef|'
-                 r'line|error|warning|region|endregion|pragma)\b.*?\n',
-                 Comment.Preproc),
-                (r'\b(extern)(\s+)(alias)\b', bygroups(Keyword, Text,
-                 Keyword)),
+                (r'(#)([ \t]*)(if|endif|else|elif|define|undef|'
+                 r'line|error|warning|region|endregion|pragma)\b',
+                 bygroups(Comment.Preproc, Whitespace, Comment.Preproc), 'preproc'),
+                (r'\b(extern)(\s+)(alias)\b', bygroups(Keyword, Whitespace, Keyword)),
                 (r'(abstract|and|as|base|catch|def|delegate|'
                  r'enum|event|extern|false|finally|'
                  r'fun|implements|interface|internal|'
@@ -237,17 +242,22 @@ class NemerleLexer(RegexLexer):
                 (r'(bool|byte|char|decimal|double|float|int|long|object|sbyte|'
                  r'short|string|uint|ulong|ushort|void|array|list)\b\??',
                  Keyword.Type),
-                (r'(:>?)\s*(' + cs_ident + r'\??)',
-                 bygroups(Punctuation, Keyword.Type)),
+                (r'(:>?)(\s*)(' + cs_ident + r'\??)',
+                 bygroups(Punctuation, Whitespace, Keyword.Type)),
                 (r'(class|struct|variant|module)(\s+)',
-                 bygroups(Keyword, Text), 'class'),
-                (r'(namespace|using)(\s+)', bygroups(Keyword, Text),
+                 bygroups(Keyword, Whitespace), 'class'),
+                (r'(namespace|using)(\s+)', bygroups(Keyword, Whitespace),
                  'namespace'),
                 (cs_ident, Name),
             ],
             'class': [
                 (cs_ident, Name.Class, '#pop')
             ],
+            'preproc': [
+                (r'\w+', Comment.Preproc),
+                (r'[ \t]+', Whitespace),
+                (r'\n', Whitespace, '#pop')
+            ],
             'namespace': [
                 (r'(?=\()', Text, '#pop'),  # using (resource)
                 ('(' + cs_ident + r'|\.)+', Name.Namespace, '#pop')
@@ -317,11 +327,11 @@ class BooLexer(RegexLexer):
 
     tokens = {
         'root': [
-            (r'\s+', Text),
+            (r'\s+', Whitespace),
             (r'(#|//).*$', Comment.Single),
             (r'/[*]', Comment.Multiline, 'comment'),
             (r'[]{}:(),.;[]', Punctuation),
-            (r'\\\n', Text),
+            (r'(\\)(\n)', bygroups(Text, Whitespace)),
             (r'\\', Text),
             (r'(in|is|and|or|not)\b', Operator.Word),
             (r'/(\\\\|\\[^\\]|[^/\\\s])/', String.Regex),
@@ -335,9 +345,9 @@ class BooLexer(RegexLexer):
              r'is|isa|not|or|otherwise|pass|raise|ref|try|unless|when|'
              r'while|from|as)\b', Keyword),
             (r'def(?=\s+\(.*?\))', Keyword),
-            (r'(def)(\s+)', bygroups(Keyword, Text), 'funcname'),
-            (r'(class)(\s+)', bygroups(Keyword, Text), 'classname'),
-            (r'(namespace)(\s+)', bygroups(Keyword, Text), 'namespace'),
+            (r'(def)(\s+)', bygroups(Keyword, Whitespace), 'funcname'),
+            (r'(class)(\s+)', bygroups(Keyword, Whitespace), 'classname'),
+            (r'(namespace)(\s+)', bygroups(Keyword, Whitespace), 'namespace'),
             (r'(?<!\.)(true|false|null|self|__eval__|__switch__|array|'
              r'assert|checked|enumerate|filter|getter|len|lock|map|'
              r'matrix|max|min|normalArrayIndexing|print|property|range|'
@@ -392,17 +402,18 @@ class VbNetLexer(RegexLexer):
     tokens = {
         'root': [
             (r'^\s*<.*?>', Name.Attribute),
-            (r'\s+', Text),
-            (r'\n', Text),
-            (r'rem\b.*?\n', Comment),
-            (r"'.*?\n", Comment),
+            (r'\s+', Whitespace),
+            (r'\n', Whitespace),
+            (r'(rem\b.*?)(\n)', bygroups(Comment, Whitespace)),
+            (r"('.*?)(\n)", bygroups(Comment, Whitespace)),
             (r'#If\s.*?\sThen|#ElseIf\s.*?\sThen|#Else|#End\s+If|#Const|'
              r'#ExternalSource.*?\n|#End\s+ExternalSource|'
              r'#Region.*?\n|#End\s+Region|#ExternalChecksum',
              Comment.Preproc),
             (r'[(){}!#,.:]', Punctuation),
-            (r'Option\s+(Strict|Explicit|Compare)\s+'
-             r'(On|Off|Binary|Text)', Keyword.Declaration),
+            (r'(Option)(\s+)(Strict|Explicit|Compare)(\s+)'
+             r'(On|Off|Binary|Text)', bygroups(Keyword.Declaration, Whitespace,
+                 Keyword.Declaration, Whitespace, Keyword.Declaration)),
             (words((
                 'AddHandler', 'Alias', 'ByRef', 'ByVal', 'Call', 'Case',
                 'Catch', 'CBool', 'CByte', 'CChar', 'CDate', 'CDec', 'CDbl',
@@ -426,11 +437,11 @@ class VbNetLexer(RegexLexer):
             (r'(?<!\.)End\b', Keyword, 'end'),
             (r'(?<!\.)(Dim|Const)\b', Keyword, 'dim'),
             (r'(?<!\.)(Function|Sub|Property)(\s+)',
-             bygroups(Keyword, Text), 'funcname'),
+             bygroups(Keyword, Whitespace), 'funcname'),
             (r'(?<!\.)(Class|Structure|Enum)(\s+)',
-             bygroups(Keyword, Text), 'classname'),
+             bygroups(Keyword, Whitespace), 'classname'),
             (r'(?<!\.)(Module|Namespace|Imports)(\s+)',
-             bygroups(Keyword, Text), 'namespace'),
+             bygroups(Keyword, Whitespace), 'namespace'),
             (r'(?<!\.)(Boolean|Byte|Char|Date|Decimal|Double|Integer|Long|'
              r'Object|SByte|Short|Single|String|Variant|UInteger|ULong|'
              r'UShort)\b', Keyword.Type),
@@ -440,7 +451,7 @@ class VbNetLexer(RegexLexer):
              r'<=|>=|<>|[-&*/\\^+=<>\[\]]',
              Operator),
             ('"', String, 'string'),
-            (r'_\n', Text),  # Line continuation  (must be before Name)
+            (r'(_)(\n)', bygroups(Text, Whitespace)),  # Line continuation  (must be before Name)
             (uni_name + '[%&@!#$]?', Name),
             ('#.*?#', Literal.Date),
             (r'(\d+\.\d*|\d*\.\d+)(F[+-]?[0-9]+)?', Number.Float),
@@ -469,7 +480,7 @@ class VbNetLexer(RegexLexer):
             default('#pop'),
         ],
         'end': [
-            (r'\s+', Text),
+            (r'\s+', Whitespace),
             (r'(Function|Sub|Property|Class|Structure|Enum|Module|Namespace)\b',
              Keyword, '#pop'),
             default('#pop'),
@@ -606,13 +617,13 @@ class FSharpLexer(RegexLexer):
             (r'\\U[0-9a-fA-F]{8}', String.Escape),
         ],
         'root': [
-            (r'\s+', Text),
+            (r'\s+', Whitespace),
             (r'\(\)|\[\]', Name.Builtin.Pseudo),
             (r'\b(?<!\.)([A-Z][\w\']*)(?=\s*\.)',
              Name.Namespace, 'dotted'),
             (r'\b([A-Z][\w\']*)', Name),
-            (r'///.*?\n', String.Doc),
-            (r'//.*?\n', Comment.Single),
+            (r'(///.*?)(\n)', bygroups(String.Doc, Whitespace)),
+            (r'(//.*?)(\n)', bygroups(Comment.Single, Whitespace)),
             (r'\(\*(?!\))', Comment, 'comment'),
 
             (r'@"', String, 'lstring'),
@@ -620,21 +631,22 @@ class FSharpLexer(RegexLexer):
             (r'"', String, 'string'),
 
             (r'\b(open|module)(\s+)([\w.]+)',
-             bygroups(Keyword, Text, Name.Namespace)),
+             bygroups(Keyword, Whitespace, Name.Namespace)),
             (r'\b(let!?)(\s+)(\w+)',
-             bygroups(Keyword, Text, Name.Variable)),
+             bygroups(Keyword, Whitespace, Name.Variable)),
             (r'\b(type)(\s+)(\w+)',
-             bygroups(Keyword, Text, Name.Class)),
+             bygroups(Keyword, Whitespace, Name.Class)),
             (r'\b(member|override)(\s+)(\w+)(\.)(\w+)',
-             bygroups(Keyword, Text, Name, Punctuation, Name.Function)),
+             bygroups(Keyword, Whitespace, Name, Punctuation, Name.Function)),
             (r'\b(%s)\b' % '|'.join(keywords), Keyword),
             (r'``([^`\n\r\t]|`[^`\n\r\t])+``', Name),
             (r'(%s)' % '|'.join(keyopts), Operator),
             (r'(%s|%s)?%s' % (infix_syms, prefix_syms, operators), Operator),
             (r'\b(%s)\b' % '|'.join(word_operators), Operator.Word),
             (r'\b(%s)\b' % '|'.join(primitives), Keyword.Type),
-            (r'#[ \t]*(if|endif|else|line|nowarn|light|\d+)\b.*?\n',
-             Comment.Preproc),
+            (r'(#)([ \t]*)(if|endif|else|line|nowarn|light|\d+)\b(.*?)(\n)',
+             bygroups(Comment.Preproc, Whitespace, Comment.Preproc, 
+                 Comment.Preproc, Whitespace)),
 
             (r"[^\W\d][\w']*", Name),
 
@@ -655,7 +667,7 @@ class FSharpLexer(RegexLexer):
             (r'[~?][a-z][\w\']*:', Name.Variable),
         ],
         'dotted': [
-            (r'\s+', Text),
+            (r'\s+', Whitespace),
             (r'\.', Punctuation),
             (r'[A-Z][\w\']*(?=\s*\.)', Name.Namespace),
             (r'[A-Z][\w\']*', Name, '#pop'),