Added support and documentation for magic method and variable tokens. Reflected

this change in an updated version of my older style, Lovelace. Added Python lexing
for Python's magic methods and variables.

4 files changed, 80 insertions, 1 deletions

diff --git a/doc/docs/tokens.rst b/doc/docs/tokens.rst
index 6455a501..f9ed3d92 100644
--- a/doc/docs/tokens.rst
+++ b/doc/docs/tokens.rst
@@ -174,6 +174,10 @@ Name Tokens
 `Name.Function`
     Token type for function names.
 
+`Name.Function.Magic`
+    same as `Name.Function` but for function names that have an implicit use in
+    a language (e.g. ``__init__`` method in Python).
+
 `Name.Label`
     Token type for label names (e.g. in languages that support ``goto``).
 
@@ -201,6 +205,10 @@ Name Tokens
 `Name.Variable.Instance`
     same as `Name.Variable` but for instance variables.
 
+`Name.Variable.Magic`
+    same as `Name.Variable` but for variable names that have an implicit use in
+    a language (e.g. ``__doc__`` in Python).
+
 
 Literals
 ========
diff --git a/pygments/lexers/python.py b/pygments/lexers/python.py
index ccbd4c15..84f61242 100644
--- a/pygments/lexers/python.py
+++ b/pygments/lexers/python.py
@@ -69,6 +69,8 @@ class PythonLexer(RegexLexer):
             (r'(import)((?:\s|\\\s)+)', bygroups(Keyword.Namespace, Text),
              'import'),
             include('builtins'),
+            include('magicfuncs'),
+            include('magicvars'),
             include('backtick'),
             ('(?:[rR]|[uU][rR]|[rR][uU])"""', String.Double, 'tdqs'),
             ("(?:[rR]|[uU][rR]|[rR][uU])'''", String.Single, 'tsqs'),
@@ -123,6 +125,37 @@ class PythonLexer(RegexLexer):
                 'ZeroDivisionError'), prefix=r'(?<!\.)', suffix=r'\b'),
              Name.Exception),
         ],
+        'magicfuncs': [
+            (words((
+                '__abs__', '__add__', '__and__', '__call__', '__cmp__', '__coerce__',
+                '__complex__', '__contains__', '__del__', '__delattr__', '__delete__',
+                '__delitem__', '__delslice__', '__div__', '__divmod__', '__enter__',
+                '__eq__', '__exit__', '__float__', '__floordiv__', '__ge__', '__get__',
+                '__getattr__', '__getattribute__', '__getitem__', '__getslice__', '__gt__',
+                '__hash__', '__hex__', '__iadd__', '__iand__', '__idiv__', '__ifloordiv__',
+                '__ilshift__', '__imod__', '__imul__', '__index__', '__init__',
+                '__instancecheck__', '__int__', '__invert__', '__iop__', '__ior__',
+                '__ipow__', '__irshift__', '__isub__', '__iter__', '__itruediv__',
+                '__ixor__', '__le__', '__len__', '__long__', '__lshift__', '__lt__',
+                '__missing__', '__mod__', '__mul__', '__ne__', '__neg__', '__new__',
+                '__nonzero__', '__oct__', '__op__', '__or__', '__pos__', '__pow__',
+                '__radd__', '__rand__', '__rcmp__', '__rdiv__', '__rdivmod__', '__repr__',
+                '__reversed__', '__rfloordiv__', '__rlshift__', '__rmod__', '__rmul__',
+                '__rop__', '__ror__', '__rpow__', '__rrshift__', '__rshift__', '__rsub__',
+                '__rtruediv__', '__rxor__', '__set__', '__setattr__', '__setitem__',
+                '__setslice__', '__str__', '__sub__', '__subclasscheck__', '__truediv__',
+                '__unicode__', '__xor__'), suffix=r'\b'),
+             Name.Function.Magic),
+        ],
+        'magicvars': [
+            (words((
+                '__bases__', '__class__', '__closure__', '__code__', '__defaults__',
+                '__dict__', '__doc__', '__file__', '__func__', '__globals__',
+                '__metaclass__', '__module__', '__mro__', '__name__', '__self__',
+                '__slots__', '__weakref__'),
+                suffix=r'\b'),
+             Name.Variable.Magic),
+        ],
         'numbers': [
             (r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?j?', Number.Float),
             (r'\d+[eE][+-]?[0-9]+j?', Number.Float),
@@ -140,7 +173,9 @@ class PythonLexer(RegexLexer):
             ('[a-zA-Z_]\w*', Name),
         ],
         'funcname': [
-            ('[a-zA-Z_]\w*', Name.Function, '#pop')
+            include('magicfuncs'),
+            ('[a-zA-Z_]\w*', Name.Function, '#pop'),
+            default('#pop'),
         ],
         'classname': [
             ('[a-zA-Z_]\w*', Name.Class, '#pop')
@@ -263,6 +298,38 @@ class Python3Lexer(RegexLexer):
             prefix=r'(?<!\.)', suffix=r'\b'),
          Name.Exception),
     ]
+    tokens['magicfuncs'] = [
+        (words((
+            '__abs__', '__add__', '__aenter__', '__aexit__', '__aiter__', '__and__',
+            '__anext__', '__await__', '__bool__', '__bytes__', '__call__',
+            '__complex__', '__contains__', '__del__', '__delattr__', '__delete__',
+            '__delitem__', '__dir__', '__divmod__', '__enter__', '__eq__', '__exit__',
+            '__float__', '__floordiv__', '__format__', '__ge__', '__get__',
+            '__getattr__', '__getattribute__', '__getitem__', '__gt__', '__hash__',
+            '__iadd__', '__iand__', '__ifloordiv__', '__ilshift__', '__imatmul__',
+            '__imod__', '__import__', '__imul__', '__index__', '__init__',
+            '__instancecheck__', '__int__', '__invert__', '__ior__', '__ipow__',
+            '__irshift__', '__isub__', '__iter__', '__itruediv__', '__ixor__',
+            '__le__', '__len__', '__length_hint__', '__lshift__', '__lt__',
+            '__matmul__', '__missing__', '__mod__', '__mul__', '__ne__', '__neg__',
+            '__new__', '__next__', '__or__', '__pos__', '__pow__', '__prepare__',
+            '__radd__', '__rand__', '__rdivmod__', '__repr__', '__reversed__',
+            '__rfloordiv__', '__rlshift__', '__rmatmul__', '__rmod__', '__rmul__',
+            '__ror__', '__round__', '__rpow__', '__rrshift__', '__rshift__',
+            '__rsub__', '__rtruediv__', '__rxor__', '__set__', '__setattr__',
+            '__setitem__', '__str__', '__sub__', '__subclasscheck__', '__truediv__',
+            '__xor__'), suffix=r'\b'),
+         Name.Function.Magic),
+    ]
+    tokens['magicvars'] = [
+        (words((
+            '__annotations__', '__bases__', '__class__', '__closure__', '__code__',
+            '__defaults__', '__dict__', '__doc__', '__file__', '__func__',
+            '__globals__', '__kwdefaults__', '__module__', '__mro__', '__name__',
+            '__objclass__', '__qualname__', '__self__', '__slots__', '__weakref__'),
+            suffix=r'\b'),
+         Name.Variable.Magic),
+    ]
     tokens['numbers'] = [
         (r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?', Number.Float),
         (r'0[oO][0-7]+', Number.Oct),
diff --git a/pygments/styles/lovelace.py b/pygments/styles/lovelace.py
index 31bd5505..04560f86 100644
--- a/pygments/styles/lovelace.py
+++ b/pygments/styles/lovelace.py
@@ -59,11 +59,13 @@ class LovelaceStyle(Style):
         Name.Entity:         _ESCAPE_LIME,
         Name.Exception:      _EXCEPT_YELLOW,
         Name.Function:       _FUN_BROWN,
+        Name.Function.Magic: _DOC_ORANGE,
         Name.Label:          _LABEL_CYAN,
         Name.Namespace:      _LABEL_CYAN,
         Name.Tag:            _KW_BLUE,
         Name.Variable:       '#b04040',
         Name.Variable.Global:_EXCEPT_YELLOW,
+        Name.Variable.Magic: _DOC_ORANGE,
 
         String:              _STR_RED,
         String.Char:         _OW_PURPLE,
diff --git a/pygments/token.py b/pygments/token.py
index bfdfc114..a1ba615f 100644
--- a/pygments/token.py
+++ b/pygments/token.py
@@ -139,6 +139,7 @@ STANDARD_TYPES = {
     Name.Entity:                   'ni',
     Name.Exception:                'ne',
     Name.Function:                 'nf',
+    Name.Function.Magic:           'fm',
     Name.Property:                 'py',
     Name.Label:                    'nl',
     Name.Namespace:                'nn',
@@ -148,6 +149,7 @@ STANDARD_TYPES = {
     Name.Variable.Class:           'vc',
     Name.Variable.Global:          'vg',
     Name.Variable.Instance:        'vi',
+    Name.Variable.Magic:           'vm',
 
     Literal:                       'l',
     Literal.Date:                  'ld',