merge

7 files changed, 132 insertions, 6 deletions

diff --git a/CHANGES b/CHANGES
index 9cd6c4fb..e88582c8 100644
--- a/CHANGES
+++ b/CHANGES
@@ -18,6 +18,9 @@ Version 2.2
 
 - Added `lexers.find_lexer_class_by_name()` (#1203)
 
+- Added new token types and lexing for magic methods and variables in Python
+  and PHP.
+
 
 Version 2.1.1
 -------------
diff --git a/doc/docs/tokens.rst b/doc/docs/tokens.rst
index 6455a501..96a6d003 100644
--- a/doc/docs/tokens.rst
+++ b/doc/docs/tokens.rst
@@ -174,6 +174,10 @@ Name Tokens
 `Name.Function`
     Token type for function names.
 
+`Name.Function.Magic`
+    same as `Name.Function` but for special function names that have an implicit use
+    in a language (e.g. ``__init__`` method in Python).
+
 `Name.Label`
     Token type for label names (e.g. in languages that support ``goto``).
 
@@ -201,6 +205,10 @@ Name Tokens
 `Name.Variable.Instance`
     same as `Name.Variable` but for instance variables.
 
+`Name.Variable.Magic`
+    same as `Name.Variable` but for special variable names that have an implicit use
+    in a language (e.g. ``__doc__`` in Python).
+
 
 Literals
 ========
diff --git a/pygments/lexers/php.py b/pygments/lexers/php.py
index 75b662cb..257dd94f 100644
--- a/pygments/lexers/php.py
+++ b/pygments/lexers/php.py
@@ -11,7 +11,8 @@
 
 import re
 
-from pygments.lexer import RegexLexer, include, bygroups, default, using, this
+from pygments.lexer import RegexLexer, include, bygroups, default, using, \
+    this, words
 from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
     Number, Punctuation, Other
 from pygments.util import get_bool_opt, get_list_opt, iteritems
@@ -162,13 +163,14 @@ class PhpLexer(RegexLexer):
              r'FALSE|print|for|require|continue|foreach|require_once|'
              r'declare|return|default|static|do|switch|die|stdClass|'
              r'echo|else|TRUE|elseif|var|empty|if|xor|enddeclare|include|'
-             r'virtual|endfor|include_once|while|endforeach|global|__FILE__|'
-             r'endif|list|__LINE__|endswitch|new|__sleep|endwhile|not|'
-             r'array|__wakeup|E_ALL|NULL|final|php_user_filter|interface|'
+             r'virtual|endfor|include_once|while|endforeach|global|'
+             r'endif|list|endswitch|new|endwhile|not|'
+             r'array|E_ALL|NULL|final|php_user_filter|interface|'
              r'implements|public|private|protected|abstract|clone|try|'
              r'catch|throw|this|use|namespace|trait|yield|'
              r'finally)\b', Keyword),
             (r'(true|false|null)\b', Keyword.Constant),
+            include('magicconstants'),
             (r'\$\{\$+' + _ident_inner + '\}', Name.Variable),
             (r'\$+' + _ident_inner, Name.Variable),
             (_ident_inner, Name.Other),
@@ -182,11 +184,29 @@ class PhpLexer(RegexLexer):
             (r'`([^`\\]*(?:\\.[^`\\]*)*)`', String.Backtick),
             (r'"', String.Double, 'string'),
         ],
+        'magicfuncs': [
+            # source: http://php.net/manual/en/language.oop5.magic.php
+            (words((
+                '__construct', '__destruct', '__call', '__callStatic', '__get', '__set',
+                '__isset', '__unset', '__sleep', '__wakeup', '__toString', '__invoke',
+                '__set_state', '__clone', '__debugInfo',), suffix=r'\b'),
+             Name.Function.Magic),
+        ],
+        'magicconstants': [
+            # source: http://php.net/manual/en/language.constants.predefined.php
+            (words((
+                '__LINE__', '__FILE__', '__DIR__', '__FUNCTION__', '__CLASS__',
+                '__TRAIT__', '__METHOD__', '__NAMESPACE__',),
+                suffix=r'\b'),
+             Name.Constant),
+        ],
         'classname': [
             (_ident_inner, Name.Class, '#pop')
         ],
         'functionname': [
-            (_ident_inner, Name.Function, '#pop')
+            include('magicfuncs'),
+            (_ident_inner, Name.Function, '#pop'),
+            default('#pop')
         ],
         'string': [
             (r'"', String.Double, '#pop'),
diff --git a/pygments/lexers/python.py b/pygments/lexers/python.py
index 9433f7fd..57af00e2 100644
--- a/pygments/lexers/python.py
+++ b/pygments/lexers/python.py
@@ -69,6 +69,8 @@ class PythonLexer(RegexLexer):
             (r'(import)((?:\s|\\\s)+)', bygroups(Keyword.Namespace, Text),
              'import'),
             include('builtins'),
+            include('magicfuncs'),
+            include('magicvars'),
             include('backtick'),
             ('(?:[rR]|[uU][rR]|[rR][uU])"""', String.Double, 'tdqs'),
             ("(?:[rR]|[uU][rR]|[rR][uU])'''", String.Single, 'tsqs'),
@@ -123,6 +125,37 @@ class PythonLexer(RegexLexer):
                 'ZeroDivisionError'), prefix=r'(?<!\.)', suffix=r'\b'),
              Name.Exception),
         ],
+        'magicfuncs': [
+            (words((
+                '__abs__', '__add__', '__and__', '__call__', '__cmp__', '__coerce__',
+                '__complex__', '__contains__', '__del__', '__delattr__', '__delete__',
+                '__delitem__', '__delslice__', '__div__', '__divmod__', '__enter__',
+                '__eq__', '__exit__', '__float__', '__floordiv__', '__ge__', '__get__',
+                '__getattr__', '__getattribute__', '__getitem__', '__getslice__', '__gt__',
+                '__hash__', '__hex__', '__iadd__', '__iand__', '__idiv__', '__ifloordiv__',
+                '__ilshift__', '__imod__', '__imul__', '__index__', '__init__',
+                '__instancecheck__', '__int__', '__invert__', '__iop__', '__ior__',
+                '__ipow__', '__irshift__', '__isub__', '__iter__', '__itruediv__',
+                '__ixor__', '__le__', '__len__', '__long__', '__lshift__', '__lt__',
+                '__missing__', '__mod__', '__mul__', '__ne__', '__neg__', '__new__',
+                '__nonzero__', '__oct__', '__op__', '__or__', '__pos__', '__pow__',
+                '__radd__', '__rand__', '__rcmp__', '__rdiv__', '__rdivmod__', '__repr__',
+                '__reversed__', '__rfloordiv__', '__rlshift__', '__rmod__', '__rmul__',
+                '__rop__', '__ror__', '__rpow__', '__rrshift__', '__rshift__', '__rsub__',
+                '__rtruediv__', '__rxor__', '__set__', '__setattr__', '__setitem__',
+                '__setslice__', '__str__', '__sub__', '__subclasscheck__', '__truediv__',
+                '__unicode__', '__xor__'), suffix=r'\b'),
+             Name.Function.Magic),
+        ],
+        'magicvars': [
+            (words((
+                '__bases__', '__class__', '__closure__', '__code__', '__defaults__',
+                '__dict__', '__doc__', '__file__', '__func__', '__globals__',
+                '__metaclass__', '__module__', '__mro__', '__name__', '__self__',
+                '__slots__', '__weakref__'),
+                suffix=r'\b'),
+             Name.Variable.Magic),
+        ],
         'numbers': [
             (r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?j?', Number.Float),
             (r'\d+[eE][+-]?[0-9]+j?', Number.Float),
@@ -140,7 +173,9 @@ class PythonLexer(RegexLexer):
             ('[a-zA-Z_]\w*', Name),
         ],
         'funcname': [
-            ('[a-zA-Z_]\w*', Name.Function, '#pop')
+            include('magicfuncs'),
+            ('[a-zA-Z_]\w*', Name.Function, '#pop'),
+            default('#pop'),
         ],
         'classname': [
             ('[a-zA-Z_]\w*', Name.Class, '#pop')
@@ -283,6 +318,38 @@ class Python3Lexer(RegexLexer):
             prefix=r'(?<!\.)', suffix=r'\b'),
          Name.Exception),
     ]
+    tokens['magicfuncs'] = [
+        (words((
+            '__abs__', '__add__', '__aenter__', '__aexit__', '__aiter__', '__and__',
+            '__anext__', '__await__', '__bool__', '__bytes__', '__call__',
+            '__complex__', '__contains__', '__del__', '__delattr__', '__delete__',
+            '__delitem__', '__dir__', '__divmod__', '__enter__', '__eq__', '__exit__',
+            '__float__', '__floordiv__', '__format__', '__ge__', '__get__',
+            '__getattr__', '__getattribute__', '__getitem__', '__gt__', '__hash__',
+            '__iadd__', '__iand__', '__ifloordiv__', '__ilshift__', '__imatmul__',
+            '__imod__', '__import__', '__imul__', '__index__', '__init__',
+            '__instancecheck__', '__int__', '__invert__', '__ior__', '__ipow__',
+            '__irshift__', '__isub__', '__iter__', '__itruediv__', '__ixor__',
+            '__le__', '__len__', '__length_hint__', '__lshift__', '__lt__',
+            '__matmul__', '__missing__', '__mod__', '__mul__', '__ne__', '__neg__',
+            '__new__', '__next__', '__or__', '__pos__', '__pow__', '__prepare__',
+            '__radd__', '__rand__', '__rdivmod__', '__repr__', '__reversed__',
+            '__rfloordiv__', '__rlshift__', '__rmatmul__', '__rmod__', '__rmul__',
+            '__ror__', '__round__', '__rpow__', '__rrshift__', '__rshift__',
+            '__rsub__', '__rtruediv__', '__rxor__', '__set__', '__setattr__',
+            '__setitem__', '__str__', '__sub__', '__subclasscheck__', '__truediv__',
+            '__xor__'), suffix=r'\b'),
+         Name.Function.Magic),
+    ]
+    tokens['magicvars'] = [
+        (words((
+            '__annotations__', '__bases__', '__class__', '__closure__', '__code__',
+            '__defaults__', '__dict__', '__doc__', '__file__', '__func__',
+            '__globals__', '__kwdefaults__', '__module__', '__mro__', '__name__',
+            '__objclass__', '__qualname__', '__self__', '__slots__', '__weakref__'),
+            suffix=r'\b'),
+         Name.Variable.Magic),
+    ]
     tokens['numbers'] = [
         (r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?', Number.Float),
         (r'0[oO][0-7]+', Number.Oct),
diff --git a/pygments/styles/lovelace.py b/pygments/styles/lovelace.py
index 4009274c..712f3e5c 100644
--- a/pygments/styles/lovelace.py
+++ b/pygments/styles/lovelace.py
@@ -62,11 +62,13 @@ class LovelaceStyle(Style):
         Name.Entity:         _ESCAPE_LIME,
         Name.Exception:      _EXCEPT_YELLOW,
         Name.Function:       _FUN_BROWN,
+        Name.Function.Magic: _DOC_ORANGE,
         Name.Label:          _LABEL_CYAN,
         Name.Namespace:      _LABEL_CYAN,
         Name.Tag:            _KW_BLUE,
         Name.Variable:       '#b04040',
         Name.Variable.Global:_EXCEPT_YELLOW,
+        Name.Variable.Magic: _DOC_ORANGE,
 
         String:              _STR_RED,
         String.Char:         _OW_PURPLE,
diff --git a/pygments/token.py b/pygments/token.py
index fa3b1e11..097ff064 100644
--- a/pygments/token.py
+++ b/pygments/token.py
@@ -147,6 +147,7 @@ STANDARD_TYPES = {
     Name.Entity:                   'ni',
     Name.Exception:                'ne',
     Name.Function:                 'nf',
+    Name.Function.Magic:           'fm',
     Name.Property:                 'py',
     Name.Label:                    'nl',
     Name.Namespace:                'nn',
@@ -156,6 +157,7 @@ STANDARD_TYPES = {
     Name.Variable.Class:           'vc',
     Name.Variable.Global:          'vg',
     Name.Variable.Instance:        'vi',
+    Name.Variable.Magic:           'vm',
 
     Literal:                       'l',
     Literal.Date:                  'ld',
diff --git a/tests/examplefiles/test.php b/tests/examplefiles/test.php
index 2ce4023e..794961c1 100644
--- a/tests/examplefiles/test.php
+++ b/tests/examplefiles/test.php
@@ -505,6 +505,30 @@ function &byref() {
     return $x;
 }
 
+// Test highlighting of magic methods and variables
+class MagicClass {
+  public $magic_str;
+  public $ordinary_str;
+
+  public function __construct($some_var) {
+    $this->magic_str = __FILE__;
+    $this->ordinary_str = $some_var;
+  }
+
+  public function __toString() {
+    return $this->magic_str;
+  }
+
+  public function nonMagic() {
+    return $this->ordinary_str;
+  }
+}
+
+$magic = new MagicClass(__DIR__);
+__toString();
+$magic->nonMagic();
+$magic->__toString();
+
      echo <<<EOF
 
      Test the heredocs...