split up lexers.compiled into multiple submodules

1 files changed, 196 insertions, 0 deletions

diff --git a/pygments/lexers/python.py b/pygments/lexers/python.py
new file mode 100644
index 00000000..aea29355
--- /dev/null
+++ b/pygments/lexers/python.py
@@ -0,0 +1,196 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.lexers.python
+    ~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for Python and related languages.
+
+    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include, bygroups, using, \
+    this, inherit, default, words, combined
+from pygments.util import get_bool_opt
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation, Error
+
+__all__ = ['CythonLexer']
+
+
+class CythonLexer(RegexLexer):
+    """
+    For Pyrex and `Cython <http://cython.org>`_ source code.
+
+    .. versionadded:: 1.1
+    """
+
+    name = 'Cython'
+    aliases = ['cython', 'pyx', 'pyrex']
+    filenames = ['*.pyx', '*.pxd', '*.pxi']
+    mimetypes = ['text/x-cython', 'application/x-cython']
+
+    tokens = {
+        'root': [
+            (r'\n', Text),
+            (r'^(\s*)("""(?:.|\n)*?""")', bygroups(Text, String.Doc)),
+            (r"^(\s*)('''(?:.|\n)*?''')", bygroups(Text, String.Doc)),
+            (r'[^\S\n]+', Text),
+            (r'#.*$', Comment),
+            (r'[]{}:(),;[]', Punctuation),
+            (r'\\\n', Text),
+            (r'\\', Text),
+            (r'(in|is|and|or|not)\b', Operator.Word),
+            (r'(<)([a-zA-Z0-9.?]+)(>)',
+             bygroups(Punctuation, Keyword.Type, Punctuation)),
+            (r'!=|==|<<|>>|[-~+/*%=<>&^|.?]', Operator),
+            (r'(from)(\d+)(<=)(\s+)(<)(\d+)(:)',
+             bygroups(Keyword, Number.Integer, Operator, Name, Operator,
+                      Name, Punctuation)),
+            include('keywords'),
+            (r'(def|property)(\s+)', bygroups(Keyword, Text), 'funcname'),
+            (r'(cp?def)(\s+)', bygroups(Keyword, Text), 'cdef'),
+            (r'(class|struct)(\s+)', bygroups(Keyword, Text), 'classname'),
+            (r'(from)(\s+)', bygroups(Keyword, Text), 'fromimport'),
+            (r'(c?import)(\s+)', bygroups(Keyword, Text), 'import'),
+            include('builtins'),
+            include('backtick'),
+            ('(?:[rR]|[uU][rR]|[rR][uU])"""', String, 'tdqs'),
+            ("(?:[rR]|[uU][rR]|[rR][uU])'''", String, 'tsqs'),
+            ('(?:[rR]|[uU][rR]|[rR][uU])"', String, 'dqs'),
+            ("(?:[rR]|[uU][rR]|[rR][uU])'", String, 'sqs'),
+            ('[uU]?"""', String, combined('stringescape', 'tdqs')),
+            ("[uU]?'''", String, combined('stringescape', 'tsqs')),
+            ('[uU]?"', String, combined('stringescape', 'dqs')),
+            ("[uU]?'", String, combined('stringescape', 'sqs')),
+            include('name'),
+            include('numbers'),
+        ],
+        'keywords': [
+            (words((
+                'assert', 'break', 'by', 'continue', 'ctypedef', 'del', 'elif',
+                'else', 'except', 'except?', 'exec', 'finally', 'for', 'gil',
+                'global', 'if', 'include', 'lambda', 'nogil', 'pass', 'print',
+                'raise', 'return', 'try', 'while', 'yield', 'as', 'with'), suffix=r'\b'),
+             Keyword),
+            (r'(DEF|IF|ELIF|ELSE)\b', Comment.Preproc),
+        ],
+        'builtins': [
+            (words((
+                '__import__', 'abs', 'all', 'any', 'apply', 'basestring', 'bin',
+                'bool', 'buffer', 'bytearray', 'bytes', 'callable', 'chr',
+                'classmethod', 'cmp', 'coerce', 'compile', 'complex', 'delattr',
+                'dict', 'dir', 'divmod', 'enumerate', 'eval', 'execfile', 'exit',
+                'file', 'filter', 'float', 'frozenset', 'getattr', 'globals',
+                'hasattr', 'hash', 'hex', 'id', 'input', 'int', 'intern', 'isinstance',
+                'issubclass', 'iter', 'len', 'list', 'locals', 'long', 'map', 'max',
+                'min', 'next', 'object', 'oct', 'open', 'ord', 'pow', 'property',
+                'range', 'raw_input', 'reduce', 'reload', 'repr', 'reversed',
+                'round', 'set', 'setattr', 'slice', 'sorted', 'staticmethod',
+                'str', 'sum', 'super', 'tuple', 'type', 'unichr', 'unicode',
+                'vars', 'xrange', 'zip'), prefix=r'(?<!\.)', suffix=r'\b'),
+             Name.Builtin),
+            (r'(?<!\.)(self|None|Ellipsis|NotImplemented|False|True|NULL'
+             r')\b', Name.Builtin.Pseudo),
+            (words((
+                'ArithmeticError', 'AssertionError', 'AttributeError',
+                'BaseException', 'DeprecationWarning', 'EOFError', 'EnvironmentError',
+                'Exception', 'FloatingPointError', 'FutureWarning', 'GeneratorExit', 'IOError',
+                'ImportError', 'ImportWarning', 'IndentationError', 'IndexError', 'KeyError',
+                'KeyboardInterrupt', 'LookupError', 'MemoryError', 'NameError',
+                'NotImplemented', 'NotImplementedError', 'OSError', 'OverflowError',
+                'OverflowWarning', 'PendingDeprecationWarning', 'ReferenceError',
+                'RuntimeError', 'RuntimeWarning', 'StandardError', 'StopIteration',
+                'SyntaxError', 'SyntaxWarning', 'SystemError', 'SystemExit', 'TabError',
+                'TypeError', 'UnboundLocalError', 'UnicodeDecodeError',
+                'UnicodeEncodeError', 'UnicodeError', 'UnicodeTranslateError',
+                'UnicodeWarning', 'UserWarning', 'ValueError', 'Warning',
+                'ZeroDivisionError'), prefix=r'(?<!\.)', suffix=r'\b'),
+             Name.Exception),
+        ],
+        'numbers': [
+            (r'(\d+\.?\d*|\d*\.\d+)([eE][+-]?[0-9]+)?', Number.Float),
+            (r'0\d+', Number.Oct),
+            (r'0[xX][a-fA-F0-9]+', Number.Hex),
+            (r'\d+L', Number.Integer.Long),
+            (r'\d+', Number.Integer)
+        ],
+        'backtick': [
+            ('`.*?`', String.Backtick),
+        ],
+        'name': [
+            (r'@\w+', Name.Decorator),
+            ('[a-zA-Z_]\w*', Name),
+        ],
+        'funcname': [
+            ('[a-zA-Z_]\w*', Name.Function, '#pop')
+        ],
+        'cdef': [
+            (r'(public|readonly|extern|api|inline)\b', Keyword.Reserved),
+            (r'(struct|enum|union|class)\b', Keyword),
+            (r'([a-zA-Z_]\w*)(\s*)(?=[(:#=]|$)',
+             bygroups(Name.Function, Text), '#pop'),
+            (r'([a-zA-Z_]\w*)(\s*)(,)',
+             bygroups(Name.Function, Text, Punctuation)),
+            (r'from\b', Keyword, '#pop'),
+            (r'as\b', Keyword),
+            (r':', Punctuation, '#pop'),
+            (r'(?=["\'])', Text, '#pop'),
+            (r'[a-zA-Z_]\w*', Keyword.Type),
+            (r'.', Text),
+        ],
+        'classname': [
+            ('[a-zA-Z_]\w*', Name.Class, '#pop')
+        ],
+        'import': [
+            (r'(\s+)(as)(\s+)', bygroups(Text, Keyword, Text)),
+            (r'[a-zA-Z_][\w.]*', Name.Namespace),
+            (r'(\s*)(,)(\s*)', bygroups(Text, Operator, Text)),
+            default('#pop')  # all else: go back
+        ],
+        'fromimport': [
+            (r'(\s+)(c?import)\b', bygroups(Text, Keyword), '#pop'),
+            (r'[a-zA-Z_.][\w.]*', Name.Namespace),
+            # ``cdef foo from "header"``, or ``for foo from 0 < i < 10``
+            default('#pop'),
+        ],
+        'stringescape': [
+            (r'\\([\\abfnrtv"\']|\n|N{.*?}|u[a-fA-F0-9]{4}|'
+             r'U[a-fA-F0-9]{8}|x[a-fA-F0-9]{2}|[0-7]{1,3})', String.Escape)
+        ],
+        'strings': [
+            (r'%(\([a-zA-Z0-9]+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?'
+             '[hlL]?[diouxXeEfFgGcrs%]', String.Interpol),
+            (r'[^\\\'"%\n]+', String),
+            # quotes, percents and backslashes must be parsed one at a time
+            (r'[\'"\\]', String),
+            # unhandled string formatting sign
+            (r'%', String)
+            # newlines are an error (use "nl" state)
+        ],
+        'nl': [
+            (r'\n', String)
+        ],
+        'dqs': [
+            (r'"', String, '#pop'),
+            (r'\\\\|\\"|\\\n', String.Escape),  # included here again for raw strings
+            include('strings')
+        ],
+        'sqs': [
+            (r"'", String, '#pop'),
+            (r"\\\\|\\'|\\\n", String.Escape),  # included here again for raw strings
+            include('strings')
+        ],
+        'tdqs': [
+            (r'"""', String, '#pop'),
+            include('strings'),
+            include('nl')
+        ],
+        'tsqs': [
+            (r"'''", String, '#pop'),
+            include('strings'),
+            include('nl')
+        ],
+    }