path: root/mako/pyparser.py

# mako/pyparser.py
# Copyright (C) 2006-2013 the Mako authors and contributors <see AUTHORS file>
#
# This module is part of Mako and is released under
# the MIT License: http://www.opensource.org/licenses/mit-license.php

"""Handles parsing of Python code.

Parsing to AST is done via _ast on Python > 2.5, otherwise the compiler
module is used.
"""

from mako import exceptions, util, compat
from mako.compat import StringIO, arg_stringname
import operator

if compat.py3k:
    # words that cannot be assigned to (notably
    # smaller than the total keys in __builtins__)
    reserved = set(['True', 'False', 'None', 'print'])

    # the "id" attribute on a function node
    arg_id = operator.attrgetter('arg')
else:
    # words that cannot be assigned to (notably
    # smaller than the total keys in __builtins__)
    reserved = set(['True', 'False', 'None'])

    # the "id" attribute on a function node
    arg_id = operator.attrgetter('id')


try:
    import _ast
    util.restore__ast(_ast)
    from mako import _ast_util
except ImportError:
    _ast = None
    from compiler import parse as compiler_parse
    from compiler import visitor



def parse(code, mode='exec', **exception_kwargs):
    """Parse an expression into AST"""


    try:
        if _ast:
            return _ast_util.parse(code, '<unknown>', mode)
        else:
            if isinstance(code, compat.text_type):
                code = code.encode('ascii', 'backslashreplace')
            return compiler_parse(code, mode)
    except Exception:
        raise exceptions.SyntaxException(
                    "(%s) %s (%r)" % (
                        compat.exception_as().__class__.__name__,
                        compat.exception_as(),
                        code[0:50]
                    ), **exception_kwargs)


if _ast:
    class FindIdentifiers(_ast_util.NodeVisitor):

        def __init__(self, listener, **exception_kwargs):
            self.in_function = False
            self.in_assign_targets = False
            self.local_ident_stack = set()
            self.listener = listener
            self.exception_kwargs = exception_kwargs

        def _add_declared(self, name):
            if not self.in_function:
                self.listener.declared_identifiers.add(name)
            else:
                self.local_ident_stack.add(name)

        def visit_ClassDef(self, node):
            self._add_declared(node.name)

        def visit_Assign(self, node):

            # flip around the visiting of Assign so the expression gets
            # evaluated first, in the case of a clause like "x=x+5" (x
            # is undeclared)

            self.visit(node.value)
            in_a = self.in_assign_targets
            self.in_assign_targets = True
            for n in node.targets:
                self.visit(n)
            self.in_assign_targets = in_a

        if compat.py3k:

            # ExceptHandler is in Python 2, but this block only works in
            # Python 3 (and is required there)

            def visit_ExceptHandler(self, node):
                if node.name is not None:
                    self._add_declared(node.name)
                if node.type is not None:
                    self.listener.undeclared_identifiers.add(node.type.id)
                for statement in node.body:
                    self.visit(statement)

        def visit_Lambda(self, node, *args):
            self._visit_function(node, True)

        def visit_FunctionDef(self, node):
            self._add_declared(node.name)
            self._visit_function(node, False)

        def _expand_tuples(self, args):
            for arg in args:
                if isinstance(arg, _ast.Tuple):
                    for n in arg.elts:
                        yield n
                else:
                    yield arg

        def _visit_function(self, node, islambda):

            # push function state onto stack.  dont log any more
            # identifiers as "declared" until outside of the function,
            # but keep logging identifiers as "undeclared". track
            # argument names in each function header so they arent
            # counted as "undeclared"

            inf = self.in_function
            self.in_function = True

            local_ident_stack = self.local_ident_stack
            self.local_ident_stack = local_ident_stack.union([
                arg_id(arg) for arg in self._expand_tuples(node.args.args)
            ])
            if islambda:
                self.visit(node.body)
            else:
                for n in node.body:
                    self.visit(n)
            self.in_function = inf
            self.local_ident_stack = local_ident_stack

        def visit_For(self, node):

            # flip around visit

            self.visit(node.iter)
            self.visit(node.target)
            for statement in node.body:
                self.visit(statement)
            for statement in node.orelse:
                self.visit(statement)

        def visit_Name(self, node):
            if isinstance(node.ctx, _ast.Store):
                # this is eqiuvalent to visit_AssName in
                # compiler
                self._add_declared(node.id)
            elif node.id not in reserved and node.id \
                not in self.listener.declared_identifiers and node.id \
                not in self.local_ident_stack:
                self.listener.undeclared_identifiers.add(node.id)

        def visit_Import(self, node):
            for name in node.names:
                if name.asname is not None:
                    self._add_declared(name.asname)
                else:
                    self._add_declared(name.name.split('.')[0])

        def visit_ImportFrom(self, node):
            for name in node.names:
                if name.asname is not None:
                    self._add_declared(name.asname)
                else:
                    if name.name == '*':
                        raise exceptions.CompileException(
                          "'import *' is not supported, since all identifier "
                          "names must be explicitly declared.  Please use the "
                          "form 'from <modulename> import <name1>, <name2>, "
                          "...' instead.", **self.exception_kwargs)
                    self._add_declared(name.name)


    class FindTuple(_ast_util.NodeVisitor):

        def __init__(self, listener, code_factory, **exception_kwargs):
            self.listener = listener
            self.exception_kwargs = exception_kwargs
            self.code_factory = code_factory

        def visit_Tuple(self, node):
            for n in node.elts:
                p = self.code_factory(n, **self.exception_kwargs)
                self.listener.codeargs.append(p)
                self.listener.args.append(ExpressionGenerator(n).value())
                self.listener.declared_identifiers = \
                    self.listener.declared_identifiers.union(
                                                    p.declared_identifiers)
                self.listener.undeclared_identifiers = \
                    self.listener.undeclared_identifiers.union(
                                                    p.undeclared_identifiers)


    class ParseFunc(_ast_util.NodeVisitor):

        def __init__(self, listener, **exception_kwargs):
            self.listener = listener
            self.exception_kwargs = exception_kwargs

        def visit_FunctionDef(self, node):
            self.listener.funcname = node.name
            argnames = [arg_id(arg) for arg in node.args.args]
            if node.args.vararg:
                argnames.append(arg_stringname(node.args.vararg))
            if node.args.kwarg:
                argnames.append(arg_stringname(node.args.kwarg))
            self.listener.argnames = argnames
            self.listener.defaults = node.args.defaults  # ast
            self.listener.varargs = node.args.vararg
            self.listener.kwargs = node.args.kwarg

    class ExpressionGenerator(object):

        def __init__(self, astnode):
            self.generator = _ast_util.SourceGenerator(' ' * 4)
            self.generator.visit(astnode)

        def value(self):
            return ''.join(self.generator.result)
else:
    class FindIdentifiers(object):

        def __init__(self, listener, **exception_kwargs):
            self.in_function = False
            self.local_ident_stack = set()
            self.listener = listener
            self.exception_kwargs = exception_kwargs

        def _add_declared(self, name):
            if not self.in_function:
                self.listener.declared_identifiers.add(name)
            else:
                self.local_ident_stack.add(name)

        def visitClass(self, node, *args):
            self._add_declared(node.name)

        def visitAssName(self, node, *args):
            self._add_declared(node.name)

        def visitAssign(self, node, *args):

            # flip around the visiting of Assign so the expression gets
            # evaluated first, in the case of a clause like "x=x+5" (x
            # is undeclared)
            self.visit(node.expr, *args)
            for n in node.nodes:
                self.visit(n, *args)

        def visitLambda(self, node, *args):
            self._visit_function(node, args)

        def visitFunction(self, node, *args):
            self._add_declared(node.name)
            self._visit_function(node, args)

        def _expand_tuples(self, args):
            for arg in args:
                if isinstance(arg, tuple):
                    for n in arg:
                        yield n
                else:
                    yield arg

        def _visit_function(self, node, args):

            # push function state onto stack.  dont log any more
            # identifiers as "declared" until outside of the function,
            # but keep logging identifiers as "undeclared". track
            # argument names in each function header so they arent
            # counted as "undeclared"

            inf = self.in_function
            self.in_function = True

            local_ident_stack = self.local_ident_stack
            self.local_ident_stack = local_ident_stack.union([
                arg for arg in self._expand_tuples(node.argnames)
            ])

            for n in node.getChildNodes():
                self.visit(n, *args)
            self.in_function = inf
            self.local_ident_stack = local_ident_stack

        def visitFor(self, node, *args):

            # flip around visit

            self.visit(node.list, *args)
            self.visit(node.assign, *args)
            self.visit(node.body, *args)

        def visitName(self, node, *args):
            if node.name not in reserved and node.name \
                not in self.listener.declared_identifiers and node.name \
                not in self.local_ident_stack:
                self.listener.undeclared_identifiers.add(node.name)

        def visitImport(self, node, *args):
            for mod, alias in node.names:
                if alias is not None:
                    self._add_declared(alias)
                else:
                    self._add_declared(mod.split('.')[0])

        def visitFrom(self, node, *args):
            for mod, alias in node.names:
                if alias is not None:
                    self._add_declared(alias)
                else:
                    if mod == '*':
                        raise exceptions.CompileException(
                        "'import *' is not supported, since all identifier "
                        "names must be explicitly declared.  Please use the "
                        "form 'from <modulename> import <name1>, <name2>, "
                        "...' instead.", **self.exception_kwargs)
                    self._add_declared(mod)

        def visit(self, expr):
            visitor.walk(expr, self)  # , walker=walker())


    class FindTuple(object):

        def __init__(self, listener, code_factory, **exception_kwargs):
            self.listener = listener
            self.exception_kwargs = exception_kwargs
            self.code_factory = code_factory

        def visitTuple(self, node, *args):
            for n in node.nodes:
                p = self.code_factory(n, **self.exception_kwargs)
                self.listener.codeargs.append(p)
                self.listener.args.append(ExpressionGenerator(n).value())
                self.listener.declared_identifiers = \
                    self.listener.declared_identifiers.union(
                                                      p.declared_identifiers)
                self.listener.undeclared_identifiers = \
                    self.listener.undeclared_identifiers.union(
                                                      p.undeclared_identifiers)

        def visit(self, expr):
            visitor.walk(expr, self)  # , walker=walker())


    class ParseFunc(object):

        def __init__(self, listener, **exception_kwargs):
            self.listener = listener
            self.exception_kwargs = exception_kwargs

        def visitFunction(self, node, *args):
            self.listener.funcname = node.name
            self.listener.argnames = node.argnames
            self.listener.defaults = node.defaults
            self.listener.varargs = node.varargs
            self.listener.kwargs = node.kwargs

        def visit(self, expr):
            visitor.walk(expr, self)


    class ExpressionGenerator(object):

        """given an AST node, generates an equivalent literal Python
        expression."""

        def __init__(self, astnode):
            self.buf = StringIO()
            visitor.walk(astnode, self)  # , walker=walker())

        def value(self):
            return self.buf.getvalue()

        def operator(self, op, node, *args):
            self.buf.write('(')
            self.visit(node.left, *args)
            self.buf.write(' %s ' % op)
            self.visit(node.right, *args)
            self.buf.write(')')

        def booleanop(self, op, node, *args):
            self.visit(node.nodes[0])
            for n in node.nodes[1:]:
                self.buf.write(' ' + op + ' ')
                self.visit(n, *args)

        def visitConst(self, node, *args):
            self.buf.write(repr(node.value))

        def visitAssName(self, node, *args):

            # TODO: figure out OP_ASSIGN, other OP_s

            self.buf.write(node.name)

        def visitName(self, node, *args):
            self.buf.write(node.name)

        def visitMul(self, node, *args):
            self.operator('*', node, *args)

        def visitAnd(self, node, *args):
            self.booleanop('and', node, *args)

        def visitOr(self, node, *args):
            self.booleanop('or', node, *args)

        def visitBitand(self, node, *args):
            self.booleanop('&', node, *args)

        def visitBitor(self, node, *args):
            self.booleanop('|', node, *args)

        def visitBitxor(self, node, *args):
            self.booleanop('^', node, *args)

        def visitAdd(self, node, *args):
            self.operator('+', node, *args)

        def visitGetattr(self, node, *args):
            self.visit(node.expr, *args)
            self.buf.write('.%s' % node.attrname)

        def visitSub(self, node, *args):
            self.operator('-', node, *args)

        def visitNot(self, node, *args):
            self.buf.write('not ')
            self.visit(node.expr)

        def visitDiv(self, node, *args):
            self.operator('/', node, *args)

        def visitFloorDiv(self, node, *args):
            self.operator('//', node, *args)

        def visitSubscript(self, node, *args):
            self.visit(node.expr)
            self.buf.write('[')
            [self.visit(x) for x in node.subs]
            self.buf.write(']')

        def visitUnarySub(self, node, *args):
            self.buf.write('-')
            self.visit(node.expr)

        def visitUnaryAdd(self, node, *args):
            self.buf.write('-')
            self.visit(node.expr)

        def visitSlice(self, node, *args):
            self.visit(node.expr)
            self.buf.write('[')
            if node.lower is not None:
                self.visit(node.lower)
            self.buf.write(':')
            if node.upper is not None:
                self.visit(node.upper)
            self.buf.write(']')

        def visitDict(self, node):
            self.buf.write('{')
            c = node.getChildren()
            for i in range(0, len(c), 2):
                self.visit(c[i])
                self.buf.write(': ')
                self.visit(c[i + 1])
                if i < len(c) - 2:
                    self.buf.write(', ')
            self.buf.write('}')

        def visitTuple(self, node):
            self.buf.write('(')
            c = node.getChildren()
            for i in range(0, len(c)):
                self.visit(c[i])
                if i < len(c) - 1:
                    self.buf.write(', ')
            self.buf.write(')')

        def visitList(self, node):
            self.buf.write('[')
            c = node.getChildren()
            for i in range(0, len(c)):
                self.visit(c[i])
                if i < len(c) - 1:
                    self.buf.write(', ')
            self.buf.write(']')

        def visitListComp(self, node):
            self.buf.write('[')
            self.visit(node.expr)
            self.buf.write(' ')
            for n in node.quals:
                self.visit(n)
            self.buf.write(']')

        def visitListCompFor(self, node):
            self.buf.write(' for ')
            self.visit(node.assign)
            self.buf.write(' in ')
            self.visit(node.list)
            for n in node.ifs:
                self.visit(n)

        def visitListCompIf(self, node):
            self.buf.write(' if ')
            self.visit(node.test)

        def visitCompare(self, node):
            self.visit(node.expr)
            for tup in node.ops:
                self.buf.write(tup[0])
                self.visit(tup[1])

        def visitCallFunc(self, node, *args):
            self.visit(node.node)
            self.buf.write('(')
            if len(node.args):
                self.visit(node.args[0])
                for a in node.args[1:]:
                    self.buf.write(', ')
                    self.visit(a)
            self.buf.write(')')

        def visitLambda(self, node, *args):
            self.buf.write('lambda ')

            argnames = list(node.argnames)

            kw = arg = None
            if node.kwargs > 0:
                kw = argnames.pop(-1)
            if node.varargs > 0:
                arg = argnames.pop(-1)

            if arg:
                argnames.append("*%s" % arg)
            if kw:
                argnames.append("**%s" % kw)

            self.buf.write(", ".join(argnames))

            self.buf.write(': ')
            self.visit(node.code)


    class walker(visitor.ASTVisitor):

        def dispatch(self, node, *args):
            print('Node:', str(node))

            # print "dir:", dir(node)

            return visitor.ASTVisitor.dispatch(self, node, *args)