From 1695e3889159537e0677528b52210b7575a2a95a Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Sun, 31 Jul 2016 21:48:16 +0200 Subject: correctly tie one PyCodeObject instance to one function to allow for proper tracing and profiling --- Cython/Compiler/Code.py | 67 ++++++++++++++++++++++++++ Cython/Compiler/ExprNodes.py | 86 ++-------------------------------- Cython/Compiler/FusedNode.py | 5 -- Cython/Compiler/Nodes.py | 18 +++---- Cython/Compiler/ParseTreeTransforms.py | 4 +- 5 files changed, 81 insertions(+), 99 deletions(-) diff --git a/Cython/Compiler/Code.py b/Cython/Compiler/Code.py index a27908486..6b1e374ae 100644 --- a/Cython/Compiler/Code.py +++ b/Cython/Compiler/Code.py @@ -1020,6 +1020,7 @@ class GlobalState(object): self.num_const_index = {} self.py_constants = [] self.cached_cmethods = {} + self.code_objects = {} writer.set_global_state(self) self.rootwriter = writer @@ -1132,6 +1133,69 @@ class GlobalState(object): def get_cached_constants_writer(self): return self.parts['cached_constants'] + def get_pycode_object_cname(self, func_name, func_node, obj_type): + if func_node in self.code_objects: + return self.code_objects[func_node] + + code_object_cname = self.code_objects[func_node] = self.get_py_const( + obj_type, 'codeobj', cleanup_level=2).cname + + code = self.get_cached_constants_writer() + code.mark_pos(func_node.pos) + + func_name = code.intern_identifier(func_name) + # FIXME: better way to get the module file path at module init time? Encoding to use? + file_path = StringEncoding.bytes_literal(func_node.pos[0].get_filenametable_entry().encode('utf8'), 'utf8') + file_path_const = code.get_py_string_const(file_path, identifier=False, is_str=True) + + flags = [] + if func_node.star_arg: + flags.append('CO_VARARGS') + if func_node.starstar_arg: + flags.append('CO_VARKEYWORDS') + + args = list(func_node.args) + num_kwonly_args = func_node.num_kwonly_args if func_node.entry.type.is_pyobject else 0 + local_vars = [arg for arg in func_node.local_scope.var_entries if arg.name] + varnames = [ + code.intern_identifier(arg.name) + for arg in args + local_vars] + + varnames_cname = Naming.quick_temp_cname + code.putln("{") + code.putln("PyObject* %s = PyTuple_New(%d); %s" % ( + varnames_cname, + len(varnames), + code.error_goto_if_null(varnames_cname, func_node.pos))) + code.put_gotref(varnames_cname) + + for i, varname in enumerate(varnames): + code.putln("Py_INCREF(%s); PyTuple_SET_ITEM(%s, %d, %s);" % (varname, varnames_cname, i, varname)) + + code.putln("%s = (PyObject*)__Pyx_PyCode_New(%d, %d, %d, 0, %s, %s, %s, %s, %s, %s, %s, %s, %s, %d, %s);" % ( + code_object_cname, + len(func_node.args) - num_kwonly_args, # argcount + num_kwonly_args, # kwonlyargcount (Py3 only) + len(varnames), # nlocals + '|'.join(flags) or '0', # flags + Naming.empty_bytes, # code + Naming.empty_tuple, # consts + Naming.empty_tuple, # names (FIXME) + varnames_cname, # varnames + Naming.empty_tuple, # freevars (FIXME) + Naming.empty_tuple, # cellvars (FIXME) + file_path_const, # filename + func_name, # name + func_node.pos[1], # firstlineno + Naming.empty_bytes, # lnotab + )) + + code.put_decref_clear(varnames_cname, obj_type) + code.putln(code.error_goto_if_null(code_object_cname, func_node.pos)) + code.putln("}") + + return code_object_cname + def get_int_const(self, str_value, longness=False): py_type = longness and 'long' or 'int' try: @@ -1646,6 +1710,9 @@ class CCodeWriter(object): def get_py_const(self, type, prefix='', cleanup_level=None): return self.globalstate.get_py_const(type, prefix, cleanup_level).cname + def get_pycode_object_const(self, func_name, def_node, obj_type): + return self.globalstate.get_pycode_object_cname(func_name, def_node, obj_type) + def get_string_const(self, text): return self.globalstate.get_string_const(text).cname diff --git a/Cython/Compiler/ExprNodes.py b/Cython/Compiler/ExprNodes.py index 7e71cdae6..05932fc1e 100644 --- a/Cython/Compiler/ExprNodes.py +++ b/Cython/Compiler/ExprNodes.py @@ -8624,13 +8624,10 @@ class PyCFunctionNode(ExprNode, ModuleNameMixin): # binding bool # def_node DefNode the Python function node # module_name EncodedString Name of defining module - # code_object CodeObjectNode the PyCodeObject creator node - subexprs = ['code_object', 'defaults_tuple', 'defaults_kwdict', - 'annotations_dict'] + subexprs = ['defaults_tuple', 'defaults_kwdict', 'annotations_dict'] self_object = None - code_object = None binding = False def_node = None defaults = None @@ -8652,8 +8649,7 @@ class PyCFunctionNode(ExprNode, ModuleNameMixin): def_node=node, pymethdef_cname=node.entry.pymethdef_cname, binding=binding or node.specialized_cpdefs, - specialized_cpdefs=node.specialized_cpdefs, - code_object=CodeObjectNode(node)) + specialized_cpdefs=node.specialized_cpdefs) def analyse_types(self, env): if self.binding: @@ -8819,11 +8815,6 @@ class PyCFunctionNode(ExprNode, ModuleNameMixin): UtilityCode.load_cached("CythonFunction", "CythonFunction.c")) constructor = "__Pyx_CyFunction_NewEx" - if self.code_object: - code_object_result = self.code_object.py_result() - else: - code_object_result = 'NULL' - flags = [] if def_node.is_staticmethod: flags.append('__Pyx_CYFUNCTION_STATICMETHOD') @@ -8848,7 +8839,7 @@ class PyCFunctionNode(ExprNode, ModuleNameMixin): self.self_result_code(), self.get_py_mod_name(code), Naming.moddict_cname, - code_object_result, + code.get_pycode_object_const(def_node.name, def_node, py_object_type), code.error_goto_if_null(self.result(), self.pos))) code.put_gotref(self.py_result()) @@ -8901,75 +8892,6 @@ class InnerFunctionNode(PyCFunctionNode): return "NULL" -class CodeObjectNode(ExprNode): - # Create a PyCodeObject for a CyFunction instance. - # - # def_node DefNode the Python function node - # varnames TupleNode a tuple with all local variable names - - subexprs = ['varnames'] - is_temp = False - result_code = None - - def __init__(self, def_node): - ExprNode.__init__(self, def_node.pos, def_node=def_node) - args = list(def_node.args) - # if we have args/kwargs, then the first two in var_entries are those - local_vars = [arg for arg in def_node.local_scope.var_entries if arg.name] - self.varnames = TupleNode( - def_node.pos, - args=[IdentifierStringNode(arg.pos, value=arg.name) - for arg in args + local_vars], - is_temp=0, - is_literal=1) - - def may_be_none(self): - return False - - def calculate_result_code(self, code=None): - if self.result_code is None: - self.result_code = code.get_py_const(py_object_type, 'codeobj', cleanup_level=2) - return self.result_code - - def generate_result_code(self, code): - if self.result_code is None: - self.result_code = code.get_py_const(py_object_type, 'codeobj', cleanup_level=2) - - code = code.get_cached_constants_writer() - code.mark_pos(self.pos) - func = self.def_node - func_name = code.get_py_string_const( - func.name, identifier=True, is_str=False, unicode_value=func.name) - # FIXME: better way to get the module file path at module init time? Encoding to use? - file_path = StringEncoding.bytes_literal(func.pos[0].get_filenametable_entry().encode('utf8'), 'utf8') - file_path_const = code.get_py_string_const(file_path, identifier=False, is_str=True) - - flags = [] - if self.def_node.star_arg: - flags.append('CO_VARARGS') - if self.def_node.starstar_arg: - flags.append('CO_VARKEYWORDS') - - code.putln("%s = (PyObject*)__Pyx_PyCode_New(%d, %d, %d, 0, %s, %s, %s, %s, %s, %s, %s, %s, %s, %d, %s); %s" % ( - self.result_code, - len(func.args) - func.num_kwonly_args, # argcount - func.num_kwonly_args, # kwonlyargcount (Py3 only) - len(self.varnames.args), # nlocals - '|'.join(flags) or '0', # flags - Naming.empty_bytes, # code - Naming.empty_tuple, # consts - Naming.empty_tuple, # names (FIXME) - self.varnames.result(), # varnames - Naming.empty_tuple, # freevars (FIXME) - Naming.empty_tuple, # cellvars (FIXME) - file_path_const, # filename - func_name, # name - self.pos[1], # firstlineno - Naming.empty_bytes, # lnotab - code.error_goto_if_null(self.result_code, self.pos), - )) - - class DefaultLiteralArgNode(ExprNode): # CyFunction's literal argument default value # @@ -9071,7 +8993,7 @@ class LambdaNode(InnerFunctionNode): name = StringEncoding.EncodedString('') def analyse_declarations(self, env): - self.lambda_name = self.def_node.lambda_name = env.next_id('lambda') + self.lambda_name = self.def_node.lambda_name = StringEncoding.EncodedString(env.next_id('lambda')) self.def_node.no_assignment_synthesis = True self.def_node.pymethdef_required = True self.def_node.analyse_declarations(env) diff --git a/Cython/Compiler/FusedNode.py b/Cython/Compiler/FusedNode.py index 5a363027d..d8e419f45 100644 --- a/Cython/Compiler/FusedNode.py +++ b/Cython/Compiler/FusedNode.py @@ -739,14 +739,11 @@ class FusedCFuncDefNode(StatListNode): self.defaults_tuple = TupleNode(self.pos, args=args) self.defaults_tuple = self.defaults_tuple.analyse_types(env, skip_children=True).coerce_to_pyobject(env) self.defaults_tuple = ProxyNode(self.defaults_tuple) - self.code_object = ProxyNode(self.specialized_pycfuncs[0].code_object) fused_func = self.resulting_fused_function.arg fused_func.defaults_tuple = CloneNode(self.defaults_tuple) - fused_func.code_object = CloneNode(self.code_object) for i, pycfunc in enumerate(self.specialized_pycfuncs): - pycfunc.code_object = CloneNode(self.code_object) pycfunc = self.specialized_pycfuncs[i] = pycfunc.analyse_types(env) pycfunc.defaults_tuple = CloneNode(self.defaults_tuple) return self @@ -792,7 +789,6 @@ class FusedCFuncDefNode(StatListNode): if self.py_func: self.defaults_tuple.generate_evaluation_code(code) - self.code_object.generate_evaluation_code(code) for stat in self.stats: code.mark_pos(stat.pos) @@ -815,7 +811,6 @@ class FusedCFuncDefNode(StatListNode): # Dispose of results self.resulting_fused_function.generate_disposal_code(code) self.defaults_tuple.generate_disposal_code(code) - self.code_object.generate_disposal_code(code) for default in self.defaults: if default is not None: diff --git a/Cython/Compiler/Nodes.py b/Cython/Compiler/Nodes.py index b81ccb824..5ffc07100 100644 --- a/Cython/Compiler/Nodes.py +++ b/Cython/Compiler/Nodes.py @@ -1624,7 +1624,6 @@ class FuncDefNode(StatNode, BlockNode): star_arg = None starstar_arg = None is_cyfunction = False - code_object = None def analyse_default_values(self, env): default_seen = 0 @@ -1821,10 +1820,16 @@ class FuncDefNode(StatNode, BlockNode): elif lenv.nogil and lenv.has_with_gil_block: code.declare_gilstate() + if isinstance(self, DefNode) and self.is_wrapper: + trace_name = self.entry.name + " (wrapper)" + else: + trace_name = self.entry.name + if profile or linetrace: tempvardecl_code.put_trace_declarations() - code_object = self.code_object.calculate_result_code(code) if self.code_object else None - code.put_trace_frame_init(code_object) + code_object_cname = code.get_pycode_object_const( + EncodedString(trace_name), self, py_object_type) + code.put_trace_frame_init(code_object_cname) # ----- set up refnanny if use_refnanny: @@ -1892,12 +1897,7 @@ class FuncDefNode(StatNode, BlockNode): if profile or linetrace: # this looks a bit late, but if we don't get here due to a # fatal error before hand, it's not really worth tracing - if isinstance(self, DefNode) and self.is_wrapper: - trace_name = self.entry.name + " (wrapper)" - else: - trace_name = self.entry.name - code.put_trace_call( - trace_name, self.pos, nogil=not code.funcstate.gil_owned) + code.put_trace_call(trace_name, self.pos, nogil=not code.funcstate.gil_owned) code.funcstate.can_trace = True # ----- Fetch arguments self.generate_argument_parsing_code(env, code) diff --git a/Cython/Compiler/ParseTreeTransforms.py b/Cython/Compiler/ParseTreeTransforms.py index 71c228df0..86eecc9e1 100644 --- a/Cython/Compiler/ParseTreeTransforms.py +++ b/Cython/Compiler/ParseTreeTransforms.py @@ -1715,12 +1715,10 @@ if VALUE is not None: if genv.is_closure_scope: rhs = node.py_cfunc_node = ExprNodes.InnerFunctionNode( node.pos, def_node=node, - pymethdef_cname=node.entry.pymethdef_cname, - code_object=ExprNodes.CodeObjectNode(node)) + pymethdef_cname=node.entry.pymethdef_cname) else: binding = self.current_directives.get('binding') rhs = ExprNodes.PyCFunctionNode.from_defnode(node, binding) - node.code_object = rhs.code_object if env.is_py_class_scope: rhs.binding = True -- cgit v1.2.1