diff options
Diffstat (limited to 'Cython/Compiler/Code.py')
| -rw-r--r-- | Cython/Compiler/Code.py | 746 |
1 files changed, 435 insertions, 311 deletions
diff --git a/Cython/Compiler/Code.py b/Cython/Compiler/Code.py index d0b4756e5..089685496 100644 --- a/Cython/Compiler/Code.py +++ b/Cython/Compiler/Code.py @@ -1,4 +1,4 @@ -# cython: language_level = 2 +# cython: language_level=3str # cython: auto_pickle=False # # Code output module @@ -13,27 +13,21 @@ cython.declare(os=object, re=object, operator=object, textwrap=object, DebugFlags=object, basestring=object, defaultdict=object, closing=object, partial=object) +import hashlib +import operator import os import re import shutil -import sys -import operator import textwrap from string import Template from functools import partial -from contextlib import closing +from contextlib import closing, contextmanager from collections import defaultdict -try: - import hashlib -except ImportError: - import md5 as hashlib - from . import Naming from . import Options from . import DebugFlags from . import StringEncoding -from . import Version from .. import Utils from .Scanning import SourceDescriptor from ..StringIOTree import StringIOTree @@ -43,8 +37,6 @@ try: except ImportError: from builtins import str as basestring -KEYWORDS_MUST_BE_BYTES = sys.version_info < (2, 7) - non_portable_builtins_map = { # builtins that have different names in different Python versions @@ -101,20 +93,18 @@ uncachable_builtins = [ '__build_class__', 'ascii', # might deserve an implementation in Cython #'exec', # implemented in Cython - ## - Py2.7+ - 'memoryview', ## - platform specific 'WindowsError', ## - others '_', # e.g. used by gettext ] -special_py_methods = set([ +special_py_methods = cython.declare(frozenset, frozenset(( '__cinit__', '__dealloc__', '__richcmp__', '__next__', '__await__', '__aiter__', '__anext__', '__getreadbuffer__', '__getwritebuffer__', '__getsegcount__', - '__getcharbuffer__', '__getbuffer__', '__releasebuffer__' -]) + '__getcharbuffer__', '__getbuffer__', '__releasebuffer__', +))) modifier_output_mapper = { 'inline': 'CYTHON_INLINE' @@ -203,6 +193,28 @@ def get_utility_dir(): Cython_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) return os.path.join(Cython_dir, "Utility") +read_utilities_hook = None +""" +Override the hook for reading a utilities file that contains code fragments used +by the codegen. + +The hook functions takes the path of the utilities file, and returns a list +of strings, one per line. + +The default behavior is to open a file relative to get_utility_dir(). +""" + +def read_utilities_from_utility_dir(path): + """ + Read all lines of the file at the provided path from a path relative + to get_utility_dir(). + """ + filename = os.path.join(get_utility_dir(), path) + with closing(Utils.open_source_file(filename, encoding='UTF-8')) as f: + return f.readlines() + +# by default, read utilities from the utility directory. +read_utilities_hook = read_utilities_from_utility_dir class UtilityCodeBase(object): """ @@ -224,6 +236,15 @@ class UtilityCodeBase(object): [definitions] + ##### MyUtility ##### + #@subsitute: tempita + + [requires tempita substitution + - context can't be specified here though so only + tempita utility that requires no external context + will benefit from this tag + - only necessary when @required from non-tempita code] + for prototypes and implementation respectively. For non-python or -cython files backslashes should be used instead. 5 to 30 comment characters may be used on either side. @@ -242,8 +263,7 @@ class UtilityCodeBase(object): return code = '\n'.join(lines) - if tags and 'substitute' in tags and tags['substitute'] == set(['naming']): - del tags['substitute'] + if tags and 'substitute' in tags and 'naming' in tags['substitute']: try: code = Template(code).substitute(vars(Naming)) except (KeyError, ValueError) as e: @@ -259,15 +279,11 @@ class UtilityCodeBase(object): utility[1] = code else: all_tags = utility[2] - if KEYWORDS_MUST_BE_BYTES: - type = type.encode('ASCII') all_tags[type] = code if tags: all_tags = utility[2] for name, values in tags.items(): - if KEYWORDS_MUST_BE_BYTES: - name = name.encode('ASCII') all_tags.setdefault(name, set()).update(values) @classmethod @@ -276,7 +292,6 @@ class UtilityCodeBase(object): if utilities: return utilities - filename = os.path.join(get_utility_dir(), path) _, ext = os.path.splitext(path) if ext in ('.pyx', '.py', '.pxd', '.pxi'): comment = '#' @@ -292,8 +307,7 @@ class UtilityCodeBase(object): {'C': comment}).match match_type = re.compile(r'(.+)[.](proto(?:[.]\S+)?|impl|init|cleanup)$').match - with closing(Utils.open_source_file(filename, encoding='UTF-8')) as f: - all_lines = f.readlines() + all_lines = read_utilities_hook(path) utilities = defaultdict(lambda: [None, None, {}]) lines = [] @@ -335,43 +349,22 @@ class UtilityCodeBase(object): return utilities @classmethod - def load(cls, util_code_name, from_file=None, **kwargs): + def load(cls, util_code_name, from_file, **kwargs): """ Load utility code from a file specified by from_file (relative to - Cython/Utility) and name util_code_name. If from_file is not given, - load it from the file util_code_name.*. There should be only one - file matched by this pattern. + Cython/Utility) and name util_code_name. """ + if '::' in util_code_name: from_file, util_code_name = util_code_name.rsplit('::', 1) - if not from_file: - utility_dir = get_utility_dir() - prefix = util_code_name + '.' - try: - listing = os.listdir(utility_dir) - except OSError: - # XXX the code below assumes as 'zipimport.zipimporter' instance - # XXX should be easy to generalize, but too lazy right now to write it - import zipfile - global __loader__ - loader = __loader__ - archive = loader.archive - with closing(zipfile.ZipFile(archive)) as fileobj: - listing = [os.path.basename(name) - for name in fileobj.namelist() - if os.path.join(archive, name).startswith(utility_dir)] - files = [filename for filename in listing - if filename.startswith(prefix)] - if not files: - raise ValueError("No match found for utility code " + util_code_name) - if len(files) > 1: - raise ValueError("More than one filename match found for utility code " + util_code_name) - from_file = files[0] - + assert from_file utilities = cls.load_utilities_from_file(from_file) proto, impl, tags = utilities[util_code_name] if tags: + if "substitute" in tags and "tempita" in tags["substitute"]: + if not issubclass(cls, TempitaUtilityCode): + return TempitaUtilityCode.load(util_code_name, from_file, **kwargs) orig_kwargs = kwargs.copy() for name, values in tags.items(): if name in kwargs: @@ -385,6 +378,12 @@ class UtilityCodeBase(object): # dependencies are rarely unique, so use load_cached() when we can values = [cls.load_cached(dep, from_file) for dep in sorted(values)] + elif name == 'substitute': + # don't want to pass "naming" or "tempita" to the constructor + # since these will have been handled + values = values - {'naming', 'tempita'} + if not values: + continue elif not values: values = None elif len(values) == 1: @@ -404,11 +403,11 @@ class UtilityCodeBase(object): return cls(**kwargs) @classmethod - def load_cached(cls, utility_code_name, from_file=None, __cache={}): + def load_cached(cls, utility_code_name, from_file, __cache={}): """ Calls .load(), but using a per-type cache based on utility name and file name. """ - key = (cls, from_file, utility_code_name) + key = (utility_code_name, from_file, cls) try: return __cache[key] except KeyError: @@ -417,7 +416,7 @@ class UtilityCodeBase(object): return code @classmethod - def load_as_string(cls, util_code_name, from_file=None, **kwargs): + def load_as_string(cls, util_code_name, from_file, **kwargs): """ Load a utility code as a string. Returns (proto, implementation) """ @@ -437,7 +436,7 @@ class UtilityCodeBase(object): return "<%s(%s)>" % (type(self).__name__, self.name) def get_tree(self, **kwargs): - pass + return None def __deepcopy__(self, memodict=None): # No need to deep-copy utility code since it's essentially immutable. @@ -566,7 +565,7 @@ class UtilityCode(UtilityCodeBase): r'([a-zA-Z_]+),' # type cname r'\s*"([^"]+)",' # method name r'\s*([^),]+)' # object cname - r'((?:,\s*[^),]+)*)' # args* + r'((?:,[^),]+)*)' # args* r'\)', externalise, impl) assert 'CALL_UNBOUND_METHOD(' not in impl @@ -692,6 +691,7 @@ class LazyUtilityCode(UtilityCodeBase): class FunctionState(object): # return_label string function return point label # error_label string error catch point label + # error_without_exception boolean Can go to the error label without an exception (e.g. __next__ can return NULL) # continue_label string loop continue point label # break_label string loop break point label # return_from_error_cleanup_label string @@ -740,6 +740,8 @@ class FunctionState(object): self.should_declare_error_indicator = False self.uses_error_indicator = False + self.error_without_exception = False + # safety checks def validate_exit(self): @@ -770,9 +772,9 @@ class FunctionState(object): self.yield_labels.append(num_and_label) return num_and_label - def new_error_label(self): + def new_error_label(self, prefix=""): old_err_lbl = self.error_label - self.error_label = self.new_label('error') + self.error_label = self.new_label(prefix + 'error') return old_err_lbl def get_loop_labels(self): @@ -784,11 +786,11 @@ class FunctionState(object): (self.continue_label, self.break_label) = labels - def new_loop_labels(self): + def new_loop_labels(self, prefix=""): old_labels = self.get_loop_labels() self.set_loop_labels( - (self.new_label("continue"), - self.new_label("break"))) + (self.new_label(prefix + "continue"), + self.new_label(prefix + "break"))) return old_labels def get_all_labels(self): @@ -829,14 +831,14 @@ class FunctionState(object): allocated and released one of the same type). Type is simply registered and handed back, but will usually be a PyrexType. - If type.is_pyobject, manage_ref comes into play. If manage_ref is set to + If type.needs_refcounting, manage_ref comes into play. If manage_ref is set to True, the temp will be decref-ed on return statements and in exception handling clauses. Otherwise the caller has to deal with any reference counting of the variable. - If not type.is_pyobject, then manage_ref will be ignored, but it + If not type.needs_refcounting, then manage_ref will be ignored, but it still has to be passed. It is recommended to pass False by convention - if it is known that type will never be a Python object. + if it is known that type will never be a reference counted type. static=True marks the temporary declaration with "static". This is only used when allocating backing store for a module-level @@ -846,14 +848,16 @@ class FunctionState(object): A C string referring to the variable is returned. """ - if type.is_const and not type.is_reference: - type = type.const_base_type + if type.is_cv_qualified and not type.is_reference: + type = type.cv_base_type elif type.is_reference and not type.is_fake_reference: type = type.ref_base_type elif type.is_cfunction: from . import PyrexTypes type = PyrexTypes.c_ptr_type(type) # A function itself isn't an l-value - if not type.is_pyobject and not type.is_memoryviewslice: + elif type.is_cpp_class and not type.is_fake_reference and self.scope.directives['cpp_locals']: + self.scope.use_utility_code(UtilityCode.load_cached("OptionalLocals", "CppSupport.cpp")) + if not type.needs_refcounting: # Make manage_ref canonical, so that manage_ref will always mean # a decref is needed. manage_ref = False @@ -906,17 +910,17 @@ class FunctionState(object): for name, type, manage_ref, static in self.temps_allocated: freelist = self.temps_free.get((type, manage_ref)) if freelist is None or name not in freelist[1]: - used.append((name, type, manage_ref and type.is_pyobject)) + used.append((name, type, manage_ref and type.needs_refcounting)) return used def temps_holding_reference(self): """Return a list of (cname,type) tuples of temp names and their type - that are currently in use. This includes only temps of a - Python object type which owns its reference. + that are currently in use. This includes only temps + with a reference counted type which owns its reference. """ return [(name, type) for name, type, manage_ref in self.temps_in_use() - if manage_ref and type.is_pyobject] + if manage_ref and type.needs_refcounting] def all_managed_temps(self): """Return a list of (cname, type) tuples of refcount-managed Python objects. @@ -1121,10 +1125,10 @@ class GlobalState(object): 'h_code', 'filename_table', 'utility_code_proto_before_types', - 'numeric_typedefs', # Let these detailed individual parts stay!, - 'complex_type_declarations', # as the proper solution is to make a full DAG... - 'type_declarations', # More coarse-grained blocks would simply hide - 'utility_code_proto', # the ugliness, not fix it + 'numeric_typedefs', # Let these detailed individual parts stay!, + 'complex_type_declarations', # as the proper solution is to make a full DAG... + 'type_declarations', # More coarse-grained blocks would simply hide + 'utility_code_proto', # the ugliness, not fix it 'module_declarations', 'typeinfo', 'before_global_var', @@ -1132,19 +1136,34 @@ class GlobalState(object): 'string_decls', 'decls', 'late_includes', - 'all_the_rest', + 'module_state', + 'module_state_clear', + 'module_state_traverse', + 'module_state_defines', # redefines names used in module_state/_clear/_traverse + 'module_code', # user code goes here 'pystring_table', 'cached_builtins', 'cached_constants', - 'init_globals', + 'init_constants', + 'init_globals', # (utility code called at init-time) 'init_module', 'cleanup_globals', 'cleanup_module', 'main_method', + 'utility_code_pragmas', # silence some irrelevant warnings in utility code 'utility_code_def', + 'utility_code_pragmas_end', # clean-up the utility_code_pragmas 'end' ] + # h files can only have a much smaller list of sections + h_code_layout = [ + 'h_code', + 'utility_code_proto_before_types', + 'type_declarations', + 'utility_code_proto', + 'end' + ] def __init__(self, writer, module_node, code_config, common_utility_include_dir=None): self.filename_table = {} @@ -1156,8 +1175,8 @@ class GlobalState(object): self.code_config = code_config self.common_utility_include_dir = common_utility_include_dir self.parts = {} - self.module_node = module_node # because some utility code generation needs it - # (generating backwards-compatible Get/ReleaseBuffer + self.module_node = module_node # because some utility code generation needs it + # (generating backwards-compatible Get/ReleaseBuffer self.const_cnames_used = {} self.string_const_index = {} @@ -1173,8 +1192,10 @@ class GlobalState(object): def initialize_main_c_code(self): rootwriter = self.rootwriter - for part in self.code_layout: - self.parts[part] = rootwriter.insertion_point() + for i, part in enumerate(self.code_layout): + w = self.parts[part] = rootwriter.insertion_point() + if i > 0: + w.putln("/* #### Code section: %s ### */" % part) if not Options.cache_builtins: del self.parts['cached_builtins'] @@ -1188,13 +1209,18 @@ class GlobalState(object): w.putln("") w.putln("static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(void) {") w.put_declare_refcount_context() - w.put_setup_refcount_context("__Pyx_InitCachedConstants") + w.put_setup_refcount_context(StringEncoding.EncodedString("__Pyx_InitCachedConstants")) w = self.parts['init_globals'] w.enter_cfunc_scope() w.putln("") w.putln("static CYTHON_SMALL_CODE int __Pyx_InitGlobals(void) {") + w = self.parts['init_constants'] + w.enter_cfunc_scope() + w.putln("") + w.putln("static CYTHON_SMALL_CODE int __Pyx_InitConstants(void) {") + if not Options.generate_cleanup_code: del self.parts['cleanup_globals'] else: @@ -1213,6 +1239,11 @@ class GlobalState(object): code.putln("") code.putln("/* --- Runtime support code --- */") + def initialize_main_h_code(self): + rootwriter = self.rootwriter + for part in self.h_code_layout: + self.parts[part] = rootwriter.insertion_point() + def finalize_main_c_code(self): self.close_global_decls() @@ -1224,6 +1255,18 @@ class GlobalState(object): code.put(util.format_code(util.impl)) code.putln("") + # + # utility code pragmas + # + code = self.parts['utility_code_pragmas'] + util = UtilityCode.load_cached("UtilityCodePragmas", "ModuleSetupCode.c") + code.putln(util.format_code(util.impl)) + code.putln("") + code = self.parts['utility_code_pragmas_end'] + util = UtilityCode.load_cached("UtilityCodePragmasEnd", "ModuleSetupCode.c") + code.putln(util.format_code(util.impl)) + code.putln("") + def __getitem__(self, key): return self.parts[key] @@ -1253,13 +1296,14 @@ class GlobalState(object): w.putln("}") w.exit_cfunc_scope() - w = self.parts['init_globals'] - w.putln("return 0;") - if w.label_used(w.error_label): - w.put_label(w.error_label) - w.putln("return -1;") - w.putln("}") - w.exit_cfunc_scope() + for part in ['init_globals', 'init_constants']: + w = self.parts[part] + w.putln("return 0;") + if w.label_used(w.error_label): + w.put_label(w.error_label) + w.putln("return -1;") + w.putln("}") + w.exit_cfunc_scope() if Options.generate_cleanup_code: w = self.parts['cleanup_globals'] @@ -1306,8 +1350,11 @@ class GlobalState(object): return const # create a new Python object constant const = self.new_py_const(type, prefix) - if cleanup_level is not None \ - and cleanup_level <= Options.generate_cleanup_code: + if (cleanup_level is not None + and cleanup_level <= Options.generate_cleanup_code + # Note that this function is used for all argument defaults + # which aren't just Python objects + and type.needs_refcounting): cleanup_writer = self.parts['cleanup_globals'] cleanup_writer.putln('Py_CLEAR(%s);' % const.cname) if dedup_key is not None: @@ -1376,23 +1423,33 @@ class GlobalState(object): value = bytes_value.decode('ASCII', 'ignore') return self.new_const_cname(value=value) - def new_num_const_cname(self, value, py_type): + def unique_const_cname(self, format_str): # type: (str) -> str + used = self.const_cnames_used + cname = value = format_str.format(sep='', counter='') + while cname in used: + counter = used[value] = used[value] + 1 + cname = format_str.format(sep='_', counter=counter) + used[cname] = 1 + return cname + + def new_num_const_cname(self, value, py_type): # type: (str, str) -> str if py_type == 'long': value += 'L' py_type = 'int' prefix = Naming.interned_prefixes[py_type] - cname = "%s%s" % (prefix, value) - cname = cname.replace('+', '_').replace('-', 'neg_').replace('.', '_') + + value = value.replace('.', '_').replace('+', '_').replace('-', 'neg_') + if len(value) > 42: + # update tests/run/large_integer_T5290.py in case the amount is changed + cname = self.unique_const_cname( + prefix + "large{counter}_" + value[:18] + "_xxx_" + value[-18:]) + else: + cname = "%s%s" % (prefix, value) return cname def new_const_cname(self, prefix='', value=''): value = replace_identifier('_', value)[:32].strip('_') - used = self.const_cnames_used - name_suffix = value - while name_suffix in used: - counter = used[value] = used[value] + 1 - name_suffix = '%s_%d' % (value, counter) - used[name_suffix] = 1 + name_suffix = self.unique_const_cname(value + "{sep}{counter}") if prefix: prefix = Naming.interned_prefixes[prefix] else: @@ -1460,26 +1517,38 @@ class GlobalState(object): consts = [(len(c.cname), c.cname, c) for c in self.py_constants] consts.sort() - decls_writer = self.parts['decls'] for _, cname, c in consts: - decls_writer.putln( - "static %s;" % c.type.declaration_code(cname)) + self.parts['module_state'].putln("%s;" % c.type.declaration_code(cname)) + self.parts['module_state_defines'].putln( + "#define %s %s->%s" % (cname, Naming.modulestateglobal_cname, cname)) + if not c.type.needs_refcounting: + # Note that py_constants is used for all argument defaults + # which aren't necessarily PyObjects, so aren't appropriate + # to clear. + continue + self.parts['module_state_clear'].putln( + "Py_CLEAR(clear_module_state->%s);" % cname) + self.parts['module_state_traverse'].putln( + "Py_VISIT(traverse_module_state->%s);" % cname) def generate_cached_methods_decls(self): if not self.cached_cmethods: return decl = self.parts['decls'] - init = self.parts['init_globals'] + init = self.parts['init_constants'] cnames = [] for (type_cname, method_name), cname in sorted(self.cached_cmethods.items()): cnames.append(cname) method_name_cname = self.get_interned_identifier(StringEncoding.EncodedString(method_name)).cname - decl.putln('static __Pyx_CachedCFunction %s = {0, &%s, 0, 0, 0};' % ( - cname, method_name_cname)) + decl.putln('static __Pyx_CachedCFunction %s = {0, 0, 0, 0, 0};' % ( + cname)) # split type reference storage as it might not be static init.putln('%s.type = (PyObject*)&%s;' % ( cname, type_cname)) + # method name string isn't static in limited api + init.putln('%s.method_name = &%s;' % ( + cname, method_name_cname)) if Options.generate_cleanup_code: cleanup = self.parts['cleanup_globals'] @@ -1517,13 +1586,18 @@ class GlobalState(object): decls_writer.putln("static Py_UNICODE %s[] = { %s };" % (cname, utf16_array)) decls_writer.putln("#endif") + init_constants = self.parts['init_constants'] if py_strings: self.use_utility_code(UtilityCode.load_cached("InitStrings", "StringTools.c")) py_strings.sort() w = self.parts['pystring_table'] w.putln("") - w.putln("static __Pyx_StringTabEntry %s[] = {" % Naming.stringtab_cname) - for c_cname, _, py_string in py_strings: + w.putln("static int __Pyx_CreateStringTabAndInitStrings(void) {") + # the stringtab is a function local rather than a global to + # ensure that it doesn't conflict with module state + w.putln("__Pyx_StringTabEntry %s[] = {" % Naming.stringtab_cname) + for py_string_args in py_strings: + c_cname, _, py_string = py_string_args if not py_string.is_str or not py_string.encoding or \ py_string.encoding in ('ASCII', 'USASCII', 'US-ASCII', 'UTF8', 'UTF-8'): @@ -1531,8 +1605,15 @@ class GlobalState(object): else: encoding = '"%s"' % py_string.encoding.lower() - decls_writer.putln( - "static PyObject *%s;" % py_string.cname) + self.parts['module_state'].putln("PyObject *%s;" % py_string.cname) + self.parts['module_state_defines'].putln("#define %s %s->%s" % ( + py_string.cname, + Naming.modulestateglobal_cname, + py_string.cname)) + self.parts['module_state_clear'].putln("Py_CLEAR(clear_module_state->%s);" % + py_string.cname) + self.parts['module_state_traverse'].putln("Py_VISIT(traverse_module_state->%s);" % + py_string.cname) if py_string.py3str_cstring: w.putln("#if PY_MAJOR_VERSION >= 3") w.putln("{&%s, %s, sizeof(%s), %s, %d, %d, %d}," % ( @@ -1556,22 +1637,27 @@ class GlobalState(object): w.putln("#endif") w.putln("{0, 0, 0, 0, 0, 0, 0}") w.putln("};") + w.putln("return __Pyx_InitStrings(%s);" % Naming.stringtab_cname) + w.putln("}") - init_globals = self.parts['init_globals'] - init_globals.putln( - "if (__Pyx_InitStrings(%s) < 0) %s" % ( - Naming.stringtab_cname, - init_globals.error_goto(self.module_pos))) + init_constants.putln( + "if (__Pyx_CreateStringTabAndInitStrings() < 0) %s;" % + init_constants.error_goto(self.module_pos)) def generate_num_constants(self): consts = [(c.py_type, c.value[0] == '-', len(c.value), c.value, c.value_code, c) for c in self.num_const_index.values()] consts.sort() - decls_writer = self.parts['decls'] - init_globals = self.parts['init_globals'] + init_constants = self.parts['init_constants'] for py_type, _, _, value, value_code, c in consts: cname = c.cname - decls_writer.putln("static PyObject *%s;" % cname) + self.parts['module_state'].putln("PyObject *%s;" % cname) + self.parts['module_state_defines'].putln("#define %s %s->%s" % ( + cname, Naming.modulestateglobal_cname, cname)) + self.parts['module_state_clear'].putln( + "Py_CLEAR(clear_module_state->%s);" % cname) + self.parts['module_state_traverse'].putln( + "Py_VISIT(traverse_module_state->%s);" % cname) if py_type == 'float': function = 'PyFloat_FromDouble(%s)' elif py_type == 'long': @@ -1582,9 +1668,9 @@ class GlobalState(object): function = "PyInt_FromLong(%sL)" else: function = "PyInt_FromLong(%s)" - init_globals.putln('%s = %s; %s' % ( + init_constants.putln('%s = %s; %s' % ( cname, function % value_code, - init_globals.error_goto_if_null(cname, self.module_pos))) + init_constants.error_goto_if_null(cname, self.module_pos))) # The functions below are there in a transition phase only # and will be deprecated. They are called from Nodes.BlockNode. @@ -1759,10 +1845,21 @@ class CCodeWriter(object): return self.buffer.getvalue() def write(self, s): - # also put invalid markers (lineno 0), to indicate that those lines - # have no Cython source code correspondence - cython_lineno = self.last_marked_pos[1] if self.last_marked_pos else 0 - self.buffer.markers.extend([cython_lineno] * s.count('\n')) + if '\n' in s: + self._write_lines(s) + else: + self._write_to_buffer(s) + + def _write_lines(self, s): + # Cygdb needs to know which Cython source line corresponds to which C line. + # Therefore, we write this information into "self.buffer.markers" and then write it from there + # into cython_debug/cython_debug_info_* (see ModuleNode._serialize_lineno_map). + filename_line = self.last_marked_pos[:2] if self.last_marked_pos else (None, 0) + self.buffer.markers.extend([filename_line] * s.count('\n')) + + self._write_to_buffer(s) + + def _write_to_buffer(self, s): self.buffer.write(s) def insertion_point(self): @@ -1804,13 +1901,37 @@ class CCodeWriter(object): @funccontext_property def yield_labels(self): pass + def label_interceptor(self, new_labels, orig_labels, skip_to_label=None, pos=None, trace=True): + """ + Helper for generating multiple label interceptor code blocks. + + @param new_labels: the new labels that should be intercepted + @param orig_labels: the original labels that we should dispatch to after the interception + @param skip_to_label: a label to skip to before starting the code blocks + @param pos: the node position to mark for each interceptor block + @param trace: add a trace line for the pos marker or not + """ + for label, orig_label in zip(new_labels, orig_labels): + if not self.label_used(label): + continue + if skip_to_label: + # jump over the whole interception block + self.put_goto(skip_to_label) + skip_to_label = None + + if pos is not None: + self.mark_pos(pos, trace=trace) + self.put_label(label) + yield (label, orig_label) + self.put_goto(orig_label) + # Functions delegated to function scope def new_label(self, name=None): return self.funcstate.new_label(name) - def new_error_label(self): return self.funcstate.new_error_label() + def new_error_label(self, *args): return self.funcstate.new_error_label(*args) def new_yield_label(self, *args): return self.funcstate.new_yield_label(*args) def get_loop_labels(self): return self.funcstate.get_loop_labels() def set_loop_labels(self, labels): return self.funcstate.set_loop_labels(labels) - def new_loop_labels(self): return self.funcstate.new_loop_labels() + def new_loop_labels(self, *args): return self.funcstate.new_loop_labels(*args) def get_all_labels(self): return self.funcstate.get_all_labels() def set_all_labels(self, labels): return self.funcstate.set_all_labels(labels) def all_new_labels(self): return self.funcstate.all_new_labels() @@ -1822,6 +1943,7 @@ class CCodeWriter(object): self.funcstate = FunctionState(self, scope=scope) def exit_cfunc_scope(self): + self.funcstate.validate_exit() self.funcstate = None # constant handling @@ -1865,13 +1987,13 @@ class CCodeWriter(object): self.emit_marker() if self.code_config.emit_linenums and self.last_marked_pos: source_desc, line, _ = self.last_marked_pos - self.write('\n#line %s "%s"\n' % (line, source_desc.get_escaped_description())) + self._write_lines('\n#line %s "%s"\n' % (line, source_desc.get_escaped_description())) if code: if safe: self.put_safe(code) else: self.put(code) - self.write("\n") + self._write_lines("\n") self.bol = 1 def mark_pos(self, pos, trace=True): @@ -1885,13 +2007,13 @@ class CCodeWriter(object): pos, trace = self.last_pos self.last_marked_pos = pos self.last_pos = None - self.write("\n") + self._write_lines("\n") if self.code_config.emit_code_comments: self.indent() - self.write("/* %s */\n" % self._build_marker(pos)) + self._write_lines("/* %s */\n" % self._build_marker(pos)) if trace and self.funcstate and self.funcstate.can_trace and self.globalstate.directives['linetrace']: self.indent() - self.write('__Pyx_TraceLine(%d,%d,%s)\n' % ( + self._write_lines('__Pyx_TraceLine(%d,%d,%s)\n' % ( pos[1], not self.funcstate.gil_owned, self.error_goto(pos))) def _build_marker(self, pos): @@ -1912,7 +2034,7 @@ class CCodeWriter(object): include_dir = self.globalstate.common_utility_include_dir if include_dir and len(code) > 1024: include_file = "%s_%s.h" % ( - name, hashlib.md5(code.encode('utf8')).hexdigest()) + name, hashlib.sha1(code.encode('utf8')).hexdigest()) path = os.path.join(include_dir, include_file) if not os.path.exists(path): tmp_path = '%s.tmp%s' % (path, os.getpid()) @@ -1968,7 +2090,7 @@ class CCodeWriter(object): self.putln("}") def indent(self): - self.write(" " * self.level) + self._write_to_buffer(" " * self.level) def get_py_version_hex(self, pyversion): return "0x%02X%02X%02X%02X" % (tuple(pyversion) + (0,0,0,0))[:4] @@ -1990,26 +2112,33 @@ class CCodeWriter(object): if entry.visibility == "private" and not entry.used: #print "...private and not used, skipping", entry.cname ### return - if storage_class: - self.put("%s " % storage_class) if not entry.cf_used: self.put('CYTHON_UNUSED ') - self.put(entry.type.declaration_code( - entry.cname, dll_linkage=dll_linkage)) + if storage_class: + self.put("%s " % storage_class) + if entry.is_cpp_optional: + self.put(entry.type.cpp_optional_declaration_code( + entry.cname, dll_linkage=dll_linkage)) + else: + self.put(entry.type.declaration_code( + entry.cname, dll_linkage=dll_linkage)) if entry.init is not None: self.put_safe(" = %s" % entry.type.literal_code(entry.init)) elif entry.type.is_pyobject: self.put(" = NULL") self.putln(";") + self.funcstate.scope.use_entry_utility_code(entry) def put_temp_declarations(self, func_context): for name, type, manage_ref, static in func_context.temps_allocated: - decl = type.declaration_code(name) + if type.is_cpp_class and not type.is_fake_reference and func_context.scope.directives['cpp_locals']: + decl = type.cpp_optional_declaration_code(name) + else: + decl = type.declaration_code(name) if type.is_pyobject: self.putln("%s = NULL;" % decl) elif type.is_memoryviewslice: - from . import MemoryView - self.putln("%s = %s;" % (decl, MemoryView.memslice_entry_init)) + self.putln("%s = %s;" % (decl, type.literal_code(type.default_value))) else: self.putln("%s%s;" % (static and "static " or "", decl)) @@ -2024,7 +2153,7 @@ class CCodeWriter(object): self.putln("%sint %s = 0;" % (unused, Naming.clineno_cname)) def put_generated_by(self): - self.putln("/* Generated by Cython %s */" % Version.watermark) + self.putln(Utils.GENERATED_BY_MARKER) self.putln("") def put_h_guard(self, guard): @@ -2047,7 +2176,7 @@ class CCodeWriter(object): def entry_as_pyobject(self, entry): type = entry.type if (not entry.is_self_arg and not entry.type.is_complete() - or entry.type.is_extension_type): + or entry.type.is_extension_type): return "(PyObject *)" + entry.cname else: return entry.cname @@ -2056,123 +2185,89 @@ class CCodeWriter(object): from .PyrexTypes import py_object_type, typecast return typecast(py_object_type, type, cname) - def put_gotref(self, cname): - self.putln("__Pyx_GOTREF(%s);" % cname) + def put_gotref(self, cname, type): + type.generate_gotref(self, cname) - def put_giveref(self, cname): - self.putln("__Pyx_GIVEREF(%s);" % cname) + def put_giveref(self, cname, type): + type.generate_giveref(self, cname) - def put_xgiveref(self, cname): - self.putln("__Pyx_XGIVEREF(%s);" % cname) + def put_xgiveref(self, cname, type): + type.generate_xgiveref(self, cname) - def put_xgotref(self, cname): - self.putln("__Pyx_XGOTREF(%s);" % cname) + def put_xgotref(self, cname, type): + type.generate_xgotref(self, cname) def put_incref(self, cname, type, nanny=True): - if nanny: - self.putln("__Pyx_INCREF(%s);" % self.as_pyobject(cname, type)) - else: - self.putln("Py_INCREF(%s);" % self.as_pyobject(cname, type)) + # Note: original put_Memslice_Incref/Decref also added in some utility code + # this is unnecessary since the relevant utility code is loaded anyway if a memoryview is used + # and so has been removed. However, it's potentially a feature that might be useful here + type.generate_incref(self, cname, nanny=nanny) - def put_decref(self, cname, type, nanny=True): - self._put_decref(cname, type, nanny, null_check=False, clear=False) + def put_xincref(self, cname, type, nanny=True): + type.generate_xincref(self, cname, nanny=nanny) - def put_var_gotref(self, entry): - if entry.type.is_pyobject: - self.putln("__Pyx_GOTREF(%s);" % self.entry_as_pyobject(entry)) + def put_decref(self, cname, type, nanny=True, have_gil=True): + type.generate_decref(self, cname, nanny=nanny, have_gil=have_gil) - def put_var_giveref(self, entry): - if entry.type.is_pyobject: - self.putln("__Pyx_GIVEREF(%s);" % self.entry_as_pyobject(entry)) + def put_xdecref(self, cname, type, nanny=True, have_gil=True): + type.generate_xdecref(self, cname, nanny=nanny, have_gil=have_gil) - def put_var_xgotref(self, entry): - if entry.type.is_pyobject: - self.putln("__Pyx_XGOTREF(%s);" % self.entry_as_pyobject(entry)) + def put_decref_clear(self, cname, type, clear_before_decref=False, nanny=True, have_gil=True): + type.generate_decref_clear(self, cname, clear_before_decref=clear_before_decref, + nanny=nanny, have_gil=have_gil) - def put_var_xgiveref(self, entry): - if entry.type.is_pyobject: - self.putln("__Pyx_XGIVEREF(%s);" % self.entry_as_pyobject(entry)) + def put_xdecref_clear(self, cname, type, clear_before_decref=False, nanny=True, have_gil=True): + type.generate_xdecref_clear(self, cname, clear_before_decref=clear_before_decref, + nanny=nanny, have_gil=have_gil) - def put_var_incref(self, entry, nanny=True): - if entry.type.is_pyobject: - if nanny: - self.putln("__Pyx_INCREF(%s);" % self.entry_as_pyobject(entry)) - else: - self.putln("Py_INCREF(%s);" % self.entry_as_pyobject(entry)) + def put_decref_set(self, cname, type, rhs_cname): + type.generate_decref_set(self, cname, rhs_cname) - def put_var_xincref(self, entry): - if entry.type.is_pyobject: - self.putln("__Pyx_XINCREF(%s);" % self.entry_as_pyobject(entry)) + def put_xdecref_set(self, cname, type, rhs_cname): + type.generate_xdecref_set(self, cname, rhs_cname) - def put_decref_clear(self, cname, type, nanny=True, clear_before_decref=False): - self._put_decref(cname, type, nanny, null_check=False, - clear=True, clear_before_decref=clear_before_decref) + def put_incref_memoryviewslice(self, slice_cname, type, have_gil): + # TODO ideally this would just be merged into "put_incref" + type.generate_incref_memoryviewslice(self, slice_cname, have_gil=have_gil) - def put_xdecref(self, cname, type, nanny=True, have_gil=True): - self._put_decref(cname, type, nanny, null_check=True, - have_gil=have_gil, clear=False) + def put_var_incref_memoryviewslice(self, entry, have_gil): + self.put_incref_memoryviewslice(entry.cname, entry.type, have_gil=have_gil) - def put_xdecref_clear(self, cname, type, nanny=True, clear_before_decref=False): - self._put_decref(cname, type, nanny, null_check=True, - clear=True, clear_before_decref=clear_before_decref) + def put_var_gotref(self, entry): + self.put_gotref(entry.cname, entry.type) - def _put_decref(self, cname, type, nanny=True, null_check=False, - have_gil=True, clear=False, clear_before_decref=False): - if type.is_memoryviewslice: - self.put_xdecref_memoryviewslice(cname, have_gil=have_gil) - return + def put_var_giveref(self, entry): + self.put_giveref(entry.cname, entry.type) - prefix = '__Pyx' if nanny else 'Py' - X = 'X' if null_check else '' + def put_var_xgotref(self, entry): + self.put_xgotref(entry.cname, entry.type) - if clear: - if clear_before_decref: - if not nanny: - X = '' # CPython doesn't have a Py_XCLEAR() - self.putln("%s_%sCLEAR(%s);" % (prefix, X, cname)) - else: - self.putln("%s_%sDECREF(%s); %s = 0;" % ( - prefix, X, self.as_pyobject(cname, type), cname)) - else: - self.putln("%s_%sDECREF(%s);" % ( - prefix, X, self.as_pyobject(cname, type))) + def put_var_xgiveref(self, entry): + self.put_xgiveref(entry.cname, entry.type) - def put_decref_set(self, cname, rhs_cname): - self.putln("__Pyx_DECREF_SET(%s, %s);" % (cname, rhs_cname)) + def put_var_incref(self, entry, **kwds): + self.put_incref(entry.cname, entry.type, **kwds) - def put_xdecref_set(self, cname, rhs_cname): - self.putln("__Pyx_XDECREF_SET(%s, %s);" % (cname, rhs_cname)) + def put_var_xincref(self, entry, **kwds): + self.put_xincref(entry.cname, entry.type, **kwds) - def put_var_decref(self, entry): - if entry.type.is_pyobject: - self.putln("__Pyx_XDECREF(%s);" % self.entry_as_pyobject(entry)) + def put_var_decref(self, entry, **kwds): + self.put_decref(entry.cname, entry.type, **kwds) - def put_var_xdecref(self, entry, nanny=True): - if entry.type.is_pyobject: - if nanny: - self.putln("__Pyx_XDECREF(%s);" % self.entry_as_pyobject(entry)) - else: - self.putln("Py_XDECREF(%s);" % self.entry_as_pyobject(entry)) - - def put_var_decref_clear(self, entry): - self._put_var_decref_clear(entry, null_check=False) - - def put_var_xdecref_clear(self, entry): - self._put_var_decref_clear(entry, null_check=True) - - def _put_var_decref_clear(self, entry, null_check): - if entry.type.is_pyobject: - if entry.in_closure: - # reset before DECREF to make sure closure state is - # consistent during call to DECREF() - self.putln("__Pyx_%sCLEAR(%s);" % ( - null_check and 'X' or '', - entry.cname)) - else: - self.putln("__Pyx_%sDECREF(%s); %s = 0;" % ( - null_check and 'X' or '', - self.entry_as_pyobject(entry), - entry.cname)) + def put_var_xdecref(self, entry, **kwds): + self.put_xdecref(entry.cname, entry.type, **kwds) + + def put_var_decref_clear(self, entry, **kwds): + self.put_decref_clear(entry.cname, entry.type, clear_before_decref=entry.in_closure, **kwds) + + def put_var_decref_set(self, entry, rhs_cname, **kwds): + self.put_decref_set(entry.cname, entry.type, rhs_cname, **kwds) + + def put_var_xdecref_set(self, entry, rhs_cname, **kwds): + self.put_xdecref_set(entry.cname, entry.type, rhs_cname, **kwds) + + def put_var_xdecref_clear(self, entry, **kwds): + self.put_xdecref_clear(entry.cname, entry.type, clear_before_decref=entry.in_closure, **kwds) def put_var_decrefs(self, entries, used_only = 0): for entry in entries: @@ -2190,19 +2285,6 @@ class CCodeWriter(object): for entry in entries: self.put_var_xdecref_clear(entry) - def put_incref_memoryviewslice(self, slice_cname, have_gil=False): - from . import MemoryView - self.globalstate.use_utility_code(MemoryView.memviewslice_init_code) - self.putln("__PYX_INC_MEMVIEW(&%s, %d);" % (slice_cname, int(have_gil))) - - def put_xdecref_memoryviewslice(self, slice_cname, have_gil=False): - from . import MemoryView - self.globalstate.use_utility_code(MemoryView.memviewslice_init_code) - self.putln("__PYX_XDEC_MEMVIEW(&%s, %d);" % (slice_cname, int(have_gil))) - - def put_xgiveref_memoryviewslice(self, slice_cname): - self.put_xgiveref("%s.memview" % slice_cname) - def put_init_to_py_none(self, cname, type, nanny=True): from .PyrexTypes import py_object_type, typecast py_none = typecast(type, py_object_type, "Py_None") @@ -2220,8 +2302,11 @@ class CCodeWriter(object): self.put_giveref('Py_None') def put_pymethoddef(self, entry, term, allow_skip=True, wrapper_code_writer=None): + is_reverse_number_slot = False if entry.is_special or entry.name == '__getattribute__': - if entry.name not in special_py_methods: + from . import TypeSlots + is_reverse_number_slot = True + if entry.name not in special_py_methods and not TypeSlots.is_reverse_number_slot(entry.name): if entry.name == '__getattr__' and not self.globalstate.directives['fast_getattr']: pass # Python's typeobject.c will automatically fill in our slot @@ -2233,37 +2318,60 @@ class CCodeWriter(object): method_flags = entry.signature.method_flags() if not method_flags: return - from . import TypeSlots - if entry.is_special or TypeSlots.is_reverse_number_slot(entry.name): + if entry.is_special: method_flags += [TypeSlots.method_coexist] func_ptr = wrapper_code_writer.put_pymethoddef_wrapper(entry) if wrapper_code_writer else entry.func_cname # Add required casts, but try not to shadow real warnings. - cast = '__Pyx_PyCFunctionFast' if 'METH_FASTCALL' in method_flags else 'PyCFunction' - if 'METH_KEYWORDS' in method_flags: - cast += 'WithKeywords' + cast = entry.signature.method_function_type() if cast != 'PyCFunction': func_ptr = '(void*)(%s)%s' % (cast, func_ptr) + entry_name = entry.name.as_c_string_literal() + if is_reverse_number_slot: + # Unlike most special functions, reverse number operator slots are actually generated here + # (to ensure that they can be looked up). However, they're sometimes guarded by the preprocessor + # so a bit of extra logic is needed + slot = TypeSlots.get_slot_table(self.globalstate.directives).get_slot_by_method_name(entry.name) + preproc_guard = slot.preprocessor_guard_code() + if preproc_guard: + self.putln(preproc_guard) self.putln( - '{"%s", (PyCFunction)%s, %s, %s}%s' % ( - entry.name, + '{%s, (PyCFunction)%s, %s, %s}%s' % ( + entry_name, func_ptr, "|".join(method_flags), entry.doc_cname if entry.doc else '0', term)) + if is_reverse_number_slot and preproc_guard: + self.putln("#endif") def put_pymethoddef_wrapper(self, entry): func_cname = entry.func_cname if entry.is_special: - method_flags = entry.signature.method_flags() - if method_flags and 'METH_NOARGS' in method_flags: + method_flags = entry.signature.method_flags() or [] + from .TypeSlots import method_noargs + if method_noargs in method_flags: # Special NOARGS methods really take no arguments besides 'self', but PyCFunction expects one. func_cname = Naming.method_wrapper_prefix + func_cname - self.putln("static PyObject *%s(PyObject *self, CYTHON_UNUSED PyObject *arg) {return %s(self);}" % ( - func_cname, entry.func_cname)) + self.putln("static PyObject *%s(PyObject *self, CYTHON_UNUSED PyObject *arg) {" % func_cname) + func_call = "%s(self)" % entry.func_cname + if entry.name == "__next__": + self.putln("PyObject *res = %s;" % func_call) + # tp_iternext can return NULL without an exception + self.putln("if (!res && !PyErr_Occurred()) { PyErr_SetNone(PyExc_StopIteration); }") + self.putln("return res;") + else: + self.putln("return %s;" % func_call) + self.putln("}") return func_cname # GIL methods + def use_fast_gil_utility_code(self): + if self.globalstate.directives['fast_gil']: + self.globalstate.use_utility_code(UtilityCode.load_cached("FastGil", "ModuleSetupCode.c")) + else: + self.globalstate.use_utility_code(UtilityCode.load_cached("NoFastGil", "ModuleSetupCode.c")) + def put_ensure_gil(self, declare_gilstate=True, variable=None): """ Acquire the GIL. The generated code is safe even when no PyThreadState @@ -2273,10 +2381,7 @@ class CCodeWriter(object): """ self.globalstate.use_utility_code( UtilityCode.load_cached("ForceInitThreads", "ModuleSetupCode.c")) - if self.globalstate.directives['fast_gil']: - self.globalstate.use_utility_code(UtilityCode.load_cached("FastGil", "ModuleSetupCode.c")) - else: - self.globalstate.use_utility_code(UtilityCode.load_cached("NoFastGil", "ModuleSetupCode.c")) + self.use_fast_gil_utility_code() self.putln("#ifdef WITH_THREAD") if not variable: variable = '__pyx_gilstate_save' @@ -2289,41 +2394,43 @@ class CCodeWriter(object): """ Releases the GIL, corresponds to `put_ensure_gil`. """ - if self.globalstate.directives['fast_gil']: - self.globalstate.use_utility_code(UtilityCode.load_cached("FastGil", "ModuleSetupCode.c")) - else: - self.globalstate.use_utility_code(UtilityCode.load_cached("NoFastGil", "ModuleSetupCode.c")) + self.use_fast_gil_utility_code() if not variable: variable = '__pyx_gilstate_save' self.putln("#ifdef WITH_THREAD") self.putln("__Pyx_PyGILState_Release(%s);" % variable) self.putln("#endif") - def put_acquire_gil(self, variable=None): + def put_acquire_gil(self, variable=None, unknown_gil_state=True): """ Acquire the GIL. The thread's thread state must have been initialized by a previous `put_release_gil` """ - if self.globalstate.directives['fast_gil']: - self.globalstate.use_utility_code(UtilityCode.load_cached("FastGil", "ModuleSetupCode.c")) - else: - self.globalstate.use_utility_code(UtilityCode.load_cached("NoFastGil", "ModuleSetupCode.c")) + self.use_fast_gil_utility_code() self.putln("#ifdef WITH_THREAD") self.putln("__Pyx_FastGIL_Forget();") if variable: self.putln('_save = %s;' % variable) + if unknown_gil_state: + self.putln("if (_save) {") self.putln("Py_BLOCK_THREADS") + if unknown_gil_state: + self.putln("}") self.putln("#endif") - def put_release_gil(self, variable=None): + def put_release_gil(self, variable=None, unknown_gil_state=True): "Release the GIL, corresponds to `put_acquire_gil`." - if self.globalstate.directives['fast_gil']: - self.globalstate.use_utility_code(UtilityCode.load_cached("FastGil", "ModuleSetupCode.c")) - else: - self.globalstate.use_utility_code(UtilityCode.load_cached("NoFastGil", "ModuleSetupCode.c")) + self.use_fast_gil_utility_code() self.putln("#ifdef WITH_THREAD") self.putln("PyThreadState *_save;") + self.putln("_save = NULL;") + if unknown_gil_state: + # we don't *know* that we don't have the GIL (since we may be inside a nogil function, + # and Py_UNBLOCK_THREADS is unsafe without the GIL) + self.putln("if (PyGILState_Check()) {") self.putln("Py_UNBLOCK_THREADS") + if unknown_gil_state: + self.putln("}") if variable: self.putln('%s = _save;' % variable) self.putln("__Pyx_FastGIL_Remember();") @@ -2341,23 +2448,34 @@ class CCodeWriter(object): # return self.putln("if (unlikely(%s < 0)) %s" % (value, self.error_goto(pos))) return self.putln("if (%s < 0) %s" % (value, self.error_goto(pos))) - def put_error_if_unbound(self, pos, entry, in_nogil_context=False): - from . import ExprNodes + def put_error_if_unbound(self, pos, entry, in_nogil_context=False, unbound_check_code=None): if entry.from_closure: func = '__Pyx_RaiseClosureNameError' self.globalstate.use_utility_code( - ExprNodes.raise_closure_name_error_utility_code) + UtilityCode.load_cached("RaiseClosureNameError", "ObjectHandling.c")) elif entry.type.is_memoryviewslice and in_nogil_context: func = '__Pyx_RaiseUnboundMemoryviewSliceNogil' self.globalstate.use_utility_code( - ExprNodes.raise_unbound_memoryview_utility_code_nogil) + UtilityCode.load_cached("RaiseUnboundMemoryviewSliceNogil", "ObjectHandling.c")) + elif entry.type.is_cpp_class and entry.is_cglobal: + func = '__Pyx_RaiseCppGlobalNameError' + self.globalstate.use_utility_code( + UtilityCode.load_cached("RaiseCppGlobalNameError", "ObjectHandling.c")) + elif entry.type.is_cpp_class and entry.is_variable and not entry.is_member and entry.scope.is_c_class_scope: + # there doesn't seem to be a good way to detecting an instance-attribute of a C class + # (is_member is only set for class attributes) + func = '__Pyx_RaiseCppAttributeError' + self.globalstate.use_utility_code( + UtilityCode.load_cached("RaiseCppAttributeError", "ObjectHandling.c")) else: func = '__Pyx_RaiseUnboundLocalError' self.globalstate.use_utility_code( - ExprNodes.raise_unbound_local_error_utility_code) + UtilityCode.load_cached("RaiseUnboundLocalError", "ObjectHandling.c")) + if not unbound_check_code: + unbound_check_code = entry.type.check_for_null_code(entry.cname) self.putln('if (unlikely(!%s)) { %s("%s"); %s }' % ( - entry.type.check_for_null_code(entry.cname), + unbound_check_code, func, entry.name, self.error_goto(pos))) @@ -2390,7 +2508,8 @@ class CCodeWriter(object): return self.error_goto_if("!%s" % cname, pos) def error_goto_if_neg(self, cname, pos): - return self.error_goto_if("%s < 0" % cname, pos) + # Add extra parentheses to silence clang warnings about constant conditions. + return self.error_goto_if("(%s < 0)" % cname, pos) def error_goto_if_PyErr(self, pos): return self.error_goto_if("PyErr_Occurred()", pos) @@ -2402,13 +2521,14 @@ class CCodeWriter(object): self.putln('__Pyx_RefNannyDeclarations') def put_setup_refcount_context(self, name, acquire_gil=False): + name = name.as_c_string_literal() # handle unicode names if acquire_gil: self.globalstate.use_utility_code( UtilityCode.load_cached("ForceInitThreads", "ModuleSetupCode.c")) - self.putln('__Pyx_RefNannySetupContext("%s", %d);' % (name, acquire_gil and 1 or 0)) + self.putln('__Pyx_RefNannySetupContext(%s, %d);' % (name, acquire_gil and 1 or 0)) - def put_finish_refcount_context(self): - self.putln("__Pyx_RefNannyFinishContext();") + def put_finish_refcount_context(self, nogil=False): + self.putln("__Pyx_RefNannyFinishContextNogil()" if nogil else "__Pyx_RefNannyFinishContext();") def put_add_traceback(self, qualified_name, include_cline=True): """ @@ -2416,14 +2536,16 @@ class CCodeWriter(object): qualified_name should be the qualified name of the function. """ + qualified_name = qualified_name.as_c_string_literal() # handle unicode names format_tuple = ( qualified_name, Naming.clineno_cname if include_cline else 0, Naming.lineno_cname, Naming.filename_cname, ) + self.funcstate.uses_error_indicator = True - self.putln('__Pyx_AddTraceback("%s", %s, %s, %s);' % format_tuple) + self.putln('__Pyx_AddTraceback(%s, %s, %s, %s);' % format_tuple) def put_unraisable(self, qualified_name, nogil=False): """ @@ -2504,16 +2626,16 @@ class PyrexCodeWriter(object): def dedent(self): self.level -= 1 + class PyxCodeWriter(object): """ - Can be used for writing out some Cython code. To use the indenter - functionality, the Cython.Compiler.Importer module will have to be used - to load the code to support python 2.4 + Can be used for writing out some Cython code. """ def __init__(self, buffer=None, indent_level=0, context=None, encoding='ascii'): self.buffer = buffer or StringIOTree() self.level = indent_level + self.original_level = indent_level self.context = context self.encoding = encoding @@ -2524,22 +2646,19 @@ class PyxCodeWriter(object): def dedent(self, levels=1): self.level -= levels + @contextmanager def indenter(self, line): """ - Instead of - - with pyx_code.indenter("for i in range(10):"): - pyx_code.putln("print i") - - write - - if pyx_code.indenter("for i in range(10);"): - pyx_code.putln("print i") - pyx_code.dedent() + with pyx_code.indenter("for i in range(10):"): + pyx_code.putln("print i") """ self.putln(line) self.indent() - return True + yield + self.dedent() + + def empty(self): + return self.buffer.empty() def getvalue(self): result = self.buffer.getvalue() @@ -2554,7 +2673,7 @@ class PyxCodeWriter(object): self._putln(line) def _putln(self, line): - self.buffer.write("%s%s\n" % (self.level * " ", line)) + self.buffer.write(u"%s%s\n" % (self.level * u" ", line)) def put_chunk(self, chunk, context=None): context = context or self.context @@ -2566,8 +2685,13 @@ class PyxCodeWriter(object): self._putln(line) def insertion_point(self): - return PyxCodeWriter(self.buffer.insertion_point(), self.level, - self.context) + return type(self)(self.buffer.insertion_point(), self.level, self.context) + + def reset(self): + # resets the buffer so that nothing gets written. Most useful + # for abandoning all work in a specific insertion point + self.buffer.reset() + self.level = self.original_level def named_insertion_point(self, name): setattr(self, name, self.insertion_point()) |