diff options
| author | Mark Florisson <markflorisson88@gmail.com> | 2011-07-25 21:13:10 +0200 |
|---|---|---|
| committer | Mark Florisson <markflorisson88@gmail.com> | 2011-09-30 14:55:17 +0100 |
| commit | 83e7a363fcc757ebab94b544b0ac64c270e73e39 (patch) | |
| tree | 4d032bf6fc912b739f07bbbd69f8ba28915107d2 | |
| parent | aad49345e18a162ab0caa332b5379729b76f90fe (diff) | |
| download | cython-83e7a363fcc757ebab94b544b0ac64c270e73e39.tar.gz | |
MemoryViewSlice indexing and object coercion + MemoryView indexing
| -rw-r--r-- | Cython/Compiler/Buffer.py | 89 | ||||
| -rw-r--r-- | Cython/Compiler/Code.py | 116 | ||||
| -rw-r--r-- | Cython/Compiler/CythonScope.py | 22 | ||||
| -rwxr-xr-x | Cython/Compiler/ExprNodes.py | 99 | ||||
| -rw-r--r-- | Cython/Compiler/Main.py | 8 | ||||
| -rw-r--r-- | Cython/Compiler/MemoryView.py | 96 | ||||
| -rw-r--r-- | Cython/Compiler/ModuleNode.py | 12 | ||||
| -rw-r--r-- | Cython/Compiler/Nodes.py | 40 | ||||
| -rwxr-xr-x | Cython/Compiler/PyrexTypes.py | 77 | ||||
| -rw-r--r-- | Cython/Compiler/Symtab.py | 4 | ||||
| -rw-r--r-- | Cython/Compiler/UtilityCode.py | 15 | ||||
| -rw-r--r-- | Cython/Utility/MemoryView.pyx | 165 | ||||
| -rw-r--r-- | Cython/Utility/MemoryView_C.c | 126 | ||||
| -rw-r--r-- | Cython/Utils.py | 8 | ||||
| -rw-r--r-- | tests/run/bufaccess.pyx | 319 | ||||
| -rw-r--r-- | tests/run/cythonarray.pyx | 38 | ||||
| -rw-r--r-- | tests/run/memslice.pyx | 1102 | ||||
| -rw-r--r-- | tests/run/memslice_indexing.pyx | 40 | ||||
| -rw-r--r-- | tests/run/mockbuffers.pxi | 313 |
19 files changed, 2119 insertions, 570 deletions
diff --git a/Cython/Compiler/Buffer.py b/Cython/Compiler/Buffer.py index 39a053fd9..6825a13ca 100644 --- a/Cython/Compiler/Buffer.py +++ b/Cython/Compiler/Buffer.py @@ -215,6 +215,50 @@ class BufferEntry(object): def _for_all_ndim(self, s): return [s % (self.cname, i) for i in range(self.type.ndim)] + def generate_buffer_lookup_code(self, code, index_cnames): + # Create buffer lookup and return it + # This is done via utility macros/inline functions, which vary + # according to the access mode used. + params = [] + nd = self.type.ndim + mode = self.type.mode + if mode == 'full': + for i, s, o in zip(index_cnames, + self.get_buf_stridevars(), + self.get_buf_suboffsetvars()): + params.append(i) + params.append(s) + params.append(o) + funcname = "__Pyx_BufPtrFull%dd" % nd + funcgen = buf_lookup_full_code + else: + if mode == 'strided': + funcname = "__Pyx_BufPtrStrided%dd" % nd + funcgen = buf_lookup_strided_code + elif mode == 'c': + funcname = "__Pyx_BufPtrCContig%dd" % nd + funcgen = buf_lookup_c_code + elif mode == 'fortran': + funcname = "__Pyx_BufPtrFortranContig%dd" % nd + funcgen = buf_lookup_fortran_code + else: + assert False + for i, s in zip(index_cnames, self.get_buf_stridevars()): + params.append(i) + params.append(s) + + # Make sure the utility code is available + if funcname not in code.globalstate.utility_codes: + code.globalstate.utility_codes.add(funcname) + protocode = code.globalstate['utility_code_proto'] + defcode = code.globalstate['utility_code_def'] + funcgen(protocode, defcode, name=funcname, nd=nd) + + buf_ptr_type_code = self.buf_ptr_type.declaration_code("") + ptrcode = "%s(%s, %s, %s)" % (funcname, buf_ptr_type_code, self.buf_ptr, + ", ".join(params)) + return ptrcode + def get_flags(buffer_aux, buffer_type): flags = 'PyBUF_FORMAT' @@ -412,7 +456,7 @@ def put_buffer_lookup_code(entry, index_signeds, index_cnames, directives, cast = "(size_t)" code.putln("if (%s) %s = %d;" % ( code.unlikely("%s >= %s%s" % (cname, cast, shape)), - tmp_cname, dim)) + tmp_cname, dim)) code.globalstate.use_utility_code(raise_indexerror_code) code.putln("if (%s) {" % code.unlikely("%s != -1" % tmp_cname)) code.putln('__Pyx_RaiseBufferIndexError(%s);' % tmp_cname) @@ -426,48 +470,7 @@ def put_buffer_lookup_code(entry, index_signeds, index_cnames, directives, if signed != 0: code.putln("if (%s < 0) %s += %s;" % (cname, cname, shape)) - # Create buffer lookup and return it - # This is done via utility macros/inline functions, which vary - # according to the access mode used. - params = [] - nd = entry.type.ndim - mode = entry.type.mode - if mode == 'full': - for i, s, o in zip(index_cnames, - entry.get_buf_stridevars(), - entry.get_buf_suboffsetvars()): - params.append(i) - params.append(s) - params.append(o) - funcname = "__Pyx_BufPtrFull%dd" % nd - funcgen = buf_lookup_full_code - else: - if mode == 'strided': - funcname = "__Pyx_BufPtrStrided%dd" % nd - funcgen = buf_lookup_strided_code - elif mode == 'c': - funcname = "__Pyx_BufPtrCContig%dd" % nd - funcgen = buf_lookup_c_code - elif mode == 'fortran': - funcname = "__Pyx_BufPtrFortranContig%dd" % nd - funcgen = buf_lookup_fortran_code - else: - assert False - for i, s in zip(index_cnames, entry.get_buf_stridevars()): - params.append(i) - params.append(s) - - # Make sure the utility code is available - if funcname not in code.globalstate.utility_codes: - code.globalstate.utility_codes.add(funcname) - protocode = code.globalstate['utility_code_proto'] - defcode = code.globalstate['utility_code_def'] - funcgen(protocode, defcode, name=funcname, nd=nd) - - buf_ptr_type_code = entry.buf_ptr_type.declaration_code("") - ptrcode = "%s(%s, %s, %s)" % (funcname, buf_ptr_type_code, entry.buf_ptr, - ", ".join(params)) - return ptrcode + return entry.generate_buffer_lookup_code(code, index_cnames) def use_bufstruct_declare_code(env): diff --git a/Cython/Compiler/Code.py b/Cython/Compiler/Code.py index f17f08cf6..595afc8fa 100644 --- a/Cython/Compiler/Code.py +++ b/Cython/Compiler/Code.py @@ -23,7 +23,6 @@ import DebugFlags import Errors from Cython import Tempita as tempita -from Cython.Utils import none_or_sub try: from __builtin__ import basestring except ImportError: @@ -162,10 +161,10 @@ class UtilityCodeBase(object): kwargs['impl'] = impl if 'name' not in kwargs: - if from_file: - kwargs['name'] = os.path.splitext(from_file)[0] - else: - kwargs['name'] = util_code_name + kwargs['name'] = util_code_name + + if 'file' not in kwargs and from_file: + kwargs['file'] = from_file return cls(**kwargs) @@ -188,13 +187,8 @@ class UtilityCodeBase(object): proto, impl = utilities[util_code_name] if context is not None: - if '__name' not in context: - context['__name'] = util_code_name - - if proto: - proto = tempita.sub(proto, **context) - if impl: - impl = tempita.sub(impl, **context) + proto = sub_tempita(proto, context, from_file, util_code_name) + impl = sub_tempita(impl, context, from_file, util_code_name) if cls.is_cython_utility: # Remember line numbers @@ -204,19 +198,56 @@ class UtilityCodeBase(object): load_as_string = classmethod(load_as_string) + def none_or_sub(self, s, context, tempita): + """ + Format a string in this utility code with context. If None, do nothing. + """ + if s is None: + return None + + if tempita: + return sub_tempita(s, context, self.file, self.name) + + return s % context + def __str__(self): return "<%s(%s)" % (type(self).__name__, self.name) +def sub_tempita(s, context, file, name): + "Run tempita on string s with context context." + if not s: + return None + + if file: + context['__name'] = "%s:%s" % (file, name) + elif name: + context['__name'] = name + + return tempita.sub(s, **context) + + class UtilityCode(UtilityCodeBase): - # Stores utility code to add during code generation. - # - # See GlobalState.put_utility_code. - # - # hashes/equals by instance + """ + Stores utility code to add during code generation. + + See GlobalState.put_utility_code. + + hashes/equals by instance + + proto C prototypes + impl implemenation code + init code to call on module initialization + requires utility code dependencies + proto_block the place in the resulting file where the prototype should + end up + name name of the utility code (or None) + file filename of the utility code file this utility was loaded + from (or None) + """ def __init__(self, proto=None, impl=None, init=None, cleanup=None, requires=None, - proto_block='utility_code_proto', name=None): + proto_block='utility_code_proto', name=None, file=None): # proto_block: Which code block to dump prototype in. See GlobalState. self.proto = proto self.impl = impl @@ -227,11 +258,13 @@ class UtilityCode(UtilityCodeBase): self.specialize_list = [] self.proto_block = proto_block self.name = name + self.file = file def get_tree(self): pass - def specialize(self, pyrex_type=None, **data): + + def specialize(self, pyrex_type=None, tempita=False, **data): # Dicts aren't hashable... if pyrex_type is not None: data['type'] = pyrex_type.declaration_code('') @@ -244,12 +277,15 @@ class UtilityCode(UtilityCodeBase): requires = None else: requires = [r.specialize(data) for r in self.requires] + s = self._cache[key] = UtilityCode( - none_or_sub(self.proto, data), - none_or_sub(self.impl, data), - none_or_sub(self.init, data), - none_or_sub(self.cleanup, data), - requires, self.proto_block) + self.none_or_sub(self.proto, data, tempita), + self.none_or_sub(self.impl, data, tempita), + self.none_or_sub(self.init, data, tempita), + self.none_or_sub(self.cleanup, data, tempita), + requires, + self.proto_block) + self.specialize_list.append(s) return s @@ -275,6 +311,29 @@ class UtilityCode(UtilityCodeBase): self.cleanup(writer, output.module_pos) +class ContentHashingUtilityCode(UtilityCode): + "UtilityCode that hashes and compares based on self.proto and self.impl" + + def __hash__(self): + return hash((self.proto, self.impl)) + + def __eq__(self, other): + return (self.proto, self.impl) == (other.proto, other.impl) + + +class LazyUtilityCode(UtilityCodeBase): + """ + Utility code that calls a callback with the root code writer when + available. Useful when you only have 'env' but not 'code'. + """ + + def __init__(self, callback): + self.callback = callback + + def put_code(self, globalstate): + utility = self.callback(globalstate.rootwriter) + globalstate.use_utility_code(utility) + class FunctionState(object): # return_label string function return point label @@ -1538,6 +1597,15 @@ class CCodeWriter(object): for entry in entries: self.put_var_xdecref_clear(entry) + def put_incref_memoryviewslice(self, slice_cname, have_gil=False): + self.putln("__PYX_INC_MEMVIEW(&%s, %d);" % (slice_cname, int(have_gil))) + + def put_xdecref_memoryviewslice(self, slice_cname, have_gil=False): + self.putln("__PYX_XDEC_MEMVIEW(&%s, %d);" % (slice_cname, int(have_gil))) + + def put_xgiveref_memoryviewslice(self, slice_cname): + self.put_xgiveref("%s.memview" % slice_cname) + def put_init_to_py_none(self, cname, type, nanny=True): from PyrexTypes import py_object_type, typecast py_none = typecast(type, py_object_type, "Py_None") diff --git a/Cython/Compiler/CythonScope.py b/Cython/Compiler/CythonScope.py index d1c404ec5..2e3cc44a7 100644 --- a/Cython/Compiler/CythonScope.py +++ b/Cython/Compiler/CythonScope.py @@ -10,10 +10,12 @@ import MemoryView class CythonScope(ModuleScope): is_cython_builtin = 1 - def __init__(self): + def __init__(self, context): ModuleScope.__init__(self, u'cython', None, None) self.pxd_file_loaded = True self.populate_cython_scope() + # The Main.Context object + self.context = context def lookup_type(self, name): # This function should go away when types are all first-level objects. @@ -80,6 +82,10 @@ class CythonScope(ModuleScope): # The view sub-scope # self.viewscope = viewscope = ModuleScope(u'cython.view', self, None) + + # Hacky monkey patch + self.viewscope.global_scope = self.global_scope + self.declare_module('view', viewscope, None) viewscope.is_cython_builtin = True viewscope.pxd_file_loaded = True @@ -93,7 +99,7 @@ def create_cython_scope(context, create_testscope): # One could in fact probably make it a singleton, # but not sure yet whether any code mutates it (which would kill reusing # it across different contexts) - scope = CythonScope() + scope = CythonScope(context) if create_testscope: scope.test_cythonscope() @@ -129,6 +135,12 @@ cython_test_extclass_utility_code = \ cythonview_testscope_utility_code = load_testscope_utility("View.TestScope") view_utility_code = MemoryView.load_memview_cy_utility( - "View.MemoryView", requires=(Buffer.GetAndReleaseBufferUtilityCode(),)) - -cython_array_utility_code = MemoryView.load_memview_cy_utility("CythonArray") + "View.MemoryView", context=MemoryView.context, + requires=[Buffer.GetAndReleaseBufferUtilityCode(), + MemoryView.memviewslice_declare_code], +) + +cython_array_utility_code = MemoryView.load_memview_cy_utility( + "CythonArray", + context=MemoryView.context, + requires=[view_utility_code]) diff --git a/Cython/Compiler/ExprNodes.py b/Cython/Compiler/ExprNodes.py index fb7ab564e..ab6dc7036 100755 --- a/Cython/Compiler/ExprNodes.py +++ b/Cython/Compiler/ExprNodes.py @@ -587,7 +587,12 @@ class ExprNode(Node): if dst_type.is_memoryviewslice: import MemoryView if not src.type.is_memoryviewslice: - src = CoerceToMemViewSliceNode(src, dst_type, env) + if src.type.is_pyobject: + src = CoerceToMemViewSliceNode(src, dst_type, env) + else: + error(self.pos, + "Cannot convert '%s' to memoryviewslice" % + (src_type,)) elif not MemoryView.src_conforms_to_dst(src.type, dst_type): error(self.pos, "Memoryview '%s' not conformable to memoryview '%s'." % (src.type, dst_type)) @@ -1282,6 +1287,7 @@ class NameNode(AtomicExprNode): # cf_is_null boolean Is uninitialized before this node # cf_maybe_null boolean Maybe uninitialized before this node # allow_null boolean Don't raise UnboundLocalError + # nogil boolean Whether it is used in a nogil context is_name = True is_cython_module = False @@ -1293,6 +1299,7 @@ class NameNode(AtomicExprNode): cf_maybe_null = True cf_is_null = False allow_null = False + nogil = False def create_analysed_rvalue(pos, env, entry): node = NameNode(pos) @@ -1452,6 +1459,7 @@ class NameNode(AtomicExprNode): self.is_used_as_rvalue = 1 def nogil_check(self, env): + self.nogil = True if self.is_used_as_rvalue: entry = self.entry if entry.is_builtin: @@ -1655,9 +1663,16 @@ class NameNode(AtomicExprNode): else: if self.type.is_memoryviewslice: import MemoryView - MemoryView.gen_acquire_memoryviewslice(rhs, self.type, + MemoryView.put_acquire_memoryviewslice(rhs, self.type, self.entry.is_cglobal, self.result(), self.pos, code) - # self.generate_acquire_memoryviewslice(rhs, code) + + if isinstance(rhs, CoerceToMemViewSliceNode): + # We had a new reference, the lhs now has another, + # dispose of ours. + # code.put_xdecref_memoryviewslice(rhs.result()) + code.put_decref("%s.memview" % rhs.result(), + cython_memoryview_ptr_type, + nanny=False) elif self.type.is_buffer: # Generate code for doing the buffer release/acquisition. @@ -1736,12 +1751,17 @@ class NameNode(AtomicExprNode): '__Pyx_DelAttrString(%s, "%s")' % ( Naming.module_cname, self.entry.name)) - elif self.entry.type.is_pyobject: + elif self.entry.type.is_pyobject or self.entry.type.is_memoryviewslice: if not self.cf_is_null: if self.cf_maybe_null: code.put_error_if_unbound(self.pos, self.entry) - code.put_decref(self.result(), self.ctype()) - code.putln('%s = NULL;' % self.result()) + + if self.entry.type.is_pyobject: + code.put_decref(self.result(), self.ctype()) + code.putln('%s = NULL;' % self.result()) + else: + code.put_xdecref_memoryviewslice(self.entry.cname, + have_gil=not self.nogil) else: error(self.pos, "Deletion of C names not supported") @@ -2347,10 +2367,12 @@ class IndexNode(ExprNode): buffer_access = False memoryviewslice_access = False - if (self.base.type.is_memoryviewslice and not self.indices and + is_memslice = self.base.type.is_memoryviewslice + + if (is_memslice and not self.indices and isinstance(self.index, EllipsisNode)): memoryviewslice_access = True - elif self.base.type.is_buffer or self.base.type.is_memoryviewslice: + elif self.base.type.is_buffer or is_memslice: if self.indices: indices = self.indices else: @@ -2358,6 +2380,7 @@ class IndexNode(ExprNode): indices = self.index.args else: indices = [self.index] + if len(indices) == self.base.type.ndim: buffer_access = True skip_child_analysis = True @@ -2365,14 +2388,9 @@ class IndexNode(ExprNode): x.analyse_types(env) if not x.type.is_int: buffer_access = False + if buffer_access: assert hasattr(self.base, "entry") # Must be a NameNode-like node - -# if self.base.type.is_memoryviewslice: -# assert hasattr(self.base, "entry") -# if self.indices or not isinstance(self.index, EllipsisNode): -# error(self.pos, "Memoryviews currently support ellipsis indexing only.") -# else: memoryviewslice_access = True # On cloning, indices is cloned. Otherwise, unpack index into indices assert not (buffer_access and isinstance(self.index, CloneNode)) @@ -2386,7 +2404,10 @@ class IndexNode(ExprNode): if getting and self.type.is_pyobject: self.is_temp = True - if setting: + + if setting and self.base.type.is_memoryviewslice: + self.type.writable_needed = True + elif setting: if not self.base.entry.type.writable: error(self.pos, "Writing to readonly buffer") else: @@ -3990,8 +4011,8 @@ class AttributeNode(ExprNode): import MemoryView MemoryView.put_assign_to_memviewslice(select_code, rhs.result(), self.type, pos=self.pos, code=code) - if rhs.is_temp: - code.put_xdecref_clear("%s.memview" % rhs.result(), cython_memoryview_ptr_type) + #if rhs.is_temp: + # code.put_xdecref_clear("%s.memview" % rhs.result(), cython_memoryview_ptr_type) if not self.type.is_memoryviewslice: code.putln( "%s = %s;" % ( @@ -7647,46 +7668,14 @@ class CoerceToMemViewSliceNode(CoercionNode): self.type = dst_type self.env = env self.is_temp = 1 + self.arg = arg def generate_result_code(self, code): - import MemoryView, Buffer - memviewobj = code.funcstate.allocate_temp(PyrexTypes.py_object_type, manage_ref=True) - buf_flag = MemoryView.get_buf_flag(self.type.axes) - code.putln("%s = (PyObject *) __pyx_memoryview_new(%s, %s);" % - (memviewobj, self.arg.py_result(), buf_flag)) - code.putln(code.error_goto_if_PyErr(self.pos)) - ndim = len(self.type.axes) - spec_int_arr = code.funcstate.allocate_temp( - PyrexTypes.c_array_type(PyrexTypes.c_int_type, ndim), - manage_ref=False) - specs_code = MemoryView.specs_to_code(self.type.axes) - for idx, cspec in enumerate(specs_code): - code.putln("%s[%d] = %s;" % (spec_int_arr, idx, cspec)) - - code.globalstate.use_utility_code(Buffer.acquire_utility_code) - code.globalstate.use_utility_code(MemoryView.memviewslice_init_code) - dtype_typeinfo = Buffer.get_type_information_cname(code, self.type.dtype) - - MemoryView.put_init_entry(self.result(), code) - code.putln("{") - code.putln("__Pyx_BufFmt_StackElem __pyx_stack[%d];" % - self.type.dtype.struct_nesting_depth()) - result = self.result() - if self.type.is_c_contig: - c_or_f_flag = "__Pyx_IS_C_CONTIG" - elif self.type.is_f_contig: - c_or_f_flag = "__Pyx_IS_F_CONTIG" - else: - c_or_f_flag = "0" - code.putln(code.error_goto_if("-1 == __Pyx_ValidateAndInit_memviewslice(" - "(struct __pyx_memoryview_obj *) %(memviewobj)s," - " %(spec_int_arr)s, %(c_or_f_flag)s, %(ndim)d," - " &%(dtype_typeinfo)s, __pyx_stack, &%(result)s)" % locals(), self.pos)) - code.putln("}") - code.put_gotref( - code.as_pyobject("%s.memview" % self.result(), cython_memoryview_ptr_type)) - code.funcstate.release_temp(memviewobj) - code.funcstate.release_temp(spec_int_arr) + self.type.create_from_py_utility_code(self.env) + code.putln("%s = %s(%s);" % (self.result(), + self.type.from_py_function, + self.arg.py_result())) + class CastNode(CoercionNode): # Wrap a node in a C type cast. diff --git a/Cython/Compiler/Main.py b/Cython/Compiler/Main.py index 0f8da7504..2f5da8b88 100644 --- a/Cython/Compiler/Main.py +++ b/Cython/Compiler/Main.py @@ -430,7 +430,7 @@ def create_default_resultobj(compilation_source, options): def run_pipeline(source, options, full_module_name = None): import Pipeline - # Set up context + context = options.create_context() # Set up source object @@ -438,6 +438,7 @@ def run_pipeline(source, options, full_module_name = None): abs_path = os.path.abspath(source) source_ext = os.path.splitext(source)[1] full_module_name = full_module_name or context.extract_module_name(source, options) + if options.relative_path_in_code_position_comments: rel_path = full_module_name.replace('.', os.sep) + source_ext if not abs_path.endswith(rel_path): @@ -520,7 +521,7 @@ class CompilationOptions(object): def create_context(self): return Context(self.include_path, self.compiler_directives, - self.cplus, self.language_level, options=self) + self.cplus, self.language_level, options=self) class CompilationResult(object): @@ -584,7 +585,8 @@ def compile_multiple(sources, options): a CompilationResultSet. Performs timestamp checking and/or recursion if these are specified in the options. """ - context = options.create_context() + # run_pipeline creates the context + # context = options.create_context() sources = [os.path.abspath(source) for source in sources] processed = set() results = CompilationResultSet() diff --git a/Cython/Compiler/MemoryView.py b/Cython/Compiler/MemoryView.py index 5ebfabd85..c0d45f27b 100644 --- a/Cython/Compiler/MemoryView.py +++ b/Cython/Compiler/MemoryView.py @@ -46,17 +46,21 @@ _spec_to_const = { 'follow' : MEMVIEW_FOLLOW, } +_spec_to_abbrev = { + 'direct' : 'd', + 'ptr' : 'p', + 'full' : 'f', + 'contig' : 'c', + 'strided' : 's', + 'follow' : '_', +} + memview_name = u'memoryview' memview_typeptr_cname = '__pyx_memoryview_type' memview_objstruct_cname = '__pyx_memoryview_obj' memviewslice_cname = u'__Pyx_memviewslice' -def specs_to_code(specs): - arr = [] - for access, packing in specs: - arr.append("(%s | %s)" % (_spec_to_const[access], - _spec_to_const[packing])) - return arr + def put_init_entry(mv_cname, code): code.putln("%s.data = NULL;" % mv_cname) @@ -70,7 +74,7 @@ def mangle_dtype_name(dtype): def axes_to_str(axes): return "".join([access[0].upper()+packing[0] for (access, packing) in axes]) -def gen_acquire_memoryviewslice(rhs, lhs_type, lhs_is_cglobal, lhs_result, lhs_pos, code): +def put_acquire_memoryviewslice(rhs, lhs_type, lhs_is_cglobal, lhs_result, lhs_pos, code): # import MemoryView assert rhs.type.is_memoryviewslice @@ -80,32 +84,17 @@ def gen_acquire_memoryviewslice(rhs, lhs_type, lhs_is_cglobal, lhs_result, lhs_p else: rhstmp = code.funcstate.allocate_temp(lhs_type, manage_ref=False) code.putln("%s = %s;" % (rhstmp, rhs.result_as(lhs_type))) - code.putln(code.error_goto_if_null("%s.memview" % rhstmp, lhs_pos)) - - if not rhs.result_in_temp(): - code.put_incref("%s.memview" % rhstmp, cython_memoryview_ptr_type) - - if lhs_is_cglobal: - code.put_gotref("%s.memview" % lhs_result) - - #XXX: this is here because self.lhs_of_first_assignment is not set correctly, - # once that is working this should take that flag into account. - # See NameNode.generate_assignment_code - code.put_xdecref("%s.memview" % lhs_result, cython_memoryview_ptr_type) - - if lhs_is_cglobal: - code.put_giveref("%s.memview" % rhstmp) + code.putln(code.error_goto_if_null("%s.memview" % rhstmp, lhs_pos)) put_assign_to_memviewslice(lhs_result, rhstmp, lhs_type, lhs_pos, code=code) - if rhs.result_in_temp() or not pretty_rhs: - code.putln("%s.memview = 0;" % rhstmp) - if not pretty_rhs: code.funcstate.release_temp(rhstmp) def put_assign_to_memviewslice(lhs_cname, rhs_cname, memviewslicetype, pos, code): + code.put_xdecref_memoryviewslice(lhs_cname) + code.put_incref_memoryviewslice(rhs_cname) code.putln("%s.memview = %s.memview;" % (lhs_cname, rhs_cname)) code.putln("%s.data = %s.data;" % (lhs_cname, rhs_cname)) @@ -195,6 +184,45 @@ class MemoryViewSliceBufferEntry(Buffer.BufferEntry): def get_buf_shapevars(self): return self._for_all_ndim("%s.shape[%d]") + def generate_buffer_lookup_code(self, code, index_cnames): + bufp = self.buf_ptr + type_decl = self.type.dtype.declaration_code("") + + for dim, (access, packing) in enumerate(self.type.axes): + shape = "%s.shape[%d]" % (self.cname, dim) + stride = "%s.strides[%d]" % (self.cname, dim) + suboffset = "%s.suboffsets[%d]" % (self.cname, dim) + index = index_cnames[dim] + + if access == 'full' and packing in ('strided', 'follow'): + code.globalstate.use_utility_code(memviewslice_index_helpers) + bufp = ('__pyx_memviewslice_index_full(%s, %s, %s)' % + (bufp, index, stride, suboffset)) + + elif access == 'full' and packing == 'contig': + # We can skip stride multiplication with the cast + code.globalstate.use_utility_code(memviewslice_index_helpers) + bufp = '((char *) ((%s *) %s) + %s)' % (type_decl, bufp, index) + bufp = ('__pyx_memviewslice_index_full_contig(%s, %s)' % + (bufp, suboffset)) + + elif access == 'ptr' and packing in ('strided', 'follow'): + bufp = ("(*((char **) %s + %s * %s) + %s)" % + (bufp, index, stride, suboffset)) + + elif access == 'ptr' and packing == 'contig': + bufp = "(*((char **) %s) + %s)" % (bufp, suboffset) + + elif access == 'direct' and packing in ('strided', 'follow'): + bufp = "(%s + %s * %s)" % (bufp, index, stride) + + else: + assert (access, packing) == ('direct', 'contig'), (access, packing) + bufp = '((char *) (((%s *) %s) + %s))' % (type_decl, bufp, index) + + bufp = '( /* dim=%d */ %s )' % (dim, bufp) + + return "((%s *) %s)" % (type_decl, bufp) def get_copy_func_name(to_memview): base = "__Pyx_BufferNew_%s_From_%s_%s" @@ -370,10 +398,10 @@ class CopyFuncUtilCode(object): if self.to_memview.is_c_contig: mode = 'c' - contig_flag = 'PyBUF_C_CONTIGUOUS' + contig_flag = memview_c_contiguous elif self.to_memview.is_f_contig: mode = 'fortran' - contig_flag = "PyBUF_F_CONTIGUOUS" + contig_flag = memview_f_contiguous context = dict( copy_name=self.copy_func_name, @@ -735,11 +763,13 @@ class MemoryViewSliceTransform(CythonTransform): return node -def load_memview_cy_utility(util_code_name, **kwargs): - return CythonUtilityCode.load(util_code_name, "MemoryView.pyx", **kwargs) +def load_memview_cy_utility(util_code_name, context=None, **kwargs): + return CythonUtilityCode.load(util_code_name, "MemoryView.pyx", + context=context, **kwargs) -def load_memview_c_utility(util_code_name, **kwargs): - return UtilityCode.load(util_code_name, "MemoryView_C.c", **kwargs) +def load_memview_c_utility(util_code_name, context=None, **kwargs): + return UtilityCode.load(util_code_name, "MemoryView_C.c", + context=context, **kwargs) context = { 'memview_struct_name': memview_objstruct_cname, @@ -753,6 +783,8 @@ memviewslice_declare_code = load_memview_c_utility( memviewslice_init_code = load_memview_c_utility( "MemviewSliceInit", - context={'BUF_MAX_NDIMS': Options.buffer_max_dims}, + context=dict(context, BUF_MAX_NDIMS=Options.buffer_max_dims), requires=[memviewslice_declare_code], ) + +memviewslice_index_helpers = load_memview_c_utility("MemviewSliceIndex")
\ No newline at end of file diff --git a/Cython/Compiler/ModuleNode.py b/Cython/Compiler/ModuleNode.py index 715a6e9de..09b99e0bf 100644 --- a/Cython/Compiler/ModuleNode.py +++ b/Cython/Compiler/ModuleNode.py @@ -79,6 +79,10 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): scope.python_include_files) if merge_scope: + # Ensure that we don't generate import code for these entries! + for entry in scope.c_class_entries: + entry.type.module_name = self.full_module_name + self.scope.merge_in(scope) def analyse_declarations(self, env): @@ -345,7 +349,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): self.generate_declarations_for_modules(env, modules, globalstate) h_code.write('\n') - for utilcode in env.utility_code_list: + for utilcode in env.utility_code_list[:]: globalstate.use_utility_code(utilcode) globalstate.finalize_main_c_code() @@ -2255,7 +2259,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): # Generate type import code for extern extension types # and type ready code for non-extern ones. for entry in env.c_class_entries: - if entry.visibility == 'extern': + if entry.visibility == 'extern' and not entry.utility_code_definition: self.generate_type_import_code(env, entry.type, entry.pos, code) else: self.generate_base_type_import_code(env, entry, code) @@ -2265,8 +2269,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): def generate_base_type_import_code(self, env, entry, code): base_type = entry.type.base_type - if base_type and base_type.module_name != env.qualified_name \ - and not base_type.is_builtin_type: + if (base_type and base_type.module_name != env.qualified_name and not + base_type.is_builtin_type and not entry.utility_code_definition): self.generate_type_import_code(env, base_type, self.pos, code) def use_type_import_utility_code(self, env): diff --git a/Cython/Compiler/Nodes.py b/Cython/Compiler/Nodes.py index db30178bd..6eb0e62fa 100644 --- a/Cython/Compiler/Nodes.py +++ b/Cython/Compiler/Nodes.py @@ -1344,17 +1344,19 @@ class FuncDefNode(StatNode, BlockNode): if not entry.in_closure: code.put_var_declaration(entry) + # Initialize the return variable __pyx_r init = "" if not self.return_type.is_void: if self.return_type.is_pyobject: init = " = NULL" + elif self.return_type.is_memoryviewslice: + init = "= {0, 0}" + code.putln( "%s%s;" % (self.return_type.declaration_code(Naming.retval_cname), init)) - if self.return_type.is_memoryviewslice: - import MemoryView - MemoryView.put_init_entry(Naming.retval_cname, code) + tempvardecl_code = code.insertion_point() self.generate_keyword_list(code) @@ -1431,15 +1433,18 @@ class FuncDefNode(StatNode, BlockNode): if (acquire_gil or entry.assignments) and not entry.in_closure: code.put_var_incref(entry) if entry.type.is_memoryviewslice: - code.put_incref("%s.memview" % entry.cname, cython_memoryview_ptr_type) + code.put_incref_memoryviewslice(entry.cname, + have_gil=not lenv.nogil) + #code.put_incref("%s.memview" % entry.cname, cython_memoryview_ptr_type) # ----- Initialise local buffer auxiliary variables for entry in lenv.var_entries + lenv.arg_entries: if entry.type.is_buffer and entry.buffer_aux.buflocal_nd_var.used: Buffer.put_init_vars(entry, code) # ----- Initialise local memoryviewslices for entry in lenv.var_entries: - if entry.type.is_memoryviewslice: - MemoryView.put_init_entry(entry.cname, code) + if entry.visibility == "private" and not entry.used: + continue + # ----- Check and convert arguments self.generate_argument_type_tests(code) # ----- Acquire buffer arguments @@ -1544,7 +1549,8 @@ class FuncDefNode(StatNode, BlockNode): if not entry.used or entry.in_closure: continue if entry.type.is_memoryviewslice: - code.put_xdecref("%s.memview" % entry.cname, cython_memoryview_ptr_type) + #code.put_xdecref("%s.memview" % entry.cname, cython_memoryview_ptr_type) + code.put_xdecref_memoryviewslice(entry.cname) if entry.type.is_pyobject: code.put_var_decref(entry) @@ -1554,7 +1560,8 @@ class FuncDefNode(StatNode, BlockNode): if (acquire_gil or entry.assignments) and not entry.in_closure: code.put_var_decref(entry) if entry.type.is_memoryviewslice: - code.put_decref("%s.memview" % entry.cname, cython_memoryview_ptr_type) + code.put_xdecref_memoryviewslice(entry.cname) + #code.put_decref("%s.memview" % entry.cname, cython_memoryview_ptr_type) if self.needs_closure: code.put_decref(Naming.cur_scope_cname, lenv.scope_class.type) @@ -1567,8 +1574,9 @@ class FuncDefNode(StatNode, BlockNode): err_val = default_retval if self.return_type.is_pyobject: code.put_xgiveref(self.return_type.as_pyobject(Naming.retval_cname)) - elif self.return_type.is_memoryviewslice: - code.put_xgiveref(code.as_pyobject("%s.memview" % Naming.retval_cname,cython_memoryview_ptr_type)) + #elif self.return_type.is_memoryviewslice: + # code.put_xgiveref(code.as_pyobject("%s.memview" % Naming.retval_cname,cython_memoryview_ptr_type)) + # code.put_xgiveref_memoryviewslice(Naming.retval_cname) if self.entry.is_special and self.entry.name == "__hash__": # Returning -1 for __hash__ is supposed to signal an error @@ -4153,7 +4161,8 @@ class DelStatNode(StatNode): def analyse_expressions(self, env): for arg in self.args: arg.analyse_target_expression(env, None) - if arg.type.is_pyobject: + if arg.type.is_pyobject or (arg.is_name and + arg.type.is_memoryviewslice): pass elif arg.type.is_ptr and arg.type.base_type.is_cpp_class: self.cpp_check(env) @@ -4172,7 +4181,7 @@ class DelStatNode(StatNode): def generate_execution_code(self, code): for arg in self.args: - if arg.type.is_pyobject: + if arg.type.is_pyobject or arg.type.is_memoryviewslice: arg.generate_deletion_code(code) elif arg.type.is_ptr and arg.type.base_type.is_cpp_class: arg.generate_result_code(code) @@ -4274,14 +4283,15 @@ class ReturnStatNode(StatNode): code.put_xdecref(Naming.retval_cname, self.return_type) elif self.return_type.is_memoryviewslice: - code.put_xdecref("%s.memview" % Naming.retval_cname, - self.return_type) + code.put_xdecref_memoryviewslice(Naming.retval_cname) + #code.put_xdecref("%s.memview" % Naming.retval_cname, + # self.return_type) if self.value: self.value.generate_evaluation_code(code) if self.return_type.is_memoryviewslice: import MemoryView - MemoryView.gen_acquire_memoryviewslice(self.value, self.return_type, + MemoryView.put_acquire_memoryviewslice(self.value, self.return_type, False, Naming.retval_cname, None, code) else: self.value.make_owned_reference(code) diff --git a/Cython/Compiler/PyrexTypes.py b/Cython/Compiler/PyrexTypes.py index 0144414bd..a42a13573 100755 --- a/Cython/Compiler/PyrexTypes.py +++ b/Cython/Compiler/PyrexTypes.py @@ -2,12 +2,14 @@ # Cython/Python language types # -from Code import UtilityCode +from Code import UtilityCode, LazyUtilityCode import StringEncoding import Naming import copy from Errors import error +import cython + class BaseType(object): # # Base class for all Cython types including pseudo-types. @@ -331,6 +333,15 @@ class MemoryViewSliceType(PyrexType): has_attributes = 1 scope = None + # These are specialcased in Defnode + from_py_function = None + to_py_function = None + + exception_value = None + exception_check = None + + utility_counter = 0 + def __init__(self, base_dtype, axes): ''' MemoryViewSliceType(base, axes) @@ -375,6 +386,7 @@ class MemoryViewSliceType(PyrexType): assert not (self.is_c_contig and self.is_f_contig) self.mode = MemoryView.get_mode(axes) + self.writable_needed = False def same_as_resolved_type(self, other_type): return ((other_type.is_memoryviewslice and @@ -495,11 +507,68 @@ class MemoryViewSliceType(PyrexType): def global_init_code(self, entry, code): code.putln("%s.data = NULL;" % entry.cname) - code.put_init_to_py_none("%s.memview" % entry.cname, cython_memoryview_ptr_type, nanny=False) + code.putln("%s.memview = NULL;" % entry.cname) + #code.put_init_to_py_none("%s.memview" % entry.cname, cython_memoryview_ptr_type, nanny=False) def check_for_null_code(self, cname): return cname + '.memview' + def create_from_py_utility_code(self, env): + import MemoryView, Buffer, Code + + # We don't have 'code', so use a LazyUtilityCode with a callback. + def lazy_utility_callback(code): + context['dtype_typeinfo'] = Buffer.get_type_information_cname( + code, self.dtype) + return Code.ContentHashingUtilityCode.load( + "ObjectToMemviewSlice", "MemoryView_C.c", context) + + env.use_utility_code(Buffer.acquire_utility_code) + env.use_utility_code(MemoryView.memviewslice_init_code) + env.use_utility_code(LazyUtilityCode(lazy_utility_callback)) + + if self.is_c_contig: + c_or_f_flag = "__Pyx_IS_C_CONTIG" + elif self.is_f_contig: + c_or_f_flag = "__Pyx_IS_F_CONTIG" + else: + c_or_f_flag = "0" + + # specializing through UtilityCode.specialize is not so useful as + # specialize on too many things to include in the function name + funcname = "__Pyx_PyObject_to_MemoryviewSlice_%d" % self.utility_counter + + context = dict( + MemoryView.context, + buf_flag = MemoryView.get_buf_flag(self.axes), + ndim = self.ndim, + axes_specs = ', '.join(self.axes_specs_to_code()), + dtype_typedecl = self.dtype.declaration_code(""), + struct_nesting_depth = self.dtype.struct_nesting_depth(), + c_or_f_flag = c_or_f_flag, + funcname = funcname, + ) + + self.from_py_function = funcname + MemoryViewSliceType.utility_counter += 1 + + return True + + def axes_specs_to_code(self): + "Return a list of code constants for each axis" + import MemoryView + d = MemoryView._spec_to_const + return ["(%s | %s)" % (d[a], d[p]) for a, p in self.axes] + + def axes_specs_to_name(self): + "Return an abbreviated name for our axes" + import MemoryView + d = MemoryView._spec_to_abbrev + return "".join(["%s%s" % (d[a], d[p]) for a, p in self.axes]) + + def error_condition(self, result_code): + return "!%s.memview" % result_code + class BufferType(BaseType): # @@ -2698,8 +2767,8 @@ cython_memoryview_type = CStructOrUnionType("__pyx_memoryview_obj", "struct", cython_memoryview_ptr_type = CPtrType(cython_memoryview_type) -memoryviewslice_type = CStructOrUnionType("__Pyx_memviewslice", "struct", - None, 1, "__Pyx_memviewslice") +memoryviewslice_type = CStructOrUnionType("memoryviewslice", "struct", + None, 1, "__Pyx_memviewslice") error_type = ErrorType() unspecified_type = UnspecifiedType() diff --git a/Cython/Compiler/Symtab.py b/Cython/Compiler/Symtab.py index 71a31fc7c..f35bdb81d 100644 --- a/Cython/Compiler/Symtab.py +++ b/Cython/Compiler/Symtab.py @@ -382,6 +382,10 @@ class Scope(object): # entries[name] = entry if not shadow: entries[name] = entry + + if type.is_memoryviewslice: + entry.init = "{ 0, 0 }" + entry.scope = self entry.visibility = visibility return entry diff --git a/Cython/Compiler/UtilityCode.py b/Cython/Compiler/UtilityCode.py index 685b229be..3329a7966 100644 --- a/Cython/Compiler/UtilityCode.py +++ b/Cython/Compiler/UtilityCode.py @@ -15,7 +15,7 @@ class NonManglingModuleScope(Symtab.ModuleScope): entry.used = True return super(NonManglingModuleScope, self).add_imported_entry( name, entry, pos) - + def mangle(self, prefix, name=None): if name: if prefix in (Naming.typeobj_prefix, Naming.func_prefix, Naming.var_prefix, Naming.pyfunc_prefix): @@ -63,7 +63,8 @@ class CythonUtilityCode(Code.UtilityCodeBase): is_cython_utility = True - def __init__(self, impl, name="__pyxutil", prefix="", requires=None): + def __init__(self, impl, name="__pyxutil", prefix="", requires=None, + file=None): # 1) We need to delay the parsing/processing, so that all modules can be # imported without import loops # 2) The same utility code object can be used for multiple source files; @@ -72,6 +73,7 @@ class CythonUtilityCode(Code.UtilityCodeBase): # Hence, delay any processing until later. self.impl = impl self.name = name + self.file = file self.prefix = prefix self.requires = requires or [] @@ -113,10 +115,11 @@ class CythonUtilityCode(Code.UtilityCodeBase): def put_code(self, output): pass - def declare_in_scope(self, dest_scope, used=False): + def declare_in_scope(self, dest_scope, used=False, modname=None): """ Declare all entries from the utility code in dest_scope. Code will only - be included for used entries. + be included for used entries. If module_name is given, declare the + type entries with that name. """ tree = self.get_tree(entries_only=True) @@ -130,6 +133,10 @@ class CythonUtilityCode(Code.UtilityCodeBase): entry.utility_code_definition = self entry.used = used + if modname and entry.type.is_extension_type: + entry.qualified_name = modname + entry.type.module_name = modname + dest_scope.merge_in(tree.scope, merge_unused=True) tree.scope = dest_scope diff --git a/Cython/Utility/MemoryView.pyx b/Cython/Utility/MemoryView.pyx index 1266b5287..4df327ea2 100644 --- a/Cython/Utility/MemoryView.pyx +++ b/Cython/Utility/MemoryView.pyx @@ -10,7 +10,10 @@ cdef extern from "Python.h": PyBUF_C_CONTIGUOUS, PyBUF_F_CONTIGUOUS, PyBUF_ANY_CONTIGUOUS + PyBUF_FORMAT +cdef extern from *: + object __pyx_memoryview_new(object obj, int flags) @cname("__pyx_array") cdef class array: @@ -105,7 +108,12 @@ cdef class array: info.strides = self.strides info.suboffsets = NULL info.itemsize = self.itemsize - info.format = self.format + + if flags & PyBUF_FORMAT: + info.format = self.format + else: + info.format = NULL + # we do not need to call releasebuffer info.obj = None @@ -130,6 +138,15 @@ cdef class array: self.format = NULL self.itemsize = 0 + def __getitem__(self, index): + view = __pyx_memoryview_new(self, PyBUF_ANY_CONTIGUOUS|PyBUF_FORMAT) + return view[index] + + def __setitem__(self, index, value): + view = __pyx_memoryview_new(self, PyBUF_ANY_CONTIGUOUS|PyBUF_FORMAT) + view[index] = value + + @cname("__pyx_array_new") cdef array array_cwrapper(tuple shape, Py_ssize_t itemsize, char *format, char *mode): return array(shape, itemsize, format, mode.decode('ASCII')) @@ -137,8 +154,10 @@ cdef array array_cwrapper(tuple shape, Py_ssize_t itemsize, char *format, char * ########## View.MemoryView ########## # from cpython cimport ... -cdef extern from "pythread.h": +cdef extern from "Python.h": + int PyIndex_Check(object) +cdef extern from "pythread.h": ctypedef void *PyThread_type_lock PyThread_type_lock PyThread_allocate_lock() @@ -150,6 +169,12 @@ cdef extern from *: int __Pyx_GetBuffer(object, Py_buffer *, int) void __Pyx_ReleaseBuffer(Py_buffer *) + ctypedef struct {{memviewslice_name}}: + char *data + Py_ssize_t shape[{{max_dims}}] + Py_ssize_t strides[{{max_dims}}] + Py_ssize_t suboffsets[{{max_dims}}] + @cname('__pyx_MemviewEnum') cdef class Enum(object): @@ -173,23 +198,143 @@ cdef class memoryview(object): cdef object obj cdef Py_buffer view - cdef PyThread_type_lock acqcnt_lock + cdef PyThread_type_lock lock cdef int acquisition_count + def __cinit__(memoryview self, object obj, int flags): self.obj = obj - #self.acqcnt_lock = PyThread_allocate_lock() - #if self.acqcnt_lock == NULL: - # raise MemoryError - __Pyx_GetBuffer(obj, &self.view, flags) + self.lock = PyThread_allocate_lock() + if self.lock == NULL: + raise MemoryError + def __dealloc__(memoryview self): - #PyThread_free_lock(self.acqcnt_lock) - self.obj = None __Pyx_ReleaseBuffer(&self.view) + PyThread_free_lock(self.lock) + + @cname('__pyx_memoryview_getitem') + def __getitem__(memoryview self, object index): + # cdef Py_ssize_t idx + cdef char *itemp = <char *> self.view.buf + cdef bytes bytesitem + cdef str fmt = self.view.format + + import struct + try: + itemsize = struct.calcsize(fmt) + except struct.error: + raise TypeError("Unsupported format: %r" % fmt) + + if index is Ellipsis: + return self + + elif isinstance(index, slice): + if index == slice(None): + return self + + raise NotImplementedError + + else: + if not isinstance(index, tuple): + index = (index,) + + tup = _unellipsify(index, self.view.ndim) + + if len(tup) != self.view.ndim: + raise NotImplementedError( + "Expected %d indices (got %d)" % + (self.view.ndim, len(tup))) + + for dim, idx in enumerate(tup): + _check_index(idx) + itemp = pybuffer_index(&self.view, itemp, idx, dim + 1) + + # Do a manual and complete check here instead of this easy hack + bytesitem = itemp[:self.view.itemsize] + return struct.unpack(fmt, bytesitem) + + +@cname('__pyx_memoryviewslice') +cdef class _memoryviewslice(memoryview): + "Internal class for passing memory view slices to Python" + + # We need this to keep our shape/strides/suboffset pointers valid + cdef {{memviewslice_name}} from_slice + # Restore the original Py_buffer before releasing + cdef Py_buffer orig_view + + def __cinit__(self, object obj, int flags): + self.orig_view = self.view + + def __dealloc__(self): + self.view = self.orig_view @cname('__pyx_memoryview_new') -cdef memoryview memoryview_cwrapper(object o, int flags): +cdef memoryview_cwrapper(object o, int flags): return memoryview(o, flags) + +@cname('__pyx_memoryview_fromslice') +cdef memoryview memoryview_from_memview_cwrapper(memoryview m, int flags, + int new_ndim, {{memviewslice_name}} *memviewslice): + cdef _memoryviewslice result = _memoryviewslice(m.obj, flags) + + result.from_slice = memviewslice[0] + + result.view.shape = <Py_ssize_t *> (&result.from_slice.shape + new_ndim) + result.view.strides = <Py_ssize_t *> (&result.from_slice.strides + new_ndim) + result.view.suboffsets = <Py_ssize_t *> (&result.from_slice.suboffsets + new_ndim) + result.view.ndim = new_ndim + + return result + +cdef _check_index(index): + if not PyIndex_Check(index): + raise TypeError("Cannot index with %s" % type(index)) + +cdef tuple _unellipsify(tuple tup, int ndim): + if Ellipsis in tup: + result = [] + for idx, item in enumerate(tup): + if item is Ellipsis: + result.extend([slice(None)] * (ndim - len(tup) + 1)) + result.extend(tup[idx + 1:]) + break + + result.append(item) + + return tuple(result) + + return tup + +@cname('__pyx_pybuffer_index') +cdef char *pybuffer_index(Py_buffer *view, char *bufp, Py_ssize_t index, int dim) except NULL: + cdef Py_ssize_t shape, stride, suboffset = -1 + cdef Py_ssize_t itemsize = view.itemsize + cdef char *resultp + + if view.ndim == 0: + shape = view.len / itemsize + stride = itemsize + else: + shape = view.shape[dim] + stride = view.strides[dim] + if view.suboffsets != NULL: + suboffset = view.suboffsets[dim] + + if index < 0: + index += view.shape[dim] + if index < 0: + raise IndexError("Out of bounds in dimension %d" % dim) + + if index > shape: + raise IndexError("Out of bounds in dimension %d" % dim) + + resultp = bufp + index * stride + + if suboffset >= 0: + resultp = (<char **> resultp)[0] + suboffset + + return resultp diff --git a/Cython/Utility/MemoryView_C.c b/Cython/Utility/MemoryView_C.c index 9e949134b..54005eebd 100644 --- a/Cython/Utility/MemoryView_C.c +++ b/Cython/Utility/MemoryView_C.c @@ -12,6 +12,9 @@ typedef struct { Py_ssize_t suboffsets[{{max_dims}}]; } {{memviewslice_name}}; +/////////////// ObjectToMemviewSlice.proto /////////////// +{{# __Pyx_PyObject_to_MemoryviewSlice_<count> }} +static CYTHON_INLINE {{memviewslice_name}} {{funcname}}(PyObject *); ////////// MemviewSliceInit.proto ////////// @@ -27,6 +30,7 @@ typedef struct { #define __Pyx_IS_C_CONTIG 1 #define __Pyx_IS_F_CONTIG 2 + /* #define __PYX_MEMSLICE_GETDATA(SLICE) ((char *) SLICE->memview->view->buf) */ static int __Pyx_ValidateAndInit_memviewslice(struct __pyx_memoryview_obj *memview, @@ -38,6 +42,49 @@ static int __Pyx_init_memviewslice( int ndim, __Pyx_memviewslice *memviewslice); +#define __PYX_INC_MEMVIEW(slice, have_gil) __Pyx_INC_MEMVIEW(slice, have_gil, __LINE__) +#define __PYX_XDEC_MEMVIEW(slice, have_gil) __Pyx_XDEC_MEMVIEW(slice, have_gil, __LINE__) +static CYTHON_INLINE void __Pyx_INC_MEMVIEW({{memviewslice_name}} *, int, int); +static CYTHON_INLINE void __Pyx_XDEC_MEMVIEW({{memviewslice_name}} *, int, int); + +/////////////// MemviewSliceIndex.proto /////////////// + +static CYTHON_INLINE char *__pyx_memviewslice_index_full(char *bufp, Py_ssize_t idx, Py_ssize_t stride, Py_ssize_t suboffset); +static CYTHON_INLINE char *__pyx_memviewslice_index_full_contig(char *bufp, Py_ssize_t suboffset); + + +/////////////// ObjectToMemviewSlice /////////////// + +{{#__Pyx_PyObject_to_MemoryviewSlice_<count>}} + +static CYTHON_INLINE {{memviewslice_name}} {{funcname}}(PyObject *obj) { + {{memviewslice_name}} result; + result.memview = NULL; + result.data = NULL; + struct __pyx_memoryview_obj *memview = \ + (struct __pyx_memoryview_obj *) __pyx_memoryview_new(obj, {{buf_flag}}); + __Pyx_BufFmt_StackElem stack[{{struct_nesting_depth}}]; + int axes_specs[] = { {{axes_specs}} }; + int retcode; + + if (unlikely(!memview)) + goto __pyx_fail; + + retcode = __Pyx_ValidateAndInit_memviewslice(memview, axes_specs, + {{c_or_f_flag}}, {{ndim}}, &{{dtype_typeinfo}}, stack, &result); + + if (unlikely(retcode == -1)) + goto __pyx_fail; + + memview->acquisition_count = 1; + return result; +__pyx_fail: + Py_XDECREF(memview); + result.memview = NULL; + result.data = NULL; + return result; +} + ////////// MemviewSliceInit ////////// static int __Pyx_ValidateAndInit_memviewslice( @@ -203,8 +250,6 @@ static int __Pyx_init_memviewslice( } } - __Pyx_INCREF((PyObject *)memview); - __Pyx_GIVEREF((PyObject *)memview); memviewslice->memview = memview; memviewslice->data = (char *)buf->buf; retval = 0; @@ -220,6 +265,62 @@ no_fail: return retval; } +static CYTHON_INLINE void __Pyx_INC_MEMVIEW({{memviewslice_name}} *memslice, + int have_gil, int lineno) { + int first_time; + struct {{memview_struct_name}} *memview = memslice->memview; + if (!memview) { + char msg[50]; + snprintf(msg, 50, "memoryslice is not initialized (line %d)", lineno); + Py_FatalError(msg); + } + + PyThread_acquire_lock(memview->lock, 1); + first_time = (memview->acquisition_count++ == 0); + PyThread_release_lock(memview->lock); + + /* printf("INCREF %d: acquisition_count=%d, refcount=%d\n", lineno, + memview->acquisition_count, memview->ob_refcnt); */ + + if (first_time) { + if (have_gil) { + Py_INCREF((PyObject *) memview); + } else { + PyGILState_STATE _gilstate = PyGILState_Ensure(); + Py_INCREF((PyObject *) memview); + PyGILState_Release(_gilstate); + } + } +} + +static CYTHON_INLINE void __Pyx_XDEC_MEMVIEW({{memviewslice_name}} *memslice, + int have_gil, int lineno) { + int last_time; + struct {{memview_struct_name}} *memview = memslice->memview; + + if (!memview) { + return; + } + + PyThread_acquire_lock(memview->lock, 1); + last_time = (memview->acquisition_count-- == 1); + PyThread_release_lock(memview->lock); + + /* printf("DECREF %d: acquisition_count=%d, refcount=%d\n", lineno, + memview->acquisition_count, memview->ob_refcnt); */ + + if (last_time) { + if (have_gil) { + Py_CLEAR(memview); + } else { + PyGILState_STATE _gilstate = PyGILState_Ensure(); + Py_CLEAR(memview); + PyGILState_Release(_gilstate); + } + memslice->data = NULL; + } +} + ////////// MemviewSliceCopyTemplate ////////// static __Pyx_memviewslice {{copy_name}}(const __Pyx_memviewslice from_mvs) { @@ -259,7 +360,8 @@ static __Pyx_memviewslice {{copy_name}}(const __Pyx_memviewslice from_mvs) { } __Pyx_GOTREF(array_obj); - memview_obj = __pyx_memoryview_new((PyObject *) array_obj, {{contig_flag}}); + memview_obj = (struct __pyx_memoryview_obj *) __pyx_memoryview_new( + (PyObject *) array_obj, {{contig_flag}}); if (unlikely(!memview_obj)) { goto fail; } @@ -292,3 +394,21 @@ no_fail: } +/////////////// MemviewSliceIndex /////////////// + +static CYTHON_INLINE char *__pyx_memviewslice_index_full(char *bufp, Py_ssize_t idx, Py_ssize_t stride, Py_ssize_t suboffset) { + bufp = bufp + idx * stride; + if (suboffset >= 0) { + bufp = *((char **) bufp) + suboffset; + } + return bufp; +} + +/* The call has already done the indexing */ +static CYTHON_INLINE char *__pyx_memviewslice_index_full_contig(char *bufp, Py_ssize_t suboffset) { + if (suboffset >= 0) { + bufp = *((char **) bufp) + suboffset; + } + return bufp; +} + diff --git a/Cython/Utils.py b/Cython/Utils.py index 5dc2d1282..a5dbea36a 100644 --- a/Cython/Utils.py +++ b/Cython/Utils.py @@ -4,6 +4,7 @@ # import os, sys, re, codecs +from Cython import Tempita def replace_suffix(path, newsuf): base, _ = os.path.splitext(path) @@ -215,10 +216,3 @@ def long_literal(value): if isinstance(value, basestring): value = str_to_number(value) return not -2**31 <= value < 2**31 - -def none_or_sub(s, data): - if s is None: - return s - else: - return s % data - diff --git a/tests/run/bufaccess.pyx b/tests/run/bufaccess.pyx index 3f639b142..895b1d4f0 100644 --- a/tests/run/bufaccess.pyx +++ b/tests/run/bufaccess.pyx @@ -9,13 +9,6 @@ from __future__ import unicode_literals -from libc cimport stdlib -from libc cimport stdio -cimport cpython.buffer -cimport cython - -from cpython cimport PyObject, Py_INCREF, Py_DECREF - __test__ = {} import sys @@ -33,9 +26,8 @@ def testcase(func): __test__[func.__name__] = doctest return func -def testcas(a): - pass +include "mockbuffers.pxi" # # Buffer acquire and release tests @@ -966,246 +958,6 @@ def buffer_cast_fails(object[char, cast=True] buf): """ return buf[0] - -# -# Testcase support code (more tests below!, because of scope rules) -# - - -available_flags = ( - ('FORMAT', cpython.buffer.PyBUF_FORMAT), - ('INDIRECT', cpython.buffer.PyBUF_INDIRECT), - ('ND', cpython.buffer.PyBUF_ND), - ('STRIDES', cpython.buffer.PyBUF_STRIDES), - ('C_CONTIGUOUS', cpython.buffer.PyBUF_C_CONTIGUOUS), - ('F_CONTIGUOUS', cpython.buffer.PyBUF_F_CONTIGUOUS), - ('WRITABLE', cpython.buffer.PyBUF_WRITABLE) -) - -cdef class MockBuffer: - cdef object format, offset - cdef void* buffer - cdef int len, itemsize, ndim - cdef Py_ssize_t* strides - cdef Py_ssize_t* shape - cdef Py_ssize_t* suboffsets - cdef object label, log - - cdef readonly object recieved_flags, release_ok - cdef public object fail - - def __init__(self, label, data, shape=None, strides=None, format=None, offset=0): - # It is important not to store references to data after the constructor - # as refcounting is checked on object buffers. - self.label = label - self.release_ok = True - self.log = "" - self.offset = offset - self.itemsize = self.get_itemsize() - if format is None: format = self.get_default_format() - if shape is None: shape = (len(data),) - if strides is None: - strides = [] - cumprod = 1 - rshape = list(shape) - rshape.reverse() - for s in rshape: - strides.append(cumprod) - cumprod *= s - strides.reverse() - strides = [x * self.itemsize for x in strides] - suboffsets = [-1] * len(shape) - datashape = [len(data)] - p = data - while True: - p = p[0] - if isinstance(p, list): datashape.append(len(p)) - else: break - if len(datashape) > 1: - # indirect access - self.ndim = len(datashape) - shape = datashape - self.buffer = self.create_indirect_buffer(data, shape) - suboffsets = [0] * (self.ndim-1) + [-1] - strides = [sizeof(void*)] * (self.ndim-1) + [self.itemsize] - self.suboffsets = self.list_to_sizebuf(suboffsets) - else: - # strided and/or simple access - self.buffer = self.create_buffer(data) - self.ndim = len(shape) - self.suboffsets = NULL - - try: - format = format.encode('ASCII') - except AttributeError: - pass - self.format = format - self.len = len(data) * self.itemsize - - self.strides = self.list_to_sizebuf(strides) - self.shape = self.list_to_sizebuf(shape) - - def __dealloc__(self): - stdlib.free(self.strides) - stdlib.free(self.shape) - if self.suboffsets != NULL: - stdlib.free(self.suboffsets) - # must recursively free indirect... - else: - stdlib.free(self.buffer) - - cdef void* create_buffer(self, data): - cdef size_t n = <size_t>(len(data) * self.itemsize) - cdef char* buf = <char*>stdlib.malloc(n) - cdef char* it = buf - for value in data: - self.write(it, value) - it += self.itemsize - return buf - - cdef void* create_indirect_buffer(self, data, shape): - cdef size_t n = 0 - cdef void** buf - assert shape[0] == len(data) - if len(shape) == 1: - return self.create_buffer(data) - else: - shape = shape[1:] - n = <size_t>len(data) * sizeof(void*) - buf = <void**>stdlib.malloc(n) - for idx, subdata in enumerate(data): - buf[idx] = self.create_indirect_buffer(subdata, shape) - return buf - - cdef Py_ssize_t* list_to_sizebuf(self, l): - cdef size_t n = <size_t>len(l) * sizeof(Py_ssize_t) - cdef Py_ssize_t* buf = <Py_ssize_t*>stdlib.malloc(n) - for i, x in enumerate(l): - buf[i] = x - return buf - - def __getbuffer__(MockBuffer self, Py_buffer* buffer, int flags): - if self.fail: - raise ValueError("Failing on purpose") - - self.recieved_flags = [] - cdef int value - for name, value in available_flags: - if (value & flags) == value: - self.recieved_flags.append(name) - - buffer.buf = <void*>(<char*>self.buffer + (<int>self.offset * self.itemsize)) - buffer.obj = self - buffer.len = self.len - buffer.readonly = 0 - buffer.format = <char*>self.format - buffer.ndim = self.ndim - buffer.shape = self.shape - buffer.strides = self.strides - buffer.suboffsets = self.suboffsets - buffer.itemsize = self.itemsize - buffer.internal = NULL - if self.label: - msg = "acquired %s" % self.label - print msg - self.log += msg + "\n" - - def __releasebuffer__(MockBuffer self, Py_buffer* buffer): - if buffer.suboffsets != self.suboffsets: - self.release_ok = False - if self.label: - msg = "released %s" % self.label - print msg - self.log += msg + "\n" - - def printlog(self): - print self.log[:-1] - - def resetlog(self): - self.log = "" - - cdef int write(self, char* buf, object value) except -1: raise Exception() - cdef get_itemsize(self): - print "ERROR, not subclassed", self.__class__ - cdef get_default_format(self): - print "ERROR, not subclassed", self.__class__ - -cdef class CharMockBuffer(MockBuffer): - cdef int write(self, char* buf, object value) except -1: - (<char*>buf)[0] = <char>value - return 0 - cdef get_itemsize(self): return sizeof(char) - cdef get_default_format(self): return b"@b" - -cdef class IntMockBuffer(MockBuffer): - cdef int write(self, char* buf, object value) except -1: - (<int*>buf)[0] = <int>value - return 0 - cdef get_itemsize(self): return sizeof(int) - cdef get_default_format(self): return b"@i" - -cdef class UnsignedIntMockBuffer(MockBuffer): - cdef int write(self, char* buf, object value) except -1: - (<unsigned int*>buf)[0] = <unsigned int>value - return 0 - cdef get_itemsize(self): return sizeof(unsigned int) - cdef get_default_format(self): return b"@I" - -cdef class ShortMockBuffer(MockBuffer): - cdef int write(self, char* buf, object value) except -1: - (<short*>buf)[0] = <short>value - return 0 - cdef get_itemsize(self): return sizeof(short) - cdef get_default_format(self): return b"h" # Try without endian specifier - -cdef class UnsignedShortMockBuffer(MockBuffer): - cdef int write(self, char* buf, object value) except -1: - (<unsigned short*>buf)[0] = <unsigned short>value - return 0 - cdef get_itemsize(self): return sizeof(unsigned short) - cdef get_default_format(self): return b"@1H" # Try with repeat count - -cdef class FloatMockBuffer(MockBuffer): - cdef int write(self, char* buf, object value) except -1: - (<float*>buf)[0] = <float>(<double>value) - return 0 - cdef get_itemsize(self): return sizeof(float) - cdef get_default_format(self): return b"f" - -cdef class DoubleMockBuffer(MockBuffer): - cdef int write(self, char* buf, object value) except -1: - (<double*>buf)[0] = <double>value - return 0 - cdef get_itemsize(self): return sizeof(double) - cdef get_default_format(self): return b"d" - -cdef extern from *: - void* addr_of_pyobject "(void*)"(object) - -cdef class ObjectMockBuffer(MockBuffer): - cdef int write(self, char* buf, object value) except -1: - (<void**>buf)[0] = addr_of_pyobject(value) - return 0 - - cdef get_itemsize(self): return sizeof(void*) - cdef get_default_format(self): return b"@O" - - -cdef class IntStridedMockBuffer(IntMockBuffer): - cdef __cythonbufferdefaults__ = {"mode" : "strided"} - -cdef class ErrorBuffer: - cdef object label - - def __init__(self, label): - self.label = label - - def __getbuffer__(ErrorBuffer self, Py_buffer* buffer, int flags): - raise Exception("acquiring %s" % self.label) - - def __releasebuffer__(ErrorBuffer self, Py_buffer* buffer): - raise Exception("releasing %s" % self.label) - # # Typed buffers # @@ -1257,75 +1009,6 @@ def bufdefaults1(IntStridedMockBuffer[int, ndim=1] buf): pass -# -# Structs -# -cdef struct MyStruct: - char a - char b - long long int c - int d - int e - -cdef struct SmallStruct: - int a - int b - -cdef struct NestedStruct: - SmallStruct x - SmallStruct y - int z - -cdef packed struct PackedStruct: - char a - int b - -cdef struct NestedPackedStruct: - char a - int b - PackedStruct sub - int c - -cdef class MyStructMockBuffer(MockBuffer): - cdef int write(self, char* buf, object value) except -1: - cdef MyStruct* s - s = <MyStruct*>buf; - s.a, s.b, s.c, s.d, s.e = value - return 0 - - cdef get_itemsize(self): return sizeof(MyStruct) - cdef get_default_format(self): return b"2bq2i" - -cdef class NestedStructMockBuffer(MockBuffer): - cdef int write(self, char* buf, object value) except -1: - cdef NestedStruct* s - s = <NestedStruct*>buf; - s.x.a, s.x.b, s.y.a, s.y.b, s.z = value - return 0 - - cdef get_itemsize(self): return sizeof(NestedStruct) - cdef get_default_format(self): return b"2T{ii}i" - -cdef class PackedStructMockBuffer(MockBuffer): - cdef int write(self, char* buf, object value) except -1: - cdef PackedStruct* s - s = <PackedStruct*>buf; - s.a, s.b = value - return 0 - - cdef get_itemsize(self): return sizeof(PackedStruct) - cdef get_default_format(self): return b"^ci" - -cdef class NestedPackedStructMockBuffer(MockBuffer): - cdef int write(self, char* buf, object value) except -1: - cdef NestedPackedStruct* s - s = <NestedPackedStruct*>buf; - s.a, s.b, s.sub.a, s.sub.b, s.c = value - return 0 - - cdef get_itemsize(self): return sizeof(NestedPackedStruct) - cdef get_default_format(self): return b"ci^ci@i" - @testcase def basic_struct(object[MyStruct] buf): """ diff --git a/tests/run/cythonarray.pyx b/tests/run/cythonarray.pyx index 34049b363..540e41967 100644 --- a/tests/run/cythonarray.pyx +++ b/tests/run/cythonarray.pyx @@ -2,10 +2,8 @@ from __future__ import unicode_literals -# from cython cimport array -# cimport cython.array as array +from cython cimport array cimport cython as cy -# array = cython.array def contiguity(): ''' @@ -69,3 +67,37 @@ def dont_allocate_buffer(): cdef void callback(char *data): print "callback called %d" % <long> data + +cdef create_array(shape, mode): + cdef array result = array(shape, itemsize=sizeof(int), format='i', mode=mode) + cdef int *data = <int *> result.data + cdef int i, j, cidx, fidx + + for i in range(shape[0]): + for j in range(shape[1]): + cidx = i * shape[1] + j + fidx = i + j * shape[0] + + if mode == 'fortran': + data[fidx] = cidx + else: + data[cidx] = cidx + + return result + +def test_cython_array(): + """ + >>> test_cython_array() + 98 + 61 + 98 + 61 + """ + cdef int[:, ::1] carr = create_array((14, 10), 'c') + cdef int[::1, :] farr = create_array((14, 10), 'fortran') + + print carr[9, 8] + print carr[6, 1] + + print farr[9, 8] + print farr[6, 1] diff --git a/tests/run/memslice.pyx b/tests/run/memslice.pyx new file mode 100644 index 000000000..0d64f093b --- /dev/null +++ b/tests/run/memslice.pyx @@ -0,0 +1,1102 @@ +# Tests the buffer access syntax functionality by constructing +# mock buffer objects. +# +# Note that the buffers are mock objects created for testing +# the buffer access behaviour -- for instance there is no flag +# checking in the buffer objects (why test our test case?), rather +# what we want to test is what is passed into the flags argument. +# + +from __future__ import unicode_literals + +from cython cimport view + +__test__ = {} + +import sys +import re +exclude = []#re.compile('object').search] + +def testcase(func): + for e in exclude: + if e(func.__name__): + return func + doctest = func.__doc__ + if sys.version_info >= (3,1,1): + doctest = doctest.replace('does not have the buffer interface', + 'does not support the buffer interface') + __test__[func.__name__] = doctest + return func + + +include "mockbuffers.pxi" + +# +# Buffer acquire and release tests +# + +def nousage(): + """ + The challenge here is just compilation. + """ + cdef int[:, :] buf + +@testcase +def acquire_release(o1, o2): + """ + >>> A = IntMockBuffer("A", range(6)) + >>> B = IntMockBuffer("B", range(6)) + >>> acquire_release(A, B) + acquired A + released A + acquired B + released B + >>> acquire_release(None, None) + >>> acquire_release(None, B) + acquired B + released B + """ + cdef int[:] buf + buf = o1 + buf = o2 + +@testcase +def acquire_raise(o): + """ + Apparently, doctest won't handle mixed exceptions and print + stats, so need to circumvent this. + + >>> A = IntMockBuffer("A", range(6)) + >>> A.resetlog() + >>> acquire_raise(A) + Traceback (most recent call last): + ... + Exception: on purpose + >>> A.printlog() + acquired A + released A + + """ + cdef int[:] buf + buf = o + raise Exception("on purpose") + +@testcase +def acquire_failure1(): + """ + >>> acquire_failure1() + acquired working + 0 3 + 0 3 + released working + """ + cdef int[:] buf + buf = IntMockBuffer("working", range(4)) + print buf[0], buf[3] + try: + buf = ErrorBuffer() + assert False + except Exception: + print buf[0], buf[3] + +@testcase +def acquire_failure2(): + """ + >>> acquire_failure2() + acquired working + 0 3 + 0 3 + released working + """ + cdef int[:] buf = IntMockBuffer("working", range(4)) + print buf[0], buf[3] + try: + buf = ErrorBuffer() + assert False + except Exception: + print buf[0], buf[3] + +@testcase +def acquire_failure3(): + """ + >>> acquire_failure3() + acquired working + 0 3 + released working + acquired working + 0 3 + released working + """ + cdef int[:] buf + buf = IntMockBuffer("working", range(4)) + print buf[0], buf[3] + try: + buf = object() + assert False + except Exception: + print buf[0], buf[3] + + +@testcase +def acquire_nonbuffer1(first, second=None): + """ + >>> acquire_nonbuffer1(3) + Traceback (most recent call last): + ... + TypeError: 'int' does not have the buffer interface + >>> acquire_nonbuffer1(type) + Traceback (most recent call last): + ... + TypeError: 'type' does not have the buffer interface + >>> acquire_nonbuffer1(None, 2) + Traceback (most recent call last): + ... + TypeError: 'int' does not have the buffer interface + """ + cdef int[:] buf + buf = first + buf = second + +@testcase +def acquire_nonbuffer2(): + """ + >>> acquire_nonbuffer2() + acquired working + 0 3 + released working + acquired working + 0 3 + released working + """ + cdef int[:] buf = IntMockBuffer("working", range(4)) + print buf[0], buf[3] + try: + buf = ErrorBuffer + assert False + except Exception: + print buf[0], buf[3] + + +@testcase +def as_argument(int[:] bufarg, int n): + """ + >>> A = IntMockBuffer("A", range(6)) + >>> as_argument(A, 6) + acquired A + 0 1 2 3 4 5 END + released A + """ + cdef int i + for i in range(n): + print bufarg[i], + print 'END' + +@testcase +def as_argument_defval(int[:] bufarg=IntMockBuffer('default', range(6)), int n=6): + """ + >>> as_argument_defval() + acquired default + 0 1 2 3 4 5 END + released default + >>> A = IntMockBuffer("A", range(6)) + >>> as_argument_defval(A, 6) + acquired A + 0 1 2 3 4 5 END + released A + """ + cdef int i + for i in range(n): + print bufarg[i], + print 'END' + +@testcase +def cdef_assignment(obj, n): + """ + >>> A = IntMockBuffer("A", range(6)) + >>> cdef_assignment(A, 6) + acquired A + 0 1 2 3 4 5 END + released A + + """ + cdef int[:] buf = obj + cdef int i + for i in range(n): + print buf[i], + print 'END' + +@testcase +def forin_assignment(objs, int pick): + """ + >>> A = IntMockBuffer("A", range(6)) + >>> B = IntMockBuffer("B", range(6)) + >>> forin_assignment([A, B, A, A], 2) + acquired A + 2 + released A + acquired B + 2 + released B + acquired A + 2 + released A + acquired A + 2 + released A + """ + cdef int[:] buf + for buf in objs: + print buf[pick] + +@testcase +def cascaded_buffer_assignment(obj): + """ + >>> A = IntMockBuffer("A", range(6)) + >>> cascaded_buffer_assignment(A) + acquired A + acquired A + released A + released A + """ + cdef int[:] a, b + a = b = obj + +@testcase +def tuple_buffer_assignment1(a, b): + """ + >>> A = IntMockBuffer("A", range(6)) + >>> B = IntMockBuffer("B", range(6)) + >>> tuple_buffer_assignment1(A, B) + acquired A + acquired B + released A + released B + """ + cdef int[:] x, y + x, y = a, b + +@testcase +def tuple_buffer_assignment2(tup): + """ + >>> A = IntMockBuffer("A", range(6)) + >>> B = IntMockBuffer("B", range(6)) + >>> tuple_buffer_assignment2((A, B)) + acquired A + acquired B + released A + released B + """ + cdef int[:] x, y + x, y = tup + +@testcase +def explicitly_release_buffer(): + """ + >>> explicitly_release_buffer() + acquired A + released A + After release + """ + cdef int[:] x = IntMockBuffer("A", range(10)) + del x + print "After release" + +# +# Getting items and index bounds checking +# +@testcase +def get_int_2d(int[:, :] buf, int i, int j): + """ + >>> C = IntMockBuffer("C", range(6), (2,3)) + >>> get_int_2d(C, 1, 1) + acquired C + released C + 4 + + Check negative indexing: + >>> get_int_2d(C, -1, 0) + acquired C + released C + 3 + >>> get_int_2d(C, -1, -2) + acquired C + released C + 4 + >>> get_int_2d(C, -2, -3) + acquired C + released C + 0 + + Out-of-bounds errors: + >>> get_int_2d(C, 2, 0) + Traceback (most recent call last): + ... + IndexError: Out of bounds on buffer access (axis 0) + >>> get_int_2d(C, 0, -4) + Traceback (most recent call last): + ... + IndexError: Out of bounds on buffer access (axis 1) + """ + return buf[i, j] + +@testcase +def get_int_2d_uintindex(int[:, :] buf, unsigned int i, unsigned int j): + """ + Unsigned indexing: + >>> C = IntMockBuffer("C", range(6), (2,3)) + >>> get_int_2d_uintindex(C, 0, 0) + acquired C + released C + 0 + >>> get_int_2d_uintindex(C, 1, 2) + acquired C + released C + 5 + """ + # This is most interesting with regards to the C code + # generated. + return buf[i, j] + +@testcase +def set_int_2d(int[:, :] buf, int i, int j, int value): + """ + Uses get_int_2d to read back the value afterwards. For pure + unit test, one should support reading in MockBuffer instead. + + >>> C = IntMockBuffer("C", range(6), (2,3)) + >>> set_int_2d(C, 1, 1, 10) + acquired C + released C + >>> get_int_2d(C, 1, 1) + acquired C + released C + 10 + + Check negative indexing: + >>> set_int_2d(C, -1, 0, 3) + acquired C + released C + >>> get_int_2d(C, -1, 0) + acquired C + released C + 3 + + >>> set_int_2d(C, -1, -2, 8) + acquired C + released C + >>> get_int_2d(C, -1, -2) + acquired C + released C + 8 + + >>> set_int_2d(C, -2, -3, 9) + acquired C + released C + >>> get_int_2d(C, -2, -3) + acquired C + released C + 9 + + Out-of-bounds errors: + >>> set_int_2d(C, 2, 0, 19) + Traceback (most recent call last): + ... + IndexError: Out of bounds on buffer access (axis 0) + >>> set_int_2d(C, 0, -4, 19) + Traceback (most recent call last): + ... + IndexError: Out of bounds on buffer access (axis 1) + + """ + buf[i, j] = value + +@testcase +def list_comprehension(int[:] buf, len): + """ + >>> list_comprehension(IntMockBuffer(None, [1,2,3]), 3) + 1|2|3 + """ + cdef int i + print u"|".join([unicode(buf[i]) for i in range(len)]) + +# +# The negative_indices buffer option +# +@testcase +def no_negative_indices(object[int, negative_indices=False] buf, int idx): + """ + The most interesting thing here is to inspect the C source and + make sure optimal code is produced. + + >>> A = IntMockBuffer(None, range(6)) + >>> no_negative_indices(A, 3) + 3 + >>> no_negative_indices(A, -1) + Traceback (most recent call last): + ... + IndexError: Out of bounds on buffer access (axis 0) + """ + return buf[idx] + +@testcase +@cython.wraparound(False) +def wraparound_directive(int[:] buf, int pos_idx, int neg_idx): + """ + Again, the most interesting thing here is to inspect the C source. + + >>> A = IntMockBuffer(None, range(4)) + >>> wraparound_directive(A, 2, -1) + 5 + >>> wraparound_directive(A, -1, 2) + Traceback (most recent call last): + ... + IndexError: Out of bounds on buffer access (axis 0) + """ + cdef int byneg + with cython.wraparound(True): + byneg = buf[neg_idx] + return buf[pos_idx] + byneg + + +# +# Test which flags are passed. +# +# @testcase +# def readonly(obj): +# """ +# >>> R = UnsignedShortMockBuffer("R", range(27), shape=(3, 3, 3)) +# >>> readonly(R) +# acquired R +# 25 +# released R +# >>> [str(x) for x in R.recieved_flags] # Works in both py2 and py3 +# ['FORMAT', 'INDIRECT', 'ND', 'STRIDES'] +# """ +# cdef unsigned short int[:, :, :] buf = obj +# print buf[2, 2, 1] + +@testcase +def writable(obj): + """ + >>> R = UnsignedShortMockBuffer("R", range(27), shape=(3, 3, 3)) + >>> writable(R) + acquired R + released R + >>> [str(x) for x in R.recieved_flags] # Py2/3 + ['FORMAT', 'ND', 'STRIDES', 'WRITABLE'] + """ + cdef unsigned short int[:, :, :] buf = obj + buf[2, 2, 1] = 23 + +@testcase +def strided(int[:] buf): + """ + >>> A = IntMockBuffer("A", range(4)) + >>> strided(A) + acquired A + released A + 2 + >>> [str(x) for x in A.recieved_flags] # Py2/3 + ['FORMAT', 'ND', 'STRIDES', 'WRITABLE'] + + Check that the suboffsets were patched back prior to release. + >>> A.release_ok + True + """ + return buf[2] + +@testcase +def c_contig(int[::1] buf): + """ + >>> A = IntMockBuffer(None, range(4)) + >>> c_contig(A) + 2 + >>> [str(x) for x in A.recieved_flags] + ['FORMAT', 'ND', 'STRIDES', 'C_CONTIGUOUS'] + """ + return buf[2] + +@testcase +def c_contig_2d(int[:, ::1] buf): + """ + Multi-dim has seperate implementation + + >>> A = IntMockBuffer(None, range(12), shape=(3,4)) + >>> c_contig_2d(A) + 7 + >>> [str(x) for x in A.recieved_flags] + ['FORMAT', 'ND', 'STRIDES', 'C_CONTIGUOUS'] + """ + return buf[1, 3] + +@testcase +def f_contig(int[::1, :] buf): + """ + >>> A = IntMockBuffer(None, range(4), shape=(2, 2)) + >>> f_contig(A) + 2 + >>> [str(x) for x in A.recieved_flags] + ['FORMAT', 'ND', 'STRIDES', 'F_CONTIGUOUS'] + """ + return buf[0, 1] + +@testcase +def f_contig_2d(object[int, ndim=2, mode='fortran'] buf): + """ + Must set up strides manually to ensure Fortran ordering. + + >>> A = IntMockBuffer(None, range(12), shape=(4,3), strides=(1, 4)) + >>> f_contig_2d(A) + 7 + >>> [str(x) for x in A.recieved_flags] + ['FORMAT', 'ND', 'STRIDES', 'F_CONTIGUOUS'] + """ + return buf[3, 1] + +# +# Test compiler options for bounds checking. We create an array with a +# safe "boundary" (memory +# allocated outside of what it published) and then check whether we get back +# what we stored in the memory or an error. + +@testcase +def safe_get(int[:] buf, int idx): + """ + >>> A = IntMockBuffer(None, range(10), shape=(3,), offset=5) + + Validate our testing buffer... + >>> safe_get(A, 0) + 5 + >>> safe_get(A, 2) + 7 + >>> safe_get(A, -3) + 5 + + Access outside it. This is already done above for bounds check + testing but we include it to tell the story right. + + >>> safe_get(A, -4) + Traceback (most recent call last): + ... + IndexError: Out of bounds on buffer access (axis 0) + >>> safe_get(A, 3) + Traceback (most recent call last): + ... + IndexError: Out of bounds on buffer access (axis 0) + """ + return buf[idx] + +@testcase +@cython.boundscheck(False) # outer decorators should take precedence +@cython.boundscheck(True) +def unsafe_get(int[:] buf, int idx): + """ + Access outside of the area the buffer publishes. + >>> A = IntMockBuffer(None, range(10), shape=(3,), offset=5) + >>> unsafe_get(A, -4) + 4 + >>> unsafe_get(A, -5) + 3 + >>> unsafe_get(A, 3) + 8 + """ + return buf[idx] + +# @testcase +# @cython.boundscheck(False) +# def unsafe_get_nonegative(object[int, negative_indices=False] buf, int idx): +# """ +# Also inspect the C source to see that it is optimal... +# +# >>> A = IntMockBuffer(None, range(10), shape=(3,), offset=5) +# >>> unsafe_get_nonegative(A, -2) +# 3 +# """ +# return buf[idx] + +@testcase +def mixed_get(int[:] buf, int unsafe_idx, int safe_idx): + """ + >>> A = IntMockBuffer(None, range(10), shape=(3,), offset=5) + >>> mixed_get(A, -4, 0) + (4, 5) + >>> mixed_get(A, 0, -4) + Traceback (most recent call last): + ... + IndexError: Out of bounds on buffer access (axis 0) + """ + with cython.boundscheck(False): + one = buf[unsafe_idx] + with cython.boundscheck(True): + two = buf[safe_idx] + return (one, two) + +# +# Coercions +# +## @testcase +## def coercions(object[unsigned char] uc): +## """ +## TODO +## """ +## print type(uc[0]) +## uc[0] = -1 +## print uc[0] +## uc[0] = <int>3.14 +## print uc[0] + +## cdef char* ch = b"asfd" +## cdef object[object] objbuf +## objbuf[3] = ch + + +# +# Testing that accessing data using various types of buffer access +# all works. +# + +def printbuf_int(int[:] buf, shape): + # Utility func + cdef int i + for i in range(shape[0]): + print buf[i], + print 'END' + + +@testcase +def printbuf_int_2d(o, shape): + """ + Strided: + + >>> printbuf_int_2d(IntMockBuffer("A", range(6), (2,3)), (2,3)) + acquired A + 0 1 2 END + 3 4 5 END + released A + >>> printbuf_int_2d(IntMockBuffer("A", range(100), (3,3), strides=(20,5)), (3,3)) + acquired A + 0 5 10 END + 20 25 30 END + 40 45 50 END + released A + + Indirect: + >>> printbuf_int_2d(IntMockBuffer("A", [[1,2],[3,4]]), (2,2)) + acquired A + 1 2 END + 3 4 END + released A + """ + # should make shape builtin + cdef int[:, :] buf + buf = o + cdef int i, j + for i in range(shape[0]): + for j in range(shape[1]): + print buf[i, j], + print 'END' + +@testcase +def printbuf_float(o, shape): + """ + >>> printbuf_float(FloatMockBuffer("F", [1.0, 1.25, 0.75, 1.0]), (4,)) + acquired F + 1.0 1.25 0.75 1.0 END + released F + """ + + # should make shape builtin + cdef float[:] buf + buf = o + cdef int i, j + for i in range(shape[0]): + print buf[i], + print "END" + + +# +# Test assignments +# +@testcase +def inplace_operators(int[:] buf): + """ + >>> buf = IntMockBuffer(None, [2, 2]) + >>> inplace_operators(buf) + >>> printbuf_int(buf, (2,)) + 0 3 END + """ + cdef int j = 0 + buf[1] += 1 + buf[j] *= 2 + buf[0] -= 4 + + + +# +# Typedefs +# +# Test three layers of typedefs going through a h file for plain int, and +# simply a header file typedef for floats and unsigned. + +ctypedef int td_cy_int +cdef extern from "bufaccess.h": + ctypedef td_cy_int td_h_short # Defined as short, but Cython doesn't know this! + ctypedef float td_h_double # Defined as double + ctypedef unsigned int td_h_ushort # Defined as unsigned short +ctypedef td_h_short td_h_cy_short + +@testcase +def printbuf_td_cy_int(td_cy_int[:] buf, shape): + """ + >>> printbuf_td_cy_int(IntMockBuffer(None, range(3)), (3,)) + 0 1 2 END + >>> printbuf_td_cy_int(ShortMockBuffer(None, range(3)), (3,)) + Traceback (most recent call last): + ... + ValueError: Buffer dtype mismatch, expected 'td_cy_int' but got 'short' + """ + cdef int i + for i in range(shape[0]): + print buf[i], + print 'END' + +@testcase +def printbuf_td_h_short(object[td_h_short] buf, shape): + """ + >>> printbuf_td_h_short(ShortMockBuffer(None, range(3)), (3,)) + 0 1 2 END + >>> printbuf_td_h_short(IntMockBuffer(None, range(3)), (3,)) + Traceback (most recent call last): + ... + ValueError: Buffer dtype mismatch, expected 'td_h_short' but got 'int' + """ + cdef int i + for i in range(shape[0]): + print buf[i], + print 'END' + +@testcase +def printbuf_td_h_cy_short(object[td_h_cy_short] buf, shape): + """ + >>> printbuf_td_h_cy_short(ShortMockBuffer(None, range(3)), (3,)) + 0 1 2 END + >>> printbuf_td_h_cy_short(IntMockBuffer(None, range(3)), (3,)) + Traceback (most recent call last): + ... + ValueError: Buffer dtype mismatch, expected 'td_h_cy_short' but got 'int' + """ + cdef int i + for i in range(shape[0]): + print buf[i], + print 'END' + +@testcase +def printbuf_td_h_ushort(object[td_h_ushort] buf, shape): + """ + >>> printbuf_td_h_ushort(UnsignedShortMockBuffer(None, range(3)), (3,)) + 0 1 2 END + >>> printbuf_td_h_ushort(ShortMockBuffer(None, range(3)), (3,)) + Traceback (most recent call last): + ... + ValueError: Buffer dtype mismatch, expected 'td_h_ushort' but got 'short' + """ + cdef int i + for i in range(shape[0]): + print buf[i], + print 'END' + +@testcase +def printbuf_td_h_double(object[td_h_double] buf, shape): + """ + >>> printbuf_td_h_double(DoubleMockBuffer(None, [0.25, 1, 3.125]), (3,)) + 0.25 1.0 3.125 END + >>> printbuf_td_h_double(FloatMockBuffer(None, [0.25, 1, 3.125]), (3,)) + Traceback (most recent call last): + ... + ValueError: Buffer dtype mismatch, expected 'td_h_double' but got 'float' + """ + cdef int i + for i in range(shape[0]): + print buf[i], + print 'END' + + +# +# Object access +# +def addref(*args): + for item in args: Py_INCREF(item) +def decref(*args): + for item in args: Py_DECREF(item) + +def get_refcount(x): + return (<PyObject*>x).ob_refcnt + +@testcase +def printbuf_object(object[object] buf, shape): + """ + Only play with unique objects, interned numbers etc. will have + unpredictable refcounts. + + ObjectMockBuffer doesn't do anything about increfing/decrefing, + we to the "buffer implementor" refcounting directly in the + testcase. + + >>> a, b, c = "globally_unique_string_23234123", {4:23}, [34,3] + >>> get_refcount(a), get_refcount(b), get_refcount(c) + (2, 2, 2) + >>> A = ObjectMockBuffer(None, [a, b, c]) + >>> printbuf_object(A, (3,)) + 'globally_unique_string_23234123' 2 + {4: 23} 2 + [34, 3] 2 + """ + cdef int i + for i in range(shape[0]): + print repr(buf[i]), (<PyObject*>buf[i]).ob_refcnt + +@testcase +def assign_to_object(object[object] buf, int idx, obj): + """ + See comments on printbuf_object above. + + >>> a, b = [1, 2, 3], [4, 5, 6] + >>> get_refcount(a), get_refcount(b) + (2, 2) + >>> addref(a) + >>> A = ObjectMockBuffer(None, [1, a]) # 1, ...,otherwise it thinks nested lists... + >>> get_refcount(a), get_refcount(b) + (3, 2) + >>> assign_to_object(A, 1, b) + >>> get_refcount(a), get_refcount(b) + (2, 3) + >>> decref(b) + """ + buf[idx] = obj + +@testcase +def assign_temporary_to_object(object[object] buf): + """ + See comments on printbuf_object above. + + >>> a, b = [1, 2, 3], {4:23} + >>> get_refcount(a) + 2 + >>> addref(a) + >>> A = ObjectMockBuffer(None, [b, a]) + >>> get_refcount(a) + 3 + >>> assign_temporary_to_object(A) + >>> get_refcount(a) + 2 + + >>> printbuf_object(A, (2,)) + {4: 23} 2 + {1: 8} 2 + + To avoid leaking a reference in our testcase we need to + replace the temporary with something we can manually decref :-) + >>> assign_to_object(A, 1, a) + >>> decref(a) + """ + buf[1] = {3-2: 2+(2*4)-2} + +# +# cast option +# +@testcase +def buffer_cast(object[unsigned int, cast=True] buf, int idx): + """ + Round-trip a signed int through unsigned int buffer access. + + >>> A = IntMockBuffer(None, [-100]) + >>> buffer_cast(A, 0) + -100 + """ + cdef unsigned int data = buf[idx] + return <int>data + +@testcase +def buffer_cast_fails(object[char, cast=True] buf): + """ + Cannot cast between datatype of different sizes. + + >>> buffer_cast_fails(IntMockBuffer(None, [0])) + Traceback (most recent call last): + ... + ValueError: Item size of buffer (4 bytes) does not match size of 'char' (1 byte) + """ + return buf[0] + +# +# Typed buffers +# +@testcase +def typedbuffer1(obj): + """ + >>> typedbuffer1(IntMockBuffer("A", range(10))) + acquired A + released A + >>> typedbuffer1(None) + >>> typedbuffer1(4) + Traceback (most recent call last): + ... + TypeError: Cannot convert int to memslice.IntMockBuffer + """ + cdef IntMockBuffer[int, ndim=1] buf = obj + +@testcase +def typedbuffer2(IntMockBuffer[int, ndim=1] obj): + """ + >>> typedbuffer2(IntMockBuffer("A", range(10))) + acquired A + released A + >>> typedbuffer2(None) + >>> typedbuffer2(4) + Traceback (most recent call last): + ... + TypeError: Argument 'obj' has incorrect type (expected memslice.IntMockBuffer, got int) + """ + pass + +# +# Test __cythonbufferdefaults__ +# +@testcase +def bufdefaults1(IntStridedMockBuffer[int, ndim=1] buf): + """ + For IntStridedMockBuffer, mode should be + "strided" by defaults which should show + up in the flags. + + >>> A = IntStridedMockBuffer("A", range(10)) + >>> bufdefaults1(A) + acquired A + released A + >>> [str(x) for x in A.recieved_flags] + ['FORMAT', 'ND', 'STRIDES'] + """ + pass + + +@testcase +def basic_struct(object[MyStruct] buf): + """ + See also buffmt.pyx + + >>> basic_struct(MyStructMockBuffer(None, [(1, 2, 3, 4, 5)])) + 1 2 3 4 5 + >>> basic_struct(MyStructMockBuffer(None, [(1, 2, 3, 4, 5)], format="bbqii")) + 1 2 3 4 5 + """ + print buf[0].a, buf[0].b, buf[0].c, buf[0].d, buf[0].e + +@testcase +def nested_struct(object[NestedStruct] buf): + """ + See also buffmt.pyx + + >>> nested_struct(NestedStructMockBuffer(None, [(1, 2, 3, 4, 5)])) + 1 2 3 4 5 + >>> nested_struct(NestedStructMockBuffer(None, [(1, 2, 3, 4, 5)], format="T{ii}T{2i}i")) + 1 2 3 4 5 + """ + print buf[0].x.a, buf[0].x.b, buf[0].y.a, buf[0].y.b, buf[0].z + +@testcase +def packed_struct(object[PackedStruct] buf): + """ + See also buffmt.pyx + + >>> packed_struct(PackedStructMockBuffer(None, [(1, 2)])) + 1 2 + >>> packed_struct(PackedStructMockBuffer(None, [(1, 2)], format="T{c^i}")) + 1 2 + >>> packed_struct(PackedStructMockBuffer(None, [(1, 2)], format="T{c=i}")) + 1 2 + + """ + print buf[0].a, buf[0].b + +@testcase +def nested_packed_struct(object[NestedPackedStruct] buf): + """ + See also buffmt.pyx + + >>> nested_packed_struct(NestedPackedStructMockBuffer(None, [(1, 2, 3, 4, 5)])) + 1 2 3 4 5 + >>> nested_packed_struct(NestedPackedStructMockBuffer(None, [(1, 2, 3, 4, 5)], format="ci^ci@i")) + 1 2 3 4 5 + >>> nested_packed_struct(NestedPackedStructMockBuffer(None, [(1, 2, 3, 4, 5)], format="^c@i^ci@i")) + 1 2 3 4 5 + """ + print buf[0].a, buf[0].b, buf[0].sub.a, buf[0].sub.b, buf[0].c + +cdef struct LongComplex: + long double real + long double imag + +cdef class LongComplexMockBuffer(MockBuffer): + cdef int write(self, char* buf, object value) except -1: + cdef LongComplex* s + s = <LongComplex*>buf; + s.real, s.imag = value + return 0 + + cdef get_itemsize(self): return sizeof(LongComplex) + cdef get_default_format(self): return b"Zg" + +#cdef extern from "complex.h": +# pass + +@testcase +def complex_dtype(object[long double complex] buf): + """ + >>> complex_dtype(LongComplexMockBuffer(None, [(0, -1)])) + -1j + """ + print buf[0] + +@testcase +def complex_inplace(object[long double complex] buf): + """ + >>> complex_inplace(LongComplexMockBuffer(None, [(0, -1)])) + (1+1j) + """ + buf[0] = buf[0] + 1 + 2j + print buf[0] + +@testcase +def complex_struct_dtype(object[LongComplex] buf): + """ + Note that the format string is "Zg" rather than "2g", yet a struct + is accessed. + >>> complex_struct_dtype(LongComplexMockBuffer(None, [(0, -1)])) + 0.0 -1.0 + """ + print buf[0].real, buf[0].imag + +@testcase +def complex_struct_inplace(object[LongComplex] buf): + """ + >>> complex_struct_inplace(LongComplexMockBuffer(None, [(0, -1)])) + 1.0 1.0 + """ + buf[0].real += 1 + buf[0].imag += 2 + print buf[0].real, buf[0].imag + +# +# Nogil +# +@testcase +@cython.boundscheck(False) +def buffer_nogil(): + """ + >>> buffer_nogil() + 10 + """ + cdef int[:] buf = IntMockBuffer(None, [1,2,3]) + with nogil: + buf[1] = 10 + return buf[1] diff --git a/tests/run/memslice_indexing.pyx b/tests/run/memslice_indexing.pyx deleted file mode 100644 index 08f519e1a..000000000 --- a/tests/run/memslice_indexing.pyx +++ /dev/null @@ -1,40 +0,0 @@ -cimport cython -from cython cimport array - -from libc.stdlib cimport malloc, free - -def create_array(shape, mode='c'): - cdef array result = array(shape, itemsize=sizeof(int), format='i', mode=mode) - cdef int *data = <int *> result.data - cdef int i, j, value - - for i in range(shape[0]): - for j in range(shape[1]): - value = i * shape[0] + j - if mode == 'fortran': - data[i + j * 10] = value - else: - data[value] = value - - return result - -def slice_contig_indexing(): - """ - >>> print("disabled") - disabled - - slice_contig_indexing() - 98 - 61 - 98 - 61 - """ - cdef int[:, ::1] carr = create_array((14, 10)) - cdef int[::1, :] farr = create_array((10, 14), mode='fortran') - - print carr[9, 8] - print carr[6, 1] - - print farr[9, 8] - print farr[6, 1] - diff --git a/tests/run/mockbuffers.pxi b/tests/run/mockbuffers.pxi new file mode 100644 index 000000000..acfb813da --- /dev/null +++ b/tests/run/mockbuffers.pxi @@ -0,0 +1,313 @@ +from libc cimport stdlib +from libc cimport stdio +cimport cpython.buffer +cimport cython + +from cpython cimport PyObject, Py_INCREF, Py_DECREF + + +available_flags = ( + ('FORMAT', cpython.buffer.PyBUF_FORMAT), + ('INDIRECT', cpython.buffer.PyBUF_INDIRECT), + ('ND', cpython.buffer.PyBUF_ND), + ('STRIDES', cpython.buffer.PyBUF_STRIDES), + ('C_CONTIGUOUS', cpython.buffer.PyBUF_C_CONTIGUOUS), + ('F_CONTIGUOUS', cpython.buffer.PyBUF_F_CONTIGUOUS), + ('WRITABLE', cpython.buffer.PyBUF_WRITABLE) +) + +cdef class MockBuffer: + cdef object format, offset + cdef void* buffer + cdef int len, itemsize, ndim + cdef Py_ssize_t* strides + cdef Py_ssize_t* shape + cdef Py_ssize_t* suboffsets + cdef object label, log + + cdef readonly object recieved_flags, release_ok + cdef public object fail + + def __init__(self, label, data, shape=None, strides=None, format=None, offset=0): + # It is important not to store references to data after the constructor + # as refcounting is checked on object buffers. + self.label = label + self.release_ok = True + self.log = "" + self.offset = offset + self.itemsize = self.get_itemsize() + if format is None: format = self.get_default_format() + if shape is None: shape = (len(data),) + if strides is None: + strides = [] + cumprod = 1 + rshape = list(shape) + rshape.reverse() + for s in rshape: + strides.append(cumprod) + cumprod *= s + strides.reverse() + strides = [x * self.itemsize for x in strides] + suboffsets = [-1] * len(shape) + datashape = [len(data)] + p = data + while True: + p = p[0] + if isinstance(p, list): datashape.append(len(p)) + else: break + if len(datashape) > 1: + # indirect access + self.ndim = len(datashape) + shape = datashape + self.buffer = self.create_indirect_buffer(data, shape) + suboffsets = [0] * (self.ndim-1) + [-1] + strides = [sizeof(void*)] * (self.ndim-1) + [self.itemsize] + self.suboffsets = self.list_to_sizebuf(suboffsets) + else: + # strided and/or simple access + self.buffer = self.create_buffer(data) + self.ndim = len(shape) + self.suboffsets = NULL + + try: + format = format.encode('ASCII') + except AttributeError: + pass + self.format = format + self.len = len(data) * self.itemsize + + self.strides = self.list_to_sizebuf(strides) + self.shape = self.list_to_sizebuf(shape) + + def __dealloc__(self): + stdlib.free(self.strides) + stdlib.free(self.shape) + if self.suboffsets != NULL: + stdlib.free(self.suboffsets) + # must recursively free indirect... + else: + stdlib.free(self.buffer) + + cdef void* create_buffer(self, data) except NULL: + cdef size_t n = <size_t>(len(data) * self.itemsize) + cdef char* buf = <char*>stdlib.malloc(n) + if buf == NULL: + raise MemoryError + cdef char* it = buf + for value in data: + self.write(it, value) + it += self.itemsize + return buf + + cdef void* create_indirect_buffer(self, data, shape): + cdef size_t n = 0 + cdef void** buf + assert shape[0] == len(data) + if len(shape) == 1: + return self.create_buffer(data) + else: + shape = shape[1:] + n = <size_t>len(data) * sizeof(void*) + buf = <void**>stdlib.malloc(n) + for idx, subdata in enumerate(data): + buf[idx] = self.create_indirect_buffer(subdata, shape) + return buf + + cdef Py_ssize_t* list_to_sizebuf(self, l): + cdef size_t n = <size_t>len(l) * sizeof(Py_ssize_t) + cdef Py_ssize_t* buf = <Py_ssize_t*>stdlib.malloc(n) + for i, x in enumerate(l): + buf[i] = x + return buf + + def __getbuffer__(MockBuffer self, Py_buffer* buffer, int flags): + if self.fail: + raise ValueError("Failing on purpose") + + self.recieved_flags = [] + cdef int value + for name, value in available_flags: + if (value & flags) == value: + self.recieved_flags.append(name) + + buffer.buf = <void*>(<char*>self.buffer + (<int>self.offset * self.itemsize)) + buffer.obj = self + buffer.len = self.len + buffer.readonly = 0 + buffer.format = <char*>self.format + buffer.ndim = self.ndim + buffer.shape = self.shape + buffer.strides = self.strides + buffer.suboffsets = self.suboffsets + buffer.itemsize = self.itemsize + buffer.internal = NULL + if self.label: + msg = "acquired %s" % self.label + print msg + self.log += msg + "\n" + + def __releasebuffer__(MockBuffer self, Py_buffer* buffer): + if buffer.suboffsets != self.suboffsets: + self.release_ok = False + if self.label: + msg = "released %s" % self.label + print msg + self.log += msg + "\n" + + def printlog(self): + print self.log[:-1] + + def resetlog(self): + self.log = "" + + cdef int write(self, char* buf, object value) except -1: raise Exception() + cdef get_itemsize(self): + print "ERROR, not subclassed", self.__class__ + cdef get_default_format(self): + print "ERROR, not subclassed", self.__class__ + +cdef class CharMockBuffer(MockBuffer): + cdef int write(self, char* buf, object value) except -1: + (<char*>buf)[0] = <char>value + return 0 + cdef get_itemsize(self): return sizeof(char) + cdef get_default_format(self): return b"@b" + +cdef class IntMockBuffer(MockBuffer): + cdef int write(self, char* buf, object value) except -1: + (<int*>buf)[0] = <int>value + return 0 + cdef get_itemsize(self): return sizeof(int) + cdef get_default_format(self): return b"@i" + +cdef class UnsignedIntMockBuffer(MockBuffer): + cdef int write(self, char* buf, object value) except -1: + (<unsigned int*>buf)[0] = <unsigned int>value + return 0 + cdef get_itemsize(self): return sizeof(unsigned int) + cdef get_default_format(self): return b"@I" + +cdef class ShortMockBuffer(MockBuffer): + cdef int write(self, char* buf, object value) except -1: + (<short*>buf)[0] = <short>value + return 0 + cdef get_itemsize(self): return sizeof(short) + cdef get_default_format(self): return b"h" # Try without endian specifier + +cdef class UnsignedShortMockBuffer(MockBuffer): + cdef int write(self, char* buf, object value) except -1: + (<unsigned short*>buf)[0] = <unsigned short>value + return 0 + cdef get_itemsize(self): return sizeof(unsigned short) + cdef get_default_format(self): return b"@1H" # Try with repeat count + +cdef class FloatMockBuffer(MockBuffer): + cdef int write(self, char* buf, object value) except -1: + (<float*>buf)[0] = <float>(<double>value) + return 0 + cdef get_itemsize(self): return sizeof(float) + cdef get_default_format(self): return b"f" + +cdef class DoubleMockBuffer(MockBuffer): + cdef int write(self, char* buf, object value) except -1: + (<double*>buf)[0] = <double>value + return 0 + cdef get_itemsize(self): return sizeof(double) + cdef get_default_format(self): return b"d" + +cdef extern from *: + void* addr_of_pyobject "(void*)"(object) + +cdef class ObjectMockBuffer(MockBuffer): + cdef int write(self, char* buf, object value) except -1: + (<void**>buf)[0] = addr_of_pyobject(value) + return 0 + + cdef get_itemsize(self): return sizeof(void*) + cdef get_default_format(self): return b"@O" + + +cdef class IntStridedMockBuffer(IntMockBuffer): + cdef __cythonbufferdefaults__ = {"mode" : "strided"} + +cdef class ErrorBuffer: + cdef object label + + def __init__(self, label): + self.label = label + + def __getbuffer__(ErrorBuffer self, Py_buffer* buffer, int flags): + raise Exception("acquiring %s" % self.label) + + def __releasebuffer__(ErrorBuffer self, Py_buffer* buffer): + raise Exception("releasing %s" % self.label) + +# +# Structs +# +cdef struct MyStruct: + char a + char b + long long int c + int d + int e + +cdef struct SmallStruct: + int a + int b + +cdef struct NestedStruct: + SmallStruct x + SmallStruct y + int z + +cdef packed struct PackedStruct: + char a + int b + +cdef struct NestedPackedStruct: + char a + int b + PackedStruct sub + int c + +cdef class MyStructMockBuffer(MockBuffer): + cdef int write(self, char* buf, object value) except -1: + cdef MyStruct* s + s = <MyStruct*>buf; + s.a, s.b, s.c, s.d, s.e = value + return 0 + + cdef get_itemsize(self): return sizeof(MyStruct) + cdef get_default_format(self): return b"2bq2i" + +cdef class NestedStructMockBuffer(MockBuffer): + cdef int write(self, char* buf, object value) except -1: + cdef NestedStruct* s + s = <NestedStruct*>buf; + s.x.a, s.x.b, s.y.a, s.y.b, s.z = value + return 0 + + cdef get_itemsize(self): return sizeof(NestedStruct) + cdef get_default_format(self): return b"2T{ii}i" + +cdef class PackedStructMockBuffer(MockBuffer): + cdef int write(self, char* buf, object value) except -1: + cdef PackedStruct* s + s = <PackedStruct*>buf; + s.a, s.b = value + return 0 + + cdef get_itemsize(self): return sizeof(PackedStruct) + cdef get_default_format(self): return b"^ci" + +cdef class NestedPackedStructMockBuffer(MockBuffer): + cdef int write(self, char* buf, object value) except -1: + cdef NestedPackedStruct* s + s = <NestedPackedStruct*>buf; + s.a, s.b, s.sub.a, s.sub.b, s.c = value + return 0 + + cdef get_itemsize(self): return sizeof(NestedPackedStruct) + cdef get_default_format(self): return b"ci^ci@i" + |