diff options
author | Armin Rigo <arigo@tunes.org> | 2020-11-24 11:21:19 +0100 |
---|---|---|
committer | Armin Rigo <arigo@tunes.org> | 2020-11-24 11:21:19 +0100 |
commit | 55f18fb97be7757995755bf4e80f60f219676e23 (patch) | |
tree | 85c0edbbfbe51a852bd72196dd35919d17c24828 | |
parent | ff3dfb69b4342b1caacfef88e705de86f5bb9844 (diff) | |
parent | 097bc56353a9084b10e30de94b8f0f12487b2784 (diff) | |
download | cffi-55f18fb97be7757995755bf4e80f60f219676e23.tar.gz |
hg merge default
-rw-r--r-- | c/_cffi_backend.c | 145 | ||||
-rw-r--r-- | c/test_c.py | 23 | ||||
-rw-r--r-- | cffi/recompiler.py | 18 | ||||
-rw-r--r-- | setup.py | 31 | ||||
-rw-r--r-- | testing/cffi0/test_ffi_backend.py | 1 | ||||
-rw-r--r-- | testing/cffi1/test_re_python.py | 5 |
6 files changed, 147 insertions, 76 deletions
diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c index 88e8379..81d2651 100644 --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -80,18 +80,46 @@ * That sounds like a horribly bad idea to me, and is the reason for why * I prefer CFFI crashing cleanly. * - * Currently, we use libffi's ffi_closure_alloc() only on NetBSD. It is + * Currently, we use libffi's ffi_closure_alloc() on NetBSD. It is * known that on the NetBSD kernel, a different strategy is used which * should not be open to the fork() bug. + * + * This is also used on macOS, provided we are executing on macOS 10.15 or + * above. It's a mess because it needs runtime checks in that case. */ #ifdef __NetBSD__ -# define CFFI_TRUST_LIBFFI -#endif -#ifndef CFFI_TRUST_LIBFFI -# include "malloc_closure.h" +# define CFFI_CHECK_FFI_CLOSURE_ALLOC 1 +# define CFFI_CHECK_FFI_CLOSURE_ALLOC_MAYBE 1 +# define CFFI_CHECK_FFI_PREP_CLOSURE_LOC 1 +# define CFFI_CHECK_FFI_PREP_CLOSURE_LOC_MAYBE 1 +# define CFFI_CHECK_FFI_PREP_CIF_VAR 0 +# define CFFI_CHECK_FFI_PREP_CIF_VAR_MAYBE 0 + +#elif defined(__APPLE__) && defined(FFI_AVAILABLE_APPLE) + +# define CFFI_CHECK_FFI_CLOSURE_ALLOC __builtin_available(macos 10.15, ios 13, watchos 6, tvos 13, *) +# define CFFI_CHECK_FFI_CLOSURE_ALLOC_MAYBE 1 +# define CFFI_CHECK_FFI_PREP_CLOSURE_LOC __builtin_available(macos 10.15, ios 13, watchos 6, tvos 13, *) +# define CFFI_CHECK_FFI_PREP_CLOSURE_LOC_MAYBE 1 +# define CFFI_CHECK_FFI_PREP_CIF_VAR __builtin_available(macos 10.15, ios 13, watchos 6, tvos 13, *) +# define CFFI_CHECK_FFI_PREP_CIF_VAR_MAYBE 1 + +#else + +# define CFFI_CHECK_FFI_CLOSURE_ALLOC 0 +# define CFFI_CHECK_FFI_CLOSURE_ALLOC_MAYBE 0 +# define CFFI_CHECK_FFI_PREP_CLOSURE_LOC 0 +# define CFFI_CHECK_FFI_PREP_CLOSURE_LOC_MAYBE 0 +# define CFFI_CHECK_FFI_PREP_CIF_VAR 0 +# define CFFI_CHECK_FFI_PREP_CIF_VAR_MAYBE 0 + #endif +/* always includes this, even if it turns out not to be used on NetBSD + because calls are behind "if (0)" */ +#include "malloc_closure.h" + #if PY_MAJOR_VERSION >= 3 # define STR_OR_BYTES "bytes" @@ -1890,11 +1918,12 @@ static void cdataowninggc_dealloc(CDataObject *cd) ffi_closure *closure = ((CDataObject_closure *)cd)->closure; PyObject *args = (PyObject *)(closure->user_data); Py_XDECREF(args); -#ifdef CFFI_TRUST_LIBFFI - ffi_closure_free(closure); -#else - cffi_closure_free(closure); +#if CFFI_CHECK_FFI_CLOSURE_ALLOC_MAYBE + if (CFFI_CHECK_FFI_CLOSURE_ALLOC) { + ffi_closure_free(closure); + } else #endif + cffi_closure_free(closure); } else { Py_FatalError("cdata CDataOwningGC_Type with unexpected type flags"); @@ -5820,7 +5849,7 @@ static cif_description_t *fb_prepare_cif(PyObject *fargs, char *buffer; cif_description_t *cif_descr; struct funcbuilder_s funcbuffer; - ffi_status status; + ffi_status status = -1; funcbuffer.nb_bytes = 0; funcbuffer.bufferp = NULL; @@ -5843,20 +5872,23 @@ static cif_description_t *fb_prepare_cif(PyObject *fargs, assert(funcbuffer.bufferp == buffer + funcbuffer.nb_bytes); cif_descr = (cif_description_t *)buffer; -#if HAVE_FFI_PREP_CIF_VAR + + /* use `ffi_prep_cif_var` if necessary and available */ +#if CFFI_CHECK_FFI_PREP_CIF_VAR_MAYBE if (variadic_nargs_declared >= 0) { - status = ffi_prep_cif_var(&cif_descr->cif, fabi, - variadic_nargs_declared, funcbuffer.nargs, - funcbuffer.rtype, funcbuffer.atypes); - } else -#endif -#if !HAVE_FFI_PREP_CIF_VAR && defined(__arm64__) && defined(__APPLE__) -#error Apple Arm64 ABI requires ffi_prep_cif_var + if (CFFI_CHECK_FFI_PREP_CIF_VAR) { + status = ffi_prep_cif_var(&cif_descr->cif, fabi, + variadic_nargs_declared, funcbuffer.nargs, + funcbuffer.rtype, funcbuffer.atypes); + } + } #endif - { + + if (status == -1) { status = ffi_prep_cif(&cif_descr->cif, fabi, funcbuffer.nargs, funcbuffer.rtype, funcbuffer.atypes); } + if (status != FFI_OK) { PyErr_SetString(PyExc_SystemError, "libffi failed to build this function type"); @@ -6244,6 +6276,7 @@ static PyObject *b_callback(PyObject *self, PyObject *args) PyObject *infotuple; cif_description_t *cif_descr; ffi_closure *closure; + ffi_status status; void *closure_exec; if (!PyArg_ParseTuple(args, "O!O|OO:callback", &CTypeDescr_Type, &ct, &ob, @@ -6254,12 +6287,16 @@ static PyObject *b_callback(PyObject *self, PyObject *args) if (infotuple == NULL) return NULL; -#ifdef CFFI_TRUST_LIBFFI - closure = ffi_closure_alloc(sizeof(ffi_closure), &closure_exec); -#else - closure = cffi_closure_alloc(); - closure_exec = closure; +#if CFFI_CHECK_FFI_CLOSURE_ALLOC_MAYBE + if (CFFI_CHECK_FFI_CLOSURE_ALLOC) { + closure = ffi_closure_alloc(sizeof(ffi_closure), &closure_exec); + } else #endif + { + closure = cffi_closure_alloc(); + closure_exec = closure; + } + if (closure == NULL) { Py_DECREF(infotuple); PyErr_SetString(PyExc_MemoryError, @@ -6276,8 +6313,8 @@ static PyObject *b_callback(PyObject *self, PyObject *args) cd->head.c_type = ct; cd->head.c_data = (char *)closure_exec; cd->head.c_weakreflist = NULL; + closure->user_data = NULL; cd->closure = closure; - PyObject_GC_Track(cd); cif_descr = (cif_description_t *)ct->ct_extra; if (cif_descr == NULL) { @@ -6286,28 +6323,43 @@ static PyObject *b_callback(PyObject *self, PyObject *args) "return type or with '...'", ct->ct_name); goto error; } - /* NOTE: ffi_prep_closure() is marked as deprecated. We could just - * call ffi_prep_closure_loc() instead, which is what ffi_prep_closure() - * does. However, cffi also runs on older systems with a libffi that - * doesn't have ffi_prep_closure_loc() at all---notably, the OS X - * machines on Azure are like that (June 2020). I didn't find a way to - * version-check the included ffi.h. So you will have to live with the - * deprecation warning for now. (We could try to check for an unrelated - * macro like FFI_API which happens to be defined in libffi 3.3 and not - * before, but that sounds very obscure. And I prefer a compile-time - * warning rather than a cross-version binary compatibility problem.) - */ -#ifdef CFFI_TRUST_LIBFFI - if (ffi_prep_closure_loc(closure, &cif_descr->cif, - invoke_callback, infotuple, closure_exec) != FFI_OK) { + +#if CFFI_CHECK_FFI_PREP_CLOSURE_LOC_MAYBE + if (CFFI_CHECK_FFI_PREP_CLOSURE_LOC) { + status = ffi_prep_closure_loc(closure, &cif_descr->cif, + invoke_callback, infotuple, closure_exec); + } + else +#endif + { +#if defined(__APPLE__) && defined(FFI_AVAILABLE_APPLE) && !FFI_LEGACY_CLOSURE_API + PyErr_Format(PyExc_SystemError, "ffi_prep_closure_loc() is missing"); + goto error; #else - if (ffi_prep_closure(closure, &cif_descr->cif, - invoke_callback, infotuple) != FFI_OK) { +/* messily try to silence a gcc/clang deprecation warning here */ +# if defined(__clang__) +# pragma clang diagnostic push +# pragma clang diagnostic ignored "-Wdeprecated-declarations" +# elif defined(__GNUC__) +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wdeprecated-declarations" +# endif + status = ffi_prep_closure(closure, &cif_descr->cif, + invoke_callback, infotuple); +# if defined(__clang__) +# pragma clang diagnostic pop +# elif defined(__GNUC__) +# pragma GCC diagnostic pop +# endif #endif + } + + if (status != FFI_OK) { PyErr_SetString(PyExc_SystemError, "libffi failed to build this callback"); goto error; } + if (closure->user_data != infotuple) { /* Issue #266. Should not occur, but could, if we are using at runtime a version of libffi compiled with a different @@ -6322,16 +6374,19 @@ static PyObject *b_callback(PyObject *self, PyObject *args) "different from the 'ffi.h' file seen at compile-time)"); goto error; } + PyObject_GC_Track(cd); return (PyObject *)cd; error: closure->user_data = NULL; if (cd == NULL) { -#ifdef CFFI_TRUST_LIBFFI - ffi_closure_free(closure); -#else - cffi_closure_free(closure); +#if CFFI_CHECK_FFI_CLOSURE_ALLOC_MAYBE + if (CFFI_CHECK_FFI_CLOSURE_ALLOC) { + ffi_closure_free(closure); + } + else #endif + cffi_closure_free(closure); } else Py_DECREF(cd); diff --git a/c/test_c.py b/c/test_c.py index fa24676..bb6b586 100644 --- a/c/test_c.py +++ b/c/test_c.py @@ -1470,7 +1470,7 @@ def test_a_lot_of_callbacks(): def make_callback(m): def cb(n): return n + m - return callback(BFunc, cb, 42) # 'cb' and 'BFunc' go out of scope + return callback(BFunc, cb, 42) # 'cb' goes out of scope # flist = [make_callback(i) for i in range(BIGNUM)] for i, f in enumerate(flist): @@ -4496,3 +4496,24 @@ def test_type_available_with_correct_names(): tp = getattr(_cffi_backend, name) assert isinstance(tp, type) assert (tp.__module__, tp.__name__) == ('_cffi_backend', name) + +def test_unaligned_types(): + BByteArray = new_array_type( + new_pointer_type(new_primitive_type("unsigned char")), None) + pbuf = newp(BByteArray, 40) + buf = buffer(pbuf) + # + for name in ['short', 'int', 'long', 'long long', 'float', 'double', + 'float _Complex', 'double _Complex']: + p = new_primitive_type(name) + if name.endswith(' _Complex'): + num = cast(p, 1.23 - 4.56j) + else: + num = cast(p, 0x0123456789abcdef) + size = sizeof(p) + buf[0:40] = b"\x00" * 40 + pbuf1 = cast(new_pointer_type(p), pbuf + 1) + pbuf1[0] = num + assert pbuf1[0] == num + assert buf[0] == b'\x00' + assert buf[1 + size] == b'\x00' diff --git a/cffi/recompiler.py b/cffi/recompiler.py index 1aeae5b..86b37d7 100644 --- a/cffi/recompiler.py +++ b/cffi/recompiler.py @@ -193,6 +193,17 @@ class Recompiler: assert isinstance(op, CffiOp) self.cffi_types = tuple(self.cffi_types) # don't change any more + def _enum_fields(self, tp): + # When producing C, expand all anonymous struct/union fields. + # That's necessary to have C code checking the offsets of the + # individual fields contained in them. When producing Python, + # don't do it and instead write it like it is, with the + # corresponding fields having an empty name. Empty names are + # recognized at runtime when we import the generated Python + # file. + expand_anonymous_struct_union = not self.target_is_python + return tp.enumfields(expand_anonymous_struct_union) + def _do_collect_type(self, tp): if not isinstance(tp, model.BaseTypeByIdentity): if isinstance(tp, tuple): @@ -206,7 +217,7 @@ class Recompiler: elif isinstance(tp, model.StructOrUnion): if tp.fldtypes is not None and ( tp not in self.ffi._parser._included_declarations): - for name1, tp1, _, _ in tp.enumfields(): + for name1, tp1, _, _ in self._enum_fields(tp): self._do_collect_type(self._field_type(tp, name1, tp1)) else: for _, x in tp._get_items(): @@ -864,7 +875,7 @@ class Recompiler: prnt('{') prnt(' /* only to generate compile-time warnings or errors */') prnt(' (void)p;') - for fname, ftype, fbitsize, fqual in tp.enumfields(): + for fname, ftype, fbitsize, fqual in self._enum_fields(tp): try: if ftype.is_integer_type() or fbitsize >= 0: # accept all integers, but complain on float or double @@ -920,8 +931,7 @@ class Recompiler: flags = '|'.join(flags) or '0' c_fields = [] if reason_for_not_expanding is None: - expand_anonymous_struct_union = not self.target_is_python - enumfields = list(tp.enumfields(expand_anonymous_struct_union)) + enumfields = list(self._enum_fields(tp)) for fldname, fldtype, fbitsize, fqual in enumfields: fldtype = self._field_type(tp, fldname, fldtype) self._check_not_opaque(fldtype, @@ -149,39 +149,18 @@ if COMPILE_LIBFFI: sources.extend(os.path.join(COMPILE_LIBFFI, filename) for filename in _filenames) else: - if 'darwin' in sys.platform and macosx_deployment_target() >= (10, 15): - # use libffi from Mac OS SDK if we're targetting 10.15 (including - # on arm64). This libffi is safe against the crash-after-fork - # issue described in _cffi_backend.c. Also, arm64 uses a different - # ABI for calls to vararg functions as opposed to regular functions. - extra_compile_args += ['-iwithsysroot/usr/include/ffi'] - define_macros += [('CFFI_TRUST_LIBFFI', '1'), - ('HAVE_FFI_PREP_CIF_VAR', '1')] - libraries += ['ffi'] - else: - use_pkg_config() + use_pkg_config() ask_supports_thread() ask_supports_sync_synchronize() +if 'darwin' in sys.platform: + # priority is given to `pkg_config`, but always fall back on SDK's libffi. + extra_compile_args += ['-iwithsysroot/usr/include/ffi'] + if 'freebsd' in sys.platform: include_dirs.append('/usr/local/include') library_dirs.append('/usr/local/lib') -if 'darwin' in sys.platform: - try: - p = subprocess.Popen(['xcrun', '--show-sdk-path'], - stdout=subprocess.PIPE) - except OSError as e: - if e.errno not in [errno.ENOENT, errno.EACCES]: - raise - else: - t = p.stdout.read().decode().strip() - p.stdout.close() - if p.wait() == 0: - include_dirs.append(t + '/usr/include/ffi') - - - if __name__ == '__main__': from setuptools import setup, Distribution, Extension diff --git a/testing/cffi0/test_ffi_backend.py b/testing/cffi0/test_ffi_backend.py index 3552196..8e29bc4 100644 --- a/testing/cffi0/test_ffi_backend.py +++ b/testing/cffi0/test_ffi_backend.py @@ -179,6 +179,7 @@ class TestBitfield: setters = ['case %d: s.%s = value; break;' % iname for iname in enumerate(fnames)] lib = ffi1.verify(""" + #include <string.h> struct s1 { %s }; struct sa { char a; struct s1 b; }; #define Gofs_y offsetof(struct s1, y) diff --git a/testing/cffi1/test_re_python.py b/testing/cffi1/test_re_python.py index dce4f40..2ae0dd1 100644 --- a/testing/cffi1/test_re_python.py +++ b/testing/cffi1/test_re_python.py @@ -74,6 +74,7 @@ def setup_module(mod): int strlen(const char *); struct with_union { union { int a; char b; }; }; union with_struct { struct { int a; char b; }; }; + struct with_struct_with_union { struct { union { int x; }; } cp; }; struct NVGcolor { union { float rgba[4]; struct { float r,g,b,a; }; }; }; typedef struct selfref { struct selfref *next; } *selfref_ptr_t; """) @@ -248,6 +249,10 @@ def test_anonymous_union_inside_struct(): assert ffi.offsetof("union with_struct", "b") == INT assert ffi.sizeof("union with_struct") >= INT + 1 # + assert ffi.sizeof("struct with_struct_with_union") == INT + p = ffi.new("struct with_struct_with_union *") + assert p.cp.x == 0 + # FLOAT = ffi.sizeof("float") assert ffi.sizeof("struct NVGcolor") == FLOAT * 4 assert ffi.offsetof("struct NVGcolor", "rgba") == 0 |