hg merge default

6 files changed, 147 insertions, 76 deletions

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
index 88e8379..81d2651 100644
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -80,18 +80,46 @@
  * That sounds like a horribly bad idea to me, and is the reason for why
  * I prefer CFFI crashing cleanly.
  *
- * Currently, we use libffi's ffi_closure_alloc() only on NetBSD.  It is
+ * Currently, we use libffi's ffi_closure_alloc() on NetBSD.  It is
  * known that on the NetBSD kernel, a different strategy is used which
  * should not be open to the fork() bug.
+ *
+ * This is also used on macOS, provided we are executing on macOS 10.15 or
+ * above.  It's a mess because it needs runtime checks in that case.
  */
 #ifdef __NetBSD__
-# define CFFI_TRUST_LIBFFI
-#endif
 
-#ifndef CFFI_TRUST_LIBFFI
-# include "malloc_closure.h"
+# define CFFI_CHECK_FFI_CLOSURE_ALLOC 1
+# define CFFI_CHECK_FFI_CLOSURE_ALLOC_MAYBE 1
+# define CFFI_CHECK_FFI_PREP_CLOSURE_LOC 1
+# define CFFI_CHECK_FFI_PREP_CLOSURE_LOC_MAYBE 1
+# define CFFI_CHECK_FFI_PREP_CIF_VAR 0
+# define CFFI_CHECK_FFI_PREP_CIF_VAR_MAYBE 0
+
+#elif defined(__APPLE__) && defined(FFI_AVAILABLE_APPLE)
+
+# define CFFI_CHECK_FFI_CLOSURE_ALLOC __builtin_available(macos 10.15, ios 13, watchos 6, tvos 13, *)
+# define CFFI_CHECK_FFI_CLOSURE_ALLOC_MAYBE 1
+# define CFFI_CHECK_FFI_PREP_CLOSURE_LOC __builtin_available(macos 10.15, ios 13, watchos 6, tvos 13, *)
+# define CFFI_CHECK_FFI_PREP_CLOSURE_LOC_MAYBE 1
+# define CFFI_CHECK_FFI_PREP_CIF_VAR __builtin_available(macos 10.15, ios 13, watchos 6, tvos 13, *)
+# define CFFI_CHECK_FFI_PREP_CIF_VAR_MAYBE 1
+
+#else
+
+# define CFFI_CHECK_FFI_CLOSURE_ALLOC 0
+# define CFFI_CHECK_FFI_CLOSURE_ALLOC_MAYBE 0
+# define CFFI_CHECK_FFI_PREP_CLOSURE_LOC 0
+# define CFFI_CHECK_FFI_PREP_CLOSURE_LOC_MAYBE 0
+# define CFFI_CHECK_FFI_PREP_CIF_VAR 0
+# define CFFI_CHECK_FFI_PREP_CIF_VAR_MAYBE 0
+
 #endif
 
+/* always includes this, even if it turns out not to be used on NetBSD
+   because calls are behind "if (0)" */
+#include "malloc_closure.h"
+
 
 #if PY_MAJOR_VERSION >= 3
 # define STR_OR_BYTES "bytes"
@@ -1890,11 +1918,12 @@ static void cdataowninggc_dealloc(CDataObject *cd)
         ffi_closure *closure = ((CDataObject_closure *)cd)->closure;
         PyObject *args = (PyObject *)(closure->user_data);
         Py_XDECREF(args);
-#ifdef CFFI_TRUST_LIBFFI
-        ffi_closure_free(closure);
-#else
-        cffi_closure_free(closure);
+#if CFFI_CHECK_FFI_CLOSURE_ALLOC_MAYBE
+        if (CFFI_CHECK_FFI_CLOSURE_ALLOC) {
+            ffi_closure_free(closure);
+        } else
 #endif
+            cffi_closure_free(closure);
     }
     else {
         Py_FatalError("cdata CDataOwningGC_Type with unexpected type flags");
@@ -5820,7 +5849,7 @@ static cif_description_t *fb_prepare_cif(PyObject *fargs,
     char *buffer;
     cif_description_t *cif_descr;
     struct funcbuilder_s funcbuffer;
-    ffi_status status;
+    ffi_status status = -1;
 
     funcbuffer.nb_bytes = 0;
     funcbuffer.bufferp = NULL;
@@ -5843,20 +5872,23 @@ static cif_description_t *fb_prepare_cif(PyObject *fargs,
     assert(funcbuffer.bufferp == buffer + funcbuffer.nb_bytes);
 
     cif_descr = (cif_description_t *)buffer;
-#if HAVE_FFI_PREP_CIF_VAR
+
+    /* use `ffi_prep_cif_var` if necessary and available */
+#if CFFI_CHECK_FFI_PREP_CIF_VAR_MAYBE
     if (variadic_nargs_declared >= 0) {
-        status = ffi_prep_cif_var(&cif_descr->cif, fabi,
-                                  variadic_nargs_declared, funcbuffer.nargs,
-                                  funcbuffer.rtype, funcbuffer.atypes);
-    } else
-#endif
-#if !HAVE_FFI_PREP_CIF_VAR && defined(__arm64__) && defined(__APPLE__)
-#error Apple Arm64 ABI requires ffi_prep_cif_var
+        if (CFFI_CHECK_FFI_PREP_CIF_VAR) {
+            status = ffi_prep_cif_var(&cif_descr->cif, fabi,
+                                      variadic_nargs_declared, funcbuffer.nargs,
+                                      funcbuffer.rtype, funcbuffer.atypes);
+        }
+    }
 #endif
-    {
+
+    if (status == -1) {
         status = ffi_prep_cif(&cif_descr->cif, fabi, funcbuffer.nargs,
                               funcbuffer.rtype, funcbuffer.atypes);
     }
+
     if (status != FFI_OK) {
         PyErr_SetString(PyExc_SystemError,
                         "libffi failed to build this function type");
@@ -6244,6 +6276,7 @@ static PyObject *b_callback(PyObject *self, PyObject *args)
     PyObject *infotuple;
     cif_description_t *cif_descr;
     ffi_closure *closure;
+    ffi_status status;
     void *closure_exec;
 
     if (!PyArg_ParseTuple(args, "O!O|OO:callback", &CTypeDescr_Type, &ct, &ob,
@@ -6254,12 +6287,16 @@ static PyObject *b_callback(PyObject *self, PyObject *args)
     if (infotuple == NULL)
         return NULL;
 
-#ifdef CFFI_TRUST_LIBFFI
-    closure = ffi_closure_alloc(sizeof(ffi_closure), &closure_exec);
-#else
-    closure = cffi_closure_alloc();
-    closure_exec = closure;
+#if CFFI_CHECK_FFI_CLOSURE_ALLOC_MAYBE
+    if (CFFI_CHECK_FFI_CLOSURE_ALLOC) {
+        closure = ffi_closure_alloc(sizeof(ffi_closure), &closure_exec);
+    } else
 #endif
+    {
+        closure = cffi_closure_alloc();
+        closure_exec = closure;
+    }
+
     if (closure == NULL) {
         Py_DECREF(infotuple);
         PyErr_SetString(PyExc_MemoryError,
@@ -6276,8 +6313,8 @@ static PyObject *b_callback(PyObject *self, PyObject *args)
     cd->head.c_type = ct;
     cd->head.c_data = (char *)closure_exec;
     cd->head.c_weakreflist = NULL;
+    closure->user_data = NULL;
     cd->closure = closure;
-    PyObject_GC_Track(cd);
 
     cif_descr = (cif_description_t *)ct->ct_extra;
     if (cif_descr == NULL) {
@@ -6286,28 +6323,43 @@ static PyObject *b_callback(PyObject *self, PyObject *args)
                      "return type or with '...'", ct->ct_name);
         goto error;
     }
-    /* NOTE: ffi_prep_closure() is marked as deprecated.  We could just
-     * call ffi_prep_closure_loc() instead, which is what ffi_prep_closure()
-     * does.  However, cffi also runs on older systems with a libffi that
-     * doesn't have ffi_prep_closure_loc() at all---notably, the OS X
-     * machines on Azure are like that (June 2020).  I didn't find a way to
-     * version-check the included ffi.h.  So you will have to live with the
-     * deprecation warning for now.  (We could try to check for an unrelated
-     * macro like FFI_API which happens to be defined in libffi 3.3 and not
-     * before, but that sounds very obscure.  And I prefer a compile-time
-     * warning rather than a cross-version binary compatibility problem.)
-     */
-#ifdef CFFI_TRUST_LIBFFI
-    if (ffi_prep_closure_loc(closure, &cif_descr->cif,
-                         invoke_callback, infotuple, closure_exec) != FFI_OK) {
+
+#if CFFI_CHECK_FFI_PREP_CLOSURE_LOC_MAYBE
+    if (CFFI_CHECK_FFI_PREP_CLOSURE_LOC) {
+        status = ffi_prep_closure_loc(closure, &cif_descr->cif,
+                                      invoke_callback, infotuple, closure_exec);
+    }
+    else
+#endif
+    {
+#if defined(__APPLE__) && defined(FFI_AVAILABLE_APPLE) && !FFI_LEGACY_CLOSURE_API
+        PyErr_Format(PyExc_SystemError, "ffi_prep_closure_loc() is missing");
+        goto error;
 #else
-    if (ffi_prep_closure(closure, &cif_descr->cif,
-                         invoke_callback, infotuple) != FFI_OK) {
+/* messily try to silence a gcc/clang deprecation warning here */
+# if defined(__clang__)
+#  pragma clang diagnostic push
+#  pragma clang diagnostic ignored "-Wdeprecated-declarations"
+# elif defined(__GNUC__)
+#  pragma GCC diagnostic push
+#  pragma GCC diagnostic ignored "-Wdeprecated-declarations"
+# endif
+        status = ffi_prep_closure(closure, &cif_descr->cif,
+                                  invoke_callback, infotuple);
+# if defined(__clang__)
+#  pragma clang diagnostic pop
+# elif defined(__GNUC__)
+#  pragma GCC diagnostic pop
+# endif
 #endif
+    }
+
+    if (status != FFI_OK) {
         PyErr_SetString(PyExc_SystemError,
                         "libffi failed to build this callback");
         goto error;
     }
+
     if (closure->user_data != infotuple) {
         /* Issue #266.  Should not occur, but could, if we are using
            at runtime a version of libffi compiled with a different
@@ -6322,16 +6374,19 @@ static PyObject *b_callback(PyObject *self, PyObject *args)
             "different from the 'ffi.h' file seen at compile-time)");
         goto error;
     }
+    PyObject_GC_Track(cd);
     return (PyObject *)cd;
 
  error:
     closure->user_data = NULL;
     if (cd == NULL) {
-#ifdef CFFI_TRUST_LIBFFI
-        ffi_closure_free(closure);
-#else
-        cffi_closure_free(closure);
+#if CFFI_CHECK_FFI_CLOSURE_ALLOC_MAYBE
+        if (CFFI_CHECK_FFI_CLOSURE_ALLOC) {
+            ffi_closure_free(closure);
+        }
+        else
 #endif
+            cffi_closure_free(closure);
     }
     else
         Py_DECREF(cd);
diff --git a/c/test_c.py b/c/test_c.py
index fa24676..bb6b586 100644
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -1470,7 +1470,7 @@ def test_a_lot_of_callbacks():
     def make_callback(m):
         def cb(n):
             return n + m
-        return callback(BFunc, cb, 42)    # 'cb' and 'BFunc' go out of scope
+        return callback(BFunc, cb, 42)    # 'cb' goes out of scope
     #
     flist = [make_callback(i) for i in range(BIGNUM)]
     for i, f in enumerate(flist):
@@ -4496,3 +4496,24 @@ def test_type_available_with_correct_names():
         tp = getattr(_cffi_backend, name)
         assert isinstance(tp, type)
         assert (tp.__module__, tp.__name__) == ('_cffi_backend', name)
+
+def test_unaligned_types():
+    BByteArray = new_array_type(
+        new_pointer_type(new_primitive_type("unsigned char")), None)
+    pbuf = newp(BByteArray, 40)
+    buf = buffer(pbuf)
+    #
+    for name in ['short', 'int', 'long', 'long long', 'float', 'double',
+                 'float _Complex', 'double _Complex']:
+        p = new_primitive_type(name)
+        if name.endswith(' _Complex'):
+            num = cast(p, 1.23 - 4.56j)
+        else:
+            num = cast(p, 0x0123456789abcdef)
+        size = sizeof(p)
+        buf[0:40] = b"\x00" * 40
+        pbuf1 = cast(new_pointer_type(p), pbuf + 1)
+        pbuf1[0] = num
+        assert pbuf1[0] == num
+        assert buf[0] == b'\x00'
+        assert buf[1 + size] == b'\x00'
diff --git a/cffi/recompiler.py b/cffi/recompiler.py
index 1aeae5b..86b37d7 100644
--- a/cffi/recompiler.py
+++ b/cffi/recompiler.py
@@ -193,6 +193,17 @@ class Recompiler:
             assert isinstance(op, CffiOp)
         self.cffi_types = tuple(self.cffi_types)    # don't change any more
 
+    def _enum_fields(self, tp):
+        # When producing C, expand all anonymous struct/union fields.
+        # That's necessary to have C code checking the offsets of the
+        # individual fields contained in them.  When producing Python,
+        # don't do it and instead write it like it is, with the
+        # corresponding fields having an empty name.  Empty names are
+        # recognized at runtime when we import the generated Python
+        # file.
+        expand_anonymous_struct_union = not self.target_is_python
+        return tp.enumfields(expand_anonymous_struct_union)
+
     def _do_collect_type(self, tp):
         if not isinstance(tp, model.BaseTypeByIdentity):
             if isinstance(tp, tuple):
@@ -206,7 +217,7 @@ class Recompiler:
             elif isinstance(tp, model.StructOrUnion):
                 if tp.fldtypes is not None and (
                         tp not in self.ffi._parser._included_declarations):
-                    for name1, tp1, _, _ in tp.enumfields():
+                    for name1, tp1, _, _ in self._enum_fields(tp):
                         self._do_collect_type(self._field_type(tp, name1, tp1))
             else:
                 for _, x in tp._get_items():
@@ -864,7 +875,7 @@ class Recompiler:
         prnt('{')
         prnt('  /* only to generate compile-time warnings or errors */')
         prnt('  (void)p;')
-        for fname, ftype, fbitsize, fqual in tp.enumfields():
+        for fname, ftype, fbitsize, fqual in self._enum_fields(tp):
             try:
                 if ftype.is_integer_type() or fbitsize >= 0:
                     # accept all integers, but complain on float or double
@@ -920,8 +931,7 @@ class Recompiler:
         flags = '|'.join(flags) or '0'
         c_fields = []
         if reason_for_not_expanding is None:
-            expand_anonymous_struct_union = not self.target_is_python
-            enumfields = list(tp.enumfields(expand_anonymous_struct_union))
+            enumfields = list(self._enum_fields(tp))
             for fldname, fldtype, fbitsize, fqual in enumfields:
                 fldtype = self._field_type(tp, fldname, fldtype)
                 self._check_not_opaque(fldtype,
diff --git a/setup.py b/setup.py
index 826216d..f2e007c 100644
--- a/setup.py
+++ b/setup.py
@@ -149,39 +149,18 @@ if COMPILE_LIBFFI:
     sources.extend(os.path.join(COMPILE_LIBFFI, filename)
                    for filename in _filenames)
 else:
-    if 'darwin' in sys.platform and macosx_deployment_target() >= (10, 15):
-        # use libffi from Mac OS SDK if we're targetting 10.15 (including
-        # on arm64).  This libffi is safe against the crash-after-fork
-        # issue described in _cffi_backend.c.  Also, arm64 uses a different
-        # ABI for calls to vararg functions as opposed to regular functions.
-        extra_compile_args += ['-iwithsysroot/usr/include/ffi']
-        define_macros += [('CFFI_TRUST_LIBFFI', '1'),
-                          ('HAVE_FFI_PREP_CIF_VAR', '1')]
-        libraries += ['ffi']
-    else:
-        use_pkg_config()
+    use_pkg_config()
     ask_supports_thread()
     ask_supports_sync_synchronize()
 
+if 'darwin' in sys.platform:
+    # priority is given to `pkg_config`, but always fall back on SDK's libffi.
+    extra_compile_args += ['-iwithsysroot/usr/include/ffi']
+
 if 'freebsd' in sys.platform:
     include_dirs.append('/usr/local/include')
     library_dirs.append('/usr/local/lib')
 
-if 'darwin' in sys.platform:
-    try:
-        p = subprocess.Popen(['xcrun', '--show-sdk-path'],
-                             stdout=subprocess.PIPE)
-    except OSError as e:
-        if e.errno not in [errno.ENOENT, errno.EACCES]:
-            raise
-    else:
-        t = p.stdout.read().decode().strip()
-        p.stdout.close()
-        if p.wait() == 0:
-            include_dirs.append(t + '/usr/include/ffi')
-
-
-
 if __name__ == '__main__':
     from setuptools import setup, Distribution, Extension
 
diff --git a/testing/cffi0/test_ffi_backend.py b/testing/cffi0/test_ffi_backend.py
index 3552196..8e29bc4 100644
--- a/testing/cffi0/test_ffi_backend.py
+++ b/testing/cffi0/test_ffi_backend.py
@@ -179,6 +179,7 @@ class TestBitfield:
         setters = ['case %d: s.%s = value; break;' % iname
                    for iname in enumerate(fnames)]
         lib = ffi1.verify("""
+            #include <string.h>
             struct s1 { %s };
             struct sa { char a; struct s1 b; };
             #define Gofs_y  offsetof(struct s1, y)
diff --git a/testing/cffi1/test_re_python.py b/testing/cffi1/test_re_python.py
index dce4f40..2ae0dd1 100644
--- a/testing/cffi1/test_re_python.py
+++ b/testing/cffi1/test_re_python.py
@@ -74,6 +74,7 @@ def setup_module(mod):
     int strlen(const char *);
     struct with_union { union { int a; char b; }; };
     union with_struct { struct { int a; char b; }; };
+    struct with_struct_with_union { struct { union { int x; }; } cp; };
     struct NVGcolor { union { float rgba[4]; struct { float r,g,b,a; }; }; };
     typedef struct selfref { struct selfref *next; } *selfref_ptr_t;
     """)
@@ -248,6 +249,10 @@ def test_anonymous_union_inside_struct():
     assert ffi.offsetof("union with_struct", "b") == INT
     assert ffi.sizeof("union with_struct") >= INT + 1
     #
+    assert ffi.sizeof("struct with_struct_with_union") == INT
+    p = ffi.new("struct with_struct_with_union *")
+    assert p.cp.x == 0
+    #
     FLOAT = ffi.sizeof("float")
     assert ffi.sizeof("struct NVGcolor") == FLOAT * 4
     assert ffi.offsetof("struct NVGcolor", "rgba") == 0