diff options
author | Armin Rigo <arigo@tunes.org> | 2020-07-26 17:37:15 +0200 |
---|---|---|
committer | Armin Rigo <arigo@tunes.org> | 2020-07-26 17:37:15 +0200 |
commit | 4b7ad14fc34747bd7d7b708a9af7bdbae66791f6 (patch) | |
tree | 90cca7eb8cba9da69f437bf91ce9150733e0cbb1 | |
parent | bd1a9ff2e12d89e9e1372ac57b443ffe6e50d682 (diff) | |
parent | 85d1d8120ce6e38daaff6b58dc68476c5709f0ca (diff) | |
download | cffi-4b7ad14fc34747bd7d7b708a9af7bdbae66791f6.tar.gz |
update branch release-1.14 for 1.14.1
-rw-r--r-- | c/_cffi_backend.c | 161 | ||||
-rw-r--r-- | c/cffi1_module.c | 15 | ||||
-rw-r--r-- | c/cglob.c | 2 | ||||
-rw-r--r-- | c/ffi_obj.c | 2 | ||||
-rw-r--r-- | c/lib_obj.c | 2 | ||||
-rw-r--r-- | c/libffi_msvc/ffi.c | 37 | ||||
-rw-r--r-- | c/libffi_msvc/prep_cif.c | 5 | ||||
-rw-r--r-- | c/test_c.py | 53 | ||||
-rw-r--r-- | cffi/_embedding.h | 4 | ||||
-rw-r--r-- | cffi/cparser.py | 55 | ||||
-rw-r--r-- | cffi/model.py | 7 | ||||
-rw-r--r-- | cffi/recompiler.py | 18 | ||||
-rw-r--r-- | cffi/vengine_cpy.py | 4 | ||||
-rw-r--r-- | cffi/vengine_gen.py | 4 | ||||
-rw-r--r-- | doc/source/goals.rst | 2 | ||||
-rw-r--r-- | doc/source/installation.rst | 6 | ||||
-rw-r--r-- | doc/source/overview.rst | 2 | ||||
-rw-r--r-- | doc/source/whatsnew.rst | 33 | ||||
-rw-r--r-- | setup.py | 16 | ||||
-rw-r--r-- | testing/cffi0/backend_tests.py | 6 | ||||
-rw-r--r-- | testing/cffi0/test_ffi_backend.py | 14 | ||||
-rw-r--r-- | testing/cffi0/test_function.py | 17 | ||||
-rw-r--r-- | testing/cffi0/test_ownlib.py | 32 | ||||
-rw-r--r-- | testing/cffi0/test_parsing.py | 89 | ||||
-rw-r--r-- | testing/cffi1/test_function_args.py | 208 | ||||
-rw-r--r-- | testing/cffi1/test_recompiler.py | 37 | ||||
-rw-r--r-- | testing/embedding/test_basic.py | 6 | ||||
-rw-r--r-- | testing/embedding/withunicode.py | 26 | ||||
-rw-r--r-- | testing/udir.py | 131 |
29 files changed, 871 insertions, 123 deletions
diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c index 7c9cd26..e6adf3c 100644 --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -148,6 +148,10 @@ (PyCObject_FromVoidPtr(pointer, destructor)) #endif +#if PY_VERSION_HEX < 0x030900a4 +# define Py_SET_REFCNT(obj, val) (Py_REFCNT(obj) = (val)) +#endif + /************************************************************/ /* base type flag: exactly one of the following: */ @@ -404,10 +408,10 @@ ctypedescr_dealloc(CTypeDescrObject *ct) if (ct->ct_unique_key != NULL) { /* revive dead object temporarily for DelItem */ - Py_REFCNT(ct) = 43; + Py_SET_REFCNT(ct, 43); PyDict_DelItem(unique_cache, ct->ct_unique_key); assert(Py_REFCNT(ct) == 42); - Py_REFCNT(ct) = 0; + Py_SET_REFCNT(ct, 0); Py_DECREF(ct->ct_unique_key); } Py_XDECREF(ct->ct_itemdescr); @@ -653,7 +657,7 @@ static PyMethodDef ctypedescr_methods[] = { static PyTypeObject CTypeDescr_Type = { PyVarObject_HEAD_INIT(NULL, 0) - "_cffi_backend.CTypeDescr", + "_cffi_backend.CType", offsetof(CTypeDescrObject, ct_name), sizeof(char), (destructor)ctypedescr_dealloc, /* tp_dealloc */ @@ -2888,7 +2892,8 @@ static PyObject * convert_struct_to_owning_object(char *data, CTypeDescrObject *ct); /*forward*/ static cif_description_t * -fb_prepare_cif(PyObject *fargs, CTypeDescrObject *, ffi_abi); /*forward*/ +fb_prepare_cif(PyObject *fargs, CTypeDescrObject *, Py_ssize_t, ffi_abi); + /*forward*/ static PyObject *new_primitive_type(const char *name); /*forward*/ @@ -3081,7 +3086,7 @@ cdata_call(CDataObject *cd, PyObject *args, PyObject *kwds) #else fabi = PyLong_AS_LONG(PyTuple_GET_ITEM(signature, 0)); #endif - cif_descr = fb_prepare_cif(fvarargs, fresult, fabi); + cif_descr = fb_prepare_cif(fvarargs, fresult, nargs_declared, fabi); if (cif_descr == NULL) goto error; } @@ -3354,7 +3359,7 @@ static PyMethodDef cdata_methods[] = { static PyTypeObject CData_Type = { PyVarObject_HEAD_INIT(NULL, 0) - "_cffi_backend.CData", + "_cffi_backend._CDataBase", sizeof(CDataObject), 0, (destructor)cdata_dealloc, /* tp_dealloc */ @@ -3373,7 +3378,9 @@ static PyTypeObject CData_Type = { (setattrofunc)cdata_setattro, /* tp_setattro */ 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_CHECKTYPES, /* tp_flags */ - 0, /* tp_doc */ + "The internal base type for CData objects. Use FFI.CData to access " + "it. Always check with isinstance(): subtypes are sometimes returned " + "on CPython, for performance reasons.", /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ cdata_richcompare, /* tp_richcompare */ @@ -3396,7 +3403,7 @@ static PyTypeObject CData_Type = { static PyTypeObject CDataOwning_Type = { PyVarObject_HEAD_INIT(NULL, 0) - "_cffi_backend.CDataOwn", + "_cffi_backend.__CDataOwn", sizeof(CDataObject), 0, (destructor)cdataowning_dealloc, /* tp_dealloc */ @@ -3415,7 +3422,8 @@ static PyTypeObject CDataOwning_Type = { 0, /* inherited */ /* tp_setattro */ 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_CHECKTYPES, /* tp_flags */ - 0, /* tp_doc */ + "This is an internal subtype of _CDataBase for performance only on " + "CPython. Check with isinstance(x, ffi.CData).", /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ 0, /* inherited */ /* tp_richcompare */ @@ -3438,7 +3446,7 @@ static PyTypeObject CDataOwning_Type = { static PyTypeObject CDataOwningGC_Type = { PyVarObject_HEAD_INIT(NULL, 0) - "_cffi_backend.CDataOwnGC", + "_cffi_backend.__CDataOwnGC", sizeof(CDataObject_own_structptr), 0, (destructor)cdataowninggc_dealloc, /* tp_dealloc */ @@ -3458,7 +3466,8 @@ static PyTypeObject CDataOwningGC_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_CHECKTYPES /* tp_flags */ | Py_TPFLAGS_HAVE_GC, - 0, /* tp_doc */ + "This is an internal subtype of _CDataBase for performance only on " + "CPython. Check with isinstance(x, ffi.CData).", /* tp_doc */ (traverseproc)cdataowninggc_traverse, /* tp_traverse */ (inquiry)cdataowninggc_clear, /* tp_clear */ 0, /* inherited */ /* tp_richcompare */ @@ -3481,7 +3490,7 @@ static PyTypeObject CDataOwningGC_Type = { static PyTypeObject CDataFromBuf_Type = { PyVarObject_HEAD_INIT(NULL, 0) - "_cffi_backend.CDataFromBuf", + "_cffi_backend.__CDataFromBuf", sizeof(CDataObject_frombuf), 0, (destructor)cdatafrombuf_dealloc, /* tp_dealloc */ @@ -3501,7 +3510,8 @@ static PyTypeObject CDataFromBuf_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_CHECKTYPES /* tp_flags */ | Py_TPFLAGS_HAVE_GC, - 0, /* tp_doc */ + "This is an internal subtype of _CDataBase for performance only on " + "CPython. Check with isinstance(x, ffi.CData).", /* tp_doc */ (traverseproc)cdatafrombuf_traverse, /* tp_traverse */ (inquiry)cdatafrombuf_clear, /* tp_clear */ 0, /* inherited */ /* tp_richcompare */ @@ -3524,7 +3534,7 @@ static PyTypeObject CDataFromBuf_Type = { static PyTypeObject CDataGCP_Type = { PyVarObject_HEAD_INIT(NULL, 0) - "_cffi_backend.CDataGCP", + "_cffi_backend.__CDataGCP", sizeof(CDataObject_gcp), 0, (destructor)cdatagcp_dealloc, /* tp_dealloc */ @@ -3547,7 +3557,8 @@ static PyTypeObject CDataGCP_Type = { | Py_TPFLAGS_HAVE_FINALIZE #endif | Py_TPFLAGS_HAVE_GC, - 0, /* tp_doc */ + "This is an internal subtype of _CDataBase for performance only on " + "CPython. Check with isinstance(x, ffi.CData).", /* tp_doc */ (traverseproc)cdatagcp_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* inherited */ /* tp_richcompare */ @@ -3608,7 +3619,7 @@ cdataiter_dealloc(CDataIterObject *it) static PyTypeObject CDataIter_Type = { PyVarObject_HEAD_INIT(NULL, 0) - "_cffi_backend.CDataIter", /* tp_name */ + "_cffi_backend.__CData_iterator", /* tp_name */ sizeof(CDataIterObject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ @@ -4363,7 +4374,7 @@ static PyMethodDef dl_methods[] = { static PyTypeObject dl_type = { PyVarObject_HEAD_INIT(NULL, 0) - "_cffi_backend.Library", /* tp_name */ + "_cffi_backend.CLibrary", /* tp_name */ sizeof(DynLibObject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ @@ -4441,9 +4452,11 @@ static void *b_do_dlopen(PyObject *args, const char **p_printable_filename, { PyObject *s = PyTuple_GET_ITEM(args, 0); #ifdef MS_WIN32 - Py_UNICODE *filenameW; - if (PyArg_ParseTuple(args, "u|i:load_library", &filenameW, &flags)) + PyObject *filename_unicode; + if (PyArg_ParseTuple(args, "U|i:load_library", &filename_unicode, &flags)) { + Py_ssize_t sz1; + wchar_t *w1; #if PY_MAJOR_VERSION < 3 s = PyUnicode_AsUTF8String(s); if (s == NULL) @@ -4454,7 +4467,15 @@ static void *b_do_dlopen(PyObject *args, const char **p_printable_filename, if (*p_printable_filename == NULL) return NULL; - handle = dlopenW(filenameW); + sz1 = PyUnicode_GetSize(filename_unicode) + 1; + sz1 *= 2; /* should not be needed, but you never know */ + w1 = alloca(sizeof(wchar_t) * sz1); + sz1 = PyUnicode_AsWideChar((PyUnicodeObject *)filename_unicode, + w1, sz1 - 1); + if (sz1 < 0) + return NULL; + w1[sz1] = 0; + handle = dlopenW(w1); goto got_handle; } PyErr_Clear(); @@ -4478,6 +4499,13 @@ static void *b_do_dlopen(PyObject *args, const char **p_printable_filename, if ((flags & (RTLD_NOW | RTLD_LAZY)) == 0) flags |= RTLD_NOW; +#ifdef MS_WIN32 + if (filename_or_null == NULL) { + PyErr_SetString(PyExc_OSError, "dlopen(None) not supported on Windows"); + return NULL; + } +#endif + handle = dlopen(filename_or_null, flags); #ifdef MS_WIN32 @@ -4995,7 +5023,9 @@ static int complete_sflags(int sflags) #ifdef MS_WIN32 sflags |= SF_MSVC_BITFIELDS; #else -# if defined(__arm__) || defined(__aarch64__) +# if defined(__APPLE__) && defined(__arm64__) + sflags |= SF_GCC_X86_BITFIELDS; +# elif defined(__arm__) || defined(__aarch64__) sflags |= SF_GCC_ARM_BITFIELDS; # else sflags |= SF_GCC_X86_BITFIELDS; @@ -5783,11 +5813,14 @@ static CTypeDescrObject *fb_prepare_ctype(struct funcbuilder_s *fb, static cif_description_t *fb_prepare_cif(PyObject *fargs, CTypeDescrObject *fresult, + Py_ssize_t variadic_nargs_declared, ffi_abi fabi) + { char *buffer; cif_description_t *cif_descr; struct funcbuilder_s funcbuffer; + ffi_status status; funcbuffer.nb_bytes = 0; funcbuffer.bufferp = NULL; @@ -5810,8 +5843,21 @@ static cif_description_t *fb_prepare_cif(PyObject *fargs, assert(funcbuffer.bufferp == buffer + funcbuffer.nb_bytes); cif_descr = (cif_description_t *)buffer; - if (ffi_prep_cif(&cif_descr->cif, fabi, funcbuffer.nargs, - funcbuffer.rtype, funcbuffer.atypes) != FFI_OK) { +#if HAVE_FFI_PREP_CIF_VAR + if (variadic_nargs_declared >= 0) { + status = ffi_prep_cif_var(&cif_descr->cif, fabi, + variadic_nargs_declared, funcbuffer.nargs, + funcbuffer.rtype, funcbuffer.atypes); + } else +#endif +#if !HAVE_FFI_PREP_CIF_VAR && defined(__arm64__) && defined(__APPLE__) +#error Apple Arm64 ABI requires ffi_prep_cif_var +#endif + { + status = ffi_prep_cif(&cif_descr->cif, fabi, funcbuffer.nargs, + funcbuffer.rtype, funcbuffer.atypes); + } + if (status != FFI_OK) { PyErr_SetString(PyExc_SystemError, "libffi failed to build this function type"); goto error; @@ -5855,7 +5901,7 @@ static PyObject *new_function_type(PyObject *fargs, /* tuple */ is computed here. */ cif_description_t *cif_descr; - cif_descr = fb_prepare_cif(fargs, fresult, fabi); + cif_descr = fb_prepare_cif(fargs, fresult, -1, fabi); if (cif_descr == NULL) { if (PyErr_ExceptionMatches(PyExc_NotImplementedError)) { PyErr_Clear(); /* will get the exception if we see an @@ -6238,6 +6284,17 @@ static PyObject *b_callback(PyObject *self, PyObject *args) "return type or with '...'", ct->ct_name); goto error; } + /* NOTE: ffi_prep_closure() is marked as deprecated. We could just + * call ffi_prep_closure_loc() instead, which is what ffi_prep_closure() + * does. However, cffi also runs on older systems with a libffi that + * doesn't have ffi_prep_closure_loc() at all---notably, the OS X + * machines on Azure are like that (June 2020). I didn't find a way to + * version-check the included ffi.h. So you will have to live with the + * deprecation warning for now. (We could try to check for an unrelated + * macro like FFI_API which happens to be defined in libffi 3.3 and not + * before, but that sounds very obscure. And I prefer a compile-time + * warning rather than a cross-version binary compatibility problem.) + */ #ifdef CFFI_TRUST_LIBFFI if (ffi_prep_closure_loc(closure, &cif_descr->cif, invoke_callback, infotuple, closure_exec) != FFI_OK) { @@ -7788,6 +7845,22 @@ init_cffi_backend(void) PyObject *m, *v; int i; static char init_done = 0; + static PyTypeObject *all_types[] = { + &dl_type, + &CTypeDescr_Type, + &CField_Type, + &CData_Type, + &CDataOwning_Type, + &CDataOwningGC_Type, + &CDataFromBuf_Type, + &CDataGCP_Type, + &CDataIter_Type, + &MiniBuffer_Type, + &FFI_Type, + &Lib_Type, + &GlobSupport_Type, + NULL + }; v = PySys_GetObject("version"); if (v == NULL || !PyText_Check(v) || @@ -7813,26 +7886,22 @@ init_cffi_backend(void) INITERROR; } - if (PyType_Ready(&dl_type) < 0) - INITERROR; - if (PyType_Ready(&CTypeDescr_Type) < 0) - INITERROR; - if (PyType_Ready(&CField_Type) < 0) - INITERROR; - if (PyType_Ready(&CData_Type) < 0) - INITERROR; - if (PyType_Ready(&CDataOwning_Type) < 0) - INITERROR; - if (PyType_Ready(&CDataOwningGC_Type) < 0) - INITERROR; - if (PyType_Ready(&CDataFromBuf_Type) < 0) - INITERROR; - if (PyType_Ready(&CDataGCP_Type) < 0) - INITERROR; - if (PyType_Ready(&CDataIter_Type) < 0) - INITERROR; - if (PyType_Ready(&MiniBuffer_Type) < 0) - INITERROR; + /* readify all types and add them to the module */ + for (i = 0; all_types[i] != NULL; i++) { + PyTypeObject *tp = all_types[i]; + PyObject *tpo = (PyObject *)tp; + if (strncmp(tp->tp_name, "_cffi_backend.", 14) != 0) { + PyErr_Format(PyExc_ImportError, + "'%s' is an ill-formed type name", tp->tp_name); + INITERROR; + } + if (PyType_Ready(tp) < 0) + INITERROR; + + Py_INCREF(tpo); + if (PyModule_AddObject(m, tp->tp_name + 14, tpo) < 0) + INITERROR; + } if (!init_done) { v = PyText_FromString("_cffi_backend"); @@ -7878,10 +7947,6 @@ init_cffi_backend(void) INITERROR; } - Py_INCREF(&MiniBuffer_Type); - if (PyModule_AddObject(m, "buffer", (PyObject *)&MiniBuffer_Type) < 0) - INITERROR; - init_cffi_tls(); if (PyErr_Occurred()) INITERROR; diff --git a/c/cffi1_module.c b/c/cffi1_module.c index d688b80..06a84fe 100644 --- a/c/cffi1_module.c +++ b/c/cffi1_module.c @@ -26,11 +26,6 @@ static int init_ffi_lib(PyObject *m) int i, res; static char init_done = 0; - if (PyType_Ready(&FFI_Type) < 0) - return -1; - if (PyType_Ready(&Lib_Type) < 0) - return -1; - if (!init_done) { if (init_global_types_dict(FFI_Type.tp_dict) < 0) return -1; @@ -62,16 +57,6 @@ static int init_ffi_lib(PyObject *m) } init_done = 1; } - - x = (PyObject *)&FFI_Type; - Py_INCREF(x); - if (PyModule_AddObject(m, "FFI", x) < 0) - return -1; - x = (PyObject *)&Lib_Type; - Py_INCREF(x); - if (PyModule_AddObject(m, "Lib", x) < 0) - return -1; - return 0; } @@ -20,7 +20,7 @@ static void glob_support_dealloc(GlobSupportObject *gs) static PyTypeObject GlobSupport_Type = { PyVarObject_HEAD_INIT(NULL, 0) - "FFIGlobSupport", + "_cffi_backend.__FFIGlobSupport", sizeof(GlobSupportObject), 0, (destructor)glob_support_dealloc, /* tp_dealloc */ diff --git a/c/ffi_obj.c b/c/ffi_obj.c index 1e8cc6f..05c3a6d 100644 --- a/c/ffi_obj.c +++ b/c/ffi_obj.c @@ -1137,7 +1137,7 @@ static PyGetSetDef ffi_getsets[] = { static PyTypeObject FFI_Type = { PyVarObject_HEAD_INIT(NULL, 0) - "CompiledFFI", + "_cffi_backend.FFI", sizeof(FFIObject), 0, (destructor)ffi_dealloc, /* tp_dealloc */ diff --git a/c/lib_obj.c b/c/lib_obj.c index 0c3d7d1..38bf3d5 100644 --- a/c/lib_obj.c +++ b/c/lib_obj.c @@ -589,7 +589,7 @@ static PyMethodDef lib_methods[] = { static PyTypeObject Lib_Type = { PyVarObject_HEAD_INIT(NULL, 0) - "CompiledLib", + "_cffi_backend.Lib", sizeof(LibObject), 0, (destructor)lib_dealloc, /* tp_dealloc */ diff --git a/c/libffi_msvc/ffi.c b/c/libffi_msvc/ffi.c index 836f171..b9e324f 100644 --- a/c/libffi_msvc/ffi.c +++ b/c/libffi_msvc/ffi.c @@ -103,7 +103,7 @@ void ffi_prep_args(char *stack, extended_cif *ecif) } } #ifdef _WIN64 - else if (z > 8) + else if (z != 1 && z != 2 && z != 4 && z != 8) { /* On Win64, if a single argument takes more than 8 bytes, then it is always passed by reference. */ @@ -144,9 +144,11 @@ ffi_status ffi_prep_cif_machdep(ffi_cif *cif) /* MSVC returns small structures in registers. Put in cif->flags the value FFI_TYPE_STRUCT only if the structure is big enough; otherwise, put the 4- or 8-bytes integer type. */ - if (cif->rtype->size <= 4) + if (cif->rtype->size == 1 || + cif->rtype->size == 2 || + cif->rtype->size == 4) cif->flags = FFI_TYPE_INT; - else if (cif->rtype->size <= 8) + else if (cif->rtype->size == 8) cif->flags = FFI_TYPE_SINT64; else cif->flags = FFI_TYPE_STRUCT; @@ -287,16 +289,12 @@ ffi_closure_SYSV (ffi_closure *closure, char *argp) _asm fld DWORD PTR [eax] ; // asm ("flds (%0)" : : "r" (resp) : "st" ); } - else if (rtype == FFI_TYPE_DOUBLE) + else if (rtype == FFI_TYPE_DOUBLE || rtype == FFI_TYPE_LONGDOUBLE) { _asm mov eax, resp ; _asm fld QWORD PTR [eax] ; // asm ("fldl (%0)" : : "r" (resp) : "st", "st(1)" ); } - else if (rtype == FFI_TYPE_LONGDOUBLE) - { -// asm ("fldt (%0)" : : "r" (resp) : "st", "st(1)" ); - } else if (rtype == FFI_TYPE_SINT64) { _asm mov edx, resp ; @@ -307,6 +305,10 @@ ffi_closure_SYSV (ffi_closure *closure, char *argp) // : : "r"(resp) // : "eax", "edx"); } + else if (rtype == FFI_TYPE_STRUCT) + { + _asm mov eax, resp ; + } #else /* now, do a generic return based on the value of rtype */ if (rtype == FFI_TYPE_INT) @@ -317,14 +319,10 @@ ffi_closure_SYSV (ffi_closure *closure, char *argp) { asm ("flds (%0)" : : "r" (resp) : "st" ); } - else if (rtype == FFI_TYPE_DOUBLE) + else if (rtype == FFI_TYPE_DOUBLE || rtype == FFI_TYPE_LONGDOUBLE) { asm ("fldl (%0)" : : "r" (resp) : "st", "st(1)" ); } - else if (rtype == FFI_TYPE_LONGDOUBLE) - { - asm ("fldt (%0)" : : "r" (resp) : "st", "st(1)" ); - } else if (rtype == FFI_TYPE_SINT64) { asm ("movl 0(%0),%%eax;" @@ -332,6 +330,10 @@ ffi_closure_SYSV (ffi_closure *closure, char *argp) : : "r"(resp) : "eax", "edx"); } + else if (rtype == FFI_TYPE_STRUCT) + { + asm ("movl %0,%%eax" : : "r" (resp) : "eax"); + } #endif #endif @@ -340,6 +342,8 @@ ffi_closure_SYSV (ffi_closure *closure, char *argp) result types except for floats; we have to 'mov xmm0, rax' in the caller to correct this. */ + if (rtype == FFI_TYPE_STRUCT) + return resp; return *(void **)resp; #endif } @@ -378,7 +382,7 @@ ffi_prep_incoming_args_SYSV(char *stack, void **rvalue, /* because we're little endian, this is what it turns into. */ #ifdef _WIN64 - if (z > 8) + if (z != 1 && z != 2 && z != 4 && z != 8) { /* On Win64, if a single argument takes more than 8 bytes, then it is always passed by reference. */ @@ -447,6 +451,11 @@ ffi_prep_closure_loc (ffi_closure* closure, || cif->arg_types[3]->type == FFI_TYPE_DOUBLE)) mask |= 8; + /* if we return a non-small struct, then the first argument is a pointer + * to the return area, and all real arguments are shifted by one */ + if (cif->flags == FFI_TYPE_STRUCT) + mask = (mask & ~8) << 1; + /* 41 BB ---- mov r11d,mask */ BYTES("\x41\xBB"); INT(mask); diff --git a/c/libffi_msvc/prep_cif.c b/c/libffi_msvc/prep_cif.c index 5dacfff..df94a98 100644 --- a/c/libffi_msvc/prep_cif.c +++ b/c/libffi_msvc/prep_cif.c @@ -117,7 +117,10 @@ ffi_status ffi_prep_cif(/*@out@*/ /*@partial@*/ ffi_cif *cif, /* Make space for the return structure pointer */ if (cif->rtype->type == FFI_TYPE_STRUCT #ifdef _WIN32 - && (cif->rtype->size > 8) /* MSVC returns small structs in registers */ + && (cif->rtype->size != 1) /* MSVC returns small structs in registers */ + && (cif->rtype->size != 2) + && (cif->rtype->size != 4) + && (cif->rtype->size != 8) #endif #ifdef SPARC && (cif->abi != FFI_V9 || cif->rtype->size > 32) diff --git a/c/test_c.py b/c/test_c.py index 3b34999..383bca2 100644 --- a/c/test_c.py +++ b/c/test_c.py @@ -4,12 +4,17 @@ import pytest def _setup_path(): import os, sys if '__pypy__' in sys.builtin_module_names: - py.test.skip("_cffi_backend.c: not tested on top of pypy, " - "use pypy/module/_cffi_backend/test/ instead.") + global pytestmark + pytestmark = pytest.mark.skip( + "_cffi_backend.c: not tested on top of pypy, " + "use pypy/module/_cffi_backend/test/ instead.") + return False sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) -_setup_path() + return True +if _setup_path(): + from _cffi_backend import _testfunc, _get_types, _get_common_types from _cffi_backend import * -from _cffi_backend import _testfunc, _get_types, _get_common_types, __version__ +from _cffi_backend import __version__ # ____________________________________________________________ @@ -109,7 +114,7 @@ def test_cast_to_signed_char(): p = new_primitive_type("signed char") x = cast(p, -65 + 17*256) assert repr(x) == "<cdata 'signed char' -65>" - assert repr(type(x)) == "<%s '_cffi_backend.CData'>" % type_or_class + assert repr(type(x)) == "<%s '_cffi_backend._CDataBase'>" % type_or_class assert int(x) == -65 x = cast(p, -66 + (1<<199)*256) assert repr(x) == "<cdata 'signed char' -66>" @@ -4453,3 +4458,41 @@ def test_huge_structure(): BStruct = new_struct_type("struct foo") complete_struct_or_union(BStruct, [('a1', BArray, -1)]) assert sizeof(BStruct) == sys.maxsize + +def test_get_types(): + import _cffi_backend + CData, CType = _get_types() + assert CData is _cffi_backend._CDataBase + assert CType is _cffi_backend.CType + +def test_type_available_with_correct_names(): + import _cffi_backend + check_names = [ + 'CType', + 'CField', + 'CLibrary', + '_CDataBase', + 'FFI', + 'Lib', + 'buffer', + ] + if '__pypy__' in sys.builtin_module_names: + check_names += [ + '__CData_iterator', + '__FFIGlobSupport', + '__FFIAllocator', + '__FFIFunctionWrapper', + ] + else: + check_names += [ + '__CDataOwn', + '__CDataOwnGC', + '__CDataFromBuf', + '__CDataGCP', + '__CData_iterator', + '__FFIGlobSupport', + ] + for name in check_names: + tp = getattr(_cffi_backend, name) + assert isinstance(tp, type) + assert (tp.__module__, tp.__name__) == ('_cffi_backend', name) diff --git a/cffi/_embedding.h b/cffi/_embedding.h index 34a4a66..e49c290 100644 --- a/cffi/_embedding.h +++ b/cffi/_embedding.h @@ -361,11 +361,11 @@ PyMODINIT_FUNC _CFFI_PYTHON_STARTUP_FUNC(const void *[]); /* forward */ static struct _cffi_pypy_init_s { const char *name; - void (*func)(const void *[]); + void *func; /* function pointer */ const char *code; } _cffi_pypy_init = { _CFFI_MODULE_NAME, - (void(*)(const void *[]))_CFFI_PYTHON_STARTUP_FUNC, + _CFFI_PYTHON_STARTUP_FUNC, _CFFI_PYTHON_STARTUP_CODE, }; diff --git a/cffi/cparser.py b/cffi/cparser.py index ea27c48..74830e9 100644 --- a/cffi/cparser.py +++ b/cffi/cparser.py @@ -29,6 +29,7 @@ _r_comment = re.compile(r"/\*.*?\*/|//([^\n\\]|\\.)*?$", _r_define = re.compile(r"^\s*#\s*define\s+([A-Za-z_][A-Za-z_0-9]*)" r"\b((?:[^\n\\]|\\.)*?)$", re.DOTALL | re.MULTILINE) +_r_line_directive = re.compile(r"^[ \t]*#[ \t]*(?:line|\d+)\b.*$", re.MULTILINE) _r_partial_enum = re.compile(r"=\s*\.\.\.\s*[,}]|\.\.\.\s*\}") _r_enum_dotdotdot = re.compile(r"__dotdotdot\d+__$") _r_partial_array = re.compile(r"\[\s*\.\.\.\s*\]") @@ -163,10 +164,37 @@ def _warn_for_non_extern_non_static_global_variable(decl): "with C it should have a storage class specifier " "(usually 'extern')" % (decl.name,)) +def _remove_line_directives(csource): + # _r_line_directive matches whole lines, without the final \n, if they + # start with '#line' with some spacing allowed, or '#NUMBER'. This + # function stores them away and replaces them with exactly the string + # '#line@N', where N is the index in the list 'line_directives'. + line_directives = [] + def replace(m): + i = len(line_directives) + line_directives.append(m.group()) + return '#line@%d' % i + csource = _r_line_directive.sub(replace, csource) + return csource, line_directives + +def _put_back_line_directives(csource, line_directives): + def replace(m): + s = m.group() + if not s.startswith('#line@'): + raise AssertionError("unexpected #line directive " + "(should have been processed and removed") + return line_directives[int(s[6:])] + return _r_line_directive.sub(replace, csource) + def _preprocess(csource): + # First, remove the lines of the form '#line N "filename"' because + # the "filename" part could confuse the rest + csource, line_directives = _remove_line_directives(csource) # Remove comments. NOTE: this only work because the cdef() section - # should not contain any string literal! - csource = _r_comment.sub(' ', csource) + # should not contain any string literals (except in line directives)! + def replace_keeping_newlines(m): + return ' ' + m.group().count('\n') * '\n' + csource = _r_comment.sub(replace_keeping_newlines, csource) # Remove the "#define FOO x" lines macros = {} for match in _r_define.finditer(csource): @@ -219,7 +247,10 @@ def _preprocess(csource): csource = _r_float_dotdotdot.sub(' __dotdotdotfloat__ ', csource) # Replace all remaining "..." with the same name, "__dotdotdot__", # which is declared with a typedef for the purpose of C parsing. - return csource.replace('...', ' __dotdotdot__ '), macros + csource = csource.replace('...', ' __dotdotdot__ ') + # Finally, put back the line directives + csource = _put_back_line_directives(csource, line_directives) + return csource, macros def _common_type_names(csource): # Look in the source for what looks like usages of types from the @@ -395,7 +426,8 @@ class Parser(object): realtype = self._get_unknown_ptr_type(decl) else: realtype, quals = self._get_type_and_quals( - decl.type, name=decl.name, partial_length_ok=True) + decl.type, name=decl.name, partial_length_ok=True, + typedef_example="*(%s *)0" % (decl.name,)) self._declare('typedef ' + decl.name, realtype, quals=quals) elif decl.__class__.__name__ == 'Pragma': pass # skip pragma, only in pycparser 2.15 @@ -562,7 +594,8 @@ class Parser(object): return model.NamedPointerType(type, declname, quals) return model.PointerType(type, quals) - def _get_type_and_quals(self, typenode, name=None, partial_length_ok=False): + def _get_type_and_quals(self, typenode, name=None, partial_length_ok=False, + typedef_example=None): # first, dereference typedefs, if we have it already parsed, we're good if (isinstance(typenode, pycparser.c_ast.TypeDecl) and isinstance(typenode.type, pycparser.c_ast.IdentifierType) and @@ -579,8 +612,18 @@ class Parser(object): else: length = self._parse_constant( typenode.dim, partial_length_ok=partial_length_ok) + # a hack: in 'typedef int foo_t[...][...];', don't use '...' as + # the length but use directly the C expression that would be + # generated by recompiler.py. This lets the typedef be used in + # many more places within recompiler.py + if typedef_example is not None: + if length == '...': + length = '_cffi_array_len(%s)' % (typedef_example,) + typedef_example = "*" + typedef_example + # tp, quals = self._get_type_and_quals(typenode.type, - partial_length_ok=partial_length_ok) + partial_length_ok=partial_length_ok, + typedef_example=typedef_example) return model.ArrayType(tp, length), quals # if isinstance(typenode, pycparser.c_ast.PtrDecl): diff --git a/cffi/model.py b/cffi/model.py index 5f1b0d2..ad1c176 100644 --- a/cffi/model.py +++ b/cffi/model.py @@ -307,11 +307,14 @@ class ArrayType(BaseType): self.c_name_with_marker = ( self.item.c_name_with_marker.replace('&', brackets)) + def length_is_unknown(self): + return isinstance(self.length, str) + def resolve_length(self, newlength): return ArrayType(self.item, newlength) def build_backend_type(self, ffi, finishlist): - if self.length == '...': + if self.length_is_unknown(): raise CDefError("cannot render the type %r: unknown length" % (self,)) self.item.get_cached_btype(ffi, finishlist) # force the item BType @@ -430,7 +433,7 @@ class StructOrUnion(StructOrUnionOrEnum): fsize = fieldsize[i] ftype = self.fldtypes[i] # - if isinstance(ftype, ArrayType) and ftype.length == '...': + if isinstance(ftype, ArrayType) and ftype.length_is_unknown(): # fix the length to match the total size BItemType = ftype.item.get_cached_btype(ffi, finishlist) nlen, nrest = divmod(fsize, ffi.sizeof(BItemType)) diff --git a/cffi/recompiler.py b/cffi/recompiler.py index d66ff7f..1309572 100644 --- a/cffi/recompiler.py +++ b/cffi/recompiler.py @@ -1296,14 +1296,28 @@ class Recompiler: def _print_string_literal_in_array(self, s): prnt = self._prnt prnt('// # NB. this is not a string because of a size limit in MSVC') + if not isinstance(s, bytes): # unicode + s = s.encode('utf-8') # -> bytes + else: + s.decode('utf-8') # got bytes, check for valid utf-8 + try: + s.decode('ascii') + except UnicodeDecodeError: + s = b'# -*- encoding: utf8 -*-\n' + s for line in s.splitlines(True): - prnt(('// ' + line).rstrip()) + comment = line + if type('//') is bytes: # python2 + line = map(ord, line) # make a list of integers + else: # python3 + # type(line) is bytes, which enumerates like a list of integers + comment = ascii(comment)[1:-1] + prnt(('// ' + comment).rstrip()) printed_line = '' for c in line: if len(printed_line) >= 76: prnt(printed_line) printed_line = '' - printed_line += '%d,' % (ord(c),) + printed_line += '%d,' % (c,) prnt(printed_line) # ---------- diff --git a/cffi/vengine_cpy.py b/cffi/vengine_cpy.py index cb344ce..6de0df0 100644 --- a/cffi/vengine_cpy.py +++ b/cffi/vengine_cpy.py @@ -762,7 +762,7 @@ class VCPythonEngine(object): if isinstance(tp, model.ArrayType): tp_ptr = model.PointerType(tp.item) self._generate_cpy_const(False, name, tp, vartp=tp_ptr, - size_too = (tp.length == '...')) + size_too = tp.length_is_unknown()) else: tp_ptr = model.PointerType(tp) self._generate_cpy_const(False, name, tp_ptr, category='var') @@ -774,7 +774,7 @@ class VCPythonEngine(object): value = getattr(library, name) if isinstance(tp, model.ArrayType): # int a[5] is "constant" in the # sense that "a=..." is forbidden - if tp.length == '...': + if tp.length_is_unknown(): assert isinstance(value, tuple) (value, size) = value BItemType = self.ffi._get_cached_btype(tp.item) diff --git a/cffi/vengine_gen.py b/cffi/vengine_gen.py index a64ff64..2642152 100644 --- a/cffi/vengine_gen.py +++ b/cffi/vengine_gen.py @@ -565,7 +565,7 @@ class VGenericEngine(object): def _generate_gen_variable_decl(self, tp, name): if isinstance(tp, model.ArrayType): - if tp.length == '...': + if tp.length_is_unknown(): prnt = self._prnt funcname = '_cffi_sizeof_%s' % (name,) self.export_symbols.append(funcname) @@ -584,7 +584,7 @@ class VGenericEngine(object): def _loaded_gen_variable(self, tp, name, module, library): if isinstance(tp, model.ArrayType): # int a[5] is "constant" in the # sense that "a=..." is forbidden - if tp.length == '...': + if tp.length_is_unknown(): funcname = '_cffi_sizeof_%s' % (name,) BFunc = self.ffi._typeof_locked('size_t(*)(void)')[0] function = module.load_function(BFunc, funcname) diff --git a/doc/source/goals.rst b/doc/source/goals.rst index 0fda659..f66df21 100644 --- a/doc/source/goals.rst +++ b/doc/source/goals.rst @@ -65,5 +65,5 @@ everything you need to access C code and nothing more. --- the authors, Armin Rigo and Maciej Fijalkowski -.. _`issue tracker`: https://bitbucket.org/cffi/cffi/issues +.. _`issue tracker`: https://foss.heptapod.net/pypy/cffi/issues .. _`mailing list`: https://groups.google.com/forum/#!forum/python-cffi diff --git a/doc/source/installation.rst b/doc/source/installation.rst index 557b580..82fcb0e 100644 --- a/doc/source/installation.rst +++ b/doc/source/installation.rst @@ -60,8 +60,8 @@ Download and Installation: - SHA256: 2d384f4a127a15ba701207f7639d94106693b6cd64173d6c8988e2c25f3ac2b6 -* Or grab the most current version from the `Bitbucket page`_: - ``hg clone https://bitbucket.org/cffi/cffi`` +* Or grab the most current version from the `Heptapod page`_: + ``hg clone https://foss.heptapod.net/pypy/cffi`` * ``python setup.py install`` or ``python setup_base.py install`` (should work out of the box on Linux or Windows; see below for @@ -71,7 +71,7 @@ Download and Installation: install cffi yet, you need first ``python setup_base.py build_ext -f -i``) -.. _`Bitbucket page`: https://bitbucket.org/cffi/cffi +.. _`Heptapod page`: https://foss.heptapod.net/pypy/cffi Demos: diff --git a/doc/source/overview.rst b/doc/source/overview.rst index 32970fa..60e3a01 100644 --- a/doc/source/overview.rst +++ b/doc/source/overview.rst @@ -358,7 +358,7 @@ Linux for example). It can be called from Python: from _pi.lib import pi_approx approx = pi_approx(10) - assert str(pi_approximation).startswith("3.") + assert str(approx).startswith("3.") approx = pi_approx(10000) assert str(approx).startswith("3.1") diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst index 660608a..4b5199b 100644 --- a/doc/source/whatsnew.rst +++ b/doc/source/whatsnew.rst @@ -3,6 +3,25 @@ What's New ====================== +v1.14.1 +======= + +* CFFI source code is now `hosted on Heptapod`_. + +* Improved support for ``typedef int my_array_t[...];`` with an explicit + dot-dot-dot in API mode (`issue #453`_) + +* Windows (32 and 64 bits): multiple fixes for ABI-mode call to functions + that return a structure. + +* Experimental support for MacOS 11 on aarch64. + +* and a few other minor changes and bug fixes. + +.. _`hosted on Heptapod`: https://foss.heptapod.net/pypy/cffi/ +.. _`issue #453`: https://foss.heptapod.net/pypy/cffi/issues/453 + + v1.14 ===== @@ -82,8 +101,11 @@ v1.13 recursion, with ``ffi.cdef("""struct X { void(*fnptr)(struct X); };""")`` +Older Versions +============== + v1.12.3 -======= +------- * Fix for nested struct types that end in a var-sized array (#405). @@ -94,13 +116,13 @@ v1.12.3 v1.12.2 -======= +------- * Added temporary workaround to compile on CPython 3.8.0a2. v1.12.1 -======= +------- * CPython 3 on Windows: we again no longer compile with ``Py_LIMITED_API`` by default because such modules *still* cannot be used with virtualenv. @@ -116,7 +138,7 @@ v1.12.1 v1.12 -===== +----- * `Direct support for pkg-config`__. @@ -154,9 +176,6 @@ v1.12 .. _`issue #362`: https://bitbucket.org/cffi/cffi/issues/362/ -Older Versions -============== - v1.11.5 ------- @@ -71,6 +71,10 @@ def get_config(): config = Distribution().get_command_obj('config') return config +def macosx_deployment_target(): + from distutils.sysconfig import get_config_var + return tuple(map(int, get_config_var("MACOSX_DEPLOYMENT_TARGET").split('.'))) + def ask_supports_thread(): config = get_config() ok = (sys.platform != 'win32' and @@ -145,7 +149,17 @@ if COMPILE_LIBFFI: sources.extend(os.path.join(COMPILE_LIBFFI, filename) for filename in _filenames) else: - use_pkg_config() + if 'darwin' in sys.platform and macosx_deployment_target() >= (10, 15): + # use libffi from Mac OS SDK if we're targetting 10.15 (including + # on arm64). This libffi is safe against the crash-after-fork + # issue described in _cffi_backend.c. Also, arm64 uses a different + # ABI for calls to vararg functions as opposed to regular functions. + extra_compile_args += ['-iwithsysroot/usr/include/ffi'] + define_macros += [('CFFI_TRUST_LIBFFI', '1'), + ('HAVE_FFI_PREP_CIF_VAR', '1')] + libraries += ['ffi'] + else: + use_pkg_config() ask_supports_thread() ask_supports_sync_synchronize() diff --git a/testing/cffi0/backend_tests.py b/testing/cffi0/backend_tests.py index 063e9f2..ab013a1 100644 --- a/testing/cffi0/backend_tests.py +++ b/testing/cffi0/backend_tests.py @@ -1,7 +1,7 @@ import py import pytest import platform -import sys, ctypes +import sys, ctypes, ctypes.util from cffi import FFI, CDefError, FFIError, VerificationMissing from testing.support import * @@ -12,8 +12,8 @@ SIZE_OF_PTR = ctypes.sizeof(ctypes.c_void_p) SIZE_OF_WCHAR = ctypes.sizeof(ctypes.c_wchar) def needs_dlopen_none(): - if sys.platform == 'win32' and sys.version_info >= (3,): - py.test.skip("dlopen(None) cannot work on Windows for Python 3") + if sys.platform == 'win32' and not ctypes.util.find_library('c'): + py.test.skip("dlopen(None) cannot work on Windows with this runtime") class BackendTests: diff --git a/testing/cffi0/test_ffi_backend.py b/testing/cffi0/test_ffi_backend.py index 6937a79..3552196 100644 --- a/testing/cffi0/test_ffi_backend.py +++ b/testing/cffi0/test_ffi_backend.py @@ -246,7 +246,10 @@ class TestBitfield: self.check("int a:2; short b:15; char c:2; char y;", 5, 4, 8) self.check("int a:2; char b:1; char c:1; char y;", 1, 4, 4) - @pytest.mark.skipif("platform.machine().startswith(('arm', 'aarch64'))") + @pytest.mark.skipif( + "not (sys.platform == 'darwin' and platform.machine() == 'arm64')" + " and " + "platform.machine().startswith(('arm', 'aarch64'))") def test_bitfield_anonymous_no_align(self): L = FFI().alignof("long long") self.check("char y; int :1;", 0, 1, 2) @@ -260,6 +263,8 @@ class TestBitfield: self.check("char x; long long :57; char y;", L + 8, 1, L + 9) @pytest.mark.skipif( + "(sys.platform == 'darwin' and platform.machine() == 'arm64')" + " or " "not platform.machine().startswith(('arm', 'aarch64'))") def test_bitfield_anonymous_align_arm(self): L = FFI().alignof("long long") @@ -273,7 +278,10 @@ class TestBitfield: self.check("char x; long long z:57; char y;", L + 8, L, L + 8 + L) self.check("char x; long long :57; char y;", L + 8, L, L + 8 + L) - @pytest.mark.skipif("platform.machine().startswith(('arm', 'aarch64'))") + @pytest.mark.skipif( + "not (sys.platform == 'darwin' and platform.machine() == 'arm64')" + " and " + "platform.machine().startswith(('arm', 'aarch64'))") def test_bitfield_zero(self): L = FFI().alignof("long long") self.check("char y; int :0;", 0, 1, 4) @@ -285,6 +293,8 @@ class TestBitfield: self.check("int a:1; int :0; int b:1; char y;", 5, 4, 8) @pytest.mark.skipif( + "(sys.platform == 'darwin' and platform.machine() == 'arm64')" + " or " "not platform.machine().startswith(('arm', 'aarch64'))") def test_bitfield_zero_arm(self): L = FFI().alignof("long long") diff --git a/testing/cffi0/test_function.py b/testing/cffi0/test_function.py index 6312707..4f2c864 100644 --- a/testing/cffi0/test_function.py +++ b/testing/cffi0/test_function.py @@ -240,6 +240,23 @@ class TestFunction(object): else: assert "None" in printed + def test_callback_returning_struct_three_bytes(self): + if self.Backend is CTypesBackend: + py.test.skip("not supported with the ctypes backend") + ffi = FFI(backend=self.Backend()) + ffi.cdef(""" + typedef struct { + unsigned char a, b, c; + } THREEBYTES; + """) + def cb(): + return (12, 34, 56) + fptr = ffi.callback("THREEBYTES(*)(void)", cb) + tb = fptr() + assert tb.a == 12 + assert tb.b == 34 + assert tb.c == 56 + def test_passing_array(self): ffi = FFI(backend=self.Backend()) ffi.cdef(""" diff --git a/testing/cffi0/test_ownlib.py b/testing/cffi0/test_ownlib.py index 990f259..ffad879 100644 --- a/testing/cffi0/test_ownlib.py +++ b/testing/cffi0/test_ownlib.py @@ -35,6 +35,10 @@ typedef struct { long bottom; } RECT; +typedef struct { + unsigned char a, b, c; +} THREEBYTES; + EXPORT int PointInRect(RECT *prc, POINT pt) { @@ -107,6 +111,15 @@ EXPORT void modify_struct_value(RECT r) { r.left = r.right = r.top = r.bottom = 500; } + +EXPORT THREEBYTES return_three_bytes(void) +{ + THREEBYTES result; + result.a = 12; + result.b = 34; + result.c = 56; + return result; +} """ class TestOwnLib(object): @@ -397,3 +410,22 @@ class TestOwnLib(object): err = lib1.dlclose(handle) assert err == 0 + + def test_return_three_bytes(self): + if self.module is None: + py.test.skip("fix the auto-generation of the tiny test lib") + if self.__class__.Backend is CTypesBackend: + py.test.skip("not working on win32 on the ctypes backend") + ffi = FFI(backend=self.Backend()) + ffi.cdef(""" + typedef struct { + unsigned char a, b, c; + } THREEBYTES; + + THREEBYTES return_three_bytes(void); + """) + lib = ffi.dlopen(self.module) + tb = lib.return_three_bytes() + assert tb.a == 12 + assert tb.b == 34 + assert tb.c == 56 diff --git a/testing/cffi0/test_parsing.py b/testing/cffi0/test_parsing.py index 3fc3783..a5e4587 100644 --- a/testing/cffi0/test_parsing.py +++ b/testing/cffi0/test_parsing.py @@ -174,7 +174,7 @@ def test_remove_line_continuation_comments(): double // blah \\ more comments x(void); - double // blah\\\\ + double // blah // blah\\\\ y(void); double // blah\\ \ etc @@ -185,6 +185,93 @@ def test_remove_line_continuation_comments(): m.y m.z +def test_dont_remove_comment_in_line_directives(): + ffi = FFI(backend=FakeBackend()) + e = py.test.raises(CDefError, ffi.cdef, """ + \t # \t line \t 8 \t "baz.c" \t + + some syntax error here + """) + assert str(e.value) == "parse error\nbaz.c:9:14: before: syntax" + # + e = py.test.raises(CDefError, ffi.cdef, """ + #line 7 "foo//bar.c" + + some syntax error here + """) + # + assert str(e.value) == "parse error\nfoo//bar.c:8:14: before: syntax" + ffi = FFI(backend=FakeBackend()) + e = py.test.raises(CDefError, ffi.cdef, """ + \t # \t 8 \t "baz.c" \t + + some syntax error here + """) + assert str(e.value) == "parse error\nbaz.c:9:14: before: syntax" + # + e = py.test.raises(CDefError, ffi.cdef, """ + # 7 "foo//bar.c" + + some syntax error here + """) + assert str(e.value) == "parse error\nfoo//bar.c:8:14: before: syntax" + +def test_multiple_line_directives(): + ffi = FFI(backend=FakeBackend()) + e = py.test.raises(CDefError, ffi.cdef, + """ #line 5 "foo.c" + extern int xx; + #line 6 "bar.c" + extern int yy; + #line 7 "baz.c" + some syntax error here + #line 8 "yadda.c" + extern int zz; + """) + assert str(e.value) == "parse error\nbaz.c:7:14: before: syntax" + # + e = py.test.raises(CDefError, ffi.cdef, + """ # 5 "foo.c" + extern int xx; + # 6 "bar.c" + extern int yy; + # 7 "baz.c" + some syntax error here + # 8 "yadda.c" + extern int zz; + """) + assert str(e.value) == "parse error\nbaz.c:7:14: before: syntax" + +def test_commented_line_directive(): + ffi = FFI(backend=FakeBackend()) + e = py.test.raises(CDefError, ffi.cdef, """ + /* + #line 5 "foo.c" + */ + void xx(void); + + #line 6 "bar.c" + /* + #line 35 "foo.c" + */ + some syntax error + """) + # + assert str(e.value) == "parse error\nbar.c:9:14: before: syntax" + e = py.test.raises(CDefError, ffi.cdef, """ + /* + # 5 "foo.c" + */ + void xx(void); + + # 6 "bar.c" + /* + # 35 "foo.c" + */ + some syntax error + """) + assert str(e.value) == "parse error\nbar.c:9:14: before: syntax" + def test_line_continuation_in_defines(): ffi = FFI(backend=FakeBackend()) ffi.cdef(""" diff --git a/testing/cffi1/test_function_args.py b/testing/cffi1/test_function_args.py new file mode 100644 index 0000000..30c6fed --- /dev/null +++ b/testing/cffi1/test_function_args.py @@ -0,0 +1,208 @@ +import pytest, sys +try: + # comment out the following line to run this test. + # the latest on x86-64 linux: https://github.com/libffi/libffi/issues/574 + if sys.platform != 'win32': + raise ImportError("this test is skipped because it keeps finding " + "failures in libffi, instead of cffi") + + from hypothesis import given, settings, example + from hypothesis import strategies as st +except ImportError as e: + e1 = e + def test_types(): + pytest.skip(str(e1)) +else: + + from cffi import FFI + import sys, random + from .test_recompiler import verify + + ALL_PRIMITIVES = [ + 'unsigned char', + 'short', + 'int', + 'long', + 'long long', + 'float', + 'double', + #'long double', --- on x86 it can give libffi crashes + ] + def _make_struct(s): + return st.lists(s, min_size=1) + types = st.one_of(st.sampled_from(ALL_PRIMITIVES), + st.lists(st.sampled_from(ALL_PRIMITIVES), min_size=1)) + # NB. 'types' could be st.recursive instead, but it doesn't + # really seem useful + + def draw_primitive(ffi, typename): + value = random.random() * 2**40 + if typename != 'long double': + return ffi.cast(typename, value) + else: + return value + + TEST_RUN_COUNTER = 0 + + + @given(st.lists(types), types) + @settings(max_examples=100, deadline=5000) # 5000ms + def test_types(tp_args, tp_result): + global TEST_RUN_COUNTER + print(tp_args, tp_result) + cdefs = [] + structs = {} + + def build_type(tp): + if type(tp) is list: + field_types = [build_type(tp1) for tp1 in tp] + fields = ['%s f%d;' % (ftp, j) + for (j, ftp) in enumerate(field_types)] + fields = '\n '.join(fields) + name = 's%d' % len(cdefs) + cdefs.append("typedef struct {\n %s\n} %s;" % (fields, name)) + structs[name] = field_types + return name + else: + return tp + + args = [build_type(tp) for tp in tp_args] + result = build_type(tp_result) + + TEST_RUN_COUNTER += 1 + signature = "%s testfargs(%s)" % (result, + ', '.join(['%s a%d' % (arg, i) for (i, arg) in enumerate(args)]) + or 'void') + + source = list(cdefs) + + cdefs.append("%s;" % signature) + cdefs.append("extern %s testfargs_result;" % result) + for i, arg in enumerate(args): + cdefs.append("extern %s testfargs_arg%d;" % (arg, i)) + source.append("%s testfargs_result;" % result) + for i, arg in enumerate(args): + source.append("%s testfargs_arg%d;" % (arg, i)) + source.append(signature) + source.append("{") + for i, arg in enumerate(args): + source.append(" testfargs_arg%d = a%d;" % (i, i)) + source.append(" return testfargs_result;") + source.append("}") + + typedef_line = "typedef %s;" % (signature.replace('testfargs', + '(*mycallback_t)'),) + assert signature.endswith(')') + sig_callback = "%s testfcallback(mycallback_t callback)" % result + cdefs.append(typedef_line) + cdefs.append("%s;" % sig_callback) + source.append(typedef_line) + source.append(sig_callback) + source.append("{") + source.append(" return callback(%s);" % + ', '.join(["testfargs_arg%d" % i for i in range(len(args))])) + source.append("}") + + ffi = FFI() + ffi.cdef("\n".join(cdefs)) + lib = verify(ffi, 'test_function_args_%d' % TEST_RUN_COUNTER, + "\n".join(source), no_cpp=True) + + # when getting segfaults, enable this: + if False: + from testing.udir import udir + import subprocess + f = open(str(udir.join('run1.py')), 'w') + f.write('import sys; sys.path = %r\n' % (sys.path,)) + f.write('from _CFFI_test_function_args_%d import ffi, lib\n' % + TEST_RUN_COUNTER) + for i in range(len(args)): + f.write('a%d = ffi.new("%s *")\n' % (i, args[i])) + aliststr = ', '.join(['a%d[0]' % i for i in range(len(args))]) + f.write('lib.testfargs(%s)\n' % aliststr) + f.write('ffi.addressof(lib, "testfargs")(%s)\n' % aliststr) + f.close() + print("checking for segfault for direct call...") + rc = subprocess.call([sys.executable, 'run1.py'], cwd=str(udir)) + assert rc == 0, rc + + def make_arg(tp): + if tp in structs: + return [make_arg(tp1) for tp1 in structs[tp]] + else: + return draw_primitive(ffi, tp) + + passed_args = [make_arg(arg) for arg in args] + returned_value = make_arg(result) + + def write(p, v): + if type(v) is list: + for i, v1 in enumerate(v): + write(ffi.addressof(p, 'f%d' % i), v1) + else: + p[0] = v + + write(ffi.addressof(lib, 'testfargs_result'), returned_value) + + ## CALL forcing libffi + print("CALL forcing libffi") + received_return = ffi.addressof(lib, 'testfargs')(*passed_args) + ## + + _tp_long_double = ffi.typeof("long double") + def check(p, v): + if type(v) is list: + for i, v1 in enumerate(v): + check(ffi.addressof(p, 'f%d' % i), v1) + else: + if ffi.typeof(p).item is _tp_long_double: + assert ffi.cast("double", p[0]) == v + else: + assert p[0] == v + + for i, arg in enumerate(passed_args): + check(ffi.addressof(lib, 'testfargs_arg%d' % i), arg) + ret = ffi.new(result + "*", received_return) + check(ret, returned_value) + + ## CALLBACK + def expand(value): + if isinstance(value, ffi.CData): + t = ffi.typeof(value) + if t is _tp_long_double: + return float(ffi.cast("double", value)) + return [expand(getattr(value, 'f%d' % i)) + for i in range(len(t.fields))] + else: + return value + + # when getting segfaults, enable this: + if False: + from testing.udir import udir + import subprocess + f = open(str(udir.join('run1.py')), 'w') + f.write('import sys; sys.path = %r\n' % (sys.path,)) + f.write('from _CFFI_test_function_args_%d import ffi, lib\n' % + TEST_RUN_COUNTER) + f.write('def callback(*args): return ffi.new("%s *")[0]\n' % result) + f.write('fptr = ffi.callback("%s(%s)", callback)\n' % (result, + ','.join(args))) + f.write('print(lib.testfcallback(fptr))\n') + f.close() + print("checking for segfault for callback...") + rc = subprocess.call([sys.executable, 'run1.py'], cwd=str(udir)) + assert rc == 0, rc + + seen_args = [] + def callback(*args): + seen_args.append([expand(arg) for arg in args]) + return returned_value + + fptr = ffi.callback("%s(%s)" % (result, ','.join(args)), callback) + print("CALL with callback") + received_return = lib.testfcallback(fptr) + + assert len(seen_args) == 1 + assert passed_args == seen_args[0] + ret = ffi.new(result + "*", received_return) + check(ret, returned_value) diff --git a/testing/cffi1/test_recompiler.py b/testing/cffi1/test_recompiler.py index 78abaa0..397270b 100644 --- a/testing/cffi1/test_recompiler.py +++ b/testing/cffi1/test_recompiler.py @@ -38,6 +38,9 @@ def verify(ffi, module_name, source, *args, **kwds): from testing.support import extra_compile_args kwds['extra_compile_args'] = (kwds.get('extra_compile_args', []) + extra_compile_args) + if sys.platform == 'darwin': + kwds['extra_link_args'] = (kwds.get('extra_link_args', []) + + ['-stdlib=libc++']) return _verify(ffi, module_name, source, *args, **kwds) def test_set_source_no_slashes(): @@ -2119,6 +2122,40 @@ def test_typedef_array_dotdotdot(): p = ffi.new("vmat_t", 4) assert ffi.sizeof(p[3]) == 8 * ffi.sizeof("int") +def test_typedef_array_dotdotdot_usage(): + ffi = FFI() + ffi.cdef(""" + typedef int foo_t[...]; + typedef int mat_t[...][...]; + struct s { foo_t a; foo_t *b; foo_t **c; }; + int myfunc(foo_t a, foo_t *b, foo_t **c); + struct sm { mat_t a; mat_t *b; mat_t **c; }; + int myfuncm(mat_t a, mat_t *b, mat_t **c); + """) + lib = verify(ffi, "test_typedef_array_dotdotdot_usage", """ + typedef int foo_t[50]; + typedef int mat_t[6][7]; + struct s { foo_t a; foo_t *b; foo_t **c; }; + static int myfunc(foo_t a, foo_t *b, foo_t **c) { return (**c)[49]; } + struct sm { mat_t a; mat_t *b; mat_t **c; }; + static int myfuncm(mat_t a, mat_t *b, mat_t **c) { return (**c)[5][6]; } + """) + assert ffi.sizeof("foo_t") == 50 * ffi.sizeof("int") + p = ffi.new("struct s *") + assert ffi.sizeof(p[0]) == 50 * ffi.sizeof("int") + 2 * ffi.sizeof("void *") + p.a[49] = 321 + p.b = ffi.addressof(p, 'a') + p.c = ffi.addressof(p, 'b') + assert lib.myfunc(ffi.NULL, ffi.NULL, p.c) == 321 + # + assert ffi.sizeof("mat_t") == 42 * ffi.sizeof("int") + p = ffi.new("struct sm *") + assert ffi.sizeof(p[0]) == 42 * ffi.sizeof("int") + 2 * ffi.sizeof("void *") + p.a[5][6] = -321 + p.b = ffi.addressof(p, 'a') + p.c = ffi.addressof(p, 'b') + assert lib.myfuncm(ffi.NULL, ffi.NULL, p.c) == -321 + def test_call_with_custom_field_pos(): ffi = FFI() ffi.cdef(""" diff --git a/testing/embedding/test_basic.py b/testing/embedding/test_basic.py index 894ace5..8d2e776 100644 --- a/testing/embedding/test_basic.py +++ b/testing/embedding/test_basic.py @@ -206,3 +206,9 @@ class TestBasic(EmbeddingTests): self.compile('add1-test', [initerror_cffi]) output = self.execute('add1-test') assert output == "got: 0 0\n" # plus lots of info to stderr + + def test_embedding_with_unicode(self): + withunicode_cffi = self.prepare_module('withunicode') + self.compile('add1-test', [withunicode_cffi]) + output = self.execute('add1-test') + assert output == "255\n4660\n65244\ngot: 0 0\n" diff --git a/testing/embedding/withunicode.py b/testing/embedding/withunicode.py new file mode 100644 index 0000000..839c6cd --- /dev/null +++ b/testing/embedding/withunicode.py @@ -0,0 +1,26 @@ +import sys, cffi +if sys.version_info < (3,): + u_prefix = "u" +else: + u_prefix = "" + unichr = chr + + +ffi = cffi.FFI() + +ffi.embedding_api(u""" + int add1(int, int); +""") + +ffi.embedding_init_code((""" + import sys, time + for c in %s'""" + unichr(0x00ff) + unichr(0x1234) + unichr(0xfedc) + """': + sys.stdout.write(str(ord(c)) + '\\n') + sys.stdout.flush() +""") % u_prefix) + +ffi.set_source("_withunicode_cffi", """ +""") + +fn = ffi.compile(verbose=True) +print('FILENAME: %s' % (fn,)) diff --git a/testing/udir.py b/testing/udir.py index 4dd0a11..59db1c4 100644 --- a/testing/udir.py +++ b/testing/udir.py @@ -1,7 +1,134 @@ import py -import sys +import sys, os, atexit -udir = py.path.local.make_numbered_dir(prefix = 'ffi-') + +# This is copied from PyPy's vendored py lib. The latest py lib release +# (1.8.1) contains a bug and crashes if it sees another temporary directory +# in which we don't have write permission (e.g. because it's owned by someone +# else). +def make_numbered_dir(prefix='session-', rootdir=None, keep=3, + lock_timeout = 172800, # two days + min_timeout = 300): # five minutes + """ return unique directory with a number greater than the current + maximum one. The number is assumed to start directly after prefix. + if keep is true directories with a number less than (maxnum-keep) + will be removed. + """ + if rootdir is None: + rootdir = py.path.local.get_temproot() + + def parse_num(path): + """ parse the number out of a path (if it matches the prefix) """ + bn = path.basename + if bn.startswith(prefix): + try: + return int(bn[len(prefix):]) + except ValueError: + pass + + # compute the maximum number currently in use with the + # prefix + lastmax = None + while True: + maxnum = -1 + for path in rootdir.listdir(): + num = parse_num(path) + if num is not None: + maxnum = max(maxnum, num) + + # make the new directory + try: + udir = rootdir.mkdir(prefix + str(maxnum+1)) + except py.error.EEXIST: + # race condition: another thread/process created the dir + # in the meantime. Try counting again + if lastmax == maxnum: + raise + lastmax = maxnum + continue + break + + # put a .lock file in the new directory that will be removed at + # process exit + if lock_timeout: + lockfile = udir.join('.lock') + mypid = os.getpid() + if hasattr(lockfile, 'mksymlinkto'): + lockfile.mksymlinkto(str(mypid)) + else: + lockfile.write(str(mypid)) + def try_remove_lockfile(): + # in a fork() situation, only the last process should + # remove the .lock, otherwise the other processes run the + # risk of seeing their temporary dir disappear. For now + # we remove the .lock in the parent only (i.e. we assume + # that the children finish before the parent). + if os.getpid() != mypid: + return + try: + lockfile.remove() + except py.error.Error: + pass + atexit.register(try_remove_lockfile) + + # prune old directories + if keep: + for path in rootdir.listdir(): + num = parse_num(path) + if num is not None and num <= (maxnum - keep): + if min_timeout: + # NB: doing this is needed to prevent (or reduce + # a lot the chance of) the following situation: + # 'keep+1' processes call make_numbered_dir() at + # the same time, they create dirs, but then the + # last process notices the first dir doesn't have + # (yet) a .lock in it and kills it. + try: + t1 = path.lstat().mtime + t2 = lockfile.lstat().mtime + if abs(t2-t1) < min_timeout: + continue # skip directories too recent + except py.error.Error: + continue # failure to get a time, better skip + lf = path.join('.lock') + try: + t1 = lf.lstat().mtime + t2 = lockfile.lstat().mtime + if not lock_timeout or abs(t2-t1) < lock_timeout: + continue # skip directories still locked + except py.error.Error: + pass # assume that it means that there is no 'lf' + try: + path.remove(rec=1) + except KeyboardInterrupt: + raise + except: # this might be py.error.Error, WindowsError ... + pass + + # make link... + try: + username = os.environ['USER'] #linux, et al + except KeyError: + try: + username = os.environ['USERNAME'] #windows + except KeyError: + username = 'current' + + src = str(udir) + dest = src[:src.rfind('-')] + '-' + username + try: + os.unlink(dest) + except OSError: + pass + try: + os.symlink(src, dest) + except (OSError, AttributeError, NotImplementedError): + pass + + return udir + + +udir = make_numbered_dir(prefix = 'ffi-') # Windows-only workaround for some configurations: see |