diff options
38 files changed, 1671 insertions, 47 deletions
diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c index f086481..092acd4 100644 --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -6352,7 +6352,7 @@ static PyObject *_cffi_from_c_wchar_t(wchar_t x) { #endif struct _cffi_externpy_s; /* forward declaration */ -static void _cffi_call_python(struct _cffi_externpy_s *, char *args); +static void cffi_call_python(struct _cffi_externpy_s *, char *args); static void *cffi_exports[] = { NULL, @@ -6385,7 +6385,7 @@ static void *cffi_exports[] = { _cffi_to_c__Bool, _prepare_pointer_call_argument, convert_array_from_object, - _cffi_call_python, + cffi_call_python, }; static struct { const char *name; int value; } all_dlopen_flags[] = { @@ -6500,7 +6500,7 @@ init_cffi_backend(void) if (v == NULL || PyModule_AddObject(m, "_C_API", v) < 0) INITERROR; - v = PyText_FromString("1.4.2"); + v = PyText_FromString("1.4.3"); if (v == NULL || PyModule_AddObject(m, "__version__", v) < 0) INITERROR; diff --git a/c/call_python.c b/c/call_python.c index b6cfa82..1a4a812 100644 --- a/c/call_python.c +++ b/c/call_python.c @@ -94,7 +94,7 @@ static PyObject *_ffi_def_extern_decorator(PyObject *outer_args, PyObject *fn) return NULL; /* force _update_cache_to_call_python() to be called the next time - the C function invokes _cffi_call_python, to update the cache */ + the C function invokes cffi_call_python, to update the cache */ old1 = externpy->reserved1; externpy->reserved1 = Py_None; /* a non-NULL value */ Py_INCREF(Py_None); @@ -143,7 +143,15 @@ static int _update_cache_to_call_python(struct _cffi_externpy_s *externpy) return 2; /* out of memory? */ } -static void _cffi_call_python(struct _cffi_externpy_s *externpy, char *args) +#if (defined(WITH_THREAD) && !defined(_MSC_VER) && \ + !defined(__amd64__) && !defined(__x86_64__) && \ + !defined(__i386__) && !defined(__i386)) +# define read_barrier() __sync_synchronize() +#else +# define read_barrier() (void)0 +#endif + +static void cffi_call_python(struct _cffi_externpy_s *externpy, char *args) { /* Invoked by the helpers generated from extern "Python" in the cdef. @@ -164,6 +172,21 @@ static void _cffi_call_python(struct _cffi_externpy_s *externpy, char *args) at least 8 bytes in size. */ int err = 0; + + /* This read barrier is needed for _embedding.h. It is paired + with the write_barrier() there. Without this barrier, we can + in theory see the following situation: the Python + initialization code already ran (in another thread), and the + '_cffi_call_python' function pointer directed execution here; + but any number of other data could still be seen as + uninitialized below. For example, 'externpy' would still + contain NULLs even though it was correctly set up, or + 'interpreter_lock' (the GIL inside CPython) would still be seen + as NULL, or 'autoInterpreterState' (used by + PyGILState_Ensure()) would be NULL or contain bogus fields. + */ + read_barrier(); + save_errno(); /* We need the infotuple here. We could always go through diff --git a/c/cffi1_module.c b/c/cffi1_module.c index 327b20e..2fe95e0 100644 --- a/c/cffi1_module.c +++ b/c/cffi1_module.c @@ -3,7 +3,7 @@ #include "realize_c_type.c" #define CFFI_VERSION_MIN 0x2601 -#define CFFI_VERSION_MAX 0x26FF +#define CFFI_VERSION_MAX 0x27FF typedef struct FFIObject_s FFIObject; typedef struct LibObject_s LibObject; @@ -212,5 +212,12 @@ static PyObject *b_init_cffi_1_0_external_module(PyObject *self, PyObject *arg) (PyObject *)lib) < 0) return NULL; +#if PY_MAJOR_VERSION >= 3 + /* add manually 'module_name' in sys.modules: it seems that + Py_InitModule() is not enough to do that */ + if (PyDict_SetItemString(modules_dict, module_name, m) < 0) + return NULL; +#endif + return m; } diff --git a/c/test_c.py b/c/test_c.py index 72193a7..3f8a317 100644 --- a/c/test_c.py +++ b/c/test_c.py @@ -12,7 +12,7 @@ from _cffi_backend import _testfunc, _get_types, _get_common_types, __version__ # ____________________________________________________________ import sys -assert __version__ == "1.4.2", ("This test_c.py file is for testing a version" +assert __version__ == "1.4.3", ("This test_c.py file is for testing a version" " of cffi that differs from the one that we" " get from 'import _cffi_backend'") if sys.version_info < (3,): diff --git a/cffi/__init__.py b/cffi/__init__.py index 644744c..637a4f1 100644 --- a/cffi/__init__.py +++ b/cffi/__init__.py @@ -4,8 +4,8 @@ __all__ = ['FFI', 'VerificationError', 'VerificationMissing', 'CDefError', from .api import FFI, CDefError, FFIError from .ffiplatform import VerificationError, VerificationMissing -__version__ = "1.4.2" -__version_info__ = (1, 4, 2) +__version__ = "1.4.3" +__version_info__ = (1, 4, 3) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/cffi/_cffi_include.h b/cffi/_cffi_include.h index 5e8f05b..6e90dd8 100644 --- a/cffi/_cffi_include.h +++ b/cffi/_cffi_include.h @@ -146,8 +146,9 @@ extern "C" { ((Py_ssize_t(*)(CTypeDescrObject *, PyObject *, char **))_cffi_exports[23]) #define _cffi_convert_array_from_object \ ((int(*)(char *, CTypeDescrObject *, PyObject *))_cffi_exports[24]) +#define _CFFI_CPIDX 25 #define _cffi_call_python \ - ((void(*)(struct _cffi_externpy_s *, char *))_cffi_exports[25]) + ((void(*)(struct _cffi_externpy_s *, char *))_cffi_exports[_CFFI_CPIDX]) #define _CFFI_NUM_EXPORTS 26 typedef struct _ctypedescr CTypeDescrObject; @@ -206,7 +207,8 @@ static PyObject **_cffi_unpack_args(PyObject *args_tuple, Py_ssize_t expected, /********** end CPython-specific section **********/ #else _CFFI_UNUSED_FN -static void (*_cffi_call_python)(struct _cffi_externpy_s *, char *); +static void (*_cffi_call_python_org)(struct _cffi_externpy_s *, char *); +# define _cffi_call_python _cffi_call_python_org #endif diff --git a/cffi/_embedding.h b/cffi/_embedding.h new file mode 100644 index 0000000..ecf773e --- /dev/null +++ b/cffi/_embedding.h @@ -0,0 +1,513 @@ + +/***** Support code for embedding *****/ + +#if defined(_MSC_VER) +# define CFFI_DLLEXPORT __declspec(dllexport) +#elif defined(__GNUC__) +# define CFFI_DLLEXPORT __attribute__((visibility("default"))) +#else +# define CFFI_DLLEXPORT /* nothing */ +#endif + + +/* There are two global variables of type _cffi_call_python_fnptr: + + * _cffi_call_python, which we declare just below, is the one called + by ``extern "Python"`` implementations. + + * _cffi_call_python_org, which on CPython is actually part of the + _cffi_exports[] array, is the function pointer copied from + _cffi_backend. + + After initialization is complete, both are equal. However, the + first one remains equal to &_cffi_start_and_call_python until the + very end of initialization, when we are (or should be) sure that + concurrent threads also see a completely initialized world, and + only then is it changed. +*/ +#undef _cffi_call_python +typedef void (*_cffi_call_python_fnptr)(struct _cffi_externpy_s *, char *); +static void _cffi_start_and_call_python(struct _cffi_externpy_s *, char *); +static _cffi_call_python_fnptr _cffi_call_python = &_cffi_start_and_call_python; + + +#ifndef _MSC_VER + /* --- Assuming a GCC not infinitely old --- */ +# define cffi_compare_and_swap(l,o,n) __sync_bool_compare_and_swap(l,o,n) +# define cffi_write_barrier() __sync_synchronize() +# if !defined(__amd64__) && !defined(__x86_64__) && \ + !defined(__i386__) && !defined(__i386) +# define cffi_read_barrier() __sync_synchronize() +# else +# define cffi_read_barrier() (void)0 +# endif +#else + /* --- Windows threads version --- */ +# include <Windows.h> +# define cffi_compare_and_swap(l,o,n) InterlockedCompareExchangePointer(l,n,o) +# define cffi_write_barrier() InterlockedCompareExchange(&_cffi_dummy,0,0) +# define cffi_read_barrier() (void)0 +static volatile LONG _cffi_dummy; +#endif + +#ifdef WITH_THREAD +# ifndef _MSC_VER +# include <pthread.h> + static pthread_mutex_t _cffi_embed_startup_lock; +# else + static CRITICAL_SECTION _cffi_embed_startup_lock; +# endif + static char _cffi_embed_startup_lock_ready = 0; +#endif + +static void _cffi_acquire_reentrant_mutex(void) +{ + static volatile void *lock = NULL; + + while (!cffi_compare_and_swap(&lock, NULL, (void *)1)) { + /* should ideally do a spin loop instruction here, but + hard to do it portably and doesn't really matter I + think: pthread_mutex_init() should be very fast, and + this is only run at start-up anyway. */ + } + +#ifdef WITH_THREAD + if (!_cffi_embed_startup_lock_ready) { +# ifndef _MSC_VER + pthread_mutexattr_t attr; + pthread_mutexattr_init(&attr); + pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE); + pthread_mutex_init(&_cffi_embed_startup_lock, &attr); +# else + InitializeCriticalSection(&_cffi_embed_startup_lock); +# endif + _cffi_embed_startup_lock_ready = 1; + } +#endif + + while (!cffi_compare_and_swap(&lock, (void *)1, NULL)) + ; + +#ifndef _MSC_VER + pthread_mutex_lock(&_cffi_embed_startup_lock); +#else + EnterCriticalSection(&_cffi_embed_startup_lock); +#endif +} + +static void _cffi_release_reentrant_mutex(void) +{ +#ifndef _MSC_VER + pthread_mutex_unlock(&_cffi_embed_startup_lock); +#else + LeaveCriticalSection(&_cffi_embed_startup_lock); +#endif +} + + +/********** CPython-specific section **********/ +#ifndef PYPY_VERSION + + +#define _cffi_call_python_org _cffi_exports[_CFFI_CPIDX] + +PyMODINIT_FUNC _CFFI_PYTHON_STARTUP_FUNC(void); /* forward */ + +static void _cffi_py_initialize(void) +{ + /* XXX use initsigs=0, which "skips initialization registration of + signal handlers, which might be useful when Python is + embedded" according to the Python docs. But review and think + if it should be a user-controllable setting. + + XXX we should also give a way to write errors to a buffer + instead of to stderr. + */ + Py_InitializeEx(0); +} + +static int _cffi_initialize_python(void) +{ + /* This initializes Python, imports _cffi_backend, and then the + present .dll/.so is set up as a CPython C extension module. + */ + int result; + PyGILState_STATE state; + PyObject *pycode=NULL, *global_dict=NULL, *x; + +#if PY_MAJOR_VERSION >= 3 + /* see comments in _cffi_carefully_make_gil() about the + Python2/Python3 difference + */ +#else + /* Acquire the GIL. We have no threadstate here. If Python is + already initialized, it is possible that there is already one + existing for this thread, but it is not made current now. + */ + PyEval_AcquireLock(); + + _cffi_py_initialize(); + + /* The Py_InitializeEx() sometimes made a threadstate for us, but + not always. Indeed Py_InitializeEx() could be called and do + nothing. So do we have a threadstate, or not? We don't know, + but we can replace it with NULL in all cases. + */ + (void)PyThreadState_Swap(NULL); + + /* Now we can release the GIL and re-acquire immediately using the + logic of PyGILState(), which handles making or installing the + correct threadstate. + */ + PyEval_ReleaseLock(); +#endif + state = PyGILState_Ensure(); + + /* Call the initxxx() function from the present module. It will + create and initialize us as a CPython extension module, instead + of letting the startup Python code do it---it might reimport + the same .dll/.so and get maybe confused on some platforms. + It might also have troubles locating the .dll/.so again for all + I know. + */ + (void)_CFFI_PYTHON_STARTUP_FUNC(); + if (PyErr_Occurred()) + goto error; + + /* Now run the Python code provided to ffi.embedding_init_code(). + */ + pycode = Py_CompileString(_CFFI_PYTHON_STARTUP_CODE, + "<init code for '" _CFFI_MODULE_NAME "'>", + Py_file_input); + if (pycode == NULL) + goto error; + global_dict = PyDict_New(); + if (global_dict == NULL) + goto error; + if (PyDict_SetItemString(global_dict, "__builtins__", + PyThreadState_GET()->interp->builtins) < 0) + goto error; + x = PyEval_EvalCode( +#if PY_MAJOR_VERSION < 3 + (PyCodeObject *) +#endif + pycode, global_dict, global_dict); + if (x == NULL) + goto error; + Py_DECREF(x); + + /* Done! Now if we've been called from + _cffi_start_and_call_python() in an ``extern "Python"``, we can + only hope that the Python code did correctly set up the + corresponding @ffi.def_extern() function. Otherwise, the + general logic of ``extern "Python"`` functions (inside the + _cffi_backend module) will find that the reference is still + missing and print an error. + */ + result = 0; + done: + Py_XDECREF(pycode); + Py_XDECREF(global_dict); + PyGILState_Release(state); + return result; + + error:; + { + /* Print as much information as potentially useful. + Debugging load-time failures with embedding is not fun + */ + PyObject *exception, *v, *tb, *f, *modules, *mod; + PyErr_Fetch(&exception, &v, &tb); + if (exception != NULL) { + PyErr_NormalizeException(&exception, &v, &tb); + PyErr_Display(exception, v, tb); + } + Py_XDECREF(exception); + Py_XDECREF(v); + Py_XDECREF(tb); + + f = PySys_GetObject((char *)"stderr"); + if (f != NULL && f != Py_None) { + PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME + "\ncompiled with cffi version: 1.4.3" + "\n_cffi_backend module: ", f); + modules = PyImport_GetModuleDict(); + mod = PyDict_GetItemString(modules, "_cffi_backend"); + if (mod == NULL) { + PyFile_WriteString("not loaded", f); + } + else { + v = PyObject_GetAttrString(mod, "__file__"); + PyFile_WriteObject(v, f, 0); + Py_XDECREF(v); + } + PyFile_WriteString("\nsys.path: ", f); + PyFile_WriteObject(PySys_GetObject((char *)"path"), f, 0); + PyFile_WriteString("\n\n", f); + } + } + result = -1; + goto done; +} + +PyAPI_DATA(char *) _PyParser_TokenNames[]; /* from CPython */ + +static int _cffi_carefully_make_gil(void) +{ + /* This does the basic initialization of Python. It can be called + completely concurrently from unrelated threads. It assumes + that we don't hold the GIL before (if it exists), and we don't + hold it afterwards. + + What it really does is completely different in Python 2 and + Python 3. + + Python 2 + ======== + + Initialize the GIL, without initializing the rest of Python, + by calling PyEval_InitThreads(). + + PyEval_InitThreads() must not be called concurrently at all. + So we use a global variable as a simple spin lock. This global + variable must be from 'libpythonX.Y.so', not from this + cffi-based extension module, because it must be shared from + different cffi-based extension modules. We choose + _PyParser_TokenNames[0] as a completely arbitrary pointer value + that is never written to. The default is to point to the + string "ENDMARKER". We change it temporarily to point to the + next character in that string. (Yes, I know it's REALLY + obscure.) + + Python 3 + ======== + + In Python 3, PyEval_InitThreads() cannot be called before + Py_InitializeEx() any more. So this function calls + Py_InitializeEx() first. It uses the same obscure logic to + make sure we never call it concurrently. + + Arguably, this is less good on the spinlock, because + Py_InitializeEx() takes much longer to run than + PyEval_InitThreads(). But I didn't find a way around it. + */ + +#ifdef WITH_THREAD + char *volatile *lock = (char *volatile *)_PyParser_TokenNames; + char *old_value; + + while (1) { /* spin loop */ + old_value = *lock; + if (old_value[0] == 'E') { + assert(old_value[1] == 'N'); + if (cffi_compare_and_swap(lock, old_value, old_value + 1)) + break; + } + else { + assert(old_value[0] == 'N'); + /* should ideally do a spin loop instruction here, but + hard to do it portably and doesn't really matter I + think: PyEval_InitThreads() should be very fast, and + this is only run at start-up anyway. */ + } + } +#endif + +#if PY_MAJOR_VERSION >= 3 + /* Python 3: call Py_InitializeEx() */ + { + PyGILState_STATE state = PyGILState_UNLOCKED; + if (!Py_IsInitialized()) + _cffi_py_initialize(); + else + state = PyGILState_Ensure(); + + PyEval_InitThreads(); + PyGILState_Release(state); + } +#else + /* Python 2: call PyEval_InitThreads() */ +# ifdef WITH_THREAD + if (!PyEval_ThreadsInitialized()) { + PyEval_InitThreads(); /* makes the GIL */ + PyEval_ReleaseLock(); /* then release it */ + } + /* else: there is already a GIL, but we still needed to do the + spinlock dance to make sure that we see it as fully ready */ +# endif +#endif + +#ifdef WITH_THREAD + /* release the lock */ + while (!cffi_compare_and_swap(lock, old_value + 1, old_value)) + ; +#endif + + return 0; +} + +/********** end CPython-specific section **********/ + + +#else + + +/********** PyPy-specific section **********/ + +PyMODINIT_FUNC _CFFI_PYTHON_STARTUP_FUNC(const void *[]); /* forward */ + +static struct _cffi_pypy_init_s { + const char *name; + void (*func)(const void *[]); + const char *code; +} _cffi_pypy_init = { + _CFFI_MODULE_NAME, + _CFFI_PYTHON_STARTUP_FUNC, + _CFFI_PYTHON_STARTUP_CODE, +}; + +extern int pypy_carefully_make_gil(const char *); +extern int pypy_init_embedded_cffi_module(int, struct _cffi_pypy_init_s *); + +static int _cffi_carefully_make_gil(void) +{ + return pypy_carefully_make_gil(_CFFI_MODULE_NAME); +} + +static int _cffi_initialize_python(void) +{ + return pypy_init_embedded_cffi_module(0xB011, &_cffi_pypy_init); +} + +/********** end PyPy-specific section **********/ + + +#endif + + +#ifdef __GNUC__ +__attribute__((noinline)) +#endif +static _cffi_call_python_fnptr _cffi_start_python(void) +{ + /* Delicate logic to initialize Python. This function can be + called multiple times concurrently, e.g. when the process calls + its first ``extern "Python"`` functions in multiple threads at + once. It can also be called recursively, in which case we must + ignore it. We also have to consider what occurs if several + different cffi-based extensions reach this code in parallel + threads---it is a different copy of the code, then, and we + can't have any shared global variable unless it comes from + 'libpythonX.Y.so'. + + Idea: + + * _cffi_carefully_make_gil(): "carefully" call + PyEval_InitThreads() (possibly with Py_InitializeEx() first). + + * then we use a (local) custom lock to make sure that a call to this + cffi-based extension will wait if another call to the *same* + extension is running the initialization in another thread. + It is reentrant, so that a recursive call will not block, but + only one from a different thread. + + * then we grab the GIL and (Python 2) we call Py_InitializeEx(). + At this point, concurrent calls to Py_InitializeEx() are not + possible: we have the GIL. + + * do the rest of the specific initialization, which may + temporarily release the GIL but not the custom lock. + Only release the custom lock when we are done. + */ + static char called = 0; + + if (_cffi_carefully_make_gil() != 0) + return NULL; + + _cffi_acquire_reentrant_mutex(); + + /* Here the GIL exists, but we don't have it. We're only protected + from concurrency by the reentrant mutex. */ + + /* This file only initializes the embedded module once, the first + time this is called, even if there are subinterpreters. */ + if (!called) { + called = 1; /* invoke _cffi_initialize_python() only once, + but don't set '_cffi_call_python' right now, + otherwise concurrent threads won't call + this function at all (we need them to wait) */ + if (_cffi_initialize_python() == 0) { + /* now initialization is finished. Switch to the fast-path. */ + + /* We would like nobody to see the new value of + '_cffi_call_python' without also seeing the rest of the + data initialized. However, this is not possible. But + the new value of '_cffi_call_python' is the function + 'cffi_call_python()' from _cffi_backend. So: */ + cffi_write_barrier(); + /* ^^^ we put a write barrier here, and a corresponding + read barrier at the start of cffi_call_python(). This + ensures that after that read barrier, we see everything + done here before the write barrier. + */ + + assert(_cffi_call_python_org != NULL); + _cffi_call_python = (_cffi_call_python_fnptr)_cffi_call_python_org; + } + else { + /* initialization failed. Reset this to NULL, even if it was + already set to some other value. Future calls to + _cffi_start_python() are still forced to occur, and will + always return NULL from now on. */ + _cffi_call_python_org = NULL; + } + } + + _cffi_release_reentrant_mutex(); + + return (_cffi_call_python_fnptr)_cffi_call_python_org; +} + +static +void _cffi_start_and_call_python(struct _cffi_externpy_s *externpy, char *args) +{ + _cffi_call_python_fnptr fnptr; + int current_err = errno; +#ifdef _MSC_VER + int current_lasterr = GetLastError(); +#endif + fnptr = _cffi_start_python(); + if (fnptr == NULL) { + fprintf(stderr, "function %s() called, but initialization code " + "failed. Returning 0.\n", externpy->name); + memset(args, 0, externpy->size_of_result); + } +#ifdef _MSC_VER + SetLastError(current_lasterr); +#endif + errno = current_err; + + if (fnptr != NULL) + fnptr(externpy, args); +} + + +/* The cffi_start_python() function makes sure Python is initialized + and our cffi module is set up. It can be called manually from the + user C code. The same effect is obtained automatically from any + dll-exported ``extern "Python"`` function. This function returns + -1 if initialization failed, 0 if all is OK. */ +_CFFI_UNUSED_FN +static int cffi_start_python(void) +{ + if (_cffi_call_python == &_cffi_start_and_call_python) { + if (_cffi_start_python() == NULL) + return -1; + } + cffi_read_barrier(); + return 0; +} + +#undef cffi_compare_and_swap +#undef cffi_write_barrier +#undef cffi_read_barrier diff --git a/cffi/api.py b/cffi/api.py index 8af9795..9f218a7 100644 --- a/cffi/api.py +++ b/cffi/api.py @@ -74,6 +74,7 @@ class FFI(object): self._windows_unicode = None self._init_once_cache = {} self._cdef_version = None + self._embedding = None if hasattr(backend, 'set_ffi'): backend.set_ffi(self) for name in backend.__dict__: @@ -101,13 +102,21 @@ class FFI(object): If 'packed' is specified as True, all structs declared inside this cdef are packed, i.e. laid out without any field alignment at all. """ + self._cdef(csource, override=override, packed=packed) + + def embedding_api(self, csource, packed=False): + self._cdef(csource, packed=packed, dllexport=True) + if self._embedding is None: + self._embedding = '' + + def _cdef(self, csource, override=False, **options): if not isinstance(csource, str): # unicode, on Python 2 if not isinstance(csource, basestring): raise TypeError("cdef() argument must be a string") csource = csource.encode('ascii') with self._lock: self._cdef_version = object() - self._parser.parse(csource, override=override, packed=packed) + self._parser.parse(csource, override=override, **options) self._cdefsources.append(csource) if override: for cache in self._function_caches: @@ -533,6 +542,25 @@ class FFI(object): ('_UNICODE', '1')] kwds['define_macros'] = defmacros + def _apply_embedding_fix(self, kwds): + # must include an argument like "-lpython2.7" for the compiler + if '__pypy__' in sys.builtin_module_names: + pythonlib = "pypy-c" + else: + if sys.platform == "win32": + template = "python%d%d" + if sys.flags.debug: + template = template + '_d' + else: + template = "python%d.%d" + pythonlib = (template % + (sys.hexversion >> 24, (sys.hexversion >> 16) & 0xff)) + if hasattr(sys, 'abiflags'): + pythonlib += sys.abiflags + libraries = kwds.get('libraries', []) + if pythonlib not in libraries: + kwds['libraries'] = libraries + [pythonlib] + def set_source(self, module_name, source, source_extension='.c', **kwds): if hasattr(self, '_assigned_source'): raise ValueError("set_source() cannot be called several times " @@ -592,13 +620,25 @@ class FFI(object): recompile(self, module_name, source, c_file=filename, call_c_compiler=False, **kwds) - def compile(self, tmpdir='.', verbose=0): + def compile(self, tmpdir='.', verbose=0, ext=None): + """Values recognized for the ext parameter: + + - 'capi': use distutils' default to build CPython C API extensions + - 'system': use the system's default for dynamic libraries (.so/.dll) + - '.FOO': exactly .FOO + + The default is 'capi' when building a non-embedded C API extension, + and 'system' when building an embedded library. + """ from .recompiler import recompile # if not hasattr(self, '_assigned_source'): raise ValueError("set_source() must be called before compile()") + if ext not in (None, 'capi', 'system') and '.' not in ext: + raise ValueError("bad value for 'ext' argument: %r" % (ext,)) module_name, source, source_extension, kwds = self._assigned_source return recompile(self, module_name, source, tmpdir=tmpdir, + target_extention=ext, source_extension=source_extension, compiler_verbose=verbose, **kwds) @@ -626,6 +666,32 @@ class FFI(object): self._init_once_cache[tag] = (True, result) return result + def embedding_init_code(self, pysource): + if self._embedding: + raise ValueError("embedding_init_code() can only be called once") + # fix 'pysource' before it gets dumped into the C file: + # - remove empty lines at the beginning, so it starts at "line 1" + # - dedent, if all non-empty lines are indented + # - check for SyntaxErrors + import re + match = re.match(r'\s*\n', pysource) + if match: + pysource = pysource[match.end():] + lines = pysource.splitlines() or [''] + prefix = re.match(r'\s*', lines[0]).group() + for i in range(1, len(lines)): + line = lines[i] + if line.rstrip(): + while not line.startswith(prefix): + prefix = prefix[:-1] + i = len(prefix) + lines = [line[i:]+'\n' for line in lines] + pysource = ''.join(lines) + # + compile(pysource, "cffi_init", "exec") + # + self._embedding = pysource + def _load_backend_lib(backend, name, flags): if name is None: diff --git a/cffi/cparser.py b/cffi/cparser.py index e5168af..baa0e98 100644 --- a/cffi/cparser.py +++ b/cffi/cparser.py @@ -220,8 +220,7 @@ class Parser(object): self._included_declarations = set() self._anonymous_counter = 0 self._structnode2type = weakref.WeakKeyDictionary() - self._override = False - self._packed = False + self._options = None self._int_constants = {} self._recomplete = [] self._uses_new_feature = None @@ -281,16 +280,15 @@ class Parser(object): msg = 'parse error\n%s' % (msg,) raise api.CDefError(msg) - def parse(self, csource, override=False, packed=False): - prev_override = self._override - prev_packed = self._packed + def parse(self, csource, override=False, packed=False, dllexport=False): + prev_options = self._options try: - self._override = override - self._packed = packed + self._options = {'override': override, + 'packed': packed, + 'dllexport': dllexport} self._internal_parse(csource) finally: - self._override = prev_override - self._packed = prev_packed + self._options = prev_options def _internal_parse(self, csource): ast, macros, csource = self._parse(csource) @@ -376,10 +374,13 @@ class Parser(object): def _declare_function(self, tp, quals, decl): tp = self._get_type_pointer(tp, quals) - if self._inside_extern_python: - self._declare('extern_python ' + decl.name, tp) + if self._options['dllexport']: + tag = 'dllexport_python ' + elif self._inside_extern_python: + tag = 'extern_python ' else: - self._declare('function ' + decl.name, tp) + tag = 'function ' + self._declare(tag + decl.name, tp) def _parse_decl(self, decl): node = decl.type @@ -449,7 +450,7 @@ class Parser(object): prevobj, prevquals = self._declarations[name] if prevobj is obj and prevquals == quals: return - if not self._override: + if not self._options['override']: raise api.FFIError( "multiple declarations of %s (for interactive usage, " "try cdef(xx, override=True))" % (name,)) @@ -728,7 +729,7 @@ class Parser(object): if isinstance(tp, model.StructType) and tp.partial: raise NotImplementedError("%s: using both bitfields and '...;'" % (tp,)) - tp.packed = self._packed + tp.packed = self._options['packed'] if tp.completed: # must be re-completed: it is not opaque any more tp.completed = 0 self._recomplete.append(tp) diff --git a/cffi/ffiplatform.py b/cffi/ffiplatform.py index da2f5a5..6ddef60 100644 --- a/cffi/ffiplatform.py +++ b/cffi/ffiplatform.py @@ -21,12 +21,14 @@ def get_extension(srcfilename, modname, sources=(), **kwds): allsources.append(os.path.normpath(src)) return Extension(name=modname, sources=allsources, **kwds) -def compile(tmpdir, ext, compiler_verbose=0): +def compile(tmpdir, ext, compiler_verbose=0, target_extention=None, + embedding=False): """Compile a C extension module using distutils.""" saved_environ = os.environ.copy() try: - outputfilename = _build(tmpdir, ext, compiler_verbose) + outputfilename = _build(tmpdir, ext, compiler_verbose, + target_extention, embedding) outputfilename = os.path.abspath(outputfilename) finally: # workaround for a distutils bugs where some env vars can @@ -36,7 +38,32 @@ def compile(tmpdir, ext, compiler_verbose=0): os.environ[key] = value return outputfilename -def _build(tmpdir, ext, compiler_verbose=0): +def _save_val(name): + import distutils.sysconfig + config_vars = distutils.sysconfig.get_config_vars() + return config_vars.get(name, Ellipsis) + +def _restore_val(name, value): + import distutils.sysconfig + config_vars = distutils.sysconfig.get_config_vars() + config_vars[name] = value + if value is Ellipsis: + del config_vars[name] + +def _win32_hack_for_embedding(): + from distutils.msvc9compiler import MSVCCompiler + if not hasattr(MSVCCompiler, '_remove_visual_c_ref_CFFI_BAK'): + MSVCCompiler._remove_visual_c_ref_CFFI_BAK = \ + MSVCCompiler._remove_visual_c_ref + MSVCCompiler._remove_visual_c_ref = lambda self,manifest_file: manifest_file + +def _win32_unhack_for_embedding(): + from distutils.msvc9compiler import MSVCCompiler + MSVCCompiler._remove_visual_c_ref = \ + MSVCCompiler._remove_visual_c_ref_CFFI_BAK + +def _build(tmpdir, ext, compiler_verbose=0, target_extention=None, + embedding=False): # XXX compact but horrible :-( from distutils.core import Distribution import distutils.errors, distutils.log @@ -49,18 +76,41 @@ def _build(tmpdir, ext, compiler_verbose=0): options['build_temp'] = ('ffiplatform', tmpdir) # try: + if sys.platform == 'win32' and embedding: + _win32_hack_for_embedding() old_level = distutils.log.set_threshold(0) or 0 + old_SO = _save_val('SO') + old_EXT_SUFFIX = _save_val('EXT_SUFFIX') try: + if target_extention is None: + if embedding: + target_extention = 'system' + else: + target_extention = 'capi' + if target_extention == 'capi': + pass # keep the values already in 'SO' and 'EXT_SUFFIX' + else: + if target_extention == 'system': + if sys.platform == 'win32': + target_extention = '.dll' + else: + target_extention = '.so' + _restore_val('SO', target_extention) + _restore_val('EXT_SUFFIX', target_extention) distutils.log.set_verbosity(compiler_verbose) dist.run_command('build_ext') + cmd_obj = dist.get_command_obj('build_ext') + [soname] = cmd_obj.get_outputs() finally: distutils.log.set_threshold(old_level) + _restore_val('SO', old_SO) + _restore_val('EXT_SUFFIX', old_EXT_SUFFIX) + if sys.platform == 'win32' and embedding: + _win32_unhack_for_embedding() except (distutils.errors.CompileError, distutils.errors.LinkError) as e: raise VerificationError('%s: %s' % (e.__class__.__name__, e)) # - cmd_obj = dist.get_command_obj('build_ext') - [soname] = cmd_obj.get_outputs() return soname try: diff --git a/cffi/recompiler.py b/cffi/recompiler.py index e9e8f5c..742408e 100644 --- a/cffi/recompiler.py +++ b/cffi/recompiler.py @@ -3,6 +3,7 @@ from . import ffiplatform, model from .cffi_opcode import * VERSION = "0x2601" +VERSION_EMBEDDED = "0x2701" class GlobalExpr: @@ -281,6 +282,29 @@ class Recompiler: lines[i:i+1] = self._rel_readlines('parse_c_type.h') prnt(''.join(lines)) # + # if we have ffi._embedding != None, we give it here as a macro + # and include an extra file + base_module_name = self.module_name.split('.')[-1] + if self.ffi._embedding is not None: + prnt('#define _CFFI_MODULE_NAME "%s"' % (self.module_name,)) + prnt('#define _CFFI_PYTHON_STARTUP_CODE %s' % + (self._string_literal(self.ffi._embedding),)) + prnt('#ifdef PYPY_VERSION') + prnt('# define _CFFI_PYTHON_STARTUP_FUNC _cffi_pypyinit_%s' % ( + base_module_name,)) + prnt('#elif PY_MAJOR_VERSION >= 3') + prnt('# define _CFFI_PYTHON_STARTUP_FUNC PyInit_%s' % ( + base_module_name,)) + prnt('#else') + prnt('# define _CFFI_PYTHON_STARTUP_FUNC init%s' % ( + base_module_name,)) + prnt('#endif') + lines = self._rel_readlines('_embedding.h') + prnt(''.join(lines)) + version = VERSION_EMBEDDED + else: + version = VERSION + # # then paste the C source given by the user, verbatim. prnt('/************************************************************/') prnt() @@ -365,17 +389,16 @@ class Recompiler: prnt() # # the init function - base_module_name = self.module_name.split('.')[-1] prnt('#ifdef PYPY_VERSION') prnt('PyMODINIT_FUNC') prnt('_cffi_pypyinit_%s(const void *p[])' % (base_module_name,)) prnt('{') if self._num_externpy: prnt(' if (((intptr_t)p[0]) >= 0x0A03) {') - prnt(' _cffi_call_python = ' + prnt(' _cffi_call_python_org = ' '(void(*)(struct _cffi_externpy_s *, char *))p[1];') prnt(' }') - prnt(' p[0] = (const void *)%s;' % VERSION) + prnt(' p[0] = (const void *)%s;' % version) prnt(' p[1] = &_cffi_type_context;') prnt('}') # on Windows, distutils insists on putting init_cffi_xyz in @@ -394,14 +417,14 @@ class Recompiler: prnt('PyInit_%s(void)' % (base_module_name,)) prnt('{') prnt(' return _cffi_init("%s", %s, &_cffi_type_context);' % ( - self.module_name, VERSION)) + self.module_name, version)) prnt('}') prnt('#else') prnt('PyMODINIT_FUNC') prnt('init%s(void)' % (base_module_name,)) prnt('{') prnt(' _cffi_init("%s", %s, &_cffi_type_context);' % ( - self.module_name, VERSION)) + self.module_name, version)) prnt('}') prnt('#endif') @@ -1123,7 +1146,10 @@ class Recompiler: assert isinstance(tp, model.FunctionPtrType) self._do_collect_type(tp) - def _generate_cpy_extern_python_decl(self, tp, name): + def _generate_cpy_dllexport_python_collecttype(self, tp, name): + self._generate_cpy_extern_python_collecttype(tp, name) + + def _generate_cpy_extern_python_decl(self, tp, name, dllexport=False): prnt = self._prnt if isinstance(tp.result, model.VoidType): size_of_result = '0' @@ -1156,7 +1182,11 @@ class Recompiler: size_of_a = 'sizeof(%s) > %d ? sizeof(%s) : %d' % ( tp.result.get_c_name(''), size_of_a, tp.result.get_c_name(''), size_of_a) - prnt('static %s' % tp.result.get_c_name(name_and_arguments)) + if dllexport: + tag = 'CFFI_DLLEXPORT' + else: + tag = 'static' + prnt('%s %s' % (tag, tp.result.get_c_name(name_and_arguments))) prnt('{') prnt(' char a[%s];' % size_of_a) prnt(' char *p = a;') @@ -1174,6 +1204,9 @@ class Recompiler: prnt() self._num_externpy += 1 + def _generate_cpy_dllexport_python_decl(self, tp, name): + self._generate_cpy_extern_python_decl(tp, name, dllexport=True) + def _generate_cpy_extern_python_ctx(self, tp, name): if self.target_is_python: raise ffiplatform.VerificationError( @@ -1185,6 +1218,21 @@ class Recompiler: self._lsts["global"].append( GlobalExpr(name, '&_cffi_externpy__%s' % name, type_op, name)) + def _generate_cpy_dllexport_python_ctx(self, tp, name): + self._generate_cpy_extern_python_ctx(tp, name) + + def _string_literal(self, s): + def _char_repr(c): + # escape with a '\' the characters '\', '"' or (for trigraphs) '?' + if c in '\\"?': return '\\' + c + if ' ' <= c < '\x7F': return c + if c == '\n': return '\\n' + return '\\%03o' % ord(c) + lines = [] + for line in s.splitlines(True): + lines.append('"%s"' % ''.join([_char_repr(c) for c in line])) + return ' \\\n'.join(lines) + # ---------- # emitting the opcodes for individual types @@ -1311,12 +1359,15 @@ def _modname_to_file(outputdir, modname, extension): def recompile(ffi, module_name, preamble, tmpdir='.', call_c_compiler=True, c_file=None, source_extension='.c', extradir=None, - compiler_verbose=1, **kwds): + compiler_verbose=1, target_extention=None, **kwds): if not isinstance(module_name, str): module_name = module_name.encode('ascii') if ffi._windows_unicode: ffi._apply_windows_unicode(kwds) if preamble is not None: + embedding = (ffi._embedding is not None) + if embedding: + ffi._apply_embedding_fix(kwds) if c_file is None: c_file, parts = _modname_to_file(tmpdir, module_name, source_extension) @@ -1331,7 +1382,9 @@ def recompile(ffi, module_name, preamble, tmpdir='.', call_c_compiler=True, cwd = os.getcwd() try: os.chdir(tmpdir) - outputfilename = ffiplatform.compile('.', ext, compiler_verbose) + outputfilename = ffiplatform.compile('.', ext, compiler_verbose, + target_extention, + embedding=embedding) finally: os.chdir(cwd) return outputfilename diff --git a/demo/embedding.py b/demo/embedding.py new file mode 100644 index 0000000..40c419f --- /dev/null +++ b/demo/embedding.py @@ -0,0 +1,21 @@ +import cffi + +ffi = cffi.FFI() + +ffi.embedding_api(""" + int add(int, int); +""") + +ffi.embedding_init_code(""" + from _embedding_cffi import ffi + print("preparing") # printed once + + @ffi.def_extern() + def add(x, y): + print("adding %d and %d" % (x, y)) + return x + y +""") + +ffi.set_source("_embedding_cffi", "") + +ffi.compile(verbose=True) diff --git a/demo/embedding_test.c b/demo/embedding_test.c new file mode 100644 index 0000000..87d313b --- /dev/null +++ b/demo/embedding_test.c @@ -0,0 +1,19 @@ +/* Link this program with libembedding_test.so. + E.g. with gcc: + + gcc -o embedding_test embedding_test.c _embedding_cffi*.so +*/ + +#include <stdio.h> + +extern int add(int x, int y); + + +int main(void) +{ + int res = add(40, 2); + printf("result: %d\n", res); + res = add(100, -5); + printf("result: %d\n", res); + return 0; +} diff --git a/doc/source/conf.py b/doc/source/conf.py index 95b58ba..22fa388 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -47,7 +47,7 @@ copyright = u'2012-2015, Armin Rigo, Maciej Fijalkowski' # The short X.Y version. version = '1.4' # The full version, including alpha/beta/rc tags. -release = '1.4.2' +release = '1.4.3' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/doc/source/installation.rst b/doc/source/installation.rst index d3c4bc8..1add061 100644 --- a/doc/source/installation.rst +++ b/doc/source/installation.rst @@ -51,11 +51,11 @@ Requirements: Download and Installation: -* http://pypi.python.org/packages/source/c/cffi/cffi-1.4.2.tar.gz +* http://pypi.python.org/packages/source/c/cffi/cffi-1.4.3.tar.gz - - MD5: 81357fe5042d00650b85b728cc181df2 + - MD5: ... - - SHA: 76cff6f1ff5bfb2b9c6c8e2cfa8bf90b5c944394 + - SHA: ... * Or grab the most current version from the `Bitbucket page`_: ``hg clone https://bitbucket.org/cffi/cffi`` @@ -144,9 +144,10 @@ Contact `Mailing list <https://groups.google.com/forum/#!forum/python-cffi>`_ """, - version='1.4.2', + version='1.4.3', packages=['cffi'] if cpython else [], - package_data={'cffi': ['_cffi_include.h', 'parse_c_type.h']} + package_data={'cffi': ['_cffi_include.h', 'parse_c_type.h', + '_embedding.h']} if cpython else {}, zip_safe=False, diff --git a/testing/cffi0/test_version.py b/testing/cffi0/test_version.py index eb378f1..7485db1 100644 --- a/testing/cffi0/test_version.py +++ b/testing/cffi0/test_version.py @@ -53,3 +53,10 @@ def test_c_version(): content = open(p).read() #v = BACKEND_VERSIONS.get(v, v) assert (('assert __version__ == "%s"' % v) in content) + +def test_embedding_h(): + parent = os.path.dirname(os.path.dirname(cffi.__file__)) + v = cffi.__version__ + p = os.path.join(parent, 'cffi', '_embedding.h') + content = open(p).read() + assert ('cffi version: %s"' % (v,)) in content diff --git a/testing/embedding/__init__.py b/testing/embedding/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/testing/embedding/__init__.py diff --git a/testing/embedding/add1-test.c b/testing/embedding/add1-test.c new file mode 100644 index 0000000..1328d1a --- /dev/null +++ b/testing/embedding/add1-test.c @@ -0,0 +1,13 @@ +#include <stdio.h> + +extern int add1(int, int); + + +int main(void) +{ + int x, y; + x = add1(40, 2); + y = add1(100, -5); + printf("got: %d %d\n", x, y); + return 0; +} diff --git a/testing/embedding/add1.py b/testing/embedding/add1.py new file mode 100644 index 0000000..98f3c07 --- /dev/null +++ b/testing/embedding/add1.py @@ -0,0 +1,33 @@ +import cffi + +ffi = cffi.FFI() + +ffi.embedding_api(""" + int add1(int, int); +""") + +ffi.embedding_init_code(r""" + import sys, time + sys.stdout.write("preparing") + for i in range(3): + sys.stdout.flush() + time.sleep(0.02) + sys.stdout.write(".") + sys.stdout.write("\n") + + from _add1_cffi import ffi + + int(ord("A")) # check that built-ins are there + + @ffi.def_extern() + def add1(x, y): + sys.stdout.write("adding %d and %d\n" % (x, y)) + sys.stdout.flush() + return x + y +""") + +ffi.set_source("_add1_cffi", """ +""") + +fn = ffi.compile(verbose=True) +print('FILENAME: %s' % (fn,)) diff --git a/testing/embedding/add2-test.c b/testing/embedding/add2-test.c new file mode 100644 index 0000000..9620843 --- /dev/null +++ b/testing/embedding/add2-test.c @@ -0,0 +1,14 @@ +#include <stdio.h> + +extern int add1(int, int); +extern int add2(int, int, int); + + +int main(void) +{ + int x, y; + x = add1(40, 2); + y = add2(100, -5, -20); + printf("got: %d %d\n", x, y); + return 0; +} diff --git a/testing/embedding/add2.py b/testing/embedding/add2.py new file mode 100644 index 0000000..311a464 --- /dev/null +++ b/testing/embedding/add2.py @@ -0,0 +1,29 @@ +import cffi + +ffi = cffi.FFI() + +ffi.embedding_api(""" + int add2(int, int, int); +""") + +ffi.embedding_init_code(r""" + import sys + sys.stdout.write("prepADD2\n") + + assert '_add2_cffi' in sys.modules + m = sys.modules['_add2_cffi'] + import _add2_cffi + ffi = _add2_cffi.ffi + + @ffi.def_extern() + def add2(x, y, z): + sys.stdout.write("adding %d and %d and %d\n" % (x, y, z)) + sys.stdout.flush() + return x + y + z +""") + +ffi.set_source("_add2_cffi", """ +""") + +fn = ffi.compile(verbose=True) +print('FILENAME: %s' % (fn,)) diff --git a/testing/embedding/add3.py b/testing/embedding/add3.py new file mode 100644 index 0000000..1361912 --- /dev/null +++ b/testing/embedding/add3.py @@ -0,0 +1,24 @@ +import cffi + +ffi = cffi.FFI() + +ffi.embedding_api(""" + int add3(int, int, int, int); +""") + +ffi.embedding_init_code(r""" + from _add3_cffi import ffi + import sys + + @ffi.def_extern() + def add3(x, y, z, t): + sys.stdout.write("adding %d, %d, %d, %d\n" % (x, y, z, t)) + sys.stdout.flush() + return x + y + z + t +""") + +ffi.set_source("_add3_cffi", """ +""") + +fn = ffi.compile(verbose=True) +print('FILENAME: %s' % (fn,)) diff --git a/testing/embedding/add_recursive-test.c b/testing/embedding/add_recursive-test.c new file mode 100644 index 0000000..cd29b79 --- /dev/null +++ b/testing/embedding/add_recursive-test.c @@ -0,0 +1,27 @@ +#include <stdio.h> + +#ifdef _MSC_VER +# define DLLIMPORT __declspec(dllimport) +#else +# define DLLIMPORT extern +#endif + +DLLIMPORT int add_rec(int, int); +DLLIMPORT int (*my_callback)(int); + +static int some_callback(int x) +{ + printf("some_callback(%d)\n", x); + fflush(stdout); + return add_rec(x, 9); +} + +int main(void) +{ + int x, y; + my_callback = some_callback; + x = add_rec(40, 2); + y = add_rec(100, -5); + printf("got: %d %d\n", x, y); + return 0; +} diff --git a/testing/embedding/add_recursive.py b/testing/embedding/add_recursive.py new file mode 100644 index 0000000..9fa463d --- /dev/null +++ b/testing/embedding/add_recursive.py @@ -0,0 +1,33 @@ +import cffi + +ffi = cffi.FFI() + +ffi.embedding_api(""" + int (*my_callback)(int); + int add_rec(int, int); +""") + +ffi.embedding_init_code(r""" + from _add_recursive_cffi import ffi, lib + import sys + print("preparing REC") + sys.stdout.flush() + + @ffi.def_extern() + def add_rec(x, y): + print("adding %d and %d" % (x, y)) + sys.stdout.flush() + return x + y + + x = lib.my_callback(400) + print('<<< %d >>>' % (x,)) +""") + +ffi.set_source("_add_recursive_cffi", """ +/* use CFFI_DLLEXPORT: on windows, it expands to __declspec(dllexport), + which is needed to export a variable from a dll */ +CFFI_DLLEXPORT int (*my_callback)(int); +""") + +fn = ffi.compile(verbose=True) +print('FILENAME: %s' % (fn,)) diff --git a/testing/embedding/perf-test.c b/testing/embedding/perf-test.c new file mode 100644 index 0000000..2931186 --- /dev/null +++ b/testing/embedding/perf-test.c @@ -0,0 +1,86 @@ +#include <stdio.h> +#include <assert.h> +#include <sys/time.h> +#ifdef PTEST_USE_THREAD +# include <pthread.h> +# include <semaphore.h> +static sem_t done; +#endif + + +extern int add1(int, int); + + +static double time_delta(struct timeval *stop, struct timeval *start) +{ + return (stop->tv_sec - start->tv_sec) + + 1e-6 * (stop->tv_usec - start->tv_usec); +} + +static double measure(void) +{ + long long i, iterations; + int result; + struct timeval start, stop; + double elapsed; + + add1(0, 0); /* prepare off-line */ + + i = 0; + iterations = 1000; + result = gettimeofday(&start, NULL); + assert(result == 0); + + while (1) { + for (; i < iterations; i++) { + add1(((int)i) & 0xaaaaaa, ((int)i) & 0x555555); + } + result = gettimeofday(&stop, NULL); + assert(result == 0); + + elapsed = time_delta(&stop, &start); + assert(elapsed >= 0.0); + if (elapsed > 2.5) + break; + iterations = iterations * 3 / 2; + } + + return elapsed / (double)iterations; +} + +static void *start_routine(void *arg) +{ + double t = measure(); + printf("time per call: %.3g\n", t); + +#ifdef PTEST_USE_THREAD + int status = sem_post(&done); + assert(status == 0); +#endif + + return arg; +} + + +int main(void) +{ +#ifndef PTEST_USE_THREAD + start_routine(0); +#else + pthread_t th; + int i, status = sem_init(&done, 0, 0); + assert(status == 0); + + add1(0, 0); /* this is the main thread */ + + for (i = 0; i < PTEST_USE_THREAD; i++) { + status = pthread_create(&th, NULL, start_routine, NULL); + assert(status == 0); + } + for (i = 0; i < PTEST_USE_THREAD; i++) { + status = sem_wait(&done); + assert(status == 0); + } +#endif + return 0; +} diff --git a/testing/embedding/perf.py b/testing/embedding/perf.py new file mode 100644 index 0000000..a8d20f4 --- /dev/null +++ b/testing/embedding/perf.py @@ -0,0 +1,21 @@ +import cffi + +ffi = cffi.FFI() + +ffi.embedding_api(""" + int add1(int, int); +""") + +ffi.embedding_init_code(r""" + from _perf_cffi import ffi + + @ffi.def_extern() + def add1(x, y): + return x + y +""") + +ffi.set_source("_perf_cffi", """ +""") + +fn = ffi.compile(verbose=True) +print('FILENAME: %s' % (fn,)) diff --git a/testing/embedding/test_basic.py b/testing/embedding/test_basic.py new file mode 100644 index 0000000..683777a --- /dev/null +++ b/testing/embedding/test_basic.py @@ -0,0 +1,146 @@ +import py +import sys, os, re +import shutil, subprocess, time +from testing.udir import udir +import cffi + + +local_dir = os.path.dirname(os.path.abspath(__file__)) +_link_error = '?' + +def check_lib_python_found(tmpdir): + global _link_error + if _link_error == '?': + ffi = cffi.FFI() + kwds = {} + ffi._apply_embedding_fix(kwds) + ffi.set_source("_test_lib_python_found", "", **kwds) + try: + ffi.compile(tmpdir=tmpdir) + except cffi.VerificationError as e: + _link_error = e + else: + _link_error = None + if _link_error: + py.test.skip(str(_link_error)) + + +class EmbeddingTests: + _compiled_modules = {} + + def setup_method(self, meth): + check_lib_python_found(str(udir.ensure('embedding', dir=1))) + self._path = udir.join('embedding', meth.__name__) + if sys.platform == "win32": + self._compiled_modules.clear() # workaround + + def get_path(self): + return str(self._path.ensure(dir=1)) + + def _run(self, args, env=None): + print(args) + popen = subprocess.Popen(args, env=env, cwd=self.get_path(), + stdout=subprocess.PIPE, + universal_newlines=True) + output = popen.stdout.read() + err = popen.wait() + if err: + raise OSError("popen failed with exit code %r: %r" % ( + err, args)) + print(output.rstrip()) + return output + + def prepare_module(self, name): + if name not in self._compiled_modules: + path = self.get_path() + filename = '%s.py' % name + # NOTE: if you have an .egg globally installed with an older + # version of cffi, this will not work, because sys.path ends + # up with the .egg before the PYTHONPATH entries. I didn't + # find a solution to that: we could hack sys.path inside the + # script run here, but we can't hack it in the same way in + # execute(). + env = os.environ.copy() + env['PYTHONPATH'] = os.path.dirname(os.path.dirname(local_dir)) + output = self._run([sys.executable, os.path.join(local_dir, filename)], + env=env) + match = re.compile(r"\bFILENAME: (.+)").search(output) + assert match + dynamic_lib_name = match.group(1) + if sys.platform == 'win32': + assert dynamic_lib_name.endswith('_cffi.dll') + else: + assert dynamic_lib_name.endswith('_cffi.so') + self._compiled_modules[name] = dynamic_lib_name + return self._compiled_modules[name] + + def compile(self, name, modules, opt=False, threads=False, defines={}): + path = self.get_path() + filename = '%s.c' % name + shutil.copy(os.path.join(local_dir, filename), path) + shutil.copy(os.path.join(local_dir, 'thread-test.h'), path) + import distutils.ccompiler + curdir = os.getcwd() + try: + os.chdir(self.get_path()) + c = distutils.ccompiler.new_compiler() + print('compiling %s with %r' % (name, modules)) + extra_preargs = [] + if sys.platform == 'win32': + libfiles = [] + for m in modules: + m = os.path.basename(m) + assert m.endswith('.dll') + libfiles.append('Release\\%s.lib' % m[:-4]) + modules = libfiles + elif threads: + extra_preargs.append('-pthread') + objects = c.compile([filename], macros=sorted(defines.items()), debug=True) + c.link_executable(objects + modules, name, extra_preargs=extra_preargs) + finally: + os.chdir(curdir) + + def execute(self, name): + path = self.get_path() + env = os.environ.copy() + env['PYTHONPATH'] = os.path.dirname(os.path.dirname(local_dir)) + libpath = env.get('LD_LIBRARY_PATH') + if libpath: + libpath = path + ':' + libpath + else: + libpath = path + env['LD_LIBRARY_PATH'] = libpath + print('running %r in %r' % (name, path)) + executable_name = name + if sys.platform == 'win32': + executable_name = os.path.join(path, executable_name + '.exe') + popen = subprocess.Popen([executable_name], cwd=path, env=env, + stdout=subprocess.PIPE, + universal_newlines=True) + result = popen.stdout.read() + err = popen.wait() + if err: + raise OSError("%r failed with exit code %r" % (name, err)) + return result + + +class TestBasic(EmbeddingTests): + def test_basic(self): + add1_cffi = self.prepare_module('add1') + self.compile('add1-test', [add1_cffi]) + output = self.execute('add1-test') + assert output == ("preparing...\n" + "adding 40 and 2\n" + "adding 100 and -5\n" + "got: 42 95\n") + + def test_two_modules(self): + add1_cffi = self.prepare_module('add1') + add2_cffi = self.prepare_module('add2') + self.compile('add2-test', [add1_cffi, add2_cffi]) + output = self.execute('add2-test') + assert output == ("preparing...\n" + "adding 40 and 2\n" + "prepADD2\n" + "adding 100 and -5 and -20\n" + "got: 42 75\n") diff --git a/testing/embedding/test_performance.py b/testing/embedding/test_performance.py new file mode 100644 index 0000000..f9f2605 --- /dev/null +++ b/testing/embedding/test_performance.py @@ -0,0 +1,52 @@ +import sys +from testing.embedding.test_basic import EmbeddingTests + +if sys.platform == 'win32': + import py + py.test.skip("written with POSIX functions") + + +class TestPerformance(EmbeddingTests): + def test_perf_single_threaded(self): + perf_cffi = self.prepare_module('perf') + self.compile('perf-test', [perf_cffi], opt=True) + output = self.execute('perf-test') + print('='*79) + print(output.rstrip()) + print('='*79) + + def test_perf_in_1_thread(self): + perf_cffi = self.prepare_module('perf') + self.compile('perf-test', [perf_cffi], opt=True, threads=True, + defines={'PTEST_USE_THREAD': '1'}) + output = self.execute('perf-test') + print('='*79) + print(output.rstrip()) + print('='*79) + + def test_perf_in_2_threads(self): + perf_cffi = self.prepare_module('perf') + self.compile('perf-test', [perf_cffi], opt=True, threads=True, + defines={'PTEST_USE_THREAD': '2'}) + output = self.execute('perf-test') + print('='*79) + print(output.rstrip()) + print('='*79) + + def test_perf_in_4_threads(self): + perf_cffi = self.prepare_module('perf') + self.compile('perf-test', [perf_cffi], opt=True, threads=True, + defines={'PTEST_USE_THREAD': '4'}) + output = self.execute('perf-test') + print('='*79) + print(output.rstrip()) + print('='*79) + + def test_perf_in_8_threads(self): + perf_cffi = self.prepare_module('perf') + self.compile('perf-test', [perf_cffi], opt=True, threads=True, + defines={'PTEST_USE_THREAD': '8'}) + output = self.execute('perf-test') + print('='*79) + print(output.rstrip()) + print('='*79) diff --git a/testing/embedding/test_recursive.py b/testing/embedding/test_recursive.py new file mode 100644 index 0000000..b85e7ed --- /dev/null +++ b/testing/embedding/test_recursive.py @@ -0,0 +1,15 @@ +from testing.embedding.test_basic import EmbeddingTests + + +class TestRecursive(EmbeddingTests): + def test_recursive(self): + add_recursive_cffi = self.prepare_module('add_recursive') + self.compile('add_recursive-test', [add_recursive_cffi]) + output = self.execute('add_recursive-test') + assert output == ("preparing REC\n" + "some_callback(400)\n" + "adding 400 and 9\n" + "<<< 409 >>>\n" + "adding 40 and 2\n" + "adding 100 and -5\n" + "got: 42 95\n") diff --git a/testing/embedding/test_thread.py b/testing/embedding/test_thread.py new file mode 100644 index 0000000..6653572 --- /dev/null +++ b/testing/embedding/test_thread.py @@ -0,0 +1,61 @@ +from testing.embedding.test_basic import EmbeddingTests + + +class TestThread(EmbeddingTests): + def test_first_calls_in_parallel(self): + add1_cffi = self.prepare_module('add1') + self.compile('thread1-test', [add1_cffi], threads=True) + for i in range(50): + output = self.execute('thread1-test') + assert output == ("starting\n" + "preparing...\n" + + "adding 40 and 2\n" * 10 + + "done\n") + + def _take_out(self, text, content): + assert content in text + i = text.index(content) + return text[:i] + text[i+len(content):] + + def test_init_different_modules_in_different_threads(self): + add1_cffi = self.prepare_module('add1') + add2_cffi = self.prepare_module('add2') + self.compile('thread2-test', [add1_cffi, add2_cffi], threads=True) + output = self.execute('thread2-test') + output = self._take_out(output, "preparing") + output = self._take_out(output, ".") + output = self._take_out(output, ".") + # at least the 3rd dot should be after everything from ADD2 + assert output == ("starting\n" + "prepADD2\n" + "adding 1000 and 200 and 30\n" + ".\n" + "adding 40 and 2\n" + "done\n") + + def test_alt_issue(self): + add1_cffi = self.prepare_module('add1') + add2_cffi = self.prepare_module('add2') + self.compile('thread2-test', [add1_cffi, add2_cffi], + threads=True, defines={'T2TEST_AGAIN_ADD1': '1'}) + output = self.execute('thread2-test') + output = self._take_out(output, "adding 40 and 2\n") + assert output == ("starting\n" + "preparing...\n" + "adding -1 and -1\n" + "prepADD2\n" + "adding 1000 and 200 and 30\n" + "done\n") + + def test_load_in_parallel_more(self): + add2_cffi = self.prepare_module('add2') + add3_cffi = self.prepare_module('add3') + self.compile('thread3-test', [add2_cffi, add3_cffi], threads=True) + for i in range(150): + output = self.execute('thread3-test') + for j in range(10): + output = self._take_out(output, "adding 40 and 2 and 100\n") + output = self._take_out(output, "adding 1000, 200, 30, 4\n") + assert output == ("starting\n" + "prepADD2\n" + "done\n") diff --git a/testing/embedding/test_tlocal.py b/testing/embedding/test_tlocal.py new file mode 100644 index 0000000..6e7c5af --- /dev/null +++ b/testing/embedding/test_tlocal.py @@ -0,0 +1,10 @@ +from testing.embedding.test_basic import EmbeddingTests + + +class TestThreadLocal(EmbeddingTests): + def test_thread_local(self): + tlocal_cffi = self.prepare_module('tlocal') + self.compile('tlocal-test', [tlocal_cffi], threads=True) + for i in range(10): + output = self.execute('tlocal-test') + assert output == "done\n" diff --git a/testing/embedding/thread-test.h b/testing/embedding/thread-test.h new file mode 100644 index 0000000..33a7e48 --- /dev/null +++ b/testing/embedding/thread-test.h @@ -0,0 +1,62 @@ +/************************************************************/ +#ifndef _MSC_VER +/************************************************************/ + + +#include <pthread.h> +#include <semaphore.h> + + +/************************************************************/ +#else +/************************************************************/ + + +/* Very quick and dirty, just what I need for these tests. + Don't use directly in any real code! +*/ + +#include <Windows.h> +#include <assert.h> + +typedef HANDLE sem_t; +typedef HANDLE pthread_t; + +int sem_init(sem_t *sem, int pshared, unsigned int value) +{ + assert(pshared == 0); + assert(value == 0); + *sem = CreateSemaphore(NULL, 0, 999, NULL); + return *sem ? 0 : -1; +} + +int sem_post(sem_t *sem) +{ + return ReleaseSemaphore(*sem, 1, NULL) ? 0 : -1; +} + +int sem_wait(sem_t *sem) +{ + WaitForSingleObject(*sem, INFINITE); + return 0; +} + +DWORD WINAPI myThreadProc(LPVOID lpParameter) +{ + void *(* start_routine)(void *) = (void *(*)(void *))lpParameter; + start_routine(NULL); + return 0; +} + +int pthread_create(pthread_t *thread, void *attr, + void *start_routine(void *), void *arg) +{ + assert(arg == NULL); + *thread = CreateThread(NULL, 0, myThreadProc, start_routine, 0, NULL); + return *thread ? 0 : -1; +} + + +/************************************************************/ +#endif +/************************************************************/ diff --git a/testing/embedding/thread1-test.c b/testing/embedding/thread1-test.c new file mode 100644 index 0000000..70bb861 --- /dev/null +++ b/testing/embedding/thread1-test.c @@ -0,0 +1,43 @@ +#include <stdio.h> +#include <assert.h> +#include "thread-test.h" + +#define NTHREADS 10 + + +extern int add1(int, int); + +static sem_t done; + + +static void *start_routine(void *arg) +{ + int x, status; + x = add1(40, 2); + assert(x == 42); + + status = sem_post(&done); + assert(status == 0); + + return arg; +} + +int main(void) +{ + pthread_t th; + int i, status = sem_init(&done, 0, 0); + assert(status == 0); + + printf("starting\n"); + fflush(stdout); + for (i = 0; i < NTHREADS; i++) { + status = pthread_create(&th, NULL, start_routine, NULL); + assert(status == 0); + } + for (i = 0; i < NTHREADS; i++) { + status = sem_wait(&done); + assert(status == 0); + } + printf("done\n"); + return 0; +} diff --git a/testing/embedding/thread2-test.c b/testing/embedding/thread2-test.c new file mode 100644 index 0000000..62f5ec8 --- /dev/null +++ b/testing/embedding/thread2-test.c @@ -0,0 +1,57 @@ +#include <stdio.h> +#include <assert.h> +#include "thread-test.h" + +extern int add1(int, int); +extern int add2(int, int, int); + +static sem_t done; + + +static void *start_routine_1(void *arg) +{ + int x, status; + x = add1(40, 2); + assert(x == 42); + + status = sem_post(&done); + assert(status == 0); + + return arg; +} + +static void *start_routine_2(void *arg) +{ + int x, status; +#ifdef T2TEST_AGAIN_ADD1 + add1(-1, -1); +#endif + x = add2(1000, 200, 30); + assert(x == 1230); + + status = sem_post(&done); + assert(status == 0); + + return arg; +} + +int main(void) +{ + pthread_t th; + int i, status = sem_init(&done, 0, 0); + assert(status == 0); + + printf("starting\n"); + fflush(stdout); + status = pthread_create(&th, NULL, start_routine_1, NULL); + assert(status == 0); + status = pthread_create(&th, NULL, start_routine_2, NULL); + assert(status == 0); + + for (i = 0; i < 2; i++) { + status = sem_wait(&done); + assert(status == 0); + } + printf("done\n"); + return 0; +} diff --git a/testing/embedding/thread3-test.c b/testing/embedding/thread3-test.c new file mode 100644 index 0000000..fa549af --- /dev/null +++ b/testing/embedding/thread3-test.c @@ -0,0 +1,55 @@ +#include <stdio.h> +#include <assert.h> +#include "thread-test.h" + +extern int add2(int, int, int); +extern int add3(int, int, int, int); + +static sem_t done; + + +static void *start_routine_2(void *arg) +{ + int x, status; + x = add2(40, 2, 100); + assert(x == 142); + + status = sem_post(&done); + assert(status == 0); + + return arg; +} + +static void *start_routine_3(void *arg) +{ + int x, status; + x = add3(1000, 200, 30, 4); + assert(x == 1234); + + status = sem_post(&done); + assert(status == 0); + + return arg; +} + +int main(void) +{ + pthread_t th; + int i, status = sem_init(&done, 0, 0); + assert(status == 0); + + printf("starting\n"); + fflush(stdout); + for (i = 0; i < 10; i++) { + status = pthread_create(&th, NULL, start_routine_2, NULL); + assert(status == 0); + status = pthread_create(&th, NULL, start_routine_3, NULL); + assert(status == 0); + } + for (i = 0; i < 20; i++) { + status = sem_wait(&done); + assert(status == 0); + } + printf("done\n"); + return 0; +} diff --git a/testing/embedding/tlocal-test.c b/testing/embedding/tlocal-test.c new file mode 100644 index 0000000..b78a03d --- /dev/null +++ b/testing/embedding/tlocal-test.c @@ -0,0 +1,47 @@ +#include <stdio.h> +#include <assert.h> +#include "thread-test.h" + +#define NTHREADS 10 + + +extern int add1(int, int); + +static sem_t done; + + +static void *start_routine(void *arg) +{ + int i, x, expected, status; + + expected = add1(40, 2); + assert((expected % 1000) == 42); + + for (i=0; i<10; i++) { + x = add1(50, i); + assert(x == expected + 8 + i); + } + + status = sem_post(&done); + assert(status == 0); + + return arg; +} + +int main(void) +{ + pthread_t th; + int i, status = sem_init(&done, 0, 0); + assert(status == 0); + + for (i = 0; i < NTHREADS; i++) { + status = pthread_create(&th, NULL, start_routine, NULL); + assert(status == 0); + } + for (i = 0; i < NTHREADS; i++) { + status = sem_wait(&done); + assert(status == 0); + } + printf("done\n"); + return 0; +} diff --git a/testing/embedding/tlocal.py b/testing/embedding/tlocal.py new file mode 100644 index 0000000..7800dff --- /dev/null +++ b/testing/embedding/tlocal.py @@ -0,0 +1,33 @@ +import cffi + +ffi = cffi.FFI() + +ffi.embedding_api(""" + int add1(int, int); +""") + +ffi.embedding_init_code(r""" + from _tlocal_cffi import ffi + import itertools + try: + import thread + g_seen = itertools.count().next + except ImportError: + import _thread as thread # py3 + g_seen = itertools.count().__next__ + tloc = thread._local() + + @ffi.def_extern() + def add1(x, y): + try: + num = tloc.num + except AttributeError: + num = tloc.num = g_seen() * 1000 + return x + y + num +""") + +ffi.set_source("_tlocal_cffi", """ +""") + +fn = ffi.compile(verbose=True) +print('FILENAME: %s' % (fn,)) |