merge pull request #96. Thanks Cody!

5 files changed, 76 insertions, 51 deletions

diff --git a/cffi/__init__.py b/cffi/__init__.py
index 157a215..0224a15 100644
--- a/cffi/__init__.py
+++ b/cffi/__init__.py
@@ -5,8 +5,8 @@ from .api import FFI
 from .error import CDefError, FFIError, VerificationError, VerificationMissing
 from .error import PkgConfigError
 
-__version__ = "1.12.0"
-__version_info__ = (1, 12, 0)
+__version__ = "1.12.3"
+__version_info__ = (1, 12, 3)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/cffi/_cffi_include.h b/cffi/_cffi_include.h
index 6711428..37ea74f 100644
--- a/cffi/_cffi_include.h
+++ b/cffi/_cffi_include.h
@@ -8,43 +8,20 @@
    the same works for the other two macros.  Py_DEBUG implies them,
    but not the other way around.
 
-   The implementation is messy (issue #350): on Windows, with _MSC_VER,
-   we have to define Py_LIMITED_API even before including pyconfig.h.
-   In that case, we guess what pyconfig.h will do to the macros above,
-   and check our guess after the #include.
-
-   Note that on Windows, with CPython 3.x, you need virtualenv version
-   >= 16.0.0.  Older versions don't copy PYTHON3.DLL.  As a workaround
-   you can remove the definition of Py_LIMITED_API here.
-
-   See also 'py_limited_api' in cffi/setuptools_ext.py.
+   Issue #350 is still open: on Windows, the code here causes it to link
+   with PYTHON36.DLL (for example) instead of PYTHON3.DLL.  A fix was
+   attempted in 164e526a5515 and 14ce6985e1c3, but reverted: virtualenv
+   does not make PYTHON3.DLL available, and so the "correctly" compiled
+   version would not run inside a virtualenv.  We will re-apply the fix
+   after virtualenv has been fixed for some time.  For explanation, see
+   issue #355.  For a workaround if you want PYTHON3.DLL and don't worry
+   about virtualenv, see issue #350.  See also 'py_limited_api' in
+   setuptools_ext.py.
 */
 #if !defined(_CFFI_USE_EMBEDDING) && !defined(Py_LIMITED_API)
-#  ifdef _MSC_VER
-#    if !defined(_DEBUG) && !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG)
-#      define Py_LIMITED_API
-#    endif
-#    include <pyconfig.h>
-     /* sanity-check: Py_LIMITED_API will cause crashes if any of these
-        are also defined.  Normally, the Python file PC/pyconfig.h does not
-        cause any of these to be defined, with the exception that _DEBUG
-        causes Py_DEBUG.  Double-check that. */
-#    ifdef Py_LIMITED_API
-#      if defined(Py_DEBUG)
-#        error "pyconfig.h unexpectedly defines Py_DEBUG, but Py_LIMITED_API is set"
-#      endif
-#      if defined(Py_TRACE_REFS)
-#        error "pyconfig.h unexpectedly defines Py_TRACE_REFS, but Py_LIMITED_API is set"
-#      endif
-#      if defined(Py_REF_DEBUG)
-#        error "pyconfig.h unexpectedly defines Py_REF_DEBUG, but Py_LIMITED_API is set"
-#      endif
-#    endif
-#  else
-#    include <pyconfig.h>
-#    if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG)
-#      define Py_LIMITED_API
-#    endif
+#  include <pyconfig.h>
+#  if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG)
+#    define Py_LIMITED_API
 #  endif
 #endif
 
diff --git a/cffi/_embedding.h b/cffi/_embedding.h
index fa2afca..30842c1 100644
--- a/cffi/_embedding.h
+++ b/cffi/_embedding.h
@@ -145,6 +145,7 @@ static int _cffi_initialize_python(void)
     int result;
     PyGILState_STATE state;
     PyObject *pycode=NULL, *global_dict=NULL, *x;
+    PyObject *builtins;
 
     state = PyGILState_Ensure();
 
@@ -169,8 +170,10 @@ static int _cffi_initialize_python(void)
     global_dict = PyDict_New();
     if (global_dict == NULL)
         goto error;
-    if (PyDict_SetItemString(global_dict, "__builtins__",
-                             PyThreadState_GET()->interp->builtins) < 0)
+    builtins = PyEval_GetBuiltins();
+    if (builtins == NULL)
+        goto error;
+    if (PyDict_SetItemString(global_dict, "__builtins__", builtins) < 0)
         goto error;
     x = PyEval_EvalCode(
 #if PY_MAJOR_VERSION < 3
@@ -221,7 +224,7 @@ static int _cffi_initialize_python(void)
 
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.12.0"
+                               "\ncompiled with cffi version: 1.12.3"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
@@ -263,23 +266,33 @@ static int _cffi_carefully_make_gil(void)
        So we use a global variable as a simple spin lock.  This global
        variable must be from 'libpythonX.Y.so', not from this
        cffi-based extension module, because it must be shared from
-       different cffi-based extension modules.  We choose
+       different cffi-based extension modules.
+
+       In Python < 3.8, we choose
        _PyParser_TokenNames[0] as a completely arbitrary pointer value
        that is never written to.  The default is to point to the
        string "ENDMARKER".  We change it temporarily to point to the
        next character in that string.  (Yes, I know it's REALLY
        obscure.)
+
+       In Python >= 3.8, this string array is no longer writable, so
+       instead we pick PyCapsuleType.tp_version_tag.  We can't change
+       Python < 3.8 because someone might use a mixture of cffi
+       embedded modules, some of which were compiled before this file
+       changed.
     */
 
 #ifdef WITH_THREAD
+# if PY_VERSION_HEX < 0x03080000
     char *volatile *lock = (char *volatile *)_PyParser_TokenNames;
-    char *old_value;
+    char *old_value, *locked_value;
 
     while (1) {    /* spin loop */
         old_value = *lock;
+        locked_value = old_value + 1;
         if (old_value[0] == 'E') {
             assert(old_value[1] == 'N');
-            if (cffi_compare_and_swap(lock, old_value, old_value + 1))
+            if (cffi_compare_and_swap(lock, old_value, locked_value))
                 break;
         }
         else {
@@ -290,6 +303,27 @@ static int _cffi_carefully_make_gil(void)
                this is only run at start-up anyway. */
         }
     }
+# else
+    int volatile *lock = (int volatile *)&PyCapsule_Type.tp_version_tag;
+    int old_value, locked_value;
+    assert(!(PyCapsule_Type.tp_flags & Py_TPFLAGS_HAVE_VERSION_TAG));
+
+    while (1) {    /* spin loop */
+        old_value = *lock;
+        locked_value = -42;
+        if (old_value == 0) {
+            if (cffi_compare_and_swap(lock, old_value, locked_value))
+                break;
+        }
+        else {
+            assert(old_value == locked_value);
+            /* should ideally do a spin loop instruction here, but
+               hard to do it portably and doesn't really matter I
+               think: PyEval_InitThreads() should be very fast, and
+               this is only run at start-up anyway. */
+        }
+    }
+# endif
 #endif
 
     /* call Py_InitializeEx() */
@@ -306,7 +340,7 @@ static int _cffi_carefully_make_gil(void)
 
 #ifdef WITH_THREAD
     /* release the lock */
-    while (!cffi_compare_and_swap(lock, old_value + 1, old_value))
+    while (!cffi_compare_and_swap(lock, locked_value, old_value))
         ;
 #endif
 
diff --git a/cffi/cparser.py b/cffi/cparser.py
index 89a9b59..b23238a 100644
--- a/cffi/cparser.py
+++ b/cffi/cparser.py
@@ -817,12 +817,20 @@ class Parser(object):
         # or positive/negative number
         if isinstance(exprnode, pycparser.c_ast.Constant):
             s = exprnode.value
-            if s.startswith('0'):
-                if s.startswith('0x') or s.startswith('0X'):
-                    return int(s, 16)
-                return int(s, 8)
-            elif '1' <= s[0] <= '9':
-                return int(s, 10)
+            if '0' <= s[0] <= '9':
+                s = s.rstrip('uUlL')
+                try:
+                    if s.startswith('0'):
+                        return int(s, 8)
+                    else:
+                        return int(s, 10)
+                except ValueError:
+                    if len(s) > 1:
+                        if s.lower()[0:2] == '0x':
+                            return int(s, 16)
+                        elif s.lower()[0:2] == '0b':
+                            return int(s, 2)
+                raise CDefError("invalid constant %r" % (s,))
             elif s[0] == "'" and s[-1] == "'" and (
                     len(s) == 3 or (len(s) == 4 and s[1] == "\\")):
                 return ord(s[-2])
diff --git a/cffi/setuptools_ext.py b/cffi/setuptools_ext.py
index a6eb8b8..df5a518 100644
--- a/cffi/setuptools_ext.py
+++ b/cffi/setuptools_ext.py
@@ -81,8 +81,14 @@ def _set_py_limited_api(Extension, kwds):
     it doesn't so far, creating troubles.  That's why we check
     for "not hasattr(sys, 'gettotalrefcount')" (the 2.7 compatible equivalent
     of 'd' not in sys.abiflags). (http://bugs.python.org/issue28401)
+
+    On Windows, with CPython <= 3.4, it's better not to use py_limited_api
+    because virtualenv *still* doesn't copy PYTHON3.DLL on these versions.
+    For now we'll skip py_limited_api on all Windows versions to avoid an
+    inconsistent mess.
     """
-    if 'py_limited_api' not in kwds and not hasattr(sys, 'gettotalrefcount'):
+    if ('py_limited_api' not in kwds and not hasattr(sys, 'gettotalrefcount')
+            and sys.platform != 'win32'):
         import setuptools
         try:
             setuptools_major_version = int(setuptools.__version__.partition('.')[0])