musllinux test updates and wheels, py3.11 beta wheels

12 files changed, 97 insertions, 24 deletions

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index b26dfec..d30f9b0 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -33,6 +33,7 @@ jobs:
         - spec: cp38-manylinux_x86_64
         - spec: cp39-manylinux_x86_64
         - spec: cp310-manylinux_x86_64
+        - spec: cp311-manylinux_x86_64
         - spec: cp27-manylinux_i686
           cibw_version: cibuildwheel<2.0  # py2.7 is not supported on CIBW 2.0+
           manylinux_img: manylinux1  # build really old Pythons on manylinux1
@@ -42,12 +43,20 @@ jobs:
         - spec: cp38-manylinux_i686
         - spec: cp39-manylinux_i686
         - spec: cp310-manylinux_i686
+        - spec: cp311-manylinux_i686
+        - spec: cp39-musllinux_x86_64
+        - spec: cp310-musllinux_x86_64
+        - spec: cp311-musllinux_x86_64
+        - spec: cp39-musllinux_i686
+        - spec: cp310-musllinux_i686
+        - spec: cp311-musllinux_i686
     steps:
     - name: clone repo
       uses: actions/checkout@v2
 
     - name: build/test wheels
       env:
+        CFLAGS: -Dffi_call=cffistatic_ffi_call  # override name for ffi_call to break hard if we linked against someone else's libffi
         CIBW_ARCHS_LINUX: auto
         CIBW_BUILD: ${{ matrix.spec }}
         CIBW_BEFORE_BUILD: |
@@ -55,19 +64,20 @@ jobs:
           curl -L -O https://github.com/libffi/libffi/archive/v3.4.2.tar.gz && \
           tar zxf v3.4.2.tar.gz && cd libffi-3.4.2 && \
           ./autogen.sh && \
-          ./configure --without-gcc-arch --disable-docs && \
+          ./configure --without-gcc-arch --disable-docs --with-pic --enable-shared=no && \
           make && \
           make install && \
-          ldconfig
-        # TODO: update default to '' once CIBW 2.1.3 ships: https://github.com/pypa/cibuildwheel/pull/829
-        CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_img || 'manylinux2010' }}
-        CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.manylinux_img || 'manylinux2010' }}
+          ldconfig || true
+        CIBW_ENVIRONMENT_PASS_LINUX: CFLAGS  # ensure that the build container can see our overridden build config
+        CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_img || '' }}
+        CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.manylinux_img || '' }}
+        CIBW_PRERELEASE_PYTHONS: 'True'
         CIBW_TEST_REQUIRES: pytest
-        CIBW_TEST_COMMAND: python -m pytest {project}/c {project}/testing
+        CIBW_TEST_COMMAND: PYTHONUNBUFFERED=1 python -m pytest {project}
       run: |
         python -m pip install --upgrade "${{ matrix.cibw_version || 'cibuildwheel' }}"
 
-        # actually build libyaml + wheel (using env tweaks above)
+        # actually build libffi + wheel (using env tweaks above)
         python -m cibuildwheel --output-dir dist .
 
 
@@ -94,6 +104,7 @@ jobs:
         - spec: cp38-macosx_x86_64
         - spec: cp39-macosx_x86_64
         - spec: cp310-macosx_x86_64
+        - spec: cp311-macosx_x86_64
         # build for arm64 under a hacked macOS 12 self-hosted x86_64-on-arm64 runner until arm64 is fully supported
         # FIXME: ? cp38-macosx_arm64 requires special handling and fails some test_zdist tests under cibw 2.1.2, skip it (so Apple's XCode python3 won't have a wheel)
         - spec: cp39-macosx_arm64
@@ -107,6 +118,14 @@ jobs:
           runs_on: [self-hosted, macOS]
           run_wrapper: arch -arm64 bash --noprofile --norc -eo pipefail {0}
           sdkroot: macosx11.3
+
+        - spec: cp311-macosx_arm64
+          deployment_target: '11.0'
+          runs_on: [self-hosted, macOS]
+          run_wrapper: arch -arm64 bash --noprofile --norc -eo pipefail {0}
+          sdkroot: macosx11.3
+
+
     steps:
     - name: clone repo
       uses: actions/checkout@v2
@@ -119,8 +138,9 @@ jobs:
     - name: build/test wheels
       env:
         CIBW_BUILD: ${{ matrix.spec }}
+        CIBW_PRERELEASE_PYTHONS: 'True'
         CIBW_TEST_REQUIRES: pytest
-        CIBW_TEST_COMMAND: pip install pip --upgrade; cd {project}; pytest
+        CIBW_TEST_COMMAND: pip install pip --upgrade; cd {project}; PYTHONUNBUFFERED=1 pytest
       run: |
         if [[ -n "${{ matrix.deployment_target || '' }}" ]]
         then
@@ -141,7 +161,7 @@ jobs:
         if-no-files-found: error
 
   windows:
-    runs-on: windows-2016
+    runs-on: windows-2019
     strategy:
       matrix:
         include:
@@ -152,6 +172,7 @@ jobs:
         - spec: cp38-win_amd64
         - spec: cp39-win_amd64
         - spec: cp310-win_amd64
+        - spec: cp311-win_amd64
         - spec: cp27-win32
           cibw_version: cibuildwheel==1.10  # last release with proper py2.7 Windows support
         - spec: cp36-win32
@@ -159,6 +180,7 @@ jobs:
         - spec: cp38-win32
         - spec: cp39-win32
         - spec: cp310-win32
+        - spec: cp311-win32
     steps:
     - name: clone repo
       uses: actions/checkout@v2
@@ -180,6 +202,7 @@ jobs:
     - name: build/test wheels
       env:
         CIBW_BUILD: ${{ matrix.spec }}
+        CIBW_PRERELEASE_PYTHONS: 'True'
       run: |
         python -m pip install --upgrade pip
         pip install "${{ matrix.cibw_version || 'cibuildwheel'}}"
diff --git a/c/test_c.py b/c/test_c.py
index f2f1c70..cde83b8 100644
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -1,5 +1,15 @@
 import py
 import pytest
+import sys
+
+is_musl = False
+if sys.platform == 'linux':
+    try:
+        from packaging.tags import platform_tags
+        is_musl = any(t.startswith('musllinux') for t in platform_tags())
+        del platform_tags
+    except ImportError:
+        pass
 
 def _setup_path():
     import os, sys
@@ -93,7 +103,8 @@ def test_all_rtld_symbols():
     if sys.platform.startswith("linux"):
         RTLD_NODELETE
         RTLD_NOLOAD
-        RTLD_DEEPBIND
+        if not is_musl:
+            RTLD_DEEPBIND
 
 def test_new_primitive_type():
     py.test.raises(KeyError, new_primitive_type, "foo")
@@ -1296,7 +1307,7 @@ def test_read_variable_as_unknown_length_array():
 def test_write_variable():
     ## FIXME: this test assumes glibc specific behavior, it's not compliant with C standard
     ## https://bugs.pypy.org/issue1643
-    if not sys.platform.startswith("linux"):
+    if not sys.platform.startswith("linux") or is_musl:
         py.test.skip("untested")
     BVoidP = new_pointer_type(new_void_type())
     ll = find_and_load_library('c')
diff --git a/setup.py b/setup.py
index 0922f4a..4ce0007 100644
--- a/setup.py
+++ b/setup.py
@@ -156,6 +156,11 @@ if 'freebsd' in sys.platform:
     include_dirs.append('/usr/local/include')
     library_dirs.append('/usr/local/lib')
 
+forced_extra_objs = os.environ.get('CFFI_FORCE_STATIC', [])
+if forced_extra_objs:
+    forced_extra_objs = forced_extra_objs.split(';')
+
+
 if __name__ == '__main__':
     from setuptools import setup, Distribution, Extension
 
@@ -209,6 +214,7 @@ Contact
             library_dirs=library_dirs,
             extra_compile_args=extra_compile_args,
             extra_link_args=extra_link_args,
+            extra_objects=forced_extra_objs,
         )] if cpython else [],
 
         install_requires=[
diff --git a/testing/cffi0/test_function.py b/testing/cffi0/test_function.py
index b4bb23d..84d8db6 100644
--- a/testing/cffi0/test_function.py
+++ b/testing/cffi0/test_function.py
@@ -5,7 +5,7 @@ import math, os, sys
 import ctypes.util
 from cffi.backend_ctypes import CTypesBackend
 from testing.udir import udir
-from testing.support import FdWriteCapture, StdErrCapture
+from testing.support import FdWriteCapture, StdErrCapture, is_musl
 from .backend_tests import needs_dlopen_none
 
 try:
@@ -13,6 +13,12 @@ try:
 except ImportError:
     from io import StringIO
 
+try:
+    from packaging.tags import platform_tags
+    _platform_tags_cached = set(platform_tags())
+    _is_musl = any(t.startswith('musllinux') for t in _platform_tags_cached)
+except ImportError:
+    _is_musl = False
 
 lib_m = 'm'
 if sys.platform == 'win32':
@@ -20,6 +26,8 @@ if sys.platform == 'win32':
     import distutils.ccompiler
     if distutils.ccompiler.get_default_compiler() == 'msvc':
         lib_m = 'msvcrt'
+elif is_musl:
+    lib_m = 'c'
 
 class TestFunction(object):
     Backend = CTypesBackend
@@ -165,11 +173,15 @@ class TestFunction(object):
                           ffi.cast("long long", 168))
             ffi.C.fprintf(ffi.C.stderr, b"hello %p\n", ffi.NULL)
         res = fd.getvalue()
+        if is_musl:
+            nil_repr = b'0'
+        else:
+            nil_repr = b'(nil)'
         assert res == (b"hello with no arguments\n"
                        b"hello, world!\n"
                        b"hello, world2!\n"
                        b"hello int 42 long 84 long long 168\n"
-                       b"hello (nil)\n")
+                       b"hello " + nil_repr + b"\n")
 
     def test_must_specify_type_of_vararg(self):
         ffi = FFI(backend=self.Backend())
@@ -265,7 +277,7 @@ class TestFunction(object):
         assert res == 5
 
     def test_write_variable(self):
-        if not sys.platform.startswith('linux'):
+        if not sys.platform.startswith('linux') or _is_musl:
             py.test.skip("probably no symbol 'stdout' in the lib")
         ffi = FFI(backend=self.Backend())
         ffi.cdef("""
diff --git a/testing/cffi0/test_ownlib.py b/testing/cffi0/test_ownlib.py
index ffad879..bbdab8c 100644
--- a/testing/cffi0/test_ownlib.py
+++ b/testing/cffi0/test_ownlib.py
@@ -2,7 +2,7 @@ import py, sys, os
 import subprocess, weakref
 from cffi import FFI
 from cffi.backend_ctypes import CTypesBackend
-from testing.support import u
+from testing.support import u, is_musl
 
 
 SOURCE = """\
@@ -388,7 +388,7 @@ class TestOwnLib(object):
     def test_dlopen_handle(self):
         if self.module is None:
             py.test.skip("fix the auto-generation of the tiny test lib")
-        if sys.platform == 'win32':
+        if sys.platform == 'win32' or is_musl:
             py.test.skip("uses 'dl' explicitly")
         if self.__class__.Backend is CTypesBackend:
             py.test.skip("not for the ctypes backend")
diff --git a/testing/cffi0/test_parsing.py b/testing/cffi0/test_parsing.py
index a5e4587..5d93a8d 100644
--- a/testing/cffi0/test_parsing.py
+++ b/testing/cffi0/test_parsing.py
@@ -1,6 +1,7 @@
 import py, sys, re
 from cffi import FFI, FFIError, CDefError, VerificationError
 from .backend_tests import needs_dlopen_none
+from testing.support import is_musl
 
 
 class FakeBackend(object):
@@ -80,6 +81,9 @@ if sys.platform == 'win32':
     import distutils.ccompiler
     if distutils.ccompiler.get_default_compiler() == 'msvc':
         lib_m = 'msvcrt'
+elif is_musl:
+    lib_m = 'c'
+
 
 def test_simple():
     ffi = FFI(backend=FakeBackend())
diff --git a/testing/cffi0/test_unicode_literals.py b/testing/cffi0/test_unicode_literals.py
index 7b0a5cc..8838de5 100644
--- a/testing/cffi0/test_unicode_literals.py
+++ b/testing/cffi0/test_unicode_literals.py
@@ -9,6 +9,8 @@ from __future__ import unicode_literals
 #
 import sys, math
 from cffi import FFI
+from testing.support import is_musl
+
 
 lib_m = "m"
 if sys.platform == 'win32':
@@ -16,6 +18,8 @@ if sys.platform == 'win32':
     import distutils.ccompiler
     if distutils.ccompiler.get_default_compiler() == 'msvc':
         lib_m = 'msvcrt'
+elif is_musl:
+    lib_m = 'c'
 
 
 def test_cast():
diff --git a/testing/cffi0/test_verify.py b/testing/cffi0/test_verify.py
index 3a1c0b9..de1608d 100644
--- a/testing/cffi0/test_verify.py
+++ b/testing/cffi0/test_verify.py
@@ -3,7 +3,7 @@ import pytest
 import sys, os, math, weakref
 from cffi import FFI, VerificationError, VerificationMissing, model, FFIError
 from testing.support import *
-from testing.support import extra_compile_args
+from testing.support import extra_compile_args, is_musl
 
 
 lib_m = ['m']
@@ -1609,7 +1609,7 @@ def test_keepalive_ffi():
     assert func() == 42
 
 def test_FILE_stored_in_stdout():
-    if not sys.platform.startswith('linux'):
+    if not sys.platform.startswith('linux') or is_musl:
         py.test.skip("likely, we cannot assign to stdout")
     ffi = FFI()
     ffi.cdef("int printf(const char *, ...); FILE *setstdout(FILE *);")
diff --git a/testing/cffi1/test_cffi_binary.py b/testing/cffi1/test_cffi_binary.py
index 7cfbace..45421ed 100644
--- a/testing/cffi1/test_cffi_binary.py
+++ b/testing/cffi1/test_cffi_binary.py
@@ -1,10 +1,11 @@
 import py, sys, os
 import _cffi_backend
+from testing.support import is_musl
 
 def test_no_unknown_exported_symbols():
     if not hasattr(_cffi_backend, '__file__'):
         py.test.skip("_cffi_backend module is built-in")
-    if not sys.platform.startswith('linux'):
+    if not sys.platform.startswith('linux') or is_musl:
         py.test.skip("linux-only")
     g = os.popen("objdump -T '%s'" % _cffi_backend.__file__, 'r')
     for line in g:
@@ -17,6 +18,9 @@ def test_no_unknown_exported_symbols():
         name = line.split()[-1]
         if name.startswith('_') or name.startswith('.'):
             continue
-        if name not in ('init_cffi_backend', 'PyInit__cffi_backend'):
+        # a statically-linked libffi will always appear here without header hackage, ignore it if it's internal
+        if name.startswith('ffi_') and 'Base' in line:
+            continue
+        if name not in ('init_cffi_backend', 'PyInit__cffi_backend', 'cffistatic_ffi_call'):
             raise Exception("Unexpected exported name %r" % (name,))
     g.close()
diff --git a/testing/cffi1/test_re_python.py b/testing/cffi1/test_re_python.py
index 2ae0dd1..45dd70c 100644
--- a/testing/cffi1/test_re_python.py
+++ b/testing/cffi1/test_re_python.py
@@ -3,7 +3,7 @@ import py
 from cffi import FFI
 from cffi import recompiler, ffiplatform, VerificationMissing
 from testing.udir import udir
-from testing.support import u
+from testing.support import u, is_musl
 
 
 def setup_module(mod):
@@ -269,7 +269,7 @@ def test_selfref():
 def test_dlopen_handle():
     import _cffi_backend
     from re_python_pysrc import ffi
-    if sys.platform == 'win32':
+    if sys.platform == 'win32' or is_musl:
         py.test.skip("uses 'dl' explicitly")
     ffi1 = FFI()
     ffi1.cdef("""void *dlopen(const char *filename, int flags);
diff --git a/testing/cffi1/test_verify1.py b/testing/cffi1/test_verify1.py
index 33244cc..45df2b3 100644
--- a/testing/cffi1/test_verify1.py
+++ b/testing/cffi1/test_verify1.py
@@ -4,7 +4,7 @@ from cffi import FFI, FFIError, VerificationError, VerificationMissing, model
 from cffi import CDefError
 from cffi import recompiler
 from testing.support import *
-from testing.support import _verify, extra_compile_args
+from testing.support import _verify, extra_compile_args, is_musl
 import _cffi_backend
 
 lib_m = ['m']
@@ -1571,7 +1571,7 @@ def test_keepalive_ffi():
     assert func() == 42
 
 def test_FILE_stored_in_stdout():
-    if not sys.platform.startswith('linux'):
+    if not sys.platform.startswith('linux') or is_musl:
         py.test.skip("likely, we cannot assign to stdout")
     ffi = FFI()
     ffi.cdef("int printf(const char *, ...); FILE *setstdout(FILE *);")
diff --git a/testing/support.py b/testing/support.py
index 6339a94..a65375e 100644
--- a/testing/support.py
+++ b/testing/support.py
@@ -117,3 +117,12 @@ else:
         extra_compile_args = ['-Werror', '-Wall', '-Wextra', '-Wconversion',
                               '-Wno-unused-parameter',
                               '-Wno-unreachable-code']
+
+is_musl = False
+if sys.platform == 'linux':
+    try:
+        from packaging.tags import platform_tags
+        is_musl = any(t.startswith('musllinux') for t in platform_tags())
+        del platform_tags
+    except ImportError:
+        pass