From 30f5ad30542c63156e4babcab1e783ef44b09e15 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Wed, 21 Apr 2021 12:49:26 +0200 Subject: Try to make a test less flaky. --- tests/run/time_pxd.pyx | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/run/time_pxd.pyx b/tests/run/time_pxd.pyx index 91e6e8d24..f8bf9d66d 100644 --- a/tests/run/time_pxd.pyx +++ b/tests/run/time_pxd.pyx @@ -56,4 +56,9 @@ def test_localtime(): ltp = time.localtime() ltc = ctime.localtime() + if ltp.tm_sec != ltc.tm_sec: + # or three times in a row... + ltp = time.localtime() + ltc = ctime.localtime() + return ltp, ltc -- cgit v1.2.1 From 5f8c72dfa8ffb7c2a9b85552f65fc2f0fe85f2fe Mon Sep 17 00:00:00 2001 From: scoder Date: Sun, 25 Apr 2021 07:12:44 +0200 Subject: Make "__Pyx_UnicodeContainsUCS4()" work for WCHAR unicode strings with Py3.9+. (GH-4135) * Use the same fallback as for missing PEP-393 support. * Prepare for "PyUnicode_READY()" and "PyUnicode_WCHAR_KIND" to be removed in Py3.12. See https://www.python.org/dev/peps/pep-0623/ * Avoid C compiler warnings about deprecated C-API functions in Py3.9+. Closes https://github.com/cython/cython/issues/3925 --- Cython/Utility/ModuleSetupCode.c | 13 +++++++++++++ Cython/Utility/ObjectHandling.c | 4 ++-- Cython/Utility/Optimize.c | 2 +- Cython/Utility/StringTools.c | 35 ++++++++++++++++++++++++++--------- 4 files changed, 42 insertions(+), 12 deletions(-) diff --git a/Cython/Utility/ModuleSetupCode.c b/Cython/Utility/ModuleSetupCode.c index 3929bc64b..8119e083f 100644 --- a/Cython/Utility/ModuleSetupCode.c +++ b/Cython/Utility/ModuleSetupCode.c @@ -733,8 +733,15 @@ static CYTHON_INLINE PyObject * __Pyx_PyDict_GetItemStrWithError(PyObject *dict, #elif PY_VERSION_HEX > 0x03030000 && defined(PyUnicode_KIND) /* new Py3.3 unicode type (PEP 393) */ #define CYTHON_PEP393_ENABLED 1 + + #if defined(PyUnicode_IS_READY) #define __Pyx_PyUnicode_READY(op) (likely(PyUnicode_IS_READY(op)) ? \ 0 : _PyUnicode_Ready((PyObject *)(op))) + #else + // Py3.12 / PEP-623 will remove wstr type unicode strings and all of the PyUnicode_READY() machinery. + #define __Pyx_PyUnicode_READY(op) (0) + #endif + #define __Pyx_PyUnicode_GET_LENGTH(u) PyUnicode_GET_LENGTH(u) #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_READ_CHAR(u, i) #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) PyUnicode_MAX_CHAR_VALUE(u) @@ -743,7 +750,13 @@ static CYTHON_INLINE PyObject * __Pyx_PyDict_GetItemStrWithError(PyObject *dict, #define __Pyx_PyUnicode_READ(k, d, i) PyUnicode_READ(k, d, i) #define __Pyx_PyUnicode_WRITE(k, d, i, ch) PyUnicode_WRITE(k, d, i, ch) #if defined(PyUnicode_IS_READY) && defined(PyUnicode_GET_SIZE) + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x03090000 + // Avoid calling deprecated C-API functions in Py3.9+ that PEP-623 schedules for removal in Py3.12. + // https://www.python.org/dev/peps/pep-0623/ + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != (likely(PyUnicode_IS_READY(u)) ? PyUnicode_GET_LENGTH(u) : ((PyCompactUnicodeObject *)(u))->wstr_length)) + #else #define __Pyx_PyUnicode_IS_TRUE(u) (0 != (likely(PyUnicode_IS_READY(u)) ? PyUnicode_GET_LENGTH(u) : PyUnicode_GET_SIZE(u))) + #endif #else #define __Pyx_PyUnicode_IS_TRUE(u) (0 != PyUnicode_GET_LENGTH(u)) #endif diff --git a/Cython/Utility/ObjectHandling.c b/Cython/Utility/ObjectHandling.c index 903c03833..10de79d61 100644 --- a/Cython/Utility/ObjectHandling.c +++ b/Cython/Utility/ObjectHandling.c @@ -2928,9 +2928,9 @@ static CYTHON_INLINE PyObject *__Pyx_PyUnicode_ConcatInPlaceImpl(PyObject **p_le PyObject *left = *p_left; Py_ssize_t left_len, right_len, new_len; - if (unlikely(PyUnicode_READY(left) == -1)) + if (unlikely(__Pyx_PyUnicode_READY(left) == -1)) return NULL; - if (unlikely(PyUnicode_READY(right) == -1)) + if (unlikely(__Pyx_PyUnicode_READY(right) == -1)) return NULL; // Shortcuts diff --git a/Cython/Utility/Optimize.c b/Cython/Utility/Optimize.c index f9637b670..207c2d3ab 100644 --- a/Cython/Utility/Optimize.c +++ b/Cython/Utility/Optimize.c @@ -789,7 +789,7 @@ fallback: static CYTHON_INLINE double __Pyx_PyUnicode_AsDouble(PyObject *obj) { // Currently not optimised for Py2.7. #if PY_MAJOR_VERSION >= 3 && !CYTHON_COMPILING_IN_PYPY - if (unlikely(PyUnicode_READY(obj) == -1)) + if (unlikely(__Pyx_PyUnicode_READY(obj) == -1)) return (double)-1; if (likely(PyUnicode_IS_ASCII(obj))) { const char *s; diff --git a/Cython/Utility/StringTools.c b/Cython/Utility/StringTools.c index feb6c99df..aa8c06d55 100644 --- a/Cython/Utility/StringTools.c +++ b/Cython/Utility/StringTools.c @@ -118,8 +118,19 @@ static CYTHON_INLINE int __Pyx_UnicodeContainsUCS4(PyObject* unicode, Py_UCS4 ch //////////////////// PyUCS4InUnicode //////////////////// +#if PY_VERSION_HEX < 0x03090000 || (defined(PyUnicode_WCHAR_KIND) && defined(PyUnicode_AS_UNICODE)) + #if PY_VERSION_HEX < 0x03090000 -#if Py_UNICODE_SIZE == 2 +#define __Pyx_PyUnicode_AS_UNICODE(op) PyUnicode_AS_UNICODE(op) +#define __Pyx_PyUnicode_GET_SIZE(op) PyUnicode_GET_SIZE(op) +#else +// Avoid calling deprecated C-API functions in Py3.9+ that PEP-623 schedules for removal in Py3.12. +// https://www.python.org/dev/peps/pep-0623/ +#define __Pyx_PyUnicode_AS_UNICODE(op) (((PyASCIIObject *)(op))->wstr) +#define __Pyx_PyUnicode_GET_SIZE(op) ((PyCompactUnicodeObject *)(op))->wstr_length +#endif + +#if !defined(Py_UNICODE_SIZE) || Py_UNICODE_SIZE == 2 static int __Pyx_PyUnicodeBufferContainsUCS4_SP(Py_UNICODE* buffer, Py_ssize_t length, Py_UCS4 character) { /* handle surrogate pairs for Py_UNICODE buffers in 16bit Unicode builds */ Py_UNICODE high_val, low_val; @@ -147,7 +158,10 @@ static int __Pyx_PyUnicodeBufferContainsUCS4_BMP(Py_UNICODE* buffer, Py_ssize_t static CYTHON_INLINE int __Pyx_UnicodeContainsUCS4(PyObject* unicode, Py_UCS4 character) { #if CYTHON_PEP393_ENABLED const int kind = PyUnicode_KIND(unicode); - if (likely(kind != PyUnicode_WCHAR_KIND)) { + #ifdef PyUnicode_WCHAR_KIND + if (likely(kind != PyUnicode_WCHAR_KIND)) + #endif + { Py_ssize_t i; const void* udata = PyUnicode_DATA(unicode); const Py_ssize_t length = PyUnicode_GET_LENGTH(unicode); @@ -158,20 +172,23 @@ static CYTHON_INLINE int __Pyx_UnicodeContainsUCS4(PyObject* unicode, Py_UCS4 ch } #elif PY_VERSION_HEX >= 0x03090000 #error Cannot use "UChar in Unicode" in Python 3.9 without PEP-393 unicode strings. +#elif !defined(PyUnicode_AS_UNICODE) + #error Cannot use "UChar in Unicode" in Python < 3.9 without Py_UNICODE support. #endif -#if PY_VERSION_HEX < 0x03090000 -#if Py_UNICODE_SIZE == 2 - if (unlikely(character > 65535)) { + +#if PY_VERSION_HEX < 0x03090000 || (defined(PyUnicode_WCHAR_KIND) && defined(PyUnicode_AS_UNICODE)) +#if !defined(Py_UNICODE_SIZE) || Py_UNICODE_SIZE == 2 + if ((sizeof(Py_UNICODE) == 2) && unlikely(character > 65535)) { return __Pyx_PyUnicodeBufferContainsUCS4_SP( - PyUnicode_AS_UNICODE(unicode), - PyUnicode_GET_SIZE(unicode), + __Pyx_PyUnicode_AS_UNICODE(unicode), + __Pyx_PyUnicode_GET_SIZE(unicode), character); } else #endif { return __Pyx_PyUnicodeBufferContainsUCS4_BMP( - PyUnicode_AS_UNICODE(unicode), - PyUnicode_GET_SIZE(unicode), + __Pyx_PyUnicode_AS_UNICODE(unicode), + __Pyx_PyUnicode_GET_SIZE(unicode), character); } -- cgit v1.2.1 From a89a881286d3c709c35843642fbf5a02ca648418 Mon Sep 17 00:00:00 2001 From: scoder Date: Sun, 25 Apr 2021 07:12:44 +0200 Subject: Make "__Pyx_UnicodeContainsUCS4()" work for WCHAR unicode strings with Py3.9+. (GH-4135) * Use the same fallback as for missing PEP-393 support. * Prepare for "PyUnicode_READY()" and "PyUnicode_WCHAR_KIND" to be removed in Py3.12. See https://www.python.org/dev/peps/pep-0623/ * Avoid C compiler warnings about deprecated C-API functions in Py3.9+. Closes https://github.com/cython/cython/issues/3925 --- Cython/Utility/ModuleSetupCode.c | 13 +++++++++++++ Cython/Utility/StringTools.c | 35 ++++++++++++++++++++++++++--------- 2 files changed, 39 insertions(+), 9 deletions(-) diff --git a/Cython/Utility/ModuleSetupCode.c b/Cython/Utility/ModuleSetupCode.c index aa3f368c7..52ec14f2b 100644 --- a/Cython/Utility/ModuleSetupCode.c +++ b/Cython/Utility/ModuleSetupCode.c @@ -523,8 +523,15 @@ static CYTHON_INLINE void * PyThread_tss_get(Py_tss_t *key) { /* new Py3.3 unicode type (PEP 393) */ #if PY_VERSION_HEX > 0x03030000 && defined(PyUnicode_KIND) #define CYTHON_PEP393_ENABLED 1 + + #if defined(PyUnicode_IS_READY) #define __Pyx_PyUnicode_READY(op) (likely(PyUnicode_IS_READY(op)) ? \ 0 : _PyUnicode_Ready((PyObject *)(op))) + #else + // Py3.12 / PEP-623 will remove wstr type unicode strings and all of the PyUnicode_READY() machinery. + #define __Pyx_PyUnicode_READY(op) (0) + #endif + #define __Pyx_PyUnicode_GET_LENGTH(u) PyUnicode_GET_LENGTH(u) #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_READ_CHAR(u, i) #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) PyUnicode_MAX_CHAR_VALUE(u) @@ -533,7 +540,13 @@ static CYTHON_INLINE void * PyThread_tss_get(Py_tss_t *key) { #define __Pyx_PyUnicode_READ(k, d, i) PyUnicode_READ(k, d, i) #define __Pyx_PyUnicode_WRITE(k, d, i, ch) PyUnicode_WRITE(k, d, i, ch) #if defined(PyUnicode_IS_READY) && defined(PyUnicode_GET_SIZE) + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x03090000 + // Avoid calling deprecated C-API functions in Py3.9+ that PEP-623 schedules for removal in Py3.12. + // https://www.python.org/dev/peps/pep-0623/ + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != (likely(PyUnicode_IS_READY(u)) ? PyUnicode_GET_LENGTH(u) : ((PyCompactUnicodeObject *)(u))->wstr_length)) + #else #define __Pyx_PyUnicode_IS_TRUE(u) (0 != (likely(PyUnicode_IS_READY(u)) ? PyUnicode_GET_LENGTH(u) : PyUnicode_GET_SIZE(u))) + #endif #else #define __Pyx_PyUnicode_IS_TRUE(u) (0 != PyUnicode_GET_LENGTH(u)) #endif diff --git a/Cython/Utility/StringTools.c b/Cython/Utility/StringTools.c index 7a3426215..a7559ec03 100644 --- a/Cython/Utility/StringTools.c +++ b/Cython/Utility/StringTools.c @@ -66,8 +66,19 @@ static CYTHON_INLINE int __Pyx_UnicodeContainsUCS4(PyObject* unicode, Py_UCS4 ch //////////////////// PyUCS4InUnicode //////////////////// +#if PY_VERSION_HEX < 0x03090000 || (defined(PyUnicode_WCHAR_KIND) && defined(PyUnicode_AS_UNICODE)) + #if PY_VERSION_HEX < 0x03090000 -#if Py_UNICODE_SIZE == 2 +#define __Pyx_PyUnicode_AS_UNICODE(op) PyUnicode_AS_UNICODE(op) +#define __Pyx_PyUnicode_GET_SIZE(op) PyUnicode_GET_SIZE(op) +#else +// Avoid calling deprecated C-API functions in Py3.9+ that PEP-623 schedules for removal in Py3.12. +// https://www.python.org/dev/peps/pep-0623/ +#define __Pyx_PyUnicode_AS_UNICODE(op) (((PyASCIIObject *)(op))->wstr) +#define __Pyx_PyUnicode_GET_SIZE(op) ((PyCompactUnicodeObject *)(op))->wstr_length +#endif + +#if !defined(Py_UNICODE_SIZE) || Py_UNICODE_SIZE == 2 static int __Pyx_PyUnicodeBufferContainsUCS4_SP(Py_UNICODE* buffer, Py_ssize_t length, Py_UCS4 character) { /* handle surrogate pairs for Py_UNICODE buffers in 16bit Unicode builds */ Py_UNICODE high_val, low_val; @@ -95,7 +106,10 @@ static int __Pyx_PyUnicodeBufferContainsUCS4_BMP(Py_UNICODE* buffer, Py_ssize_t static CYTHON_INLINE int __Pyx_UnicodeContainsUCS4(PyObject* unicode, Py_UCS4 character) { #if CYTHON_PEP393_ENABLED const int kind = PyUnicode_KIND(unicode); - if (likely(kind != PyUnicode_WCHAR_KIND)) { + #ifdef PyUnicode_WCHAR_KIND + if (likely(kind != PyUnicode_WCHAR_KIND)) + #endif + { Py_ssize_t i; const void* udata = PyUnicode_DATA(unicode); const Py_ssize_t length = PyUnicode_GET_LENGTH(unicode); @@ -106,20 +120,23 @@ static CYTHON_INLINE int __Pyx_UnicodeContainsUCS4(PyObject* unicode, Py_UCS4 ch } #elif PY_VERSION_HEX >= 0x03090000 #error Cannot use "UChar in Unicode" in Python 3.9 without PEP-393 unicode strings. +#elif !defined(PyUnicode_AS_UNICODE) + #error Cannot use "UChar in Unicode" in Python < 3.9 without Py_UNICODE support. #endif -#if PY_VERSION_HEX < 0x03090000 -#if Py_UNICODE_SIZE == 2 - if (unlikely(character > 65535)) { + +#if PY_VERSION_HEX < 0x03090000 || (defined(PyUnicode_WCHAR_KIND) && defined(PyUnicode_AS_UNICODE)) +#if !defined(Py_UNICODE_SIZE) || Py_UNICODE_SIZE == 2 + if ((sizeof(Py_UNICODE) == 2) && unlikely(character > 65535)) { return __Pyx_PyUnicodeBufferContainsUCS4_SP( - PyUnicode_AS_UNICODE(unicode), - PyUnicode_GET_SIZE(unicode), + __Pyx_PyUnicode_AS_UNICODE(unicode), + __Pyx_PyUnicode_GET_SIZE(unicode), character); } else #endif { return __Pyx_PyUnicodeBufferContainsUCS4_BMP( - PyUnicode_AS_UNICODE(unicode), - PyUnicode_GET_SIZE(unicode), + __Pyx_PyUnicode_AS_UNICODE(unicode), + __Pyx_PyUnicode_GET_SIZE(unicode), character); } -- cgit v1.2.1 From 24057adc72f09b9c9c12f6e1f9a2ffceb8fea466 Mon Sep 17 00:00:00 2001 From: realead Date: Sun, 25 Apr 2021 23:15:16 +0200 Subject: Capture and redirect stdout/stderr for %%cython-magic to show C compiler warnings/errors (GH-3872) --- Cython/Build/IpythonMagic.py | 52 +++++++++++++++++++++-- Cython/Build/Tests/TestIpythonMagic.py | 75 ++++++++++++++++++++++++++++++++++ 2 files changed, 123 insertions(+), 4 deletions(-) diff --git a/Cython/Build/IpythonMagic.py b/Cython/Build/IpythonMagic.py index 7aa7bf666..15868d862 100644 --- a/Cython/Build/IpythonMagic.py +++ b/Cython/Build/IpythonMagic.py @@ -82,6 +82,7 @@ from ..Shadow import __version__ as cython_version from ..Compiler.Errors import CompileError from .Inline import cython_inline from .Dependencies import cythonize +from ..Utils import captured_fd PGO_CONFIG = { @@ -106,6 +107,37 @@ else: return name +def get_encoding_candidates(): + candidates = [sys.getdefaultencoding()] + for stream in (sys.stdout, sys.stdin, sys.__stdout__, sys.__stdin__): + encoding = getattr(stream, 'encoding', None) + # encoding might be None (e.g. somebody redirects stdout): + if encoding is not None and encoding not in candidates: + candidates.append(encoding) + return candidates + + +def prepare_captured(captured): + captured_bytes = captured.strip() + if not captured_bytes: + return None + for encoding in get_encoding_candidates(): + try: + return captured_bytes.decode(encoding) + except UnicodeDecodeError: + pass + # last resort: print at least the readable ascii parts correctly. + return captured_bytes.decode('latin-1') + + +def print_captured(captured, output, header_line=None): + captured = prepare_captured(captured) + if captured: + if header_line: + output.write(header_line) + output.write(captured) + + @magics_class class CythonMagics(Magics): @@ -342,13 +374,25 @@ class CythonMagics(Magics): if args.pgo: self._profile_pgo_wrapper(extension, lib_dir) + def print_compiler_output(stdout, stderr, where): + # On windows, errors are printed to stdout, we redirect both to sys.stderr. + print_captured(stdout, where, u"Content of stdout:\n") + print_captured(stderr, where, u"Content of stderr:\n") + + get_stderr = get_stdout = None try: - self._build_extension(extension, lib_dir, pgo_step_name='use' if args.pgo else None, - quiet=args.quiet) - except distutils.errors.CompileError: - # Build failed and printed error message + with captured_fd(1) as get_stdout: + with captured_fd(2) as get_stderr: + self._build_extension( + extension, lib_dir, pgo_step_name='use' if args.pgo else None, quiet=args.quiet) + except (distutils.errors.CompileError, distutils.errors.LinkError): + # Build failed, print error message from compiler/linker + print_compiler_output(get_stdout(), get_stderr(), sys.stderr) return None + # Build seems ok, but we might still want to show any warnings that occurred + print_compiler_output(get_stdout(), get_stderr(), sys.stdout) + module = imp.load_dynamic(module_name, module_path) self._import_all(module) diff --git a/Cython/Build/Tests/TestIpythonMagic.py b/Cython/Build/Tests/TestIpythonMagic.py index ed4db98cb..febb480ac 100644 --- a/Cython/Build/Tests/TestIpythonMagic.py +++ b/Cython/Build/Tests/TestIpythonMagic.py @@ -6,6 +6,7 @@ from __future__ import absolute_import import os +import io import sys from contextlib import contextmanager from Cython.Build import IpythonMagic @@ -29,6 +30,26 @@ try: except ImportError: pass + +@contextmanager +def capture_output(): + backup = sys.stdout, sys.stderr + try: + replacement = [ + io.TextIOWrapper(io.BytesIO(), encoding=sys.stdout.encoding), + io.TextIOWrapper(io.BytesIO(), encoding=sys.stderr.encoding), + ] + sys.stdout, sys.stderr = replacement + output = [] + yield output + finally: + sys.stdout, sys.stderr = backup + for wrapper in replacement: + wrapper.seek(0) # rewind + output.append(wrapper.read()) + wrapper.close() + + code = u"""\ def f(x): return 2*x @@ -48,6 +69,27 @@ def main(): main() """ +compile_error_code = u'''\ +cdef extern from *: + """ + xxx a=1; + """ + int a; +def doit(): + return a +''' + +compile_warning_code = u'''\ +cdef extern from *: + """ + #pragma message ( "CWarning" ) + int a = 42; + """ + int a; +def doit(): + return a +''' + if sys.platform == 'win32': # not using IPython's decorators here because they depend on "nose" @@ -143,6 +185,39 @@ class TestIPythonMagic(CythonTest): self.assertEqual(ip.user_ns['g'], 2 // 10) self.assertEqual(ip.user_ns['h'], 2 // 10) + def test_cython_compile_error_shown(self): + ip = self._ip + with capture_output() as out: + ip.run_cell_magic('cython', '-3', compile_error_code) + captured_out, captured_err = out + + # it could be that c-level output is captured by distutil-extension + # (and not by us) and is printed to stdout: + captured_all = captured_out + "\n" + captured_err + self.assertTrue("error" in captured_all, msg="error in " + captured_all) + + def test_cython_link_error_shown(self): + ip = self._ip + with capture_output() as out: + ip.run_cell_magic('cython', '-3 -l=xxxxxxxx', code) + captured_out, captured_err = out + + # it could be that c-level output is captured by distutil-extension + # (and not by us) and is printed to stdout: + captured_all = captured_out + "\n!" + captured_err + self.assertTrue("error" in captured_all, msg="error in " + captured_all) + + def test_cython_warning_shown(self): + ip = self._ip + with capture_output() as out: + # force rebuild, otherwise no warning as after the first success + # no build step is performed + ip.run_cell_magic('cython', '-3 -f', compile_warning_code) + captured_out, captured_err = out + + # check that warning was printed to stdout even if build hasn't failed + self.assertTrue("CWarning" in captured_out) + @skip_win32('Skip on Windows') def test_cython3_pgo(self): # The Cython cell defines the functions f() and call(). -- cgit v1.2.1