From 4d0ebf454346e70e5b7c7803ab7939f7c12a2b39 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 30 Oct 2012 01:42:39 +0100 Subject: Issue #16330: Use surrogate-related macros Patch written by Serhiy Storchaka. --- Python/fileutils.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'Python/fileutils.c') diff --git a/Python/fileutils.c b/Python/fileutils.c index 501cb8c8d6..526751d5ad 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -85,7 +85,7 @@ _Py_char2wchar(const char* arg, size_t *size) /* Only use the result if it contains no surrogate characters. */ for (tmp = res; *tmp != 0 && - (*tmp < 0xd800 || *tmp > 0xdfff); tmp++) + !Py_UNICODE_IS_SURROGATE(*tmp); tmp++) ; if (*tmp == 0) { if (size != NULL) @@ -131,7 +131,7 @@ _Py_char2wchar(const char* arg, size_t *size) memset(&mbs, 0, sizeof mbs); continue; } - if (*out >= 0xd800 && *out <= 0xdfff) { + if (Py_UNICODE_IS_SURROGATE(*out)) { /* Surrogate character. Escape the original byte sequence with surrogateescape. */ argsize -= converted; -- cgit v1.2.1 From 99f96c8a9dd3ab52c622c7fe6eb7173da86f3fa3 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 12 Nov 2012 23:04:02 +0100 Subject: Issue #16416: OS data are now always encoded/decoded to/from UTF-8/surrogateescape, instead of the locale encoding (which may be ASCII if no locale environment variable is set), to avoid inconsistencies with os.fsencode() and os.fsdecode() functions which are already using UTF-8/surrogateescape. --- Python/fileutils.c | 47 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 2 deletions(-) (limited to 'Python/fileutils.c') diff --git a/Python/fileutils.c b/Python/fileutils.c index 2e25aae075..42a532d9fa 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -8,6 +8,10 @@ #include #endif +#ifdef __APPLE__ +extern wchar_t* _Py_DecodeUTF8_surrogateescape(const char *s, Py_ssize_t size); +#endif + PyObject * _Py_device_encoding(int fd) { @@ -60,6 +64,15 @@ _Py_device_encoding(int fd) wchar_t* _Py_char2wchar(const char* arg, size_t *size) { +#ifdef __APPLE__ + wchar_t *wstr; + wstr = _Py_DecodeUTF8_surrogateescape(arg, strlen(arg)); + if (wstr == NULL) + return NULL; + if (size != NULL) + *size = wcslen(wstr); + return wstr; +#else wchar_t *res; #ifdef HAVE_BROKEN_MBSTOWCS /* Some platforms have a broken implementation of @@ -145,7 +158,7 @@ _Py_char2wchar(const char* arg, size_t *size) argsize -= converted; out++; } -#else +#else /* HAVE_MBRTOWC */ /* Cannot use C locale for escaping; manually escape as if charset is ASCII (i.e. escape all bytes > 128. This will still roundtrip correctly in the locale's charset, which must be an ASCII superset. */ @@ -160,7 +173,7 @@ _Py_char2wchar(const char* arg, size_t *size) else *out++ = 0xdc00 + *in++; *out = 0; -#endif +#endif /* HAVE_MBRTOWC */ if (size != NULL) *size = out - res; return res; @@ -168,6 +181,7 @@ oom: if (size != NULL) *size = (size_t)-1; return NULL; +#endif /* __APPLE__ */ } /* Encode a (wide) character string to the locale encoding with the @@ -184,6 +198,34 @@ oom: char* _Py_wchar2char(const wchar_t *text, size_t *error_pos) { +#ifdef __APPLE__ + Py_ssize_t len; + PyObject *unicode, *bytes = NULL; + char *cpath; + + unicode = PyUnicode_FromWideChar(text, wcslen(text)); + if (unicode == NULL) { + Py_DECREF(unicode); + return NULL; + } + + bytes = _PyUnicode_AsUTF8String(unicode, "surrogateescape"); + Py_DECREF(unicode); + if (bytes == NULL) { + PyErr_Clear(); + return NULL; + } + + len = PyBytes_GET_SIZE(bytes); + cpath = PyMem_Malloc(len+1); + if (cpath == NULL) { + Py_DECREF(bytes); + return NULL; + } + memcpy(cpath, PyBytes_AsString(bytes), len + 1); + Py_DECREF(bytes); + return cpath; +#else /* __APPLE__ */ const size_t len = wcslen(text); char *result = NULL, *bytes = NULL; size_t i, size, converted; @@ -243,6 +285,7 @@ _Py_wchar2char(const wchar_t *text, size_t *error_pos) bytes = result; } return result; +#endif /* __APPLE__ */ } /* In principle, this should use HAVE__WSTAT, and _wstat -- cgit v1.2.1 From e037652273f080b41545e058575097645959b804 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 12 Nov 2012 23:32:21 +0100 Subject: Issue #16416: Fix error handling in _Py_wchar2char() _Py_char2wchar() functions --- Python/fileutils.c | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) (limited to 'Python/fileutils.c') diff --git a/Python/fileutils.c b/Python/fileutils.c index 42a532d9fa..2cd75ce163 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -67,10 +67,12 @@ _Py_char2wchar(const char* arg, size_t *size) #ifdef __APPLE__ wchar_t *wstr; wstr = _Py_DecodeUTF8_surrogateescape(arg, strlen(arg)); - if (wstr == NULL) - return NULL; - if (size != NULL) - *size = wcslen(wstr); + if (size != NULL) { + if (wstr != NULL) + *size = wcslen(wstr); + else + *size = (size_t)-1; + } return wstr; #else wchar_t *res; @@ -204,22 +206,25 @@ _Py_wchar2char(const wchar_t *text, size_t *error_pos) char *cpath; unicode = PyUnicode_FromWideChar(text, wcslen(text)); - if (unicode == NULL) { - Py_DECREF(unicode); + if (unicode == NULL) return NULL; - } bytes = _PyUnicode_AsUTF8String(unicode, "surrogateescape"); Py_DECREF(unicode); if (bytes == NULL) { PyErr_Clear(); + if (error_pos != NULL) + *error_pos = (size_t)-1; return NULL; } len = PyBytes_GET_SIZE(bytes); cpath = PyMem_Malloc(len+1); if (cpath == NULL) { + PyErr_Clear(); Py_DECREF(bytes); + if (error_pos != NULL) + *error_pos = (size_t)-1; return NULL; } memcpy(cpath, PyBytes_AsString(bytes), len + 1); @@ -231,9 +236,6 @@ _Py_wchar2char(const wchar_t *text, size_t *error_pos) size_t i, size, converted; wchar_t c, buf[2]; - if (error_pos != NULL) - *error_pos = (size_t)-1; - /* The function works in two steps: 1. compute the length of the output buffer in bytes (size) 2. outputs the bytes */ @@ -280,8 +282,11 @@ _Py_wchar2char(const wchar_t *text, size_t *error_pos) size += 1; /* nul byte at the end */ result = PyMem_Malloc(size); - if (result == NULL) + if (result == NULL) { + if (error_pos != NULL) + *error_pos = (size_t)-1; return NULL; + } bytes = result; } return result; -- cgit v1.2.1 From 2e540f952688de0d7b1a6f5926fb0ef695518eb6 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 4 Dec 2012 01:34:47 +0100 Subject: Issue #16455: On FreeBSD and Solaris, if the locale is C, the ASCII/surrogateescape codec is now used, instead of the locale encoding, to decode the command line arguments. This change fixes inconsistencies with os.fsencode() and os.fsdecode() because these operating systems announces an ASCII locale encoding, whereas the ISO-8859-1 encoding is used in practice. --- Python/fileutils.c | 240 ++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 217 insertions(+), 23 deletions(-) (limited to 'Python/fileutils.c') diff --git a/Python/fileutils.c b/Python/fileutils.c index 2cd75ce163..9c0833e003 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -5,6 +5,7 @@ #endif #ifdef HAVE_LANGINFO_H +#include #include #endif @@ -42,7 +43,182 @@ _Py_device_encoding(int fd) Py_RETURN_NONE; } -#ifdef HAVE_STAT +#if !defined(__APPLE__) && !defined(MS_WINDOWS) +extern int _Py_normalize_encoding(const char *, char *, size_t); + +/* Workaround FreeBSD and OpenIndiana locale encoding issue with the C locale. + On these operating systems, nl_langinfo(CODESET) announces an alias of the + ASCII encoding, whereas mbstowcs() and wcstombs() functions use the + ISO-8859-1 encoding. The problem is that os.fsencode() and os.fsdecode() use + locale.getpreferredencoding() codec. For example, if command line arguments + are decoded by mbstowcs() and encoded back by os.fsencode(), we get a + UnicodeEncodeError instead of retrieving the original byte string. + + The workaround is enabled if setlocale(LC_CTYPE, NULL) returns "C", + nl_langinfo(CODESET) announces "ascii" (or an alias to ASCII), and at least + one byte in range 0x80-0xff can be decoded from the locale encoding. The + workaround is also enabled on error, for example if getting the locale + failed. + + Values of locale_is_ascii: + + 1: the workaround is used: _Py_wchar2char() uses + encode_ascii_surrogateescape() and _Py_char2wchar() uses + decode_ascii_surrogateescape() + 0: the workaround is not used: _Py_wchar2char() uses wcstombs() and + _Py_char2wchar() uses mbstowcs() + -1: unknown, need to call check_force_ascii() to get the value +*/ +static int force_ascii = -1; + +static int +check_force_ascii(void) +{ + char *loc; +#if defined(HAVE_LANGINFO_H) && defined(CODESET) + char *codeset, **alias; + char encoding[100]; + int is_ascii; + unsigned int i; + char* ascii_aliases[] = { + "ascii", + "646", + "ansi-x3.4-1968", + "ansi-x3-4-1968", + "ansi-x3.4-1986", + "cp367", + "csascii", + "ibm367", + "iso646-us", + "iso-646.irv-1991", + "iso-ir-6", + "us", + "us-ascii", + NULL + }; +#endif + + loc = setlocale(LC_CTYPE, NULL); + if (loc == NULL) + goto error; + if (strcmp(loc, "C") != 0) { + /* the LC_CTYPE locale is different than C */ + return 0; + } + +#if defined(HAVE_LANGINFO_H) && defined(CODESET) + codeset = nl_langinfo(CODESET); + if (!codeset || codeset[0] == '\0') { + /* CODESET is not set or empty */ + goto error; + } + if (!_Py_normalize_encoding(codeset, encoding, sizeof(encoding))) + goto error; + + is_ascii = 0; + for (alias=ascii_aliases; *alias != NULL; alias++) { + if (strcmp(encoding, *alias) == 0) { + is_ascii = 1; + break; + } + } + if (!is_ascii) { + /* nl_langinfo(CODESET) is not "ascii" or an alias of ASCII */ + return 0; + } + + for (i=0x80; i<0xff; i++) { + unsigned char ch; + wchar_t wch; + size_t res; + + ch = (unsigned char)i; + res = mbstowcs(&wch, (char*)&ch, 1); + if (res != (size_t)-1) { + /* decoding a non-ASCII character from the locale encoding succeed: + the locale encoding is not ASCII, force ASCII */ + return 1; + } + } + /* None of the bytes in the range 0x80-0xff can be decoded from the locale + encoding: the locale encoding is really ASCII */ + return 0; +#else + /* nl_langinfo(CODESET) is not available: always force ASCII */ + return 1; +#endif + +error: + /* if an error occured, force the ASCII encoding */ + return 1; +} + +static char* +encode_ascii_surrogateescape(const wchar_t *text, size_t *error_pos) +{ + char *result = NULL, *out; + size_t len, i; + wchar_t ch; + + if (error_pos != NULL) + *error_pos = (size_t)-1; + + len = wcslen(text); + + result = PyMem_Malloc(len + 1); /* +1 for NUL byte */ + if (result == NULL) + return NULL; + + out = result; + for (i=0; i 128. This will still roundtrip correctly in the locale's charset, which must be an ASCII superset. */ - res = PyMem_Malloc((strlen(arg)+1)*sizeof(wchar_t)); - if (!res) + res = decode_ascii_surrogateescape(arg, size); + if (res == NULL) goto oom; - in = (unsigned char*)arg; - out = res; - while(*in) - if(*in < 128) - *out++ = *in++; - else - *out++ = 0xdc00 + *in++; - *out = 0; #endif /* HAVE_MBRTOWC */ - if (size != NULL) - *size = out - res; return res; oom: if (size != NULL) @@ -236,6 +419,14 @@ _Py_wchar2char(const wchar_t *text, size_t *error_pos) size_t i, size, converted; wchar_t c, buf[2]; +#ifndef MS_WINDOWS + if (force_ascii == -1) + force_ascii = check_force_ascii(); + + if (force_ascii) + return encode_ascii_surrogateescape(text, error_pos); +#endif + /* The function works in two steps: 1. compute the length of the output buffer in bytes (size) 2. outputs the bytes */ @@ -276,7 +467,7 @@ _Py_wchar2char(const wchar_t *text, size_t *error_pos) } } if (result != NULL) { - *bytes = 0; + *bytes = '\0'; break; } @@ -320,6 +511,8 @@ _Py_wstat(const wchar_t* path, struct stat *buf) } #endif +#ifdef HAVE_STAT + /* Call _wstat() on Windows, or encode the path to the filesystem encoding and call stat() otherwise. Only fill st_mode attribute on Windows. @@ -352,6 +545,8 @@ _Py_stat(PyObject *path, struct stat *statbuf) #endif } +#endif + /* Open a file. Use _wfopen() on Windows, encode the path to the locale encoding and use fopen() otherwise. */ @@ -533,4 +728,3 @@ _Py_wgetcwd(wchar_t *buf, size_t size) #endif } -#endif -- cgit v1.2.1 From 2733e1a2064a96136de915ae8c8f13eb827df900 Mon Sep 17 00:00:00 2001 From: Philip Jenvey Date: Tue, 15 Jan 2013 13:24:12 -0800 Subject: thinko --- Python/fileutils.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'Python/fileutils.c') diff --git a/Python/fileutils.c b/Python/fileutils.c index 9c0833e003..3c04e4969e 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -60,7 +60,7 @@ extern int _Py_normalize_encoding(const char *, char *, size_t); workaround is also enabled on error, for example if getting the locale failed. - Values of locale_is_ascii: + Values of force_ascii: 1: the workaround is used: _Py_wchar2char() uses encode_ascii_surrogateescape() and _Py_char2wchar() uses -- cgit v1.2.1 From 14fce4cbce30e3887c13e2e9b727e27b8faa69ba Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 7 May 2013 23:48:56 +0200 Subject: Fix a compiler warning: in and out are unused in _Py_char2wchar() if HAVE_MBRTOWC is not defined --- Python/fileutils.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'Python/fileutils.c') diff --git a/Python/fileutils.c b/Python/fileutils.c index 3c04e4969e..bb0cd43500 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -254,9 +254,9 @@ _Py_char2wchar(const char* arg, size_t *size) wchar_t *res; size_t argsize; size_t count; +#ifdef HAVE_MBRTOWC unsigned char *in; wchar_t *out; -#ifdef HAVE_MBRTOWC mbstate_t mbs; #endif -- cgit v1.2.1 From 6bf2656e55b72945d36bba54dbbc3f2f285ebe71 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 5 Jun 2013 00:46:29 +0200 Subject: Issue #9566: Fix compiler warning on Windows 64-bit --- Python/fileutils.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'Python/fileutils.c') diff --git a/Python/fileutils.c b/Python/fileutils.c index bb0cd43500..293cb77f55 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -707,7 +707,8 @@ wchar_t* _Py_wgetcwd(wchar_t *buf, size_t size) { #ifdef MS_WINDOWS - return _wgetcwd(buf, size); + int isize = (int)Py_MIN(size, INT_MAX); + return _wgetcwd(buf, isize); #else char fname[PATH_MAX]; wchar_t *wname; -- cgit v1.2.1 From b963769c480e670e792ee3da641ded5452d83194 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 25 Jun 2013 00:37:25 +0200 Subject: If MS_WIN64 is defined, MS_WINDOWS is also defined: #ifdef can be simplified. --- Python/fileutils.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'Python/fileutils.c') diff --git a/Python/fileutils.c b/Python/fileutils.c index 293cb77f55..e75985655d 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -16,13 +16,13 @@ extern wchar_t* _Py_DecodeUTF8_surrogateescape(const char *s, Py_ssize_t size); PyObject * _Py_device_encoding(int fd) { -#if defined(MS_WINDOWS) || defined(MS_WIN64) +#if defined(MS_WINDOWS) UINT cp; #endif if (!_PyVerify_fd(fd) || !isatty(fd)) { Py_RETURN_NONE; } -#if defined(MS_WINDOWS) || defined(MS_WIN64) +#if defined(MS_WINDOWS) if (fd == 0) cp = GetConsoleCP(); else if (fd == 1 || fd == 2) -- cgit v1.2.1 From 0e6215070ac12c5aecdce573f75624eaa7ed056e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 7 Jul 2013 16:25:15 +0200 Subject: Issue #18203: Replace malloc() with PyMem_RawMalloc() at Python initialization * Replace malloc() with PyMem_RawMalloc() * Replace PyMem_Malloc() with PyMem_RawMalloc() where the GIL is not held. * _Py_char2wchar() now returns a buffer allocated by PyMem_RawMalloc(), instead of PyMem_Malloc() --- Python/fileutils.c | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) (limited to 'Python/fileutils.c') diff --git a/Python/fileutils.c b/Python/fileutils.c index e75985655d..7880ab0393 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -229,7 +229,7 @@ decode_ascii_surrogateescape(const char *arg, size_t *size) Use _Py_wchar2char() to encode the character string back to a byte string. Return a pointer to a newly allocated wide character string (use - PyMem_Free() to free the memory) and write the number of written wide + PyMem_RawFree() to free the memory) and write the number of written wide characters excluding the null character into *size if size is not NULL, or NULL on error (decoding or memory allocation error). If size is not NULL, *size is set to (size_t)-1 on memory error and (size_t)-2 on decoding @@ -283,7 +283,7 @@ _Py_char2wchar(const char* arg, size_t *size) argsize = mbstowcs(NULL, arg, 0); #endif if (argsize != (size_t)-1) { - res = (wchar_t *)PyMem_Malloc((argsize+1)*sizeof(wchar_t)); + res = (wchar_t *)PyMem_RawMalloc((argsize+1)*sizeof(wchar_t)); if (!res) goto oom; count = mbstowcs(res, arg, argsize+1); @@ -300,7 +300,7 @@ _Py_char2wchar(const char* arg, size_t *size) return res; } } - PyMem_Free(res); + PyMem_RawFree(res); } /* Conversion failed. Fall back to escaping with surrogateescape. */ #ifdef HAVE_MBRTOWC @@ -309,7 +309,7 @@ _Py_char2wchar(const char* arg, size_t *size) /* Overallocate; as multi-byte characters are in the argument, the actual output could use less memory. */ argsize = strlen(arg) + 1; - res = (wchar_t*)PyMem_Malloc(argsize*sizeof(wchar_t)); + res = (wchar_t*)PyMem_RawMalloc(argsize*sizeof(wchar_t)); if (!res) goto oom; in = (unsigned char*)arg; @@ -325,7 +325,7 @@ _Py_char2wchar(const char* arg, size_t *size) since we provide everything that we have - unless there is a bug in the C library, or I misunderstood how mbrtowc works. */ - PyMem_Free(res); + PyMem_RawFree(res); if (size != NULL) *size = (size_t)-2; return NULL; @@ -648,12 +648,12 @@ _Py_wreadlink(const wchar_t *path, wchar_t *buf, size_t bufsiz) return -1; } if (bufsiz <= r1) { - PyMem_Free(wbuf); + PyMem_RawFree(wbuf); errno = EINVAL; return -1; } wcsncpy(buf, wbuf, bufsiz); - PyMem_Free(wbuf); + PyMem_RawFree(wbuf); return (int)r1; } #endif @@ -689,12 +689,12 @@ _Py_wrealpath(const wchar_t *path, return NULL; } if (resolved_path_size <= r) { - PyMem_Free(wresolved_path); + PyMem_RawFree(wresolved_path); errno = EINVAL; return NULL; } wcsncpy(resolved_path, wresolved_path, resolved_path_size); - PyMem_Free(wresolved_path); + PyMem_RawFree(wresolved_path); return resolved_path; } #endif @@ -720,11 +720,11 @@ _Py_wgetcwd(wchar_t *buf, size_t size) if (wname == NULL) return NULL; if (size <= len) { - PyMem_Free(wname); + PyMem_RawFree(wname); return NULL; } wcsncpy(buf, wname, size); - PyMem_Free(wname); + PyMem_RawFree(wname); return buf; #endif } -- cgit v1.2.1 From 408ccb0a25aa0e5ecea25420a3cd6b92d4f8cacf Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 7 Jul 2013 16:35:54 +0200 Subject: Issue #18203: Fix decode_ascii_surrogateescape(), use PyMem_RawMalloc() as _Py_char2wchar() --- Python/fileutils.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'Python/fileutils.c') diff --git a/Python/fileutils.c b/Python/fileutils.c index 7880ab0393..698385504c 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -201,7 +201,7 @@ decode_ascii_surrogateescape(const char *arg, size_t *size) unsigned char *in; wchar_t *out; - res = PyMem_Malloc((strlen(arg)+1)*sizeof(wchar_t)); + res = PyMem_RawMalloc((strlen(arg)+1)*sizeof(wchar_t)); if (!res) return NULL; -- cgit v1.2.1