From 4d0ebf454346e70e5b7c7803ab7939f7c12a2b39 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 30 Oct 2012 01:42:39 +0100 Subject: Issue #16330: Use surrogate-related macros Patch written by Serhiy Storchaka. --- Python/fileutils.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'Python/fileutils.c') diff --git a/Python/fileutils.c b/Python/fileutils.c index 501cb8c8d6..526751d5ad 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -85,7 +85,7 @@ _Py_char2wchar(const char* arg, size_t *size) /* Only use the result if it contains no surrogate characters. */ for (tmp = res; *tmp != 0 && - (*tmp < 0xd800 || *tmp > 0xdfff); tmp++) + !Py_UNICODE_IS_SURROGATE(*tmp); tmp++) ; if (*tmp == 0) { if (size != NULL) @@ -131,7 +131,7 @@ _Py_char2wchar(const char* arg, size_t *size) memset(&mbs, 0, sizeof mbs); continue; } - if (*out >= 0xd800 && *out <= 0xdfff) { + if (Py_UNICODE_IS_SURROGATE(*out)) { /* Surrogate character. Escape the original byte sequence with surrogateescape. */ argsize -= converted; -- cgit v1.2.1 From 99f96c8a9dd3ab52c622c7fe6eb7173da86f3fa3 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 12 Nov 2012 23:04:02 +0100 Subject: Issue #16416: OS data are now always encoded/decoded to/from UTF-8/surrogateescape, instead of the locale encoding (which may be ASCII if no locale environment variable is set), to avoid inconsistencies with os.fsencode() and os.fsdecode() functions which are already using UTF-8/surrogateescape. --- Python/fileutils.c | 47 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 2 deletions(-) (limited to 'Python/fileutils.c') diff --git a/Python/fileutils.c b/Python/fileutils.c index 2e25aae075..42a532d9fa 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -8,6 +8,10 @@ #include #endif +#ifdef __APPLE__ +extern wchar_t* _Py_DecodeUTF8_surrogateescape(const char *s, Py_ssize_t size); +#endif + PyObject * _Py_device_encoding(int fd) { @@ -60,6 +64,15 @@ _Py_device_encoding(int fd) wchar_t* _Py_char2wchar(const char* arg, size_t *size) { +#ifdef __APPLE__ + wchar_t *wstr; + wstr = _Py_DecodeUTF8_surrogateescape(arg, strlen(arg)); + if (wstr == NULL) + return NULL; + if (size != NULL) + *size = wcslen(wstr); + return wstr; +#else wchar_t *res; #ifdef HAVE_BROKEN_MBSTOWCS /* Some platforms have a broken implementation of @@ -145,7 +158,7 @@ _Py_char2wchar(const char* arg, size_t *size) argsize -= converted; out++; } -#else +#else /* HAVE_MBRTOWC */ /* Cannot use C locale for escaping; manually escape as if charset is ASCII (i.e. escape all bytes > 128. This will still roundtrip correctly in the locale's charset, which must be an ASCII superset. */ @@ -160,7 +173,7 @@ _Py_char2wchar(const char* arg, size_t *size) else *out++ = 0xdc00 + *in++; *out = 0; -#endif +#endif /* HAVE_MBRTOWC */ if (size != NULL) *size = out - res; return res; @@ -168,6 +181,7 @@ oom: if (size != NULL) *size = (size_t)-1; return NULL; +#endif /* __APPLE__ */ } /* Encode a (wide) character string to the locale encoding with the @@ -184,6 +198,34 @@ oom: char* _Py_wchar2char(const wchar_t *text, size_t *error_pos) { +#ifdef __APPLE__ + Py_ssize_t len; + PyObject *unicode, *bytes = NULL; + char *cpath; + + unicode = PyUnicode_FromWideChar(text, wcslen(text)); + if (unicode == NULL) { + Py_DECREF(unicode); + return NULL; + } + + bytes = _PyUnicode_AsUTF8String(unicode, "surrogateescape"); + Py_DECREF(unicode); + if (bytes == NULL) { + PyErr_Clear(); + return NULL; + } + + len = PyBytes_GET_SIZE(bytes); + cpath = PyMem_Malloc(len+1); + if (cpath == NULL) { + Py_DECREF(bytes); + return NULL; + } + memcpy(cpath, PyBytes_AsString(bytes), len + 1); + Py_DECREF(bytes); + return cpath; +#else /* __APPLE__ */ const size_t len = wcslen(text); char *result = NULL, *bytes = NULL; size_t i, size, converted; @@ -243,6 +285,7 @@ _Py_wchar2char(const wchar_t *text, size_t *error_pos) bytes = result; } return result; +#endif /* __APPLE__ */ } /* In principle, this should use HAVE__WSTAT, and _wstat -- cgit v1.2.1 From e037652273f080b41545e058575097645959b804 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 12 Nov 2012 23:32:21 +0100 Subject: Issue #16416: Fix error handling in _Py_wchar2char() _Py_char2wchar() functions --- Python/fileutils.c | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) (limited to 'Python/fileutils.c') diff --git a/Python/fileutils.c b/Python/fileutils.c index 42a532d9fa..2cd75ce163 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -67,10 +67,12 @@ _Py_char2wchar(const char* arg, size_t *size) #ifdef __APPLE__ wchar_t *wstr; wstr = _Py_DecodeUTF8_surrogateescape(arg, strlen(arg)); - if (wstr == NULL) - return NULL; - if (size != NULL) - *size = wcslen(wstr); + if (size != NULL) { + if (wstr != NULL) + *size = wcslen(wstr); + else + *size = (size_t)-1; + } return wstr; #else wchar_t *res; @@ -204,22 +206,25 @@ _Py_wchar2char(const wchar_t *text, size_t *error_pos) char *cpath; unicode = PyUnicode_FromWideChar(text, wcslen(text)); - if (unicode == NULL) { - Py_DECREF(unicode); + if (unicode == NULL) return NULL; - } bytes = _PyUnicode_AsUTF8String(unicode, "surrogateescape"); Py_DECREF(unicode); if (bytes == NULL) { PyErr_Clear(); + if (error_pos != NULL) + *error_pos = (size_t)-1; return NULL; } len = PyBytes_GET_SIZE(bytes); cpath = PyMem_Malloc(len+1); if (cpath == NULL) { + PyErr_Clear(); Py_DECREF(bytes); + if (error_pos != NULL) + *error_pos = (size_t)-1; return NULL; } memcpy(cpath, PyBytes_AsString(bytes), len + 1); @@ -231,9 +236,6 @@ _Py_wchar2char(const wchar_t *text, size_t *error_pos) size_t i, size, converted; wchar_t c, buf[2]; - if (error_pos != NULL) - *error_pos = (size_t)-1; - /* The function works in two steps: 1. compute the length of the output buffer in bytes (size) 2. outputs the bytes */ @@ -280,8 +282,11 @@ _Py_wchar2char(const wchar_t *text, size_t *error_pos) size += 1; /* nul byte at the end */ result = PyMem_Malloc(size); - if (result == NULL) + if (result == NULL) { + if (error_pos != NULL) + *error_pos = (size_t)-1; return NULL; + } bytes = result; } return result; -- cgit v1.2.1 From 2e540f952688de0d7b1a6f5926fb0ef695518eb6 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 4 Dec 2012 01:34:47 +0100 Subject: Issue #16455: On FreeBSD and Solaris, if the locale is C, the ASCII/surrogateescape codec is now used, instead of the locale encoding, to decode the command line arguments. This change fixes inconsistencies with os.fsencode() and os.fsdecode() because these operating systems announces an ASCII locale encoding, whereas the ISO-8859-1 encoding is used in practice. --- Python/fileutils.c | 240 ++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 217 insertions(+), 23 deletions(-) (limited to 'Python/fileutils.c') diff --git a/Python/fileutils.c b/Python/fileutils.c index 2cd75ce163..9c0833e003 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -5,6 +5,7 @@ #endif #ifdef HAVE_LANGINFO_H +#include #include #endif @@ -42,7 +43,182 @@ _Py_device_encoding(int fd) Py_RETURN_NONE; } -#ifdef HAVE_STAT +#if !defined(__APPLE__) && !defined(MS_WINDOWS) +extern int _Py_normalize_encoding(const char *, char *, size_t); + +/* Workaround FreeBSD and OpenIndiana locale encoding issue with the C locale. + On these operating systems, nl_langinfo(CODESET) announces an alias of the + ASCII encoding, whereas mbstowcs() and wcstombs() functions use the + ISO-8859-1 encoding. The problem is that os.fsencode() and os.fsdecode() use + locale.getpreferredencoding() codec. For example, if command line arguments + are decoded by mbstowcs() and encoded back by os.fsencode(), we get a + UnicodeEncodeError instead of retrieving the original byte string. + + The workaround is enabled if setlocale(LC_CTYPE, NULL) returns "C", + nl_langinfo(CODESET) announces "ascii" (or an alias to ASCII), and at least + one byte in range 0x80-0xff can be decoded from the locale encoding. The + workaround is also enabled on error, for example if getting the locale + failed. + + Values of locale_is_ascii: + + 1: the workaround is used: _Py_wchar2char() uses + encode_ascii_surrogateescape() and _Py_char2wchar() uses + decode_ascii_surrogateescape() + 0: the workaround is not used: _Py_wchar2char() uses wcstombs() and + _Py_char2wchar() uses mbstowcs() + -1: unknown, need to call check_force_ascii() to get the value +*/ +static int force_ascii = -1; + +static int +check_force_ascii(void) +{ + char *loc; +#if defined(HAVE_LANGINFO_H) && defined(CODESET) + char *codeset, **alias; + char encoding[100]; + int is_ascii; + unsigned int i; + char* ascii_aliases[] = { + "ascii", + "646", + "ansi-x3.4-1968", + "ansi-x3-4-1968", + "ansi-x3.4-1986", + "cp367", + "csascii", + "ibm367", + "iso646-us", + "iso-646.irv-1991", + "iso-ir-6", + "us", + "us-ascii", + NULL + }; +#endif + + loc = setlocale(LC_CTYPE, NULL); + if (loc == NULL) + goto error; + if (strcmp(loc, "C") != 0) { + /* the LC_CTYPE locale is different than C */ + return 0; + } + +#if defined(HAVE_LANGINFO_H) && defined(CODESET) + codeset = nl_langinfo(CODESET); + if (!codeset || codeset[0] == '\0') { + /* CODESET is not set or empty */ + goto error; + } + if (!_Py_normalize_encoding(codeset, encoding, sizeof(encoding))) + goto error; + + is_ascii = 0; + for (alias=ascii_aliases; *alias != NULL; alias++) { + if (strcmp(encoding, *alias) == 0) { + is_ascii = 1; + break; + } + } + if (!is_ascii) { + /* nl_langinfo(CODESET) is not "ascii" or an alias of ASCII */ + return 0; + } + + for (i=0x80; i<0xff; i++) { + unsigned char ch; + wchar_t wch; + size_t res; + + ch = (unsigned char)i; + res = mbstowcs(&wch, (char*)&ch, 1); + if (res != (size_t)-1) { + /* decoding a non-ASCII character from the locale encoding succeed: + the locale encoding is not ASCII, force ASCII */ + return 1; + } + } + /* None of the bytes in the range 0x80-0xff can be decoded from the locale + encoding: the locale encoding is really ASCII */ + return 0; +#else + /* nl_langinfo(CODESET) is not available: always force ASCII */ + return 1; +#endif + +error: + /* if an error occured, force the ASCII encoding */ + return 1; +} + +static char* +encode_ascii_surrogateescape(const wchar_t *text, size_t *error_pos) +{ + char *result = NULL, *out; + size_t len, i; + wchar_t ch; + + if (error_pos != NULL) + *error_pos = (size_t)-1; + + len = wcslen(text); + + result = PyMem_Malloc(len + 1); /* +1 for NUL byte */ + if (result == NULL) + return NULL; + + out = result; + for (i=0; i 128. This will still roundtrip correctly in the locale's charset, which must be an ASCII superset. */ - res = PyMem_Malloc((strlen(arg)+1)*sizeof(wchar_t)); - if (!res) + res = decode_ascii_surrogateescape(arg, size); + if (res == NULL) goto oom; - in = (unsigned char*)arg; - out = res; - while(*in) - if(*in < 128) - *out++ = *in++; - else - *out++ = 0xdc00 + *in++; - *out = 0; #endif /* HAVE_MBRTOWC */ - if (size != NULL) - *size = out - res; return res; oom: if (size != NULL) @@ -236,6 +419,14 @@ _Py_wchar2char(const wchar_t *text, size_t *error_pos) size_t i, size, converted; wchar_t c, buf[2]; +#ifndef MS_WINDOWS + if (force_ascii == -1) + force_ascii = check_force_ascii(); + + if (force_ascii) + return encode_ascii_surrogateescape(text, error_pos); +#endif + /* The function works in two steps: 1. compute the length of the output buffer in bytes (size) 2. outputs the bytes */ @@ -276,7 +467,7 @@ _Py_wchar2char(const wchar_t *text, size_t *error_pos) } } if (result != NULL) { - *bytes = 0; + *bytes = '\0'; break; } @@ -320,6 +511,8 @@ _Py_wstat(const wchar_t* path, struct stat *buf) } #endif +#ifdef HAVE_STAT + /* Call _wstat() on Windows, or encode the path to the filesystem encoding and call stat() otherwise. Only fill st_mode attribute on Windows. @@ -352,6 +545,8 @@ _Py_stat(PyObject *path, struct stat *statbuf) #endif } +#endif + /* Open a file. Use _wfopen() on Windows, encode the path to the locale encoding and use fopen() otherwise. */ @@ -533,4 +728,3 @@ _Py_wgetcwd(wchar_t *buf, size_t size) #endif } -#endif -- cgit v1.2.1 From 2733e1a2064a96136de915ae8c8f13eb827df900 Mon Sep 17 00:00:00 2001 From: Philip Jenvey Date: Tue, 15 Jan 2013 13:24:12 -0800 Subject: thinko --- Python/fileutils.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'Python/fileutils.c') diff --git a/Python/fileutils.c b/Python/fileutils.c index 9c0833e003..3c04e4969e 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -60,7 +60,7 @@ extern int _Py_normalize_encoding(const char *, char *, size_t); workaround is also enabled on error, for example if getting the locale failed. - Values of locale_is_ascii: + Values of force_ascii: 1: the workaround is used: _Py_wchar2char() uses encode_ascii_surrogateescape() and _Py_char2wchar() uses -- cgit v1.2.1