diff options
Diffstat (limited to 'Objects/unicodeobject.c')
| -rw-r--r-- | Objects/unicodeobject.c | 2486 |
1 files changed, 1292 insertions, 1194 deletions
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 64a3760328..a5ae454b49 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -42,6 +42,7 @@ OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. #include "Python.h" #include "ucnhash.h" #include "bytes_methods.h" +#include "stringlib/eq.h" #ifdef MS_WINDOWS #include <windows.h> @@ -162,6 +163,14 @@ extern "C" { *_to++ = (to_type) *_iter++; \ } while (0) +#ifdef MS_WINDOWS + /* On Windows, overallocate by 50% is the best factor */ +# define OVERALLOCATE_FACTOR 2 +#else + /* On Linux, overallocate by 25% is the best factor */ +# define OVERALLOCATE_FACTOR 4 +#endif + /* This dictionary holds all interned unicode strings. Note that references to strings in this dictionary are *not* counted in the string's ob_refcnt. When the interned string reaches a refcnt of 0 the string deallocation @@ -195,7 +204,7 @@ static PyObject *unicode_empty = NULL; } while (0) /* Forward declaration */ -Py_LOCAL_INLINE(int) +static inline int _PyUnicodeWriter_WriteCharInline(_PyUnicodeWriter *writer, Py_UCS4 ch); /* List of static strings. */ @@ -263,7 +272,7 @@ raise_encode_exception(PyObject **exceptionObject, const char *reason); /* Same for linebreaks */ -static unsigned char ascii_linebreak[] = { +static const unsigned char ascii_linebreak[] = { 0, 0, 0, 0, 0, 0, 0, 0, /* 0x000A, * LINE FEED */ /* 0x000B, * LINE TABULATION */ @@ -292,6 +301,45 @@ static unsigned char ascii_linebreak[] = { #include "clinic/unicodeobject.c.h" +typedef enum { + _Py_ERROR_UNKNOWN=0, + _Py_ERROR_STRICT, + _Py_ERROR_SURROGATEESCAPE, + _Py_ERROR_REPLACE, + _Py_ERROR_IGNORE, + _Py_ERROR_BACKSLASHREPLACE, + _Py_ERROR_SURROGATEPASS, + _Py_ERROR_XMLCHARREFREPLACE, + _Py_ERROR_OTHER +} _Py_error_handler; + +static _Py_error_handler +get_error_handler(const char *errors) +{ + if (errors == NULL || strcmp(errors, "strict") == 0) { + return _Py_ERROR_STRICT; + } + if (strcmp(errors, "surrogateescape") == 0) { + return _Py_ERROR_SURROGATEESCAPE; + } + if (strcmp(errors, "replace") == 0) { + return _Py_ERROR_REPLACE; + } + if (strcmp(errors, "ignore") == 0) { + return _Py_ERROR_IGNORE; + } + if (strcmp(errors, "backslashreplace") == 0) { + return _Py_ERROR_BACKSLASHREPLACE; + } + if (strcmp(errors, "surrogatepass") == 0) { + return _Py_ERROR_SURROGATEPASS; + } + if (strcmp(errors, "xmlcharrefreplace") == 0) { + return _Py_ERROR_XMLCHARREFREPLACE; + } + return _Py_ERROR_OTHER; +} + /* The max unicode value is always 0x10FFFF while using the PEP-393 API. This function is kept for backward compatibility with the old API. */ Py_UNICODE @@ -521,6 +569,129 @@ unicode_result_unchanged(PyObject *unicode) return _PyUnicode_Copy(unicode); } +/* Implementation of the "backslashreplace" error handler for 8-bit encodings: + ASCII, Latin1, UTF-8, etc. */ +static char* +backslashreplace(_PyBytesWriter *writer, char *str, + PyObject *unicode, Py_ssize_t collstart, Py_ssize_t collend) +{ + Py_ssize_t size, i; + Py_UCS4 ch; + enum PyUnicode_Kind kind; + void *data; + + assert(PyUnicode_IS_READY(unicode)); + kind = PyUnicode_KIND(unicode); + data = PyUnicode_DATA(unicode); + + size = 0; + /* determine replacement size */ + for (i = collstart; i < collend; ++i) { + Py_ssize_t incr; + + ch = PyUnicode_READ(kind, data, i); + if (ch < 0x100) + incr = 2+2; + else if (ch < 0x10000) + incr = 2+4; + else { + assert(ch <= MAX_UNICODE); + incr = 2+8; + } + if (size > PY_SSIZE_T_MAX - incr) { + PyErr_SetString(PyExc_OverflowError, + "encoded result is too long for a Python string"); + return NULL; + } + size += incr; + } + + str = _PyBytesWriter_Prepare(writer, str, size); + if (str == NULL) + return NULL; + + /* generate replacement */ + for (i = collstart; i < collend; ++i) { + ch = PyUnicode_READ(kind, data, i); + *str++ = '\\'; + if (ch >= 0x00010000) { + *str++ = 'U'; + *str++ = Py_hexdigits[(ch>>28)&0xf]; + *str++ = Py_hexdigits[(ch>>24)&0xf]; + *str++ = Py_hexdigits[(ch>>20)&0xf]; + *str++ = Py_hexdigits[(ch>>16)&0xf]; + *str++ = Py_hexdigits[(ch>>12)&0xf]; + *str++ = Py_hexdigits[(ch>>8)&0xf]; + } + else if (ch >= 0x100) { + *str++ = 'u'; + *str++ = Py_hexdigits[(ch>>12)&0xf]; + *str++ = Py_hexdigits[(ch>>8)&0xf]; + } + else + *str++ = 'x'; + *str++ = Py_hexdigits[(ch>>4)&0xf]; + *str++ = Py_hexdigits[ch&0xf]; + } + return str; +} + +/* Implementation of the "xmlcharrefreplace" error handler for 8-bit encodings: + ASCII, Latin1, UTF-8, etc. */ +static char* +xmlcharrefreplace(_PyBytesWriter *writer, char *str, + PyObject *unicode, Py_ssize_t collstart, Py_ssize_t collend) +{ + Py_ssize_t size, i; + Py_UCS4 ch; + enum PyUnicode_Kind kind; + void *data; + + assert(PyUnicode_IS_READY(unicode)); + kind = PyUnicode_KIND(unicode); + data = PyUnicode_DATA(unicode); + + size = 0; + /* determine replacement size */ + for (i = collstart; i < collend; ++i) { + Py_ssize_t incr; + + ch = PyUnicode_READ(kind, data, i); + if (ch < 10) + incr = 2+1+1; + else if (ch < 100) + incr = 2+2+1; + else if (ch < 1000) + incr = 2+3+1; + else if (ch < 10000) + incr = 2+4+1; + else if (ch < 100000) + incr = 2+5+1; + else if (ch < 1000000) + incr = 2+6+1; + else { + assert(ch <= MAX_UNICODE); + incr = 2+7+1; + } + if (size > PY_SSIZE_T_MAX - incr) { + PyErr_SetString(PyExc_OverflowError, + "encoded result is too long for a Python string"); + return NULL; + } + size += incr; + } + + str = _PyBytesWriter_Prepare(writer, str, size); + if (str == NULL) + return NULL; + + /* generate replacement */ + for (i = collstart; i < collend; ++i) { + str += sprintf(str, "&#%d;", PyUnicode_READ(kind, data, i)); + } + return str; +} + /* --- Bloom Filters ----------------------------------------------------- */ /* stuff to implement simple "bloom filters" for Unicode characters. @@ -549,7 +720,7 @@ static BLOOM_MASK bloom_linebreak = ~(BLOOM_MASK)0; ((ch) < 128U ? ascii_linebreak[(ch)] : \ (BLOOM(bloom_linebreak, (ch)) && Py_UNICODE_ISLINEBREAK(ch))) -Py_LOCAL_INLINE(BLOOM_MASK) +static inline BLOOM_MASK make_bloom_mask(int kind, void* ptr, Py_ssize_t len) { #define BLOOM_UPDATE(TYPE, MASK, PTR, LEN) \ @@ -587,6 +758,18 @@ make_bloom_mask(int kind, void* ptr, Py_ssize_t len) #undef BLOOM_UPDATE } +static int +ensure_unicode(PyObject *obj) +{ + if (!PyUnicode_Check(obj)) { + PyErr_Format(PyExc_TypeError, + "must be str, not %.100s", + Py_TYPE(obj)->tp_name); + return -1; + } + return PyUnicode_READY(obj); +} + /* Compilation of templated routines */ #include "stringlib/asciilib.h" @@ -643,31 +826,31 @@ make_bloom_mask(int kind, void* ptr, Py_ssize_t len) static PyObject * fixup(PyObject *self, Py_UCS4 (*fixfct)(PyObject *s)); -Py_LOCAL_INLINE(Py_ssize_t) findchar(const void *s, int kind, - Py_ssize_t size, Py_UCS4 ch, - int direction) +static inline Py_ssize_t +findchar(const void *s, int kind, + Py_ssize_t size, Py_UCS4 ch, + int direction) { - int mode = (direction == 1) ? FAST_SEARCH : FAST_RSEARCH; - switch (kind) { case PyUnicode_1BYTE_KIND: - { - Py_UCS1 ch1 = (Py_UCS1) ch; - if (ch1 == ch) - return ucs1lib_fastsearch((Py_UCS1 *) s, size, &ch1, 1, 0, mode); - else - return -1; - } + if ((Py_UCS1) ch != ch) + return -1; + if (direction > 0) + return ucs1lib_find_char((Py_UCS1 *) s, size, (Py_UCS1) ch); + else + return ucs1lib_rfind_char((Py_UCS1 *) s, size, (Py_UCS1) ch); case PyUnicode_2BYTE_KIND: - { - Py_UCS2 ch2 = (Py_UCS2) ch; - if (ch2 == ch) - return ucs2lib_fastsearch((Py_UCS2 *) s, size, &ch2, 1, 0, mode); - else - return -1; - } + if ((Py_UCS2) ch != ch) + return -1; + if (direction > 0) + return ucs2lib_find_char((Py_UCS2 *) s, size, (Py_UCS2) ch); + else + return ucs2lib_rfind_char((Py_UCS2 *) s, size, (Py_UCS2) ch); case PyUnicode_4BYTE_KIND: - return ucs4lib_fastsearch((Py_UCS4 *) s, size, &ch, 1, 0, mode); + if (direction > 0) + return ucs4lib_find_char((Py_UCS4 *) s, size, ch); + else + return ucs4lib_rfind_char((Py_UCS4 *) s, size, ch); default: assert(0); return -1; @@ -865,7 +1048,7 @@ resize_copy(PyObject *unicode, Py_ssize_t length) return NULL; copy_length = _PyUnicode_WSTR_LENGTH(unicode); copy_length = Py_MIN(copy_length, length); - Py_MEMCPY(_PyUnicode_WSTR(w), _PyUnicode_WSTR(unicode), + memcpy(_PyUnicode_WSTR(w), _PyUnicode_WSTR(unicode), copy_length * sizeof(wchar_t)); return w; } @@ -1252,7 +1435,7 @@ _copy_characters(PyObject *to, Py_ssize_t to_start, if (max_char >= 128) return -1; } - Py_MEMCPY((char*)to_data + to_kind * to_start, + memcpy((char*)to_data + to_kind * to_start, (char*)from_data + from_kind * from_start, to_kind * how_many); } @@ -1845,7 +2028,7 @@ PyUnicode_FromUnicode(const Py_UNICODE *u, Py_ssize_t size) break; case PyUnicode_2BYTE_KIND: #if Py_UNICODE_SIZE == 2 - Py_MEMCPY(PyUnicode_2BYTE_DATA(unicode), u, size * 2); + memcpy(PyUnicode_2BYTE_DATA(unicode), u, size * 2); #else _PyUnicode_CONVERT_BYTES(Py_UNICODE, Py_UCS2, u, u + size, PyUnicode_2BYTE_DATA(unicode)); @@ -1858,7 +2041,7 @@ PyUnicode_FromUnicode(const Py_UNICODE *u, Py_ssize_t size) unicode_convert_wchar_to_ucs4(u, u + size, unicode); #else assert(num_surrogates == 0); - Py_MEMCPY(PyUnicode_4BYTE_DATA(unicode), u, size * 4); + memcpy(PyUnicode_4BYTE_DATA(unicode), u, size * 4); #endif break; default: @@ -1960,7 +2143,7 @@ kind_maxchar_limit(unsigned int kind) } } -Py_LOCAL_INLINE(Py_UCS4) +static inline Py_UCS4 align_maxchar(Py_UCS4 maxchar) { if (maxchar <= 127) @@ -2169,7 +2352,7 @@ _PyUnicode_Copy(PyObject *unicode) return NULL; assert(PyUnicode_KIND(copy) == PyUnicode_KIND(unicode)); - Py_MEMCPY(PyUnicode_DATA(copy), PyUnicode_DATA(unicode), + memcpy(PyUnicode_DATA(copy), PyUnicode_DATA(unicode), length * PyUnicode_KIND(unicode)); assert(_PyUnicode_CheckConsistency(copy, 1)); return copy; @@ -2275,7 +2458,7 @@ as_ucs4(PyObject *string, Py_UCS4 *target, Py_ssize_t targetsize, } else { assert(kind == PyUnicode_4BYTE_KIND); - Py_MEMCPY(target, data, len * sizeof(Py_UCS4)); + memcpy(target, data, len * sizeof(Py_UCS4)); } if (copy_null) target[len] = 0; @@ -2458,13 +2641,11 @@ unicode_fromformat_arg(_PyUnicodeWriter *writer, longflag = 1; ++f; } -#ifdef HAVE_LONG_LONG else if (f[1] == 'l' && (f[2] == 'd' || f[2] == 'u' || f[2] == 'i')) { longlongflag = 1; f += 2; } -#endif } /* handle the size_t flag. */ else if (*f == 'z' && (f[1] == 'd' || f[1] == 'u' || f[1] == 'i')) { @@ -2502,11 +2683,9 @@ unicode_fromformat_arg(_PyUnicodeWriter *writer, if (longflag) len = sprintf(buffer, "%lu", va_arg(*vargs, unsigned long)); -#ifdef HAVE_LONG_LONG else if (longlongflag) - len = sprintf(buffer, "%" PY_FORMAT_LONG_LONG "u", - va_arg(*vargs, unsigned PY_LONG_LONG)); -#endif + len = sprintf(buffer, "%llu", + va_arg(*vargs, unsigned long long)); else if (size_tflag) len = sprintf(buffer, "%" PY_FORMAT_SIZE_T "u", va_arg(*vargs, size_t)); @@ -2521,11 +2700,9 @@ unicode_fromformat_arg(_PyUnicodeWriter *writer, if (longflag) len = sprintf(buffer, "%li", va_arg(*vargs, long)); -#ifdef HAVE_LONG_LONG else if (longlongflag) - len = sprintf(buffer, "%" PY_FORMAT_LONG_LONG "i", - va_arg(*vargs, PY_LONG_LONG)); -#endif + len = sprintf(buffer, "%lli", + va_arg(*vargs, long long)); else if (size_tflag) len = sprintf(buffer, "%" PY_FORMAT_SIZE_T "i", va_arg(*vargs, Py_ssize_t)); @@ -2701,9 +2878,8 @@ PyUnicode_FromFormatV(const char *format, va_list vargs) writer.min_length = strlen(format) + 100; writer.overallocate = 1; - /* va_list may be an array (of 1 item) on some platforms (ex: AMD64). - Copy it to be able to pass a reference to a subfunction. */ - Py_VA_COPY(vargs2, vargs); + // Copy varags to be able to pass a reference to a subfunction. + va_copy(vargs2, vargs); for (f = format; *f; ) { if (*f == '%') { @@ -2739,9 +2915,11 @@ PyUnicode_FromFormatV(const char *format, va_list vargs) f = p; } } + va_end(vargs2); return _PyUnicodeWriter_Finish(&writer); fail: + va_end(vargs2); _PyUnicodeWriter_Dealloc(&writer); return NULL; } @@ -2790,7 +2968,7 @@ unicode_aswidechar(PyObject *unicode, size = res + 1; else res = size; - Py_MEMCPY(w, wstr, size * sizeof(wchar_t)); + memcpy(w, wstr, size * sizeof(wchar_t)); return res; } else @@ -2907,7 +3085,7 @@ PyUnicode_FromEncodedObject(PyObject *obj, /* Retrieve a bytes buffer view through the PEP 3118 buffer interface */ if (PyObject_GetBuffer(obj, &buffer, PyBUF_SIMPLE) < 0) { PyErr_Format(PyExc_TypeError, - "coercing to str: need a bytes-like object, %.80s found", + "decoding to str: need a bytes-like object, %.80s found", Py_TYPE(obj)->tp_name); return NULL; } @@ -2922,9 +3100,9 @@ PyUnicode_FromEncodedObject(PyObject *obj, return v; } -/* Convert encoding to lower case and replace '_' with '-' in order to - catch e.g. UTF_8. Return 0 on error (encoding is longer than lower_len-1), - 1 on success. */ +/* Normalize an encoding name: similar to encodings.normalize_encoding(), but + also convert to lowercase. Return 1 on success, or 0 on error (encoding is + longer than lower_len-1). */ int _Py_normalize_encoding(const char *encoding, char *lower, @@ -2933,30 +3111,39 @@ _Py_normalize_encoding(const char *encoding, const char *e; char *l; char *l_end; + int punct; + + assert(encoding != NULL); - if (encoding == NULL) { - /* 6 == strlen("utf-8") + 1 */ - if (lower_len < 6) - return 0; - strcpy(lower, "utf-8"); - return 1; - } e = encoding; l = lower; l_end = &lower[lower_len - 1]; - while (*e) { - if (l == l_end) - return 0; - if (Py_ISUPPER(*e)) { - *l++ = Py_TOLOWER(*e++); + punct = 0; + while (1) { + char c = *e; + if (c == 0) { + break; } - else if (*e == '_') { - *l++ = '-'; - e++; + + if (Py_ISALNUM(c) || c == '.') { + if (punct && l != lower) { + if (l == l_end) { + return 0; + } + *l++ = '_'; + } + punct = 0; + + if (l == l_end) { + return 0; + } + *l++ = Py_TOLOWER(c); } else { - *l++ = *e++; + punct = 1; } + + e++; } *l = '\0'; return 1; @@ -2970,28 +3157,51 @@ PyUnicode_Decode(const char *s, { PyObject *buffer = NULL, *unicode; Py_buffer info; - char lower[11]; /* Enough for any encoding shortcut */ + char buflower[11]; /* strlen("iso-8859-1\0") == 11, longest shortcut */ + + if (encoding == NULL) { + return PyUnicode_DecodeUTF8Stateful(s, size, errors, NULL); + } /* Shortcuts for common default encodings */ - if (_Py_normalize_encoding(encoding, lower, sizeof(lower))) { - if ((strcmp(lower, "utf-8") == 0) || - (strcmp(lower, "utf8") == 0)) - return PyUnicode_DecodeUTF8Stateful(s, size, errors, NULL); - else if ((strcmp(lower, "latin-1") == 0) || - (strcmp(lower, "latin1") == 0) || - (strcmp(lower, "iso-8859-1") == 0) || - (strcmp(lower, "iso8859-1") == 0)) - return PyUnicode_DecodeLatin1(s, size, errors); -#ifdef HAVE_MBCS - else if (strcmp(lower, "mbcs") == 0) - return PyUnicode_DecodeMBCS(s, size, errors); -#endif - else if (strcmp(lower, "ascii") == 0) - return PyUnicode_DecodeASCII(s, size, errors); - else if (strcmp(lower, "utf-16") == 0) - return PyUnicode_DecodeUTF16(s, size, errors, 0); - else if (strcmp(lower, "utf-32") == 0) - return PyUnicode_DecodeUTF32(s, size, errors, 0); + if (_Py_normalize_encoding(encoding, buflower, sizeof(buflower))) { + char *lower = buflower; + + /* Fast paths */ + if (lower[0] == 'u' && lower[1] == 't' && lower[2] == 'f') { + lower += 3; + if (*lower == '_') { + /* Match "utf8" and "utf_8" */ + lower++; + } + + if (lower[0] == '8' && lower[1] == 0) { + return PyUnicode_DecodeUTF8Stateful(s, size, errors, NULL); + } + else if (lower[0] == '1' && lower[1] == '6' && lower[2] == 0) { + return PyUnicode_DecodeUTF16(s, size, errors, 0); + } + else if (lower[0] == '3' && lower[1] == '2' && lower[2] == 0) { + return PyUnicode_DecodeUTF32(s, size, errors, 0); + } + } + else { + if (strcmp(lower, "ascii") == 0 + || strcmp(lower, "us_ascii") == 0) { + return PyUnicode_DecodeASCII(s, size, errors); + } + #ifdef MS_WINDOWS + else if (strcmp(lower, "mbcs") == 0) { + return PyUnicode_DecodeMBCS(s, size, errors); + } + #endif + else if (strcmp(lower, "latin1") == 0 + || strcmp(lower, "latin_1") == 0 + || strcmp(lower, "iso_8859_1") == 0 + || strcmp(lower, "iso8859_1") == 0) { + return PyUnicode_DecodeLatin1(s, size, errors); + } + } } /* Decode via the codec registry */ @@ -3031,6 +3241,11 @@ PyUnicode_AsDecodedObject(PyObject *unicode, return NULL; } + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "PyUnicode_AsDecodedObject() is deprecated; " + "use PyCodec_Decode() to decode from str", 1) < 0) + return NULL; + if (encoding == NULL) encoding = PyUnicode_GetDefaultEncoding(); @@ -3050,6 +3265,11 @@ PyUnicode_AsDecodedUnicode(PyObject *unicode, goto onError; } + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "PyUnicode_AsDecodedUnicode() is deprecated; " + "use PyCodec_Decode() to decode from str to str", 1) < 0) + return NULL; + if (encoding == NULL) encoding = PyUnicode_GetDefaultEncoding(); @@ -3100,6 +3320,12 @@ PyUnicode_AsEncodedObject(PyObject *unicode, goto onError; } + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "PyUnicode_AsEncodedObject() is deprecated; " + "use PyUnicode_AsEncodedString() to encode from str to bytes " + "or PyCodec_Encode() for generic encoding", 1) < 0) + return NULL; + if (encoding == NULL) encoding = PyUnicode_GetDefaultEncoding(); @@ -3163,24 +3389,22 @@ wcstombs_errorpos(const wchar_t *wstr) static int locale_error_handler(const char *errors, int *surrogateescape) { - if (errors == NULL) { - *surrogateescape = 0; - return 0; - } - - if (strcmp(errors, "strict") == 0) { + _Py_error_handler error_handler = get_error_handler(errors); + switch (error_handler) + { + case _Py_ERROR_STRICT: *surrogateescape = 0; return 0; - } - if (strcmp(errors, "surrogateescape") == 0) { + case _Py_ERROR_SURROGATEESCAPE: *surrogateescape = 1; return 0; + default: + PyErr_Format(PyExc_ValueError, + "only 'strict' and 'surrogateescape' error handlers " + "are supported, not '%s'", + errors); + return -1; } - PyErr_Format(PyExc_ValueError, - "only 'strict' and 'surrogateescape' error handlers " - "are supported, not '%s'", - errors); - return -1; } PyObject * @@ -3296,10 +3520,8 @@ encode_error: PyObject * PyUnicode_EncodeFSDefault(PyObject *unicode) { -#ifdef HAVE_MBCS - return PyUnicode_EncodeCodePage(CP_ACP, unicode, NULL); -#elif defined(__APPLE__) - return _PyUnicode_AsUTF8String(unicode, "surrogateescape"); +#if defined(__APPLE__) + return _PyUnicode_AsUTF8String(unicode, Py_FileSystemDefaultEncodeErrors); #else PyInterpreterState *interp = PyThreadState_GET()->interp; /* Bootstrap check: if the filesystem codec is implemented in Python, we @@ -3314,10 +3536,10 @@ PyUnicode_EncodeFSDefault(PyObject *unicode) if (Py_FileSystemDefaultEncoding && interp->fscodec_initialized) { return PyUnicode_AsEncodedString(unicode, Py_FileSystemDefaultEncoding, - "surrogateescape"); + Py_FileSystemDefaultEncodeErrors); } else { - return PyUnicode_EncodeLocale(unicode, "surrogateescape"); + return PyUnicode_EncodeLocale(unicode, Py_FileSystemDefaultEncodeErrors); } #endif } @@ -3328,34 +3550,56 @@ PyUnicode_AsEncodedString(PyObject *unicode, const char *errors) { PyObject *v; - char lower[11]; /* Enough for any encoding shortcut */ + char buflower[11]; /* strlen("iso_8859_1\0") == 11, longest shortcut */ if (!PyUnicode_Check(unicode)) { PyErr_BadArgument(); return NULL; } + if (encoding == NULL) { + return _PyUnicode_AsUTF8String(unicode, errors); + } + /* Shortcuts for common default encodings */ - if (_Py_normalize_encoding(encoding, lower, sizeof(lower))) { - if ((strcmp(lower, "utf-8") == 0) || - (strcmp(lower, "utf8") == 0)) - { - if (errors == NULL || strcmp(errors, "strict") == 0) - return _PyUnicode_AsUTF8String(unicode, NULL); - else + if (_Py_normalize_encoding(encoding, buflower, sizeof(buflower))) { + char *lower = buflower; + + /* Fast paths */ + if (lower[0] == 'u' && lower[1] == 't' && lower[2] == 'f') { + lower += 3; + if (*lower == '_') { + /* Match "utf8" and "utf_8" */ + lower++; + } + + if (lower[0] == '8' && lower[1] == 0) { return _PyUnicode_AsUTF8String(unicode, errors); + } + else if (lower[0] == '1' && lower[1] == '6' && lower[2] == 0) { + return _PyUnicode_EncodeUTF16(unicode, errors, 0); + } + else if (lower[0] == '3' && lower[1] == '2' && lower[2] == 0) { + return _PyUnicode_EncodeUTF32(unicode, errors, 0); + } } - else if ((strcmp(lower, "latin-1") == 0) || - (strcmp(lower, "latin1") == 0) || - (strcmp(lower, "iso-8859-1") == 0) || - (strcmp(lower, "iso8859-1") == 0)) - return _PyUnicode_AsLatin1String(unicode, errors); -#ifdef HAVE_MBCS - else if (strcmp(lower, "mbcs") == 0) - return PyUnicode_EncodeCodePage(CP_ACP, unicode, errors); + else { + if (strcmp(lower, "ascii") == 0 + || strcmp(lower, "us_ascii") == 0) { + return _PyUnicode_AsASCIIString(unicode, errors); + } +#ifdef MS_WINDOWS + else if (strcmp(lower, "mbcs") == 0) { + return PyUnicode_EncodeCodePage(CP_ACP, unicode, errors); + } #endif - else if (strcmp(lower, "ascii") == 0) - return _PyUnicode_AsASCIIString(unicode, errors); + else if (strcmp(lower, "latin1") == 0 || + strcmp(lower, "latin_1") == 0 || + strcmp(lower, "iso_8859_1") == 0 || + strcmp(lower, "iso8859_1") == 0) { + return _PyUnicode_AsLatin1String(unicode, errors); + } + } } /* Encode via the codec registry */ @@ -3407,6 +3651,11 @@ PyUnicode_AsEncodedUnicode(PyObject *unicode, goto onError; } + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "PyUnicode_AsEncodedUnicode() is deprecated; " + "use PyCodec_Encode() to encode from str to str", 1) < 0) + return NULL; + if (encoding == NULL) encoding = PyUnicode_GetDefaultEncoding(); @@ -3580,10 +3829,8 @@ PyUnicode_DecodeFSDefault(const char *s) { PyObject* PyUnicode_DecodeFSDefaultAndSize(const char *s, Py_ssize_t size) { -#ifdef HAVE_MBCS - return PyUnicode_DecodeMBCS(s, size, NULL); -#elif defined(__APPLE__) - return PyUnicode_DecodeUTF8Stateful(s, size, "surrogateescape", NULL); +#if defined(__APPLE__) + return PyUnicode_DecodeUTF8Stateful(s, size, Py_FileSystemDefaultEncodeErrors, NULL); #else PyInterpreterState *interp = PyThreadState_GET()->interp; /* Bootstrap check: if the filesystem codec is implemented in Python, we @@ -3598,10 +3845,10 @@ PyUnicode_DecodeFSDefaultAndSize(const char *s, Py_ssize_t size) if (Py_FileSystemDefaultEncoding && interp->fscodec_initialized) { return PyUnicode_Decode(s, size, Py_FileSystemDefaultEncoding, - "surrogateescape"); + Py_FileSystemDefaultEncodeErrors); } else { - return PyUnicode_DecodeLocaleAndSize(s, size, "surrogateescape"); + return PyUnicode_DecodeLocaleAndSize(s, size, Py_FileSystemDefaultEncodeErrors); } #endif } @@ -3610,6 +3857,7 @@ PyUnicode_DecodeFSDefaultAndSize(const char *s, Py_ssize_t size) int PyUnicode_FSConverter(PyObject* arg, void* addr) { + PyObject *path = NULL; PyObject *output = NULL; Py_ssize_t size; void *data; @@ -3618,24 +3866,22 @@ PyUnicode_FSConverter(PyObject* arg, void* addr) *(PyObject**)addr = NULL; return 1; } - if (PyBytes_Check(arg)) { - output = arg; - Py_INCREF(output); + path = PyOS_FSPath(arg); + if (path == NULL) { + return 0; } - else { - arg = PyUnicode_FromObject(arg); - if (!arg) - return 0; - output = PyUnicode_EncodeFSDefault(arg); - Py_DECREF(arg); - if (!output) - return 0; - if (!PyBytes_Check(output)) { - Py_DECREF(output); - PyErr_SetString(PyExc_TypeError, "encoder failed to return bytes"); + if (PyBytes_Check(path)) { + output = path; + } + else { // PyOS_FSPath() guarantees its returned value is bytes or str. + output = PyUnicode_EncodeFSDefault(path); + Py_DECREF(path); + if (!output) { return 0; } + assert(PyBytes_Check(output)); } + size = PyBytes_GET_SIZE(output); data = PyBytes_AS_STRING(output); if ((size_t)size != strlen(data)) { @@ -3651,36 +3897,60 @@ PyUnicode_FSConverter(PyObject* arg, void* addr) int PyUnicode_FSDecoder(PyObject* arg, void* addr) { + int is_buffer = 0; + PyObject *path = NULL; PyObject *output = NULL; if (arg == NULL) { Py_DECREF(*(PyObject**)addr); return 1; } - if (PyUnicode_Check(arg)) { - if (PyUnicode_READY(arg) == -1) + + is_buffer = PyObject_CheckBuffer(arg); + if (!is_buffer) { + path = PyOS_FSPath(arg); + if (path == NULL) { return 0; - output = arg; - Py_INCREF(output); + } } - else if (PyObject_CheckBuffer(arg)) { - arg = PyBytes_FromObject(arg); - if (!arg) + else { + path = arg; + Py_INCREF(arg); + } + + if (PyUnicode_Check(path)) { + if (PyUnicode_READY(path) == -1) { + Py_DECREF(path); return 0; - output = PyUnicode_DecodeFSDefaultAndSize(PyBytes_AS_STRING(arg), - PyBytes_GET_SIZE(arg)); - Py_DECREF(arg); - if (!output) + } + output = path; + } + else if (PyBytes_Check(path) || is_buffer) { + PyObject *path_bytes = NULL; + + if (!PyBytes_Check(path) && + PyErr_WarnFormat(PyExc_DeprecationWarning, 1, + "path should be string, bytes, or os.PathLike, not %.200s", + Py_TYPE(arg)->tp_name)) { + Py_DECREF(path); return 0; - if (!PyUnicode_Check(output)) { - Py_DECREF(output); - PyErr_SetString(PyExc_TypeError, "decoder failed to return unicode"); + } + path_bytes = PyBytes_FromObject(path); + Py_DECREF(path); + if (!path_bytes) { + return 0; + } + output = PyUnicode_DecodeFSDefaultAndSize(PyBytes_AS_STRING(path_bytes), + PyBytes_GET_SIZE(path_bytes)); + Py_DECREF(path_bytes); + if (!output) { return 0; } } else { PyErr_Format(PyExc_TypeError, - "path should be string or bytes, not %.200s", + "path should be string, bytes, or os.PathLike, not %.200s", Py_TYPE(arg)->tp_name); + Py_DECREF(path); return 0; } if (PyUnicode_READY(output) == -1) { @@ -3712,7 +3982,7 @@ PyUnicode_AsUTF8AndSize(PyObject *unicode, Py_ssize_t *psize) if (PyUnicode_UTF8(unicode) == NULL) { assert(!PyUnicode_IS_COMPACT_ASCII(unicode)); - bytes = _PyUnicode_AsUTF8String(unicode, "strict"); + bytes = _PyUnicode_AsUTF8String(unicode, NULL); if (bytes == NULL) return NULL; _PyUnicode_UTF8(unicode) = PyObject_MALLOC(PyBytes_GET_SIZE(bytes) + 1); @@ -3722,7 +3992,7 @@ PyUnicode_AsUTF8AndSize(PyObject *unicode, Py_ssize_t *psize) return NULL; } _PyUnicode_UTF8_LENGTH(unicode) = PyBytes_GET_SIZE(bytes); - Py_MEMCPY(_PyUnicode_UTF8(unicode), + memcpy(_PyUnicode_UTF8(unicode), PyBytes_AS_STRING(bytes), _PyUnicode_UTF8_LENGTH(unicode) + 1); Py_DECREF(bytes); @@ -3962,7 +4232,7 @@ onError: Py_CLEAR(*exceptionObject); } -#ifdef HAVE_MBCS +#ifdef MS_WINDOWS /* error handling callback helper: build arguments, call the callback and check the arguments, if no exception occurred, copy the replacement to the output @@ -3978,7 +4248,7 @@ unicode_decode_call_errorhandler_wchar( Py_ssize_t *endinpos, PyObject **exceptionObject, const char **inptr, PyObject **output, Py_ssize_t *outpos) { - static char *argparse = "O!n;decoding error handler must return (str, int) tuple"; + static const char *argparse = "O!n;decoding error handler must return (str, int) tuple"; PyObject *restuple = NULL; PyObject *repunicode = NULL; @@ -4076,7 +4346,7 @@ unicode_decode_call_errorhandler_wchar( Py_XDECREF(restuple); return -1; } -#endif /* HAVE_MBCS */ +#endif /* MS_WINDOWS */ static int unicode_decode_call_errorhandler_writer( @@ -4086,7 +4356,7 @@ unicode_decode_call_errorhandler_writer( Py_ssize_t *endinpos, PyObject **exceptionObject, const char **inptr, _PyUnicodeWriter *writer /* PyObject **output, Py_ssize_t *outpos */) { - static char *argparse = "O!n;decoding error handler must return (str, int) tuple"; + static const char *argparse = "O!n;decoding error handler must return (str, int) tuple"; PyObject *restuple = NULL; PyObject *repunicode = NULL; @@ -4692,8 +4962,9 @@ PyUnicode_DecodeUTF8Stateful(const char *s, Py_ssize_t startinpos; Py_ssize_t endinpos; const char *errmsg = ""; - PyObject *errorHandler = NULL; + PyObject *error_handler_obj = NULL; PyObject *exc = NULL; + _Py_error_handler error_handler = _Py_ERROR_UNKNOWN; if (size == 0) { if (consumed) @@ -4718,6 +4989,7 @@ PyUnicode_DecodeUTF8Stateful(const char *s, while (s < end) { Py_UCS4 ch; int kind = writer.kind; + if (kind == PyUnicode_1BYTE_KIND) { if (PyUnicode_IS_ASCII(writer.buffer)) ch = asciilib_utf8_decode(&s, end, writer.data, &writer.pos); @@ -4756,33 +5028,65 @@ PyUnicode_DecodeUTF8Stateful(const char *s, continue; } - if (unicode_decode_call_errorhandler_writer( - errors, &errorHandler, - "utf-8", errmsg, - &starts, &end, &startinpos, &endinpos, &exc, &s, - &writer)) - goto onError; + if (error_handler == _Py_ERROR_UNKNOWN) + error_handler = get_error_handler(errors); + + switch (error_handler) { + case _Py_ERROR_IGNORE: + s += (endinpos - startinpos); + break; + + case _Py_ERROR_REPLACE: + if (_PyUnicodeWriter_WriteCharInline(&writer, 0xfffd) < 0) + goto onError; + s += (endinpos - startinpos); + break; + + case _Py_ERROR_SURROGATEESCAPE: + { + Py_ssize_t i; + + if (_PyUnicodeWriter_PrepareKind(&writer, PyUnicode_2BYTE_KIND) < 0) + goto onError; + for (i=startinpos; i<endinpos; i++) { + ch = (Py_UCS4)(unsigned char)(starts[i]); + PyUnicode_WRITE(writer.kind, writer.data, writer.pos, + ch + 0xdc00); + writer.pos++; + } + s += (endinpos - startinpos); + break; + } + + default: + if (unicode_decode_call_errorhandler_writer( + errors, &error_handler_obj, + "utf-8", errmsg, + &starts, &end, &startinpos, &endinpos, &exc, &s, + &writer)) + goto onError; + } } End: if (consumed) *consumed = s - starts; - Py_XDECREF(errorHandler); + Py_XDECREF(error_handler_obj); Py_XDECREF(exc); return _PyUnicodeWriter_Finish(&writer); onError: - Py_XDECREF(errorHandler); + Py_XDECREF(error_handler_obj); Py_XDECREF(exc); _PyUnicodeWriter_Dealloc(&writer); return NULL; } -#ifdef __APPLE__ +#if defined(__APPLE__) || defined(__ANDROID__) /* Simplified UTF-8 decoder using surrogateescape error handler, - used to decode the command line arguments on Mac OS X. + used to decode the command line arguments on Mac OS X and Android. Return a pointer to a newly allocated wide character string (use PyMem_RawFree() to free the memory), or NULL on memory allocation error. */ @@ -4833,7 +5137,7 @@ _Py_DecodeUTF8_surrogateescape(const char *s, Py_ssize_t size) return unicode; } -#endif /* __APPLE__ */ +#endif /* __APPLE__ or __ANDROID__ */ /* Primary internal function which creates utf8 encoded bytes objects. @@ -5065,7 +5369,7 @@ _PyUnicode_EncodeUTF32(PyObject *str, const void *data; Py_ssize_t len; PyObject *v; - PY_UINT32_T *out; + uint32_t *out; #if PY_LITTLE_ENDIAN int native_ordering = byteorder <= 0; #else @@ -5096,7 +5400,7 @@ _PyUnicode_EncodeUTF32(PyObject *str, /* output buffer is 4-bytes aligned */ assert(_Py_IS_ALIGNED(PyBytes_AS_STRING(v), 4)); - out = (PY_UINT32_T *)PyBytes_AS_STRING(v); + out = (uint32_t *)PyBytes_AS_STRING(v); if (byteorder == 0) *out++ = 0xFEFF; if (len == 0) @@ -5162,7 +5466,7 @@ _PyUnicode_EncodeUTF32(PyObject *str, /* four bytes are reserved for each surrogate */ if (moreunits > 1) { - Py_ssize_t outpos = out - (PY_UINT32_T*) PyBytes_AS_STRING(v); + Py_ssize_t outpos = out - (uint32_t*) PyBytes_AS_STRING(v); Py_ssize_t morebytes = 4 * (moreunits - 1); if (PyBytes_GET_SIZE(v) > PY_SSIZE_T_MAX - morebytes) { /* integer overflow */ @@ -5171,11 +5475,11 @@ _PyUnicode_EncodeUTF32(PyObject *str, } if (_PyBytes_Resize(&v, PyBytes_GET_SIZE(v) + morebytes) < 0) goto error; - out = (PY_UINT32_T*) PyBytes_AS_STRING(v) + outpos; + out = (uint32_t*) PyBytes_AS_STRING(v) + outpos; } if (PyBytes_Check(rep)) { - Py_MEMCPY(out, PyBytes_AS_STRING(rep), repsize); + memcpy(out, PyBytes_AS_STRING(rep), repsize); out += moreunits; } else /* rep is unicode */ { assert(PyUnicode_KIND(rep) == PyUnicode_1BYTE_KIND); @@ -5426,36 +5730,45 @@ _PyUnicode_EncodeUTF16(PyObject *str, if (kind == PyUnicode_4BYTE_KIND) { const Py_UCS4 *in = (const Py_UCS4 *)data; const Py_UCS4 *end = in + len; - while (in < end) - if (*in++ >= 0x10000) + while (in < end) { + if (*in++ >= 0x10000) { pairs++; + } + } } - if (len > PY_SSIZE_T_MAX / 2 - pairs - (byteorder == 0)) + if (len > PY_SSIZE_T_MAX / 2 - pairs - (byteorder == 0)) { return PyErr_NoMemory(); + } nsize = len + pairs + (byteorder == 0); v = PyBytes_FromStringAndSize(NULL, nsize * 2); - if (v == NULL) + if (v == NULL) { return NULL; + } /* output buffer is 2-bytes aligned */ assert(_Py_IS_ALIGNED(PyBytes_AS_STRING(v), 2)); out = (unsigned short *)PyBytes_AS_STRING(v); - if (byteorder == 0) + if (byteorder == 0) { *out++ = 0xFEFF; - if (len == 0) + } + if (len == 0) { goto done; + } if (kind == PyUnicode_1BYTE_KIND) { ucs1lib_utf16_encode((const Py_UCS1 *)data, len, &out, native_ordering); goto done; } - if (byteorder < 0) + if (byteorder < 0) { encoding = "utf-16-le"; - else if (byteorder > 0) + } + else if (byteorder > 0) { encoding = "utf-16-be"; - else + } + else { encoding = "utf-16"; + } pos = 0; while (pos < len) { @@ -5518,7 +5831,7 @@ _PyUnicode_EncodeUTF16(PyObject *str, } if (PyBytes_Check(rep)) { - Py_MEMCPY(out, PyBytes_AS_STRING(rep), repsize); + memcpy(out, PyBytes_AS_STRING(rep), repsize); out += moreunits; } else /* rep is unicode */ { assert(PyUnicode_KIND(rep) == PyUnicode_1BYTE_KIND); @@ -5571,281 +5884,228 @@ PyUnicode_AsUTF16String(PyObject *unicode) /* --- Unicode Escape Codec ----------------------------------------------- */ -/* Helper function for PyUnicode_DecodeUnicodeEscape, determines - if all the escapes in the string make it still a valid ASCII string. - Returns -1 if any escapes were found which cause the string to - pop out of ASCII range. Otherwise returns the length of the - required buffer to hold the string. - */ -static Py_ssize_t -length_of_escaped_ascii_string(const char *s, Py_ssize_t size) -{ - const unsigned char *p = (const unsigned char *)s; - const unsigned char *end = p + size; - Py_ssize_t length = 0; - - if (size < 0) - return -1; - - for (; p < end; ++p) { - if (*p > 127) { - /* Non-ASCII */ - return -1; - } - else if (*p != '\\') { - /* Normal character */ - ++length; - } - else { - /* Backslash-escape, check next char */ - ++p; - /* Escape sequence reaches till end of string or - non-ASCII follow-up. */ - if (p >= end || *p > 127) - return -1; - switch (*p) { - case '\n': - /* backslash + \n result in zero characters */ - break; - case '\\': case '\'': case '\"': - case 'b': case 'f': case 't': - case 'n': case 'r': case 'v': case 'a': - ++length; - break; - case '0': case '1': case '2': case '3': - case '4': case '5': case '6': case '7': - case 'x': case 'u': case 'U': case 'N': - /* these do not guarantee ASCII characters */ - return -1; - default: - /* count the backslash + the other character */ - length += 2; - } - } - } - return length; -} - static _PyUnicode_Name_CAPI *ucnhash_CAPI = NULL; PyObject * -PyUnicode_DecodeUnicodeEscape(const char *s, - Py_ssize_t size, - const char *errors) +_PyUnicode_DecodeUnicodeEscape(const char *s, + Py_ssize_t size, + const char *errors, + const char **first_invalid_escape) { const char *starts = s; - Py_ssize_t startinpos; - Py_ssize_t endinpos; _PyUnicodeWriter writer; const char *end; - char* message; - Py_UCS4 chr = 0xffffffff; /* in case 'getcode' messes up */ PyObject *errorHandler = NULL; PyObject *exc = NULL; - Py_ssize_t len; - len = length_of_escaped_ascii_string(s, size); - if (len == 0) - _Py_RETURN_UNICODE_EMPTY(); + // so we can remember if we've seen an invalid escape char or not + *first_invalid_escape = NULL; - /* After length_of_escaped_ascii_string() there are two alternatives, - either the string is pure ASCII with named escapes like \n, etc. - and we determined it's exact size (common case) - or it contains \x, \u, ... escape sequences. then we create a - legacy wchar string and resize it at the end of this function. */ - _PyUnicodeWriter_Init(&writer); - if (len > 0) { - writer.min_length = len; + if (size == 0) { + _Py_RETURN_UNICODE_EMPTY(); } - else { - /* Escaped strings will always be longer than the resulting - Unicode string, so we start with size here and then reduce the - length after conversion to the true value. - (but if the error callback returns a long replacement string - we'll have to allocate more space) */ - writer.min_length = size; + /* Escaped strings will always be longer than the resulting + Unicode string, so we start with size here and then reduce the + length after conversion to the true value. + (but if the error callback returns a long replacement string + we'll have to allocate more space) */ + _PyUnicodeWriter_Init(&writer); + writer.min_length = size; + if (_PyUnicodeWriter_Prepare(&writer, size, 127) < 0) { + goto onError; } - if (size == 0) - return _PyUnicodeWriter_Finish(&writer); end = s + size; - while (s < end) { - unsigned char c; - Py_UCS4 x; - int digits; + unsigned char c = (unsigned char) *s++; + Py_UCS4 ch; + int count; + Py_ssize_t startinpos; + Py_ssize_t endinpos; + const char *message; + +#define WRITE_ASCII_CHAR(ch) \ + do { \ + assert(ch <= 127); \ + assert(writer.pos < writer.size); \ + PyUnicode_WRITE(writer.kind, writer.data, writer.pos++, ch); \ + } while(0) + +#define WRITE_CHAR(ch) \ + do { \ + if (ch <= writer.maxchar) { \ + assert(writer.pos < writer.size); \ + PyUnicode_WRITE(writer.kind, writer.data, writer.pos++, ch); \ + } \ + else if (_PyUnicodeWriter_WriteCharInline(&writer, ch) < 0) { \ + goto onError; \ + } \ + } while(0) /* Non-escape characters are interpreted as Unicode ordinals */ - if (*s != '\\') { - x = (unsigned char)*s; - s++; - if (_PyUnicodeWriter_WriteCharInline(&writer, x) < 0) - goto onError; + if (c != '\\') { + WRITE_CHAR(c); continue; } - startinpos = s-starts; + startinpos = s - starts - 1; /* \ - Escapes */ - s++; - c = *s++; - if (s > end) - c = '\0'; /* Invalid after \ */ + if (s >= end) { + message = "\\ at end of string"; + goto error; + } + c = (unsigned char) *s++; + assert(writer.pos < writer.size); switch (c) { /* \x escapes */ -#define WRITECHAR(ch) \ - do { \ - if (_PyUnicodeWriter_WriteCharInline(&writer, (ch)) < 0) \ - goto onError; \ - } while(0) - - case '\n': break; - case '\\': WRITECHAR('\\'); break; - case '\'': WRITECHAR('\''); break; - case '\"': WRITECHAR('\"'); break; - case 'b': WRITECHAR('\b'); break; + case '\n': continue; + case '\\': WRITE_ASCII_CHAR('\\'); continue; + case '\'': WRITE_ASCII_CHAR('\''); continue; + case '\"': WRITE_ASCII_CHAR('\"'); continue; + case 'b': WRITE_ASCII_CHAR('\b'); continue; /* FF */ - case 'f': WRITECHAR('\014'); break; - case 't': WRITECHAR('\t'); break; - case 'n': WRITECHAR('\n'); break; - case 'r': WRITECHAR('\r'); break; + case 'f': WRITE_ASCII_CHAR('\014'); continue; + case 't': WRITE_ASCII_CHAR('\t'); continue; + case 'n': WRITE_ASCII_CHAR('\n'); continue; + case 'r': WRITE_ASCII_CHAR('\r'); continue; /* VT */ - case 'v': WRITECHAR('\013'); break; + case 'v': WRITE_ASCII_CHAR('\013'); continue; /* BEL, not classic C */ - case 'a': WRITECHAR('\007'); break; + case 'a': WRITE_ASCII_CHAR('\007'); continue; /* \OOO (octal) escapes */ case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': - x = s[-1] - '0'; + ch = c - '0'; if (s < end && '0' <= *s && *s <= '7') { - x = (x<<3) + *s++ - '0'; - if (s < end && '0' <= *s && *s <= '7') - x = (x<<3) + *s++ - '0'; + ch = (ch<<3) + *s++ - '0'; + if (s < end && '0' <= *s && *s <= '7') { + ch = (ch<<3) + *s++ - '0'; + } } - WRITECHAR(x); - break; + WRITE_CHAR(ch); + continue; /* hex escapes */ /* \xXX */ case 'x': - digits = 2; + count = 2; message = "truncated \\xXX escape"; goto hexescape; /* \uXXXX */ case 'u': - digits = 4; + count = 4; message = "truncated \\uXXXX escape"; goto hexescape; /* \UXXXXXXXX */ case 'U': - digits = 8; + count = 8; message = "truncated \\UXXXXXXXX escape"; hexescape: - chr = 0; - if (end - s < digits) { - /* count only hex digits */ - for (; s < end; ++s) { - c = (unsigned char)*s; - if (!Py_ISXDIGIT(c)) - goto error; + for (ch = 0; count && s < end; ++s, --count) { + c = (unsigned char)*s; + ch <<= 4; + if (c >= '0' && c <= '9') { + ch += c - '0'; + } + else if (c >= 'a' && c <= 'f') { + ch += c - ('a' - 10); + } + else if (c >= 'A' && c <= 'F') { + ch += c - ('A' - 10); + } + else { + break; } - goto error; } - for (; digits--; ++s) { - c = (unsigned char)*s; - if (!Py_ISXDIGIT(c)) - goto error; - chr = (chr<<4) & ~0xF; - if (c >= '0' && c <= '9') - chr += c - '0'; - else if (c >= 'a' && c <= 'f') - chr += 10 + c - 'a'; - else - chr += 10 + c - 'A'; + if (count) { + goto error; } - if (chr == 0xffffffff && PyErr_Occurred()) - /* _decoding_error will have already written into the - target buffer. */ - break; - store: - /* when we get here, chr is a 32-bit unicode character */ - message = "illegal Unicode character"; - if (chr > MAX_UNICODE) + + /* when we get here, ch is a 32-bit unicode character */ + if (ch > MAX_UNICODE) { + message = "illegal Unicode character"; goto error; - WRITECHAR(chr); - break; + } + + WRITE_CHAR(ch); + continue; /* \N{name} */ case 'N': - message = "malformed \\N character escape"; if (ucnhash_CAPI == NULL) { /* load the unicode data module */ ucnhash_CAPI = (_PyUnicode_Name_CAPI *)PyCapsule_Import( PyUnicodeData_CAPSULE_NAME, 1); - if (ucnhash_CAPI == NULL) - goto ucnhashError; + if (ucnhash_CAPI == NULL) { + PyErr_SetString( + PyExc_UnicodeError, + "\\N escapes not supported (can't load unicodedata module)" + ); + goto onError; + } } + + message = "malformed \\N character escape"; if (*s == '{') { - const char *start = s+1; + const char *start = ++s; + size_t namelen; /* look for the closing brace */ - while (*s != '}' && s < end) + while (s < end && *s != '}') s++; - if (s > start && s < end && *s == '}') { + namelen = s - start; + if (namelen && s < end) { /* found a name. look it up in the unicode database */ - message = "unknown Unicode character name"; s++; - if (s - start - 1 <= INT_MAX && - ucnhash_CAPI->getcode(NULL, start, (int)(s-start-1), - &chr, 0)) - goto store; + ch = 0xffffffff; /* in case 'getcode' messes up */ + if (namelen <= INT_MAX && + ucnhash_CAPI->getcode(NULL, start, (int)namelen, + &ch, 0)) { + assert(ch <= MAX_UNICODE); + WRITE_CHAR(ch); + continue; + } + message = "unknown Unicode character name"; } } goto error; default: - if (s > end) { - message = "\\ at end of string"; - s--; - goto error; - } - else { - WRITECHAR('\\'); - WRITECHAR((unsigned char)s[-1]); + if (*first_invalid_escape == NULL) { + *first_invalid_escape = s-1; /* Back up one char, since we've + already incremented s. */ } - break; + WRITE_ASCII_CHAR('\\'); + WRITE_CHAR(c); + continue; } - continue; error: endinpos = s-starts; + writer.min_length = end - s + writer.pos; if (unicode_decode_call_errorhandler_writer( errors, &errorHandler, "unicodeescape", message, &starts, &end, &startinpos, &endinpos, &exc, &s, - &writer)) + &writer)) { goto onError; - continue; + } + if (_PyUnicodeWriter_Prepare(&writer, writer.min_length, 127) < 0) { + goto onError; + } + +#undef WRITE_ASCII_CHAR +#undef WRITE_CHAR } -#undef WRITECHAR Py_XDECREF(errorHandler); Py_XDECREF(exc); return _PyUnicodeWriter_Finish(&writer); - ucnhashError: - PyErr_SetString( - PyExc_UnicodeError, - "\\N escapes not supported (can't load unicodedata module)" - ); - _PyUnicodeWriter_Dealloc(&writer); - Py_XDECREF(errorHandler); - Py_XDECREF(exc); - return NULL; - onError: _PyUnicodeWriter_Dealloc(&writer); Py_XDECREF(errorHandler); @@ -5853,12 +6113,28 @@ PyUnicode_DecodeUnicodeEscape(const char *s, return NULL; } -/* Return a Unicode-Escape string version of the Unicode object. - - If quotes is true, the string is enclosed in u"" or u'' quotes as - appropriate. +PyObject * +PyUnicode_DecodeUnicodeEscape(const char *s, + Py_ssize_t size, + const char *errors) +{ + const char *first_invalid_escape; + PyObject *result = _PyUnicode_DecodeUnicodeEscape(s, size, errors, + &first_invalid_escape); + if (result == NULL) + return NULL; + if (first_invalid_escape != NULL) { + if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, + "invalid escape sequence '\\%c'", + *first_invalid_escape) < 0) { + Py_DECREF(result); + return NULL; + } + } + return result; +} -*/ +/* Return a Unicode-Escape string version of the Unicode object. */ PyObject * PyUnicode_AsUnicodeEscapeString(PyObject *unicode) @@ -5866,9 +6142,9 @@ PyUnicode_AsUnicodeEscapeString(PyObject *unicode) Py_ssize_t i, len; PyObject *repr; char *p; - int kind; + enum PyUnicode_Kind kind; void *data; - Py_ssize_t expandsize = 0; + Py_ssize_t expandsize; /* Initial allocation is based on the longest-possible character escape. @@ -5882,60 +6158,70 @@ PyUnicode_AsUnicodeEscapeString(PyObject *unicode) PyErr_BadArgument(); return NULL; } - if (PyUnicode_READY(unicode) == -1) + if (PyUnicode_READY(unicode) == -1) { return NULL; - len = PyUnicode_GET_LENGTH(unicode); - kind = PyUnicode_KIND(unicode); - data = PyUnicode_DATA(unicode); - switch (kind) { - case PyUnicode_1BYTE_KIND: expandsize = 4; break; - case PyUnicode_2BYTE_KIND: expandsize = 6; break; - case PyUnicode_4BYTE_KIND: expandsize = 10; break; } - if (len == 0) + len = PyUnicode_GET_LENGTH(unicode); + if (len == 0) { return PyBytes_FromStringAndSize(NULL, 0); + } - if (len > (PY_SSIZE_T_MAX - 2 - 1) / expandsize) + kind = PyUnicode_KIND(unicode); + data = PyUnicode_DATA(unicode); + /* 4 byte characters can take up 10 bytes, 2 byte characters can take up 6 + bytes, and 1 byte characters 4. */ + expandsize = kind * 2 + 2; + if (len > PY_SSIZE_T_MAX / expandsize) { return PyErr_NoMemory(); - - repr = PyBytes_FromStringAndSize(NULL, - 2 - + expandsize*len - + 1); - if (repr == NULL) + } + repr = PyBytes_FromStringAndSize(NULL, expandsize * len); + if (repr == NULL) { return NULL; + } p = PyBytes_AS_STRING(repr); - for (i = 0; i < len; i++) { Py_UCS4 ch = PyUnicode_READ(kind, data, i); - /* Escape backslashes */ - if (ch == '\\') { - *p++ = '\\'; - *p++ = (char) ch; - continue; - } + /* U+0000-U+00ff range */ + if (ch < 0x100) { + if (ch >= ' ' && ch < 127) { + if (ch != '\\') { + /* Copy printable US ASCII as-is */ + *p++ = (char) ch; + } + /* Escape backslashes */ + else { + *p++ = '\\'; + *p++ = '\\'; + } + } - /* Map 21-bit characters to '\U00xxxxxx' */ - else if (ch >= 0x10000) { - assert(ch <= MAX_UNICODE); - *p++ = '\\'; - *p++ = 'U'; - *p++ = Py_hexdigits[(ch >> 28) & 0x0000000F]; - *p++ = Py_hexdigits[(ch >> 24) & 0x0000000F]; - *p++ = Py_hexdigits[(ch >> 20) & 0x0000000F]; - *p++ = Py_hexdigits[(ch >> 16) & 0x0000000F]; - *p++ = Py_hexdigits[(ch >> 12) & 0x0000000F]; - *p++ = Py_hexdigits[(ch >> 8) & 0x0000000F]; - *p++ = Py_hexdigits[(ch >> 4) & 0x0000000F]; - *p++ = Py_hexdigits[ch & 0x0000000F]; - continue; - } + /* Map special whitespace to '\t', \n', '\r' */ + else if (ch == '\t') { + *p++ = '\\'; + *p++ = 't'; + } + else if (ch == '\n') { + *p++ = '\\'; + *p++ = 'n'; + } + else if (ch == '\r') { + *p++ = '\\'; + *p++ = 'r'; + } - /* Map 16-bit characters to '\uxxxx' */ - if (ch >= 256) { + /* Map non-printable US ASCII and 8-bit characters to '\xHH' */ + else { + *p++ = '\\'; + *p++ = 'x'; + *p++ = Py_hexdigits[(ch >> 4) & 0x000F]; + *p++ = Py_hexdigits[ch & 0x000F]; + } + } + /* U+0100-U+ffff range: Map 16-bit characters to '\uHHHH' */ + else if (ch < 0x10000) { *p++ = '\\'; *p++ = 'u'; *p++ = Py_hexdigits[(ch >> 12) & 0x000F]; @@ -5943,37 +6229,28 @@ PyUnicode_AsUnicodeEscapeString(PyObject *unicode) *p++ = Py_hexdigits[(ch >> 4) & 0x000F]; *p++ = Py_hexdigits[ch & 0x000F]; } + /* U+010000-U+10ffff range: Map 21-bit characters to '\U00HHHHHH' */ + else { - /* Map special whitespace to '\t', \n', '\r' */ - else if (ch == '\t') { - *p++ = '\\'; - *p++ = 't'; - } - else if (ch == '\n') { - *p++ = '\\'; - *p++ = 'n'; - } - else if (ch == '\r') { - *p++ = '\\'; - *p++ = 'r'; - } - - /* Map non-printable US ASCII to '\xhh' */ - else if (ch < ' ' || ch >= 0x7F) { + /* Make sure that the first two digits are zero */ + assert(ch <= MAX_UNICODE && MAX_UNICODE <= 0x10ffff); *p++ = '\\'; - *p++ = 'x'; - *p++ = Py_hexdigits[(ch >> 4) & 0x000F]; - *p++ = Py_hexdigits[ch & 0x000F]; + *p++ = 'U'; + *p++ = '0'; + *p++ = '0'; + *p++ = Py_hexdigits[(ch >> 20) & 0x0000000F]; + *p++ = Py_hexdigits[(ch >> 16) & 0x0000000F]; + *p++ = Py_hexdigits[(ch >> 12) & 0x0000000F]; + *p++ = Py_hexdigits[(ch >> 8) & 0x0000000F]; + *p++ = Py_hexdigits[(ch >> 4) & 0x0000000F]; + *p++ = Py_hexdigits[ch & 0x0000000F]; } - - /* Copy everything else as-is */ - else - *p++ = (char) ch; } assert(p - PyBytes_AS_STRING(repr) > 0); - if (_PyBytes_Resize(&repr, p - PyBytes_AS_STRING(repr)) < 0) + if (_PyBytes_Resize(&repr, p - PyBytes_AS_STRING(repr)) < 0) { return NULL; + } return repr; } @@ -5983,8 +6260,10 @@ PyUnicode_EncodeUnicodeEscape(const Py_UNICODE *s, { PyObject *result; PyObject *tmp = PyUnicode_FromUnicode(s, size); - if (tmp == NULL) + if (tmp == NULL) { return NULL; + } + result = PyUnicode_AsUnicodeEscapeString(tmp); Py_DECREF(tmp); return result; @@ -5998,95 +6277,107 @@ PyUnicode_DecodeRawUnicodeEscape(const char *s, const char *errors) { const char *starts = s; - Py_ssize_t startinpos; - Py_ssize_t endinpos; _PyUnicodeWriter writer; const char *end; - const char *bs; PyObject *errorHandler = NULL; PyObject *exc = NULL; - if (size == 0) + if (size == 0) { _Py_RETURN_UNICODE_EMPTY(); + } /* Escaped strings will always be longer than the resulting Unicode string, so we start with size here and then reduce the length after conversion to the true value. (But decoding error handler might have to resize the string) */ _PyUnicodeWriter_Init(&writer); - writer.min_length = size; + writer.min_length = size; + if (_PyUnicodeWriter_Prepare(&writer, size, 127) < 0) { + goto onError; + } end = s + size; while (s < end) { - unsigned char c; - Py_UCS4 x; - int i; + unsigned char c = (unsigned char) *s++; + Py_UCS4 ch; int count; + Py_ssize_t startinpos; + Py_ssize_t endinpos; + const char *message; + +#define WRITE_CHAR(ch) \ + do { \ + if (ch <= writer.maxchar) { \ + assert(writer.pos < writer.size); \ + PyUnicode_WRITE(writer.kind, writer.data, writer.pos++, ch); \ + } \ + else if (_PyUnicodeWriter_WriteCharInline(&writer, ch) < 0) { \ + goto onError; \ + } \ + } while(0) /* Non-escape characters are interpreted as Unicode ordinals */ - if (*s != '\\') { - x = (unsigned char)*s++; - if (_PyUnicodeWriter_WriteCharInline(&writer, x) < 0) - goto onError; + if (c != '\\' || s >= end) { + WRITE_CHAR(c); continue; } - startinpos = s-starts; - /* \u-escapes are only interpreted iff the number of leading - backslashes if odd */ - bs = s; - for (;s < end;) { - if (*s != '\\') - break; - x = (unsigned char)*s++; - if (_PyUnicodeWriter_WriteCharInline(&writer, x) < 0) - goto onError; + c = (unsigned char) *s++; + if (c == 'u') { + count = 4; + message = "truncated \\uXXXX escape"; + } + else if (c == 'U') { + count = 8; + message = "truncated \\UXXXXXXXX escape"; } - if (((s - bs) & 1) == 0 || - s >= end || - (*s != 'u' && *s != 'U')) { + else { + assert(writer.pos < writer.size); + PyUnicode_WRITE(writer.kind, writer.data, writer.pos++, '\\'); + WRITE_CHAR(c); continue; } - writer.pos--; - count = *s=='u' ? 4 : 8; - s++; + startinpos = s - starts - 2; - /* \uXXXX with 4 hex digits, \Uxxxxxxxx with 8 */ - for (x = 0, i = 0; i < count; ++i, ++s) { + /* \uHHHH with 4 hex digits, \U00HHHHHH with 8 */ + for (ch = 0; count && s < end; ++s, --count) { c = (unsigned char)*s; - if (!Py_ISXDIGIT(c)) { - endinpos = s-starts; - if (unicode_decode_call_errorhandler_writer( - errors, &errorHandler, - "rawunicodeescape", "truncated \\uXXXX", - &starts, &end, &startinpos, &endinpos, &exc, &s, - &writer)) - goto onError; - goto nextByte; + ch <<= 4; + if (c >= '0' && c <= '9') { + ch += c - '0'; + } + else if (c >= 'a' && c <= 'f') { + ch += c - ('a' - 10); + } + else if (c >= 'A' && c <= 'F') { + ch += c - ('A' - 10); + } + else { + break; } - x = (x<<4) & ~0xF; - if (c >= '0' && c <= '9') - x += c - '0'; - else if (c >= 'a' && c <= 'f') - x += 10 + c - 'a'; - else - x += 10 + c - 'A'; } - if (x <= MAX_UNICODE) { - if (_PyUnicodeWriter_WriteCharInline(&writer, x) < 0) - goto onError; + if (!count) { + if (ch <= MAX_UNICODE) { + WRITE_CHAR(ch); + continue; + } + message = "\\Uxxxxxxxx out of range"; } - else { - endinpos = s-starts; - if (unicode_decode_call_errorhandler_writer( - errors, &errorHandler, - "rawunicodeescape", "\\Uxxxxxxxx out of range", - &starts, &end, &startinpos, &endinpos, &exc, &s, - &writer)) - goto onError; + + endinpos = s-starts; + writer.min_length = end - s + writer.pos; + if (unicode_decode_call_errorhandler_writer( + errors, &errorHandler, + "rawunicodeescape", message, + &starts, &end, &startinpos, &endinpos, &exc, &s, + &writer)) { + goto onError; + } + if (_PyUnicodeWriter_Prepare(&writer, writer.min_length, 127) < 0) { + goto onError; } - nextByte: - ; + +#undef WRITE_CHAR } Py_XDECREF(errorHandler); Py_XDECREF(exc); @@ -6097,6 +6388,7 @@ PyUnicode_DecodeRawUnicodeEscape(const char *s, Py_XDECREF(errorHandler); Py_XDECREF(exc); return NULL; + } @@ -6105,7 +6397,6 @@ PyUnicode_AsRawUnicodeEscapeString(PyObject *unicode) { PyObject *repr; char *p; - char *q; Py_ssize_t expandsize, pos; int kind; void *data; @@ -6115,58 +6406,68 @@ PyUnicode_AsRawUnicodeEscapeString(PyObject *unicode) PyErr_BadArgument(); return NULL; } - if (PyUnicode_READY(unicode) == -1) + if (PyUnicode_READY(unicode) == -1) { return NULL; + } kind = PyUnicode_KIND(unicode); data = PyUnicode_DATA(unicode); len = PyUnicode_GET_LENGTH(unicode); + if (kind == PyUnicode_1BYTE_KIND) { + return PyBytes_FromStringAndSize(data, len); + } + /* 4 byte characters can take up 10 bytes, 2 byte characters can take up 6 bytes, and 1 byte characters 4. */ expandsize = kind * 2 + 2; - if (len > PY_SSIZE_T_MAX / expandsize) + if (len > PY_SSIZE_T_MAX / expandsize) { return PyErr_NoMemory(); - + } repr = PyBytes_FromStringAndSize(NULL, expandsize * len); - if (repr == NULL) + if (repr == NULL) { return NULL; - if (len == 0) + } + if (len == 0) { return repr; + } - p = q = PyBytes_AS_STRING(repr); + p = PyBytes_AS_STRING(repr); for (pos = 0; pos < len; pos++) { Py_UCS4 ch = PyUnicode_READ(kind, data, pos); - /* Map 32-bit characters to '\Uxxxxxxxx' */ - if (ch >= 0x10000) { - assert(ch <= MAX_UNICODE); + + /* U+0000-U+00ff range: Copy 8-bit characters as-is */ + if (ch < 0x100) { + *p++ = (char) ch; + } + /* U+0000-U+00ff range: Map 16-bit characters to '\uHHHH' */ + else if (ch < 0x10000) { *p++ = '\\'; - *p++ = 'U'; - *p++ = Py_hexdigits[(ch >> 28) & 0xf]; - *p++ = Py_hexdigits[(ch >> 24) & 0xf]; - *p++ = Py_hexdigits[(ch >> 20) & 0xf]; - *p++ = Py_hexdigits[(ch >> 16) & 0xf]; + *p++ = 'u'; *p++ = Py_hexdigits[(ch >> 12) & 0xf]; *p++ = Py_hexdigits[(ch >> 8) & 0xf]; *p++ = Py_hexdigits[(ch >> 4) & 0xf]; *p++ = Py_hexdigits[ch & 15]; } - /* Map 16-bit characters to '\uxxxx' */ - else if (ch >= 256) { + /* U+010000-U+10ffff range: Map 32-bit characters to '\U00HHHHHH' */ + else { + assert(ch <= MAX_UNICODE && MAX_UNICODE <= 0x10ffff); *p++ = '\\'; - *p++ = 'u'; + *p++ = 'U'; + *p++ = '0'; + *p++ = '0'; + *p++ = Py_hexdigits[(ch >> 20) & 0xf]; + *p++ = Py_hexdigits[(ch >> 16) & 0xf]; *p++ = Py_hexdigits[(ch >> 12) & 0xf]; *p++ = Py_hexdigits[(ch >> 8) & 0xf]; *p++ = Py_hexdigits[(ch >> 4) & 0xf]; *p++ = Py_hexdigits[ch & 15]; } - /* Copy everything else as-is */ - else - *p++ = (char) ch; } - assert(p > q); - if (_PyBytes_Resize(&repr, p - q) < 0) + assert(p > PyBytes_AS_STRING(repr)); + if (_PyBytes_Resize(&repr, p - PyBytes_AS_STRING(repr)) < 0) { return NULL; + } return repr; } @@ -6344,7 +6645,7 @@ unicode_encode_call_errorhandler(const char *errors, Py_ssize_t startpos, Py_ssize_t endpos, Py_ssize_t *newpos) { - static char *argparse = "On;encoding error handler must return (str/bytes, int) tuple"; + static const char *argparse = "On;encoding error handler must return (str/bytes, int) tuple"; Py_ssize_t len; PyObject *restuple; PyObject *resunicode; @@ -6398,25 +6699,22 @@ unicode_encode_call_errorhandler(const char *errors, static PyObject * unicode_encode_ucs1(PyObject *unicode, const char *errors, - unsigned int limit) + const Py_UCS4 limit) { /* input state */ Py_ssize_t pos=0, size; int kind; void *data; - /* output object */ - PyObject *res; /* pointer into the output */ char *str; - /* current output position */ - Py_ssize_t ressize; const char *encoding = (limit == 256) ? "latin-1" : "ascii"; const char *reason = (limit == 256) ? "ordinal not in range(256)" : "ordinal not in range(128)"; - PyObject *errorHandler = NULL; + PyObject *error_handler_obj = NULL; PyObject *exc = NULL; - /* the following variable is used for caching string comparisons - * -1=not initialized, 0=unknown, 1=strict, 2=replace, 3=ignore, 4=xmlcharrefreplace */ - int known_errorHandler = -1; + _Py_error_handler error_handler = _Py_ERROR_UNKNOWN; + PyObject *rep = NULL; + /* output object */ + _PyBytesWriter writer; if (PyUnicode_READY(unicode) == -1) return NULL; @@ -6427,186 +6725,157 @@ unicode_encode_ucs1(PyObject *unicode, replacements, if we need more, we'll resize */ if (size == 0) return PyBytes_FromStringAndSize(NULL, 0); - res = PyBytes_FromStringAndSize(NULL, size); - if (res == NULL) + + _PyBytesWriter_Init(&writer); + str = _PyBytesWriter_Alloc(&writer, size); + if (str == NULL) return NULL; - str = PyBytes_AS_STRING(res); - ressize = size; while (pos < size) { - Py_UCS4 c = PyUnicode_READ(kind, data, pos); + Py_UCS4 ch = PyUnicode_READ(kind, data, pos); /* can we encode this? */ - if (c<limit) { + if (ch < limit) { /* no overflow check, because we know that the space is enough */ - *str++ = (char)c; + *str++ = (char)ch; ++pos; } else { - Py_ssize_t requiredsize; - PyObject *repunicode; - Py_ssize_t repsize, newpos, respos, i; + Py_ssize_t newpos, i; /* startpos for collecting unencodable chars */ Py_ssize_t collstart = pos; - Py_ssize_t collend = pos; + Py_ssize_t collend = collstart + 1; /* find all unecodable characters */ + while ((collend < size) && (PyUnicode_READ(kind, data, collend) >= limit)) ++collend; + + /* Only overallocate the buffer if it's not the last write */ + writer.overallocate = (collend < size); + /* cache callback name lookup (if not done yet, i.e. it's the first error) */ - if (known_errorHandler==-1) { - if ((errors==NULL) || (!strcmp(errors, "strict"))) - known_errorHandler = 1; - else if (!strcmp(errors, "replace")) - known_errorHandler = 2; - else if (!strcmp(errors, "ignore")) - known_errorHandler = 3; - else if (!strcmp(errors, "xmlcharrefreplace")) - known_errorHandler = 4; - else - known_errorHandler = 0; - } - switch (known_errorHandler) { - case 1: /* strict */ + if (error_handler == _Py_ERROR_UNKNOWN) + error_handler = get_error_handler(errors); + + switch (error_handler) { + case _Py_ERROR_STRICT: raise_encode_exception(&exc, encoding, unicode, collstart, collend, reason); goto onError; - case 2: /* replace */ - while (collstart++ < collend) - *str++ = '?'; /* fall through */ - case 3: /* ignore */ + + case _Py_ERROR_REPLACE: + memset(str, '?', collend - collstart); + str += (collend - collstart); + /* fall through ignore error handler */ + case _Py_ERROR_IGNORE: + pos = collend; + break; + + case _Py_ERROR_BACKSLASHREPLACE: + /* subtract preallocated bytes */ + writer.min_size -= (collend - collstart); + str = backslashreplace(&writer, str, + unicode, collstart, collend); + if (str == NULL) + goto onError; pos = collend; break; - case 4: /* xmlcharrefreplace */ - respos = str - PyBytes_AS_STRING(res); - requiredsize = respos; - /* determine replacement size */ + + case _Py_ERROR_XMLCHARREFREPLACE: + /* subtract preallocated bytes */ + writer.min_size -= (collend - collstart); + str = xmlcharrefreplace(&writer, str, + unicode, collstart, collend); + if (str == NULL) + goto onError; + pos = collend; + break; + + case _Py_ERROR_SURROGATEESCAPE: for (i = collstart; i < collend; ++i) { - Py_UCS4 ch = PyUnicode_READ(kind, data, i); - Py_ssize_t incr; - if (ch < 10) - incr = 2+1+1; - else if (ch < 100) - incr = 2+2+1; - else if (ch < 1000) - incr = 2+3+1; - else if (ch < 10000) - incr = 2+4+1; - else if (ch < 100000) - incr = 2+5+1; - else if (ch < 1000000) - incr = 2+6+1; - else { - assert(ch <= MAX_UNICODE); - incr = 2+7+1; + ch = PyUnicode_READ(kind, data, i); + if (ch < 0xdc80 || 0xdcff < ch) { + /* Not a UTF-8b surrogate */ + break; } - if (requiredsize > PY_SSIZE_T_MAX - incr) - goto overflow; - requiredsize += incr; - } - if (requiredsize > PY_SSIZE_T_MAX - (size - collend)) - goto overflow; - requiredsize += size - collend; - if (requiredsize > ressize) { - if (ressize <= PY_SSIZE_T_MAX/2 && requiredsize < 2*ressize) - requiredsize = 2*ressize; - if (_PyBytes_Resize(&res, requiredsize)) - goto onError; - str = PyBytes_AS_STRING(res) + respos; - ressize = requiredsize; - } - /* generate replacement */ - for (i = collstart; i < collend; ++i) { - str += sprintf(str, "&#%d;", PyUnicode_READ(kind, data, i)); + *str++ = (char)(ch - 0xdc00); + ++pos; } - pos = collend; - break; + if (i >= collend) + break; + collstart = pos; + assert(collstart != collend); + /* fallback to general error handling */ + default: - repunicode = unicode_encode_call_errorhandler(errors, &errorHandler, - encoding, reason, unicode, &exc, - collstart, collend, &newpos); - if (repunicode == NULL || (PyUnicode_Check(repunicode) && - PyUnicode_READY(repunicode) == -1)) + rep = unicode_encode_call_errorhandler(errors, &error_handler_obj, + encoding, reason, unicode, &exc, + collstart, collend, &newpos); + if (rep == NULL) goto onError; - if (PyBytes_Check(repunicode)) { + + /* subtract preallocated bytes */ + writer.min_size -= 1; + + if (PyBytes_Check(rep)) { /* Directly copy bytes result to output. */ - repsize = PyBytes_Size(repunicode); - if (repsize > 1) { - /* Make room for all additional bytes. */ - respos = str - PyBytes_AS_STRING(res); - if (ressize > PY_SSIZE_T_MAX - repsize - 1) { - Py_DECREF(repunicode); - goto overflow; - } - if (_PyBytes_Resize(&res, ressize+repsize-1)) { - Py_DECREF(repunicode); - goto onError; - } - str = PyBytes_AS_STRING(res) + respos; - ressize += repsize-1; - } - memcpy(str, PyBytes_AsString(repunicode), repsize); - str += repsize; - pos = newpos; - Py_DECREF(repunicode); - break; - } - /* need more space? (at least enough for what we - have+the replacement+the rest of the string, so - we won't have to check space for encodable characters) */ - respos = str - PyBytes_AS_STRING(res); - repsize = PyUnicode_GET_LENGTH(repunicode); - requiredsize = respos; - if (requiredsize > PY_SSIZE_T_MAX - repsize) - goto overflow; - requiredsize += repsize; - if (requiredsize > PY_SSIZE_T_MAX - (size - collend)) - goto overflow; - requiredsize += size - collend; - if (requiredsize > ressize) { - if (ressize <= PY_SSIZE_T_MAX/2 && requiredsize < 2*ressize) - requiredsize = 2*ressize; - if (_PyBytes_Resize(&res, requiredsize)) { - Py_DECREF(repunicode); + str = _PyBytesWriter_WriteBytes(&writer, str, + PyBytes_AS_STRING(rep), + PyBytes_GET_SIZE(rep)); + if (str == NULL) goto onError; - } - str = PyBytes_AS_STRING(res) + respos; - ressize = requiredsize; } - /* check if there is anything unencodable in the replacement - and copy it to the output */ - for (i = 0; repsize-->0; ++i, ++str) { - c = PyUnicode_READ_CHAR(repunicode, i); - if (c >= limit) { - raise_encode_exception(&exc, encoding, unicode, - pos, pos+1, reason); - Py_DECREF(repunicode); + else { + assert(PyUnicode_Check(rep)); + + if (PyUnicode_READY(rep) < 0) goto onError; + + if (PyUnicode_IS_ASCII(rep)) { + /* Fast path: all characters are smaller than limit */ + assert(limit >= 128); + assert(PyUnicode_KIND(rep) == PyUnicode_1BYTE_KIND); + str = _PyBytesWriter_WriteBytes(&writer, str, + PyUnicode_DATA(rep), + PyUnicode_GET_LENGTH(rep)); + } + else { + Py_ssize_t repsize = PyUnicode_GET_LENGTH(rep); + + str = _PyBytesWriter_Prepare(&writer, str, repsize); + if (str == NULL) + goto onError; + + /* check if there is anything unencodable in the + replacement and copy it to the output */ + for (i = 0; repsize-->0; ++i, ++str) { + ch = PyUnicode_READ_CHAR(rep, i); + if (ch >= limit) { + raise_encode_exception(&exc, encoding, unicode, + pos, pos+1, reason); + goto onError; + } + *str = (char)ch; + } } - *str = (char)c; } pos = newpos; - Py_DECREF(repunicode); + Py_CLEAR(rep); } + + /* If overallocation was disabled, ensure that it was the last + write. Otherwise, we missed an optimization */ + assert(writer.overallocate || pos == size); } } - /* Resize if we allocated to much */ - size = str - PyBytes_AS_STRING(res); - if (size < ressize) { /* If this falls res will be NULL */ - assert(size >= 0); - if (_PyBytes_Resize(&res, size) < 0) - goto onError; - } - Py_XDECREF(errorHandler); + Py_XDECREF(error_handler_obj); Py_XDECREF(exc); - return res; - - overflow: - PyErr_SetString(PyExc_OverflowError, - "encoded result is too long for a Python string"); + return _PyBytesWriter_Finish(&writer, str); onError: - Py_XDECREF(res); - Py_XDECREF(errorHandler); + Py_XDECREF(rep); + _PyBytesWriter_Dealloc(&writer); + Py_XDECREF(error_handler_obj); Py_XDECREF(exc); return NULL; } @@ -6666,8 +6935,9 @@ PyUnicode_DecodeASCII(const char *s, Py_ssize_t endinpos; Py_ssize_t outpos; const char *e; - PyObject *errorHandler = NULL; + PyObject *error_handler_obj = NULL; PyObject *exc = NULL; + _Py_error_handler error_handler = _Py_ERROR_UNKNOWN; if (size == 0) _Py_RETURN_UNICODE_EMPTY(); @@ -6696,12 +6966,42 @@ PyUnicode_DecodeASCII(const char *s, PyUnicode_WRITE(kind, data, writer.pos, c); writer.pos++; ++s; + continue; } - else { + + /* byte outsize range 0x00..0x7f: call the error handler */ + + if (error_handler == _Py_ERROR_UNKNOWN) + error_handler = get_error_handler(errors); + + switch (error_handler) + { + case _Py_ERROR_REPLACE: + case _Py_ERROR_SURROGATEESCAPE: + /* Fast-path: the error handler only writes one character, + but we may switch to UCS2 at the first write */ + if (_PyUnicodeWriter_PrepareKind(&writer, PyUnicode_2BYTE_KIND) < 0) + goto onError; + kind = writer.kind; + data = writer.data; + + if (error_handler == _Py_ERROR_REPLACE) + PyUnicode_WRITE(kind, data, writer.pos, 0xfffd); + else + PyUnicode_WRITE(kind, data, writer.pos, c + 0xdc00); + writer.pos++; + ++s; + break; + + case _Py_ERROR_IGNORE: + ++s; + break; + + default: startinpos = s-starts; endinpos = startinpos + 1; if (unicode_decode_call_errorhandler_writer( - errors, &errorHandler, + errors, &error_handler_obj, "ascii", "ordinal not in range(128)", &starts, &e, &startinpos, &endinpos, &exc, &s, &writer)) @@ -6710,13 +7010,13 @@ PyUnicode_DecodeASCII(const char *s, data = writer.data; } } - Py_XDECREF(errorHandler); + Py_XDECREF(error_handler_obj); Py_XDECREF(exc); return _PyUnicodeWriter_Finish(&writer); onError: _PyUnicodeWriter_Dealloc(&writer); - Py_XDECREF(errorHandler); + Py_XDECREF(error_handler_obj); Py_XDECREF(exc); return NULL; } @@ -6759,7 +7059,7 @@ PyUnicode_AsASCIIString(PyObject *unicode) return _PyUnicode_AsASCIIString(unicode, NULL); } -#ifdef HAVE_MBCS +#ifdef MS_WINDOWS /* --- MBCS codecs for Windows -------------------------------------------- */ @@ -6771,7 +7071,7 @@ PyUnicode_AsASCIIString(PyObject *unicode) # define WC_ERR_INVALID_CHARS 0x0080 #endif -static char* +static const char* code_page_name(UINT code_page, PyObject **obj) { *obj = NULL; @@ -6879,7 +7179,7 @@ decode_code_page_errors(UINT code_page, PyObject *errorHandler = NULL; PyObject *exc = NULL; PyObject *encoding_obj = NULL; - char *encoding; + const char *encoding; DWORD err; int ret = -1; @@ -7115,7 +7415,6 @@ encode_code_page_strict(UINT code_page, PyObject **outbytes, BOOL usedDefaultChar = FALSE; BOOL *pusedDefaultChar = &usedDefaultChar; int outsize; - PyObject *exc = NULL; wchar_t *p; Py_ssize_t size; const DWORD flags = encode_code_page_flags(code_page, NULL); @@ -7224,7 +7523,7 @@ encode_code_page_errors(UINT code_page, PyObject **outbytes, PyObject *errorHandler = NULL; PyObject *exc = NULL; PyObject *encoding_obj = NULL; - char *encoding; + const char *encoding; Py_ssize_t newpos, newoutsize; PyObject *rep; int ret = -1; @@ -7479,7 +7778,7 @@ PyUnicode_AsMBCSString(PyObject *unicode) #undef NEED_RETRY -#endif /* HAVE_MBCS */ +#endif /* MS_WINDOWS */ /* --- Character Mapping Codec -------------------------------------------- */ @@ -8082,7 +8381,7 @@ static int charmap_encoding_error( PyObject *unicode, Py_ssize_t *inpos, PyObject *mapping, PyObject **exceptionObject, - int *known_errorHandler, PyObject **errorHandler, const char *errors, + _Py_error_handler *error_handler, PyObject **error_handler_obj, const char *errors, PyObject **res, Py_ssize_t *respos) { PyObject *repunicode = NULL; /* initialize to prevent gcc warning */ @@ -8129,23 +8428,15 @@ charmap_encoding_error( } /* cache callback name lookup * (if not done yet, i.e. it's the first error) */ - if (*known_errorHandler==-1) { - if ((errors==NULL) || (!strcmp(errors, "strict"))) - *known_errorHandler = 1; - else if (!strcmp(errors, "replace")) - *known_errorHandler = 2; - else if (!strcmp(errors, "ignore")) - *known_errorHandler = 3; - else if (!strcmp(errors, "xmlcharrefreplace")) - *known_errorHandler = 4; - else - *known_errorHandler = 0; - } - switch (*known_errorHandler) { - case 1: /* strict */ + if (*error_handler == _Py_ERROR_UNKNOWN) + *error_handler = get_error_handler(errors); + + switch (*error_handler) { + case _Py_ERROR_STRICT: raise_encode_exception(exceptionObject, encoding, unicode, collstartpos, collendpos, reason); return -1; - case 2: /* replace */ + + case _Py_ERROR_REPLACE: for (collpos = collstartpos; collpos<collendpos; ++collpos) { x = charmapencode_output('?', mapping, res, respos); if (x==enc_EXCEPTION) { @@ -8157,10 +8448,11 @@ charmap_encoding_error( } } /* fall through */ - case 3: /* ignore */ + case _Py_ERROR_IGNORE: *inpos = collendpos; break; - case 4: /* xmlcharrefreplace */ + + case _Py_ERROR_XMLCHARREFREPLACE: /* generate replacement (temporarily (mis)uses p) */ for (collpos = collstartpos; collpos < collendpos; ++collpos) { char buffer[2+29+1+1]; @@ -8178,8 +8470,9 @@ charmap_encoding_error( } *inpos = collendpos; break; + default: - repunicode = unicode_encode_call_errorhandler(errors, errorHandler, + repunicode = unicode_encode_call_errorhandler(errors, error_handler_obj, encoding, reason, unicode, exceptionObject, collstartpos, collendpos, &newpos); if (repunicode == NULL) @@ -8242,12 +8535,9 @@ _PyUnicode_EncodeCharmap(PyObject *unicode, Py_ssize_t size; /* current output position */ Py_ssize_t respos = 0; - PyObject *errorHandler = NULL; + PyObject *error_handler_obj = NULL; PyObject *exc = NULL; - /* the following variable is used for caching string comparisons - * -1=not initialized, 0=unknown, 1=strict, 2=replace, - * 3=ignore, 4=xmlcharrefreplace */ - int known_errorHandler = -1; + _Py_error_handler error_handler = _Py_ERROR_UNKNOWN; void *data; int kind; @@ -8278,7 +8568,7 @@ _PyUnicode_EncodeCharmap(PyObject *unicode, if (x==enc_FAILED) { /* unencodable character */ if (charmap_encoding_error(unicode, &inpos, mapping, &exc, - &known_errorHandler, &errorHandler, errors, + &error_handler, &error_handler_obj, errors, &res, &respos)) { goto onError; } @@ -8294,13 +8584,13 @@ _PyUnicode_EncodeCharmap(PyObject *unicode, goto onError; Py_XDECREF(exc); - Py_XDECREF(errorHandler); + Py_XDECREF(error_handler_obj); return res; onError: Py_XDECREF(res); Py_XDECREF(exc); - Py_XDECREF(errorHandler); + Py_XDECREF(error_handler_obj); return NULL; } @@ -8367,7 +8657,7 @@ unicode_translate_call_errorhandler(const char *errors, Py_ssize_t startpos, Py_ssize_t endpos, Py_ssize_t *newpos) { - static char *argparse = "O!n;translating error handler must return (str, int) tuple"; + static const char *argparse = "O!n;translating error handler must return (str, int) tuple"; Py_ssize_t i_newpos; PyObject *restuple; @@ -8624,7 +8914,7 @@ exit: return res; } -PyObject * +static PyObject * _PyUnicode_TranslateCharmap(PyObject *input, PyObject *mapping, const char *errors) @@ -8653,10 +8943,8 @@ _PyUnicode_TranslateCharmap(PyObject *input, kind = PyUnicode_KIND(input); size = PyUnicode_GET_LENGTH(input); - if (size == 0) { - Py_INCREF(input); - return input; - } + if (size == 0) + return PyUnicode_FromObject(input); /* allocate enough for a simple 1:1 translation without replacements, if we need more, we'll resize */ @@ -8767,14 +9055,9 @@ PyUnicode_Translate(PyObject *str, PyObject *mapping, const char *errors) { - PyObject *result; - - str = PyUnicode_FromObject(str); - if (str == NULL) + if (ensure_unicode(str) < 0) return NULL; - result = _PyUnicode_TranslateCharmap(str, mapping, errors); - Py_DECREF(str); - return result; + return _PyUnicode_TranslateCharmap(str, mapping, errors); } static Py_UCS4 @@ -8956,9 +9239,10 @@ PyUnicode_EncodeDecimal(Py_UNICODE *s, } static Py_ssize_t -any_find_slice(int direction, PyObject* s1, PyObject* s2, +any_find_slice(PyObject* s1, PyObject* s2, Py_ssize_t start, - Py_ssize_t end) + Py_ssize_t end, + int direction) { int kind1, kind2; void *buf1, *buf2; @@ -9127,54 +9411,35 @@ PyUnicode_Count(PyObject *str, Py_ssize_t end) { Py_ssize_t result; - PyObject* str_obj; - PyObject* sub_obj; int kind1, kind2; void *buf1 = NULL, *buf2 = NULL; Py_ssize_t len1, len2; - str_obj = PyUnicode_FromObject(str); - if (!str_obj) + if (ensure_unicode(str) < 0 || ensure_unicode(substr) < 0) return -1; - sub_obj = PyUnicode_FromObject(substr); - if (!sub_obj) { - Py_DECREF(str_obj); - return -1; - } - if (PyUnicode_READY(sub_obj) == -1 || PyUnicode_READY(str_obj) == -1) { - Py_DECREF(sub_obj); - Py_DECREF(str_obj); - return -1; - } - kind1 = PyUnicode_KIND(str_obj); - kind2 = PyUnicode_KIND(sub_obj); - if (kind1 < kind2) { - Py_DECREF(sub_obj); - Py_DECREF(str_obj); + kind1 = PyUnicode_KIND(str); + kind2 = PyUnicode_KIND(substr); + if (kind1 < kind2) return 0; - } - len1 = PyUnicode_GET_LENGTH(str_obj); - len2 = PyUnicode_GET_LENGTH(sub_obj); + len1 = PyUnicode_GET_LENGTH(str); + len2 = PyUnicode_GET_LENGTH(substr); ADJUST_INDICES(start, end, len1); - if (end - start < len2) { - Py_DECREF(sub_obj); - Py_DECREF(str_obj); + if (end - start < len2) return 0; - } - buf1 = PyUnicode_DATA(str_obj); - buf2 = PyUnicode_DATA(sub_obj); + buf1 = PyUnicode_DATA(str); + buf2 = PyUnicode_DATA(substr); if (kind2 != kind1) { - buf2 = _PyUnicode_AsKind(sub_obj, kind1); + buf2 = _PyUnicode_AsKind(substr, kind1); if (!buf2) goto onError; } switch (kind1) { case PyUnicode_1BYTE_KIND: - if (PyUnicode_IS_ASCII(str_obj) && PyUnicode_IS_ASCII(sub_obj)) + if (PyUnicode_IS_ASCII(str) && PyUnicode_IS_ASCII(substr)) result = asciilib_count( ((Py_UCS1*)buf1) + start, end - start, buf2, len2, PY_SSIZE_T_MAX @@ -9201,16 +9466,11 @@ PyUnicode_Count(PyObject *str, assert(0); result = 0; } - Py_DECREF(sub_obj); - Py_DECREF(str_obj); - if (kind2 != kind1) PyMem_Free(buf2); return result; onError: - Py_DECREF(sub_obj); - Py_DECREF(str_obj); if (kind2 != kind1 && buf2) PyMem_Free(buf2); return -1; @@ -9218,35 +9478,15 @@ PyUnicode_Count(PyObject *str, Py_ssize_t PyUnicode_Find(PyObject *str, - PyObject *sub, + PyObject *substr, Py_ssize_t start, Py_ssize_t end, int direction) { - Py_ssize_t result; - - str = PyUnicode_FromObject(str); - if (!str) - return -2; - sub = PyUnicode_FromObject(sub); - if (!sub) { - Py_DECREF(str); - return -2; - } - if (PyUnicode_READY(sub) == -1 || PyUnicode_READY(str) == -1) { - Py_DECREF(sub); - Py_DECREF(str); + if (ensure_unicode(str) < 0 || ensure_unicode(substr) < 0) return -2; - } - - result = any_find_slice(direction, - str, sub, start, end - ); - - Py_DECREF(str); - Py_DECREF(sub); - return result; + return any_find_slice(str, substr, start, end, direction); } Py_ssize_t @@ -9349,22 +9589,10 @@ PyUnicode_Tailmatch(PyObject *str, Py_ssize_t end, int direction) { - Py_ssize_t result; - - str = PyUnicode_FromObject(str); - if (str == NULL) + if (ensure_unicode(str) < 0 || ensure_unicode(substr) < 0) return -1; - substr = PyUnicode_FromObject(substr); - if (substr == NULL) { - Py_DECREF(str); - return -1; - } - result = tailmatch(str, substr, - start, end, direction); - Py_DECREF(str); - Py_DECREF(substr); - return result; + return tailmatch(str, substr, start, end, direction); } /* Apply fixfct filter to the Unicode object self and return a @@ -9666,20 +9894,10 @@ case_operation(PyObject *self, PyObject * PyUnicode_Join(PyObject *separator, PyObject *seq) { - PyObject *sep = NULL; - Py_ssize_t seplen; - PyObject *res = NULL; /* the result */ - PyObject *fseq; /* PySequence_Fast(seq) */ - Py_ssize_t seqlen; /* len(fseq) -- number of items in sequence */ + PyObject *res; + PyObject *fseq; + Py_ssize_t seqlen; PyObject **items; - PyObject *item; - Py_ssize_t sz, i, res_offset; - Py_UCS4 maxchar; - Py_UCS4 item_maxchar; - int use_memcpy; - unsigned char *res_data = NULL, *sep_data = NULL; - PyObject *last_obj; - unsigned int kind = 0; fseq = PySequence_Fast(seq, "can only join an iterable"); if (fseq == NULL) { @@ -9690,21 +9908,39 @@ PyUnicode_Join(PyObject *separator, PyObject *seq) * so we are sure that fseq won't be mutated. */ + items = PySequence_Fast_ITEMS(fseq); seqlen = PySequence_Fast_GET_SIZE(fseq); + res = _PyUnicode_JoinArray(separator, items, seqlen); + Py_DECREF(fseq); + return res; +} + +PyObject * +_PyUnicode_JoinArray(PyObject *separator, PyObject **items, Py_ssize_t seqlen) +{ + PyObject *res = NULL; /* the result */ + PyObject *sep = NULL; + Py_ssize_t seplen; + PyObject *item; + Py_ssize_t sz, i, res_offset; + Py_UCS4 maxchar; + Py_UCS4 item_maxchar; + int use_memcpy; + unsigned char *res_data = NULL, *sep_data = NULL; + PyObject *last_obj; + unsigned int kind = 0; + /* If empty sequence, return u"". */ if (seqlen == 0) { - Py_DECREF(fseq); _Py_RETURN_UNICODE_EMPTY(); } /* If singleton sequence with an exact Unicode, return that. */ last_obj = NULL; - items = PySequence_Fast_ITEMS(fseq); if (seqlen == 1) { if (PyUnicode_CheckExact(items[0])) { res = items[0]; Py_INCREF(res); - Py_DECREF(fseq); return res; } seplen = 0; @@ -9804,7 +10040,7 @@ PyUnicode_Join(PyObject *separator, PyObject *seq) /* Copy item, and maybe the separator. */ if (i && seplen != 0) { - Py_MEMCPY(res_data, + memcpy(res_data, sep_data, kind * seplen); res_data += kind * seplen; @@ -9812,7 +10048,7 @@ PyUnicode_Join(PyObject *separator, PyObject *seq) itemlen = PyUnicode_GET_LENGTH(item); if (itemlen != 0) { - Py_MEMCPY(res_data, + memcpy(res_data, PyUnicode_DATA(item), kind * itemlen); res_data += kind * itemlen; @@ -9841,13 +10077,11 @@ PyUnicode_Join(PyObject *separator, PyObject *seq) assert(res_offset == PyUnicode_GET_LENGTH(res)); } - Py_DECREF(fseq); Py_XDECREF(sep); assert(_PyUnicode_CheckConsistency(res, 1)); return res; onError: - Py_DECREF(fseq); Py_XDECREF(sep); Py_XDECREF(res); return NULL; @@ -9972,13 +10206,8 @@ PyUnicode_Splitlines(PyObject *string, int keepends) { PyObject *list; - string = PyUnicode_FromObject(string); - if (string == NULL) + if (ensure_unicode(string) < 0) return NULL; - if (PyUnicode_READY(string) == -1) { - Py_DECREF(string); - return NULL; - } switch (PyUnicode_KIND(string)) { case PyUnicode_1BYTE_KIND: @@ -10005,7 +10234,6 @@ PyUnicode_Splitlines(PyObject *string, int keepends) assert(0); list = 0; } - Py_DECREF(string); return list; } @@ -10566,28 +10794,27 @@ unicode_casefold(PyObject *self) } -/* Argument converter. Coerces to a single unicode character */ +/* Argument converter. Accepts a single Unicode character. */ static int convert_uc(PyObject *obj, void *addr) { Py_UCS4 *fillcharloc = (Py_UCS4 *)addr; - PyObject *uniobj; - uniobj = PyUnicode_FromObject(obj); - if (uniobj == NULL) { - PyErr_SetString(PyExc_TypeError, - "The fill character cannot be converted to Unicode"); + if (!PyUnicode_Check(obj)) { + PyErr_Format(PyExc_TypeError, + "The fill character must be a unicode character, " + "not %.100s", Py_TYPE(obj)->tp_name); return 0; } - if (PyUnicode_GET_LENGTH(uniobj) != 1) { + if (PyUnicode_READY(obj) < 0) + return 0; + if (PyUnicode_GET_LENGTH(obj) != 1) { PyErr_SetString(PyExc_TypeError, "The fill character must be exactly one character long"); - Py_DECREF(uniobj); return 0; } - *fillcharloc = PyUnicode_READ_CHAR(uniobj, 0); - Py_DECREF(uniobj); + *fillcharloc = PyUnicode_READ_CHAR(obj, 0); return 1; } @@ -10737,7 +10964,7 @@ unicode_compare(PyObject *str1, PyObject *str2) #undef COMPARE } -Py_LOCAL(int) +static int unicode_compare_eq(PyObject *str1, PyObject *str2) { int kind; @@ -10781,15 +11008,6 @@ PyUnicode_Compare(PyObject *left, PyObject *right) } int -_PyUnicode_CompareWithId(PyObject *left, _Py_Identifier *right) -{ - PyObject *right_str = _PyUnicode_FromId(right); /* borrowed */ - if (right_str == NULL) - return -1; - return PyUnicode_Compare(left, right_str); -} - -int PyUnicode_CompareWithASCIIString(PyObject* uni, const char* str) { Py_ssize_t i; @@ -10873,6 +11091,12 @@ _PyUnicode_EqualToASCIIString(PyObject *unicode, const char *str) { size_t len; assert(_PyUnicode_CHECK(unicode)); + assert(str); +#ifndef NDEBUG + for (const char *p = str; *p; p++) { + assert((unsigned char)*p < 128); + } +#endif if (PyUnicode_READY(unicode) == -1) { /* Memory error or bad data */ PyErr_Clear(); @@ -10893,6 +11117,11 @@ _PyUnicode_EqualToASCIIId(PyObject *left, _Py_Identifier *right) assert(_PyUnicode_CHECK(left)); assert(right->string); +#ifndef NDEBUG + for (const char *p = right->string; *p; p++) { + assert((unsigned char)*p < 128); + } +#endif if (PyUnicode_READY(left) == -1) { /* memory error or bad data */ @@ -10990,59 +11219,49 @@ PyUnicode_RichCompare(PyObject *left, PyObject *right, int op) } int -PyUnicode_Contains(PyObject *container, PyObject *element) +_PyUnicode_EQ(PyObject *aa, PyObject *bb) +{ + return unicode_eq(aa, bb); +} + +int +PyUnicode_Contains(PyObject *str, PyObject *substr) { - PyObject *str, *sub; int kind1, kind2; void *buf1, *buf2; Py_ssize_t len1, len2; int result; - /* Coerce the two arguments */ - sub = PyUnicode_FromObject(element); - if (!sub) { + if (!PyUnicode_Check(substr)) { PyErr_Format(PyExc_TypeError, - "'in <string>' requires string as left operand, not %s", - element->ob_type->tp_name); + "'in <string>' requires string as left operand, not %.100s", + Py_TYPE(substr)->tp_name); return -1; } - - str = PyUnicode_FromObject(container); - if (!str) { - Py_DECREF(sub); + if (PyUnicode_READY(substr) == -1) + return -1; + if (ensure_unicode(str) < 0) return -1; - } kind1 = PyUnicode_KIND(str); - kind2 = PyUnicode_KIND(sub); - if (kind1 < kind2) { - Py_DECREF(sub); - Py_DECREF(str); + kind2 = PyUnicode_KIND(substr); + if (kind1 < kind2) return 0; - } len1 = PyUnicode_GET_LENGTH(str); - len2 = PyUnicode_GET_LENGTH(sub); - if (len1 < len2) { - Py_DECREF(sub); - Py_DECREF(str); + len2 = PyUnicode_GET_LENGTH(substr); + if (len1 < len2) return 0; - } buf1 = PyUnicode_DATA(str); - buf2 = PyUnicode_DATA(sub); + buf2 = PyUnicode_DATA(substr); if (len2 == 1) { Py_UCS4 ch = PyUnicode_READ(kind2, buf2, 0); result = findchar((const char *)buf1, kind1, len1, ch, 1) != -1; - Py_DECREF(sub); - Py_DECREF(str); return result; } if (kind2 != kind1) { - buf2 = _PyUnicode_AsKind(sub, kind1); - if (!buf2) { - Py_DECREF(sub); - Py_DECREF(str); + buf2 = _PyUnicode_AsKind(substr, kind1); + if (!buf2) return -1; - } } switch (kind1) { @@ -11060,9 +11279,6 @@ PyUnicode_Contains(PyObject *container, PyObject *element) assert(0); } - Py_DECREF(str); - Py_DECREF(sub); - if (kind2 != kind1) PyMem_Free(buf2); @@ -11074,56 +11290,40 @@ PyUnicode_Contains(PyObject *container, PyObject *element) PyObject * PyUnicode_Concat(PyObject *left, PyObject *right) { - PyObject *u = NULL, *v = NULL, *w; + PyObject *result; Py_UCS4 maxchar, maxchar2; - Py_ssize_t u_len, v_len, new_len; + Py_ssize_t left_len, right_len, new_len; - /* Coerce the two arguments */ - u = PyUnicode_FromObject(left); - if (u == NULL) - goto onError; - v = PyUnicode_FromObject(right); - if (v == NULL) - goto onError; + if (ensure_unicode(left) < 0 || ensure_unicode(right) < 0) + return NULL; /* Shortcuts */ - if (v == unicode_empty) { - Py_DECREF(v); - return u; - } - if (u == unicode_empty) { - Py_DECREF(u); - return v; - } + if (left == unicode_empty) + return PyUnicode_FromObject(right); + if (right == unicode_empty) + return PyUnicode_FromObject(left); - u_len = PyUnicode_GET_LENGTH(u); - v_len = PyUnicode_GET_LENGTH(v); - if (u_len > PY_SSIZE_T_MAX - v_len) { + left_len = PyUnicode_GET_LENGTH(left); + right_len = PyUnicode_GET_LENGTH(right); + if (left_len > PY_SSIZE_T_MAX - right_len) { PyErr_SetString(PyExc_OverflowError, "strings are too large to concat"); - goto onError; + return NULL; } - new_len = u_len + v_len; + new_len = left_len + right_len; - maxchar = PyUnicode_MAX_CHAR_VALUE(u); - maxchar2 = PyUnicode_MAX_CHAR_VALUE(v); + maxchar = PyUnicode_MAX_CHAR_VALUE(left); + maxchar2 = PyUnicode_MAX_CHAR_VALUE(right); maxchar = Py_MAX(maxchar, maxchar2); /* Concat the two Unicode strings */ - w = PyUnicode_New(new_len, maxchar); - if (w == NULL) - goto onError; - _PyUnicode_FastCopyCharacters(w, 0, u, 0, u_len); - _PyUnicode_FastCopyCharacters(w, u_len, v, 0, v_len); - Py_DECREF(u); - Py_DECREF(v); - assert(_PyUnicode_CheckConsistency(w, 1)); - return w; - - onError: - Py_XDECREF(u); - Py_XDECREF(v); - return NULL; + result = PyUnicode_New(new_len, maxchar); + if (result == NULL) + return NULL; + _PyUnicode_FastCopyCharacters(result, 0, left, 0, left_len); + _PyUnicode_FastCopyCharacters(result, left_len, right, 0, right_len); + assert(_PyUnicode_CheckConsistency(result, 1)); + return result; } void @@ -11214,6 +11414,25 @@ PyUnicode_AppendAndDel(PyObject **pleft, PyObject *right) Py_XDECREF(right); } +/* +Wraps stringlib_parse_args_finds() and additionally ensures that the +first argument is a unicode object. +*/ + +static inline int +parse_args_finds_unicode(const char * function_name, PyObject *args, + PyObject **substring, + Py_ssize_t *start, Py_ssize_t *end) +{ + if(stringlib_parse_args_finds(function_name, args, substring, + start, end)) { + if (ensure_unicode(*substring) < 0) + return 0; + return 1; + } + return 0; +} + PyDoc_STRVAR(count__doc__, "S.count(sub[, start[, end]]) -> int\n\ \n\ @@ -11232,31 +11451,26 @@ unicode_count(PyObject *self, PyObject *args) void *buf1, *buf2; Py_ssize_t len1, len2, iresult; - if (!stringlib_parse_args_finds_unicode("count", args, &substring, - &start, &end)) + if (!parse_args_finds_unicode("count", args, &substring, &start, &end)) return NULL; kind1 = PyUnicode_KIND(self); kind2 = PyUnicode_KIND(substring); - if (kind1 < kind2) { - Py_DECREF(substring); + if (kind1 < kind2) return PyLong_FromLong(0); - } + len1 = PyUnicode_GET_LENGTH(self); len2 = PyUnicode_GET_LENGTH(substring); ADJUST_INDICES(start, end, len1); - if (end - start < len2) { - Py_DECREF(substring); + if (end - start < len2) return PyLong_FromLong(0); - } + buf1 = PyUnicode_DATA(self); buf2 = PyUnicode_DATA(substring); if (kind2 != kind1) { buf2 = _PyUnicode_AsKind(substring, kind1); - if (!buf2) { - Py_DECREF(substring); + if (!buf2) return NULL; - } } switch (kind1) { case PyUnicode_1BYTE_KIND: @@ -11286,8 +11500,6 @@ unicode_count(PyObject *self, PyObject *args) if (kind2 != kind1) PyMem_Free(buf2); - Py_DECREF(substring); - return result; } @@ -11421,22 +11633,13 @@ unicode_find(PyObject *self, PyObject *args) Py_ssize_t end = 0; Py_ssize_t result; - if (!stringlib_parse_args_finds_unicode("find", args, &substring, - &start, &end)) + if (!parse_args_finds_unicode("find", args, &substring, &start, &end)) return NULL; - if (PyUnicode_READY(self) == -1) { - Py_DECREF(substring); - return NULL; - } - if (PyUnicode_READY(substring) == -1) { - Py_DECREF(substring); + if (PyUnicode_READY(self) == -1) return NULL; - } - - result = any_find_slice(1, self, substring, start, end); - Py_DECREF(substring); + result = any_find_slice(self, substring, start, end, 1); if (result == -2) return NULL; @@ -11509,22 +11712,13 @@ unicode_index(PyObject *self, PyObject *args) Py_ssize_t start = 0; Py_ssize_t end = 0; - if (!stringlib_parse_args_finds_unicode("index", args, &substring, - &start, &end)) + if (!parse_args_finds_unicode("index", args, &substring, &start, &end)) return NULL; - if (PyUnicode_READY(self) == -1) { - Py_DECREF(substring); - return NULL; - } - if (PyUnicode_READY(substring) == -1) { - Py_DECREF(substring); + if (PyUnicode_READY(self) == -1) return NULL; - } - result = any_find_slice(1, self, substring, start, end); - - Py_DECREF(substring); + result = any_find_slice(self, substring, start, end, 1); if (result == -2) return NULL; @@ -12038,7 +12232,7 @@ unicode_lower(PyObject *self) #define BOTHSTRIP 2 /* Arrays indexed by above */ -static const char *stripformat[] = {"|O:lstrip", "|O:rstrip", "|O:strip"}; +static const char * const stripformat[] = {"|O:lstrip", "|O:rstrip", "|O:strip"}; #define STRIPNAME(i) (stripformat[i]+3) @@ -12319,11 +12513,11 @@ unicode_repeat(PyObject *str, Py_ssize_t len) Py_ssize_t done = PyUnicode_GET_LENGTH(str); const Py_ssize_t char_size = PyUnicode_KIND(str); char *to = (char *) PyUnicode_DATA(u); - Py_MEMCPY(to, PyUnicode_DATA(str), + memcpy(to, PyUnicode_DATA(str), PyUnicode_GET_LENGTH(str) * char_size); while (done < nchars) { n = (done <= nchars-done) ? done : nchars-done; - Py_MEMCPY(to + (done * char_size), to, n * char_size); + memcpy(to + (done * char_size), to, n * char_size); done += n; } } @@ -12333,40 +12527,15 @@ unicode_repeat(PyObject *str, Py_ssize_t len) } PyObject * -PyUnicode_Replace(PyObject *obj, - PyObject *subobj, - PyObject *replobj, +PyUnicode_Replace(PyObject *str, + PyObject *substr, + PyObject *replstr, Py_ssize_t maxcount) { - PyObject *self; - PyObject *str1; - PyObject *str2; - PyObject *result; - - self = PyUnicode_FromObject(obj); - if (self == NULL) - return NULL; - str1 = PyUnicode_FromObject(subobj); - if (str1 == NULL) { - Py_DECREF(self); - return NULL; - } - str2 = PyUnicode_FromObject(replobj); - if (str2 == NULL) { - Py_DECREF(self); - Py_DECREF(str1); + if (ensure_unicode(str) < 0 || ensure_unicode(substr) < 0 || + ensure_unicode(replstr) < 0) return NULL; - } - if (PyUnicode_READY(self) == -1 || - PyUnicode_READY(str1) == -1 || - PyUnicode_READY(str2) == -1) - result = NULL; - else - result = replace(self, str1, str2, maxcount); - Py_DECREF(self); - Py_DECREF(str1); - Py_DECREF(str2); - return result; + return replace(str, substr, replstr, maxcount); } PyDoc_STRVAR(replace__doc__, @@ -12382,28 +12551,12 @@ unicode_replace(PyObject *self, PyObject *args) PyObject *str1; PyObject *str2; Py_ssize_t maxcount = -1; - PyObject *result; - if (!PyArg_ParseTuple(args, "OO|n:replace", &str1, &str2, &maxcount)) + if (!PyArg_ParseTuple(args, "UU|n:replace", &str1, &str2, &maxcount)) return NULL; if (PyUnicode_READY(self) == -1) return NULL; - str1 = PyUnicode_FromObject(str1); - if (str1 == NULL) - return NULL; - str2 = PyUnicode_FromObject(str2); - if (str2 == NULL) { - Py_DECREF(str1); - return NULL; - } - if (PyUnicode_READY(str1) == -1 || PyUnicode_READY(str2) == -1) - result = NULL; - else - result = replace(self, str1, str2, maxcount); - - Py_DECREF(str1); - Py_DECREF(str2); - return result; + return replace(self, str1, str2, maxcount); } static PyObject * @@ -12588,22 +12741,13 @@ unicode_rfind(PyObject *self, PyObject *args) Py_ssize_t end = 0; Py_ssize_t result; - if (!stringlib_parse_args_finds_unicode("rfind", args, &substring, - &start, &end)) + if (!parse_args_finds_unicode("rfind", args, &substring, &start, &end)) return NULL; - if (PyUnicode_READY(self) == -1) { - Py_DECREF(substring); - return NULL; - } - if (PyUnicode_READY(substring) == -1) { - Py_DECREF(substring); + if (PyUnicode_READY(self) == -1) return NULL; - } - - result = any_find_slice(-1, self, substring, start, end); - Py_DECREF(substring); + result = any_find_slice(self, substring, start, end, -1); if (result == -2) return NULL; @@ -12625,22 +12769,13 @@ unicode_rindex(PyObject *self, PyObject *args) Py_ssize_t end = 0; Py_ssize_t result; - if (!stringlib_parse_args_finds_unicode("rindex", args, &substring, - &start, &end)) + if (!parse_args_finds_unicode("rindex", args, &substring, &start, &end)) return NULL; - if (PyUnicode_READY(self) == -1) { - Py_DECREF(substring); - return NULL; - } - if (PyUnicode_READY(substring) == -1) { - Py_DECREF(substring); + if (PyUnicode_READY(self) == -1) return NULL; - } - result = any_find_slice(-1, self, substring, start, end); - - Py_DECREF(substring); + result = any_find_slice(self, substring, start, end, -1); if (result == -2) return NULL; @@ -12680,24 +12815,10 @@ unicode_rjust(PyObject *self, PyObject *args) PyObject * PyUnicode_Split(PyObject *s, PyObject *sep, Py_ssize_t maxsplit) { - PyObject *result; - - s = PyUnicode_FromObject(s); - if (s == NULL) + if (ensure_unicode(s) < 0 || (sep != NULL && ensure_unicode(sep) < 0)) return NULL; - if (sep != NULL) { - sep = PyUnicode_FromObject(sep); - if (sep == NULL) { - Py_DECREF(s); - return NULL; - } - } - - result = split(s, sep, maxsplit); - Py_DECREF(s); - Py_XDECREF(sep); - return result; + return split(s, sep, maxsplit); } PyDoc_STRVAR(split__doc__, @@ -12722,35 +12843,26 @@ unicode_split(PyObject *self, PyObject *args, PyObject *kwds) if (substring == Py_None) return split(self, NULL, maxcount); - else if (PyUnicode_Check(substring)) + + if (PyUnicode_Check(substring)) return split(self, substring, maxcount); - else - return PyUnicode_Split(self, substring, maxcount); + + PyErr_Format(PyExc_TypeError, + "must be str or None, not %.100s", + Py_TYPE(substring)->tp_name); + return NULL; } PyObject * -PyUnicode_Partition(PyObject *str_in, PyObject *sep_in) +PyUnicode_Partition(PyObject *str_obj, PyObject *sep_obj) { - PyObject* str_obj; - PyObject* sep_obj; PyObject* out; int kind1, kind2; void *buf1, *buf2; Py_ssize_t len1, len2; - str_obj = PyUnicode_FromObject(str_in); - if (!str_obj) - return NULL; - sep_obj = PyUnicode_FromObject(sep_in); - if (!sep_obj) { - Py_DECREF(str_obj); + if (ensure_unicode(str_obj) < 0 || ensure_unicode(sep_obj) < 0) return NULL; - } - if (PyUnicode_READY(sep_obj) == -1 || PyUnicode_READY(str_obj) == -1) { - Py_DECREF(sep_obj); - Py_DECREF(str_obj); - return NULL; - } kind1 = PyUnicode_KIND(str_obj); kind2 = PyUnicode_KIND(sep_obj); @@ -12764,8 +12876,6 @@ PyUnicode_Partition(PyObject *str_in, PyObject *sep_in) out = PyTuple_Pack(3, str_obj, unicode_empty, unicode_empty); Py_DECREF(unicode_empty); } - Py_DECREF(sep_obj); - Py_DECREF(str_obj); return out; } buf1 = PyUnicode_DATA(str_obj); @@ -12773,7 +12883,7 @@ PyUnicode_Partition(PyObject *str_in, PyObject *sep_in) if (kind2 != kind1) { buf2 = _PyUnicode_AsKind(sep_obj, kind1); if (!buf2) - goto onError; + return NULL; } switch (kind1) { @@ -12794,39 +12904,23 @@ PyUnicode_Partition(PyObject *str_in, PyObject *sep_in) out = 0; } - Py_DECREF(sep_obj); - Py_DECREF(str_obj); if (kind2 != kind1) PyMem_Free(buf2); return out; - onError: - Py_DECREF(sep_obj); - Py_DECREF(str_obj); - if (kind2 != kind1 && buf2) - PyMem_Free(buf2); - return NULL; } PyObject * -PyUnicode_RPartition(PyObject *str_in, PyObject *sep_in) +PyUnicode_RPartition(PyObject *str_obj, PyObject *sep_obj) { - PyObject* str_obj; - PyObject* sep_obj; PyObject* out; int kind1, kind2; void *buf1, *buf2; Py_ssize_t len1, len2; - str_obj = PyUnicode_FromObject(str_in); - if (!str_obj) + if (ensure_unicode(str_obj) < 0 || ensure_unicode(sep_obj) < 0) return NULL; - sep_obj = PyUnicode_FromObject(sep_in); - if (!sep_obj) { - Py_DECREF(str_obj); - return NULL; - } kind1 = PyUnicode_KIND(str_obj); kind2 = PyUnicode_KIND(sep_obj); @@ -12840,8 +12934,6 @@ PyUnicode_RPartition(PyObject *str_in, PyObject *sep_in) out = PyTuple_Pack(3, unicode_empty, unicode_empty, str_obj); Py_DECREF(unicode_empty); } - Py_DECREF(sep_obj); - Py_DECREF(str_obj); return out; } buf1 = PyUnicode_DATA(str_obj); @@ -12849,7 +12941,7 @@ PyUnicode_RPartition(PyObject *str_in, PyObject *sep_in) if (kind2 != kind1) { buf2 = _PyUnicode_AsKind(sep_obj, kind1); if (!buf2) - goto onError; + return NULL; } switch (kind1) { @@ -12870,18 +12962,10 @@ PyUnicode_RPartition(PyObject *str_in, PyObject *sep_in) out = 0; } - Py_DECREF(sep_obj); - Py_DECREF(str_obj); if (kind2 != kind1) PyMem_Free(buf2); return out; - onError: - Py_DECREF(sep_obj); - Py_DECREF(str_obj); - if (kind2 != kind1 && buf2) - PyMem_Free(buf2); - return NULL; } PyDoc_STRVAR(partition__doc__, @@ -12913,24 +12997,10 @@ unicode_rpartition(PyObject *self, PyObject *separator) PyObject * PyUnicode_RSplit(PyObject *s, PyObject *sep, Py_ssize_t maxsplit) { - PyObject *result; - - s = PyUnicode_FromObject(s); - if (s == NULL) + if (ensure_unicode(s) < 0 || (sep != NULL && ensure_unicode(sep) < 0)) return NULL; - if (sep != NULL) { - sep = PyUnicode_FromObject(sep); - if (sep == NULL) { - Py_DECREF(s); - return NULL; - } - } - - result = rsplit(s, sep, maxsplit); - Py_DECREF(s); - Py_XDECREF(sep); - return result; + return rsplit(s, sep, maxsplit); } PyDoc_STRVAR(rsplit__doc__, @@ -12955,10 +13025,14 @@ unicode_rsplit(PyObject *self, PyObject *args, PyObject *kwds) if (substring == Py_None) return rsplit(self, NULL, maxcount); - else if (PyUnicode_Check(substring)) + + if (PyUnicode_Check(substring)) return rsplit(self, substring, maxcount); - else - return PyUnicode_RSplit(self, substring, maxcount); + + PyErr_Format(PyExc_TypeError, + "must be str or None, not %.100s", + Py_TYPE(substring)->tp_name); + return NULL; } PyDoc_STRVAR(splitlines__doc__, @@ -13239,11 +13313,15 @@ unicode_startswith(PyObject *self, if (PyTuple_Check(subobj)) { Py_ssize_t i; for (i = 0; i < PyTuple_GET_SIZE(subobj); i++) { - substring = PyUnicode_FromObject(PyTuple_GET_ITEM(subobj, i)); - if (substring == NULL) + substring = PyTuple_GET_ITEM(subobj, i); + if (!PyUnicode_Check(substring)) { + PyErr_Format(PyExc_TypeError, + "tuple for startswith must only contain str, " + "not %.100s", + Py_TYPE(substring)->tp_name); return NULL; + } result = tailmatch(self, substring, start, end, -1); - Py_DECREF(substring); if (result == -1) return NULL; if (result) { @@ -13253,15 +13331,13 @@ unicode_startswith(PyObject *self, /* nothing matched */ Py_RETURN_FALSE; } - substring = PyUnicode_FromObject(subobj); - if (substring == NULL) { - if (PyErr_ExceptionMatches(PyExc_TypeError)) - PyErr_Format(PyExc_TypeError, "startswith first arg must be str or " - "a tuple of str, not %s", Py_TYPE(subobj)->tp_name); + if (!PyUnicode_Check(subobj)) { + PyErr_Format(PyExc_TypeError, + "startswith first arg must be str or " + "a tuple of str, not %.100s", Py_TYPE(subobj)->tp_name); return NULL; } - result = tailmatch(self, substring, start, end, -1); - Py_DECREF(substring); + result = tailmatch(self, subobj, start, end, -1); if (result == -1) return NULL; return PyBool_FromLong(result); @@ -13291,12 +13367,15 @@ unicode_endswith(PyObject *self, if (PyTuple_Check(subobj)) { Py_ssize_t i; for (i = 0; i < PyTuple_GET_SIZE(subobj); i++) { - substring = PyUnicode_FromObject( - PyTuple_GET_ITEM(subobj, i)); - if (substring == NULL) + substring = PyTuple_GET_ITEM(subobj, i); + if (!PyUnicode_Check(substring)) { + PyErr_Format(PyExc_TypeError, + "tuple for endswith must only contain str, " + "not %.100s", + Py_TYPE(substring)->tp_name); return NULL; + } result = tailmatch(self, substring, start, end, +1); - Py_DECREF(substring); if (result == -1) return NULL; if (result) { @@ -13305,61 +13384,67 @@ unicode_endswith(PyObject *self, } Py_RETURN_FALSE; } - substring = PyUnicode_FromObject(subobj); - if (substring == NULL) { - if (PyErr_ExceptionMatches(PyExc_TypeError)) - PyErr_Format(PyExc_TypeError, "endswith first arg must be str or " - "a tuple of str, not %s", Py_TYPE(subobj)->tp_name); + if (!PyUnicode_Check(subobj)) { + PyErr_Format(PyExc_TypeError, + "endswith first arg must be str or " + "a tuple of str, not %.100s", Py_TYPE(subobj)->tp_name); return NULL; } - result = tailmatch(self, substring, start, end, +1); - Py_DECREF(substring); + result = tailmatch(self, subobj, start, end, +1); if (result == -1) return NULL; return PyBool_FromLong(result); } -Py_LOCAL_INLINE(void) +static inline void _PyUnicodeWriter_Update(_PyUnicodeWriter *writer) { - if (!writer->readonly) + writer->maxchar = PyUnicode_MAX_CHAR_VALUE(writer->buffer); + writer->data = PyUnicode_DATA(writer->buffer); + + if (!writer->readonly) { + writer->kind = PyUnicode_KIND(writer->buffer); writer->size = PyUnicode_GET_LENGTH(writer->buffer); + } else { + /* use a value smaller than PyUnicode_1BYTE_KIND() so + _PyUnicodeWriter_PrepareKind() will copy the buffer. */ + writer->kind = PyUnicode_WCHAR_KIND; + assert(writer->kind <= PyUnicode_1BYTE_KIND); + /* Copy-on-write mode: set buffer size to 0 so * _PyUnicodeWriter_Prepare() will copy (and enlarge) the buffer on * next write. */ writer->size = 0; } - writer->maxchar = PyUnicode_MAX_CHAR_VALUE(writer->buffer); - writer->data = PyUnicode_DATA(writer->buffer); - writer->kind = PyUnicode_KIND(writer->buffer); } void _PyUnicodeWriter_Init(_PyUnicodeWriter *writer) { memset(writer, 0, sizeof(*writer)); -#ifdef Py_DEBUG - writer->kind = 5; /* invalid kind */ -#endif + + /* ASCII is the bare minimum */ writer->min_char = 127; + + /* use a value smaller than PyUnicode_1BYTE_KIND() so + _PyUnicodeWriter_PrepareKind() will copy the buffer. */ + writer->kind = PyUnicode_WCHAR_KIND; + assert(writer->kind <= PyUnicode_1BYTE_KIND); } int _PyUnicodeWriter_PrepareInternal(_PyUnicodeWriter *writer, Py_ssize_t length, Py_UCS4 maxchar) { -#ifdef MS_WINDOWS - /* On Windows, overallocate by 50% is the best factor */ -# define OVERALLOCATE_FACTOR 2 -#else - /* On Linux, overallocate by 25% is the best factor */ -# define OVERALLOCATE_FACTOR 4 -#endif Py_ssize_t newlen; PyObject *newbuffer; - assert(length > 0); + assert(maxchar <= MAX_UNICODE); + + /* ensure that the _PyUnicodeWriter_Prepare macro was used */ + assert((maxchar > writer->maxchar && length >= 0) + || length > 0); if (length > PY_SSIZE_T_MAX - writer->pos) { PyErr_NoMemory(); @@ -13425,9 +13510,32 @@ _PyUnicodeWriter_PrepareInternal(_PyUnicodeWriter *writer, #undef OVERALLOCATE_FACTOR } -Py_LOCAL_INLINE(int) +int +_PyUnicodeWriter_PrepareKindInternal(_PyUnicodeWriter *writer, + enum PyUnicode_Kind kind) +{ + Py_UCS4 maxchar; + + /* ensure that the _PyUnicodeWriter_PrepareKind macro was used */ + assert(writer->kind < kind); + + switch (kind) + { + case PyUnicode_1BYTE_KIND: maxchar = 0xff; break; + case PyUnicode_2BYTE_KIND: maxchar = 0xffff; break; + case PyUnicode_4BYTE_KIND: maxchar = 0x10ffff; break; + default: + assert(0 && "invalid kind"); + return -1; + } + + return _PyUnicodeWriter_PrepareInternal(writer, 0, maxchar); +} + +static inline int _PyUnicodeWriter_WriteCharInline(_PyUnicodeWriter *writer, Py_UCS4 ch) { + assert(ch <= MAX_UNICODE); if (_PyUnicodeWriter_Prepare(writer, 1, ch) < 0) return -1; PyUnicode_WRITE(writer->kind, writer->data, writer->pos, ch); @@ -13540,7 +13648,7 @@ _PyUnicodeWriter_WriteASCIIString(_PyUnicodeWriter *writer, const Py_UCS1 *str = (const Py_UCS1 *)ascii; Py_UCS1 *data = writer->data; - Py_MEMCPY(data + writer->pos, str, len); + memcpy(data + writer->pos, str, len); break; } case PyUnicode_2BYTE_KIND: @@ -13585,27 +13693,30 @@ PyObject * _PyUnicodeWriter_Finish(_PyUnicodeWriter *writer) { PyObject *str; + if (writer->pos == 0) { Py_CLEAR(writer->buffer); _Py_RETURN_UNICODE_EMPTY(); } + + str = writer->buffer; + writer->buffer = NULL; + if (writer->readonly) { - str = writer->buffer; - writer->buffer = NULL; assert(PyUnicode_GET_LENGTH(str) == writer->pos); return str; } - if (PyUnicode_GET_LENGTH(writer->buffer) != writer->pos) { - PyObject *newbuffer; - newbuffer = resize_compact(writer->buffer, writer->pos); - if (newbuffer == NULL) { - Py_CLEAR(writer->buffer); + + if (PyUnicode_GET_LENGTH(str) != writer->pos) { + PyObject *str2; + str2 = resize_compact(str, writer->pos); + if (str2 == NULL) { + Py_DECREF(str); return NULL; } - writer->buffer = newbuffer; + str = str2; } - str = writer->buffer; - writer->buffer = NULL; + assert(_PyUnicode_CheckConsistency(str, 1)); return unicode_result_ready(str); } @@ -14747,13 +14858,10 @@ PyUnicode_Format(PyObject *format, PyObject *args) return NULL; } - ctx.fmtstr = PyUnicode_FromObject(format); - if (ctx.fmtstr == NULL) - return NULL; - if (PyUnicode_READY(ctx.fmtstr) == -1) { - Py_DECREF(ctx.fmtstr); + if (ensure_unicode(format) < 0) return NULL; - } + + ctx.fmtstr = format; ctx.fmtdata = PyUnicode_DATA(ctx.fmtstr); ctx.fmtkind = PyUnicode_KIND(ctx.fmtstr); ctx.fmtcnt = PyUnicode_GET_LENGTH(ctx.fmtstr); @@ -14813,11 +14921,9 @@ PyUnicode_Format(PyObject *format, PyObject *args) if (ctx.args_owned) { Py_DECREF(ctx.args); } - Py_DECREF(ctx.fmtstr); return _PyUnicodeWriter_Finish(&ctx.writer); onError: - Py_DECREF(ctx.fmtstr); _PyUnicodeWriter_Dealloc(&ctx.writer); if (ctx.args_owned) { Py_DECREF(ctx.args); @@ -14934,7 +15040,7 @@ unicode_subtype_new(PyTypeObject *type, PyObject *args, PyObject *kwds) _PyUnicode_WSTR(self) = (wchar_t *)data; } - Py_MEMCPY(data, PyUnicode_DATA(unicode), + memcpy(data, PyUnicode_DATA(unicode), kind * (length + 1)); assert(_PyUnicode_CheckConsistency(self, 1)); #ifdef Py_DEBUG @@ -15095,26 +15201,18 @@ PyUnicode_InternInPlace(PyObject **p) return; } } - /* It might be that the GetItem call fails even - though the key is present in the dictionary, - namely when this happens during a stack overflow. */ Py_ALLOW_RECURSION - t = PyDict_GetItem(interned, s); + t = PyDict_SetDefault(interned, s, s); Py_END_ALLOW_RECURSION - - if (t) { - Py_INCREF(t); - Py_SETREF(*p, t); + if (t == NULL) { + PyErr_Clear(); return; } - - PyThreadState_GET()->recursion_critical = 1; - if (PyDict_SetItem(interned, s, s) < 0) { - PyErr_Clear(); - PyThreadState_GET()->recursion_critical = 0; + if (t != s) { + Py_INCREF(t); + Py_SETREF(*p, t); return; } - PyThreadState_GET()->recursion_critical = 0; /* The two references in interned are not counted by refcnt. The deallocator will take care of this */ Py_REFCNT(s) -= 2; |