diff options
Diffstat (limited to 'Modules/_pickle.c')
| -rw-r--r-- | Modules/_pickle.c | 387 |
1 files changed, 244 insertions, 143 deletions
diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 18eaa38bdb..ae801f74ef 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -319,6 +319,7 @@ typedef struct PicklerObject { objects to support self-referential objects pickling. */ PyObject *pers_func; /* persistent_id() method, can be NULL */ + PyObject *dispatch_table; /* private dispatch_table, can be NULL */ PyObject *arg; PyObject *write; /* write() method of the output stream. */ @@ -386,8 +387,8 @@ static PyTypeObject Unpickler_Type; /************************************************************************* - A custom hashtable mapping void* to longs. This is used by the pickler for - memoization. Using a custom hashtable rather than PyDict allows us to skip + A custom hashtable mapping void* to Python ints. This is used by the pickler + for memoization. Using a custom hashtable rather than PyDict allows us to skip a bunch of unnecessary object creation. This makes a huge performance difference. */ @@ -605,7 +606,7 @@ PyMemoTable_Set(PyMemoTable *self, PyObject *key, Py_ssize_t value) /*************************************************************************/ /* Helpers for creating the argument tuple passed to functions. This has the - performance advantage of calling PyTuple_New() only once. + performance advantage of calling PyTuple_New() only once. XXX(avassalotti): Inline directly in _Pickler_FastCall() and _Unpickler_FastCall(). */ @@ -764,6 +765,7 @@ _Pickler_New(void) return NULL; self->pers_func = NULL; + self->dispatch_table = NULL; self->arg = NULL; self->write = NULL; self->proto = 0; @@ -813,7 +815,7 @@ _Pickler_SetProtocol(PicklerObject *self, PyObject *proto_obj, fix_imports = PyObject_IsTrue(fix_imports_obj); if (fix_imports == -1) return -1; - + self->proto = proto; self->bin = proto > 0; self->fix_imports = fix_imports && proto < 3; @@ -826,8 +828,9 @@ _Pickler_SetProtocol(PicklerObject *self, PyObject *proto_obj, static int _Pickler_SetOutputStream(PicklerObject *self, PyObject *file) { + _Py_IDENTIFIER(write); assert(file != NULL); - self->write = PyObject_GetAttrString(file, "write"); + self->write = _PyObject_GetAttrId(file, &PyId_write); if (self->write == NULL) { if (PyErr_ExceptionMatches(PyExc_AttributeError)) PyErr_SetString(PyExc_TypeError, @@ -909,7 +912,7 @@ _Unpickler_ReadFromFile(UnpicklerObject *self, Py_ssize_t n) Py_ssize_t read_size, prefetched_size = 0; assert(self->read != NULL); - + if (_Unpickler_SkipConsumed(self) < 0) return -1; @@ -1037,7 +1040,7 @@ _Unpickler_Readline(UnpicklerObject *self, char **result) self->next_read_idx = num_read; return _Unpickler_CopyLine(self, self->input_buffer, num_read, result); } - + /* If we get here, we've run off the end of the input string. Return the remaining string and let the caller figure it out. */ *result = self->input_buffer + self->next_read_idx; @@ -1173,15 +1176,19 @@ _Unpickler_New(void) static int _Unpickler_SetInputStream(UnpicklerObject *self, PyObject *file) { - self->peek = PyObject_GetAttrString(file, "peek"); + _Py_IDENTIFIER(peek); + _Py_IDENTIFIER(read); + _Py_IDENTIFIER(readline); + + self->peek = _PyObject_GetAttrId(file, &PyId_peek); if (self->peek == NULL) { if (PyErr_ExceptionMatches(PyExc_AttributeError)) PyErr_Clear(); else return -1; } - self->read = PyObject_GetAttrString(file, "read"); - self->readline = PyObject_GetAttrString(file, "readline"); + self->read = _PyObject_GetAttrId(file, &PyId_read); + self->readline = _PyObject_GetAttrId(file, &PyId_readline); if (self->readline == NULL || self->read == NULL) { if (PyErr_ExceptionMatches(PyExc_AttributeError)) PyErr_SetString(PyExc_TypeError, @@ -1543,7 +1550,10 @@ save_long(PicklerObject *self, PyObject *obj) PyErr_Clear(); } else - return save_int(self, val); +#if SIZEOF_LONG > 4 + if (val <= 0x7fffffffL && val >= -0x80000000L) +#endif + return save_int(self, val); if (self->proto >= 2) { /* Linear-time pickling. */ @@ -1571,8 +1581,8 @@ save_long(PicklerObject *self, PyObject *obj) * need another byte even if there aren't any leftovers: * the most-significant bit of the most-significant byte * acts like a sign bit, and it's usually got a sense - * opposite of the one we need. The exception is longs - * of the form -(2**(8*j-1)) for j > 0. Such a long is + * opposite of the one we need. The exception is ints + * of the form -(2**(8*j-1)) for j > 0. Such an int is * its own 256's-complement, so has the right sign bit * even without the extra byte. That's a pain to check * for in advance, though, so we always grab an extra @@ -1581,7 +1591,7 @@ save_long(PicklerObject *self, PyObject *obj) nbytes = (nbits >> 3) + 1; if (nbytes > 0x7fffffffL) { PyErr_SetString(PyExc_OverflowError, - "long too large to pickle"); + "int too large to pickle"); goto error; } repr = PyBytes_FromStringAndSize(NULL, (Py_ssize_t)nbytes); @@ -1593,12 +1603,12 @@ save_long(PicklerObject *self, PyObject *obj) 1 /* little endian */ , 1 /* signed */ ); if (i < 0) goto error; - /* If the long is negative, this may be a byte more than + /* If the int is negative, this may be a byte more than * needed. This is so iff the MSB is all redundant sign * bits. */ if (sign < 0 && - nbytes > 1 && + nbytes > 1 && pdata[nbytes - 1] == 0xff && (pdata[nbytes - 2] & 0x80) != 0) { nbytes--; @@ -1664,7 +1674,7 @@ save_float(PicklerObject *self, PyObject *obj) return -1; if (_Pickler_Write(self, pdata, 9) < 0) return -1; - } + } else { int result = -1; char *buf = NULL; @@ -1778,7 +1788,7 @@ save_bytes(PicklerObject *self, PyObject *obj) } else { PyErr_SetString(PyExc_OverflowError, - "cannot serialize a bytes object larger than 4GB"); + "cannot serialize a bytes object larger than 4 GiB"); return -1; /* string too large */ } @@ -1798,90 +1808,65 @@ save_bytes(PicklerObject *self, PyObject *obj) /* A copy of PyUnicode_EncodeRawUnicodeEscape() that also translates backslash and newline characters to \uXXXX escapes. */ static PyObject * -raw_unicode_escape(const Py_UNICODE *s, Py_ssize_t size) +raw_unicode_escape(PyObject *obj) { PyObject *repr, *result; char *p; - char *q; + Py_ssize_t i, size, expandsize; + void *data; + unsigned int kind; - static const char *hexdigits = "0123456789abcdef"; + if (PyUnicode_READY(obj)) + return NULL; + + size = PyUnicode_GET_LENGTH(obj); + data = PyUnicode_DATA(obj); + kind = PyUnicode_KIND(obj); + if (kind == PyUnicode_4BYTE_KIND) + expandsize = 10; + else + expandsize = 6; -#ifdef Py_UNICODE_WIDE - const Py_ssize_t expandsize = 10; -#else - const Py_ssize_t expandsize = 6; -#endif - if (size > PY_SSIZE_T_MAX / expandsize) return PyErr_NoMemory(); - repr = PyByteArray_FromStringAndSize(NULL, expandsize * size); if (repr == NULL) return NULL; if (size == 0) goto done; - p = q = PyByteArray_AS_STRING(repr); - while (size-- > 0) { - Py_UNICODE ch = *s++; -#ifdef Py_UNICODE_WIDE + p = PyByteArray_AS_STRING(repr); + for (i=0; i < size; i++) { + Py_UCS4 ch = PyUnicode_READ(kind, data, i); /* Map 32-bit characters to '\Uxxxxxxxx' */ if (ch >= 0x10000) { *p++ = '\\'; *p++ = 'U'; - *p++ = hexdigits[(ch >> 28) & 0xf]; - *p++ = hexdigits[(ch >> 24) & 0xf]; - *p++ = hexdigits[(ch >> 20) & 0xf]; - *p++ = hexdigits[(ch >> 16) & 0xf]; - *p++ = hexdigits[(ch >> 12) & 0xf]; - *p++ = hexdigits[(ch >> 8) & 0xf]; - *p++ = hexdigits[(ch >> 4) & 0xf]; - *p++ = hexdigits[ch & 15]; + *p++ = Py_hexdigits[(ch >> 28) & 0xf]; + *p++ = Py_hexdigits[(ch >> 24) & 0xf]; + *p++ = Py_hexdigits[(ch >> 20) & 0xf]; + *p++ = Py_hexdigits[(ch >> 16) & 0xf]; + *p++ = Py_hexdigits[(ch >> 12) & 0xf]; + *p++ = Py_hexdigits[(ch >> 8) & 0xf]; + *p++ = Py_hexdigits[(ch >> 4) & 0xf]; + *p++ = Py_hexdigits[ch & 15]; } - else -#else - /* Map UTF-16 surrogate pairs to '\U00xxxxxx' */ - if (ch >= 0xD800 && ch < 0xDC00) { - Py_UNICODE ch2; - Py_UCS4 ucs; - - ch2 = *s++; - size--; - if (ch2 >= 0xDC00 && ch2 <= 0xDFFF) { - ucs = (((ch & 0x03FF) << 10) | (ch2 & 0x03FF)) + 0x00010000; - *p++ = '\\'; - *p++ = 'U'; - *p++ = hexdigits[(ucs >> 28) & 0xf]; - *p++ = hexdigits[(ucs >> 24) & 0xf]; - *p++ = hexdigits[(ucs >> 20) & 0xf]; - *p++ = hexdigits[(ucs >> 16) & 0xf]; - *p++ = hexdigits[(ucs >> 12) & 0xf]; - *p++ = hexdigits[(ucs >> 8) & 0xf]; - *p++ = hexdigits[(ucs >> 4) & 0xf]; - *p++ = hexdigits[ucs & 0xf]; - continue; - } - /* Fall through: isolated surrogates are copied as-is */ - s--; - size++; - } -#endif /* Map 16-bit characters to '\uxxxx' */ - if (ch >= 256 || ch == '\\' || ch == '\n') { + else if (ch >= 256 || ch == '\\' || ch == '\n') { *p++ = '\\'; *p++ = 'u'; - *p++ = hexdigits[(ch >> 12) & 0xf]; - *p++ = hexdigits[(ch >> 8) & 0xf]; - *p++ = hexdigits[(ch >> 4) & 0xf]; - *p++ = hexdigits[ch & 15]; + *p++ = Py_hexdigits[(ch >> 12) & 0xf]; + *p++ = Py_hexdigits[(ch >> 8) & 0xf]; + *p++ = Py_hexdigits[(ch >> 4) & 0xf]; + *p++ = Py_hexdigits[ch & 15]; } /* Copy everything else as-is */ else *p++ = (char) ch; } - size = p - q; + size = p - PyByteArray_AS_STRING(repr); - done: +done: result = PyBytes_FromStringAndSize(PyByteArray_AS_STRING(repr), size); Py_DECREF(repr); return result; @@ -1896,16 +1881,14 @@ save_unicode(PicklerObject *self, PyObject *obj) if (self->bin) { char pdata[5]; - encoded = PyUnicode_EncodeUTF8(PyUnicode_AS_UNICODE(obj), - PyUnicode_GET_SIZE(obj), - "surrogatepass"); + encoded = PyUnicode_AsEncodedString(obj, "utf-8", "surrogatepass"); if (encoded == NULL) goto error; size = PyBytes_GET_SIZE(encoded); if (size > 0xffffffffL) { PyErr_SetString(PyExc_OverflowError, - "cannot serialize a string larger than 4GB"); + "cannot serialize a string larger than 4 GiB"); goto error; /* string too large */ } @@ -1924,8 +1907,7 @@ save_unicode(PicklerObject *self, PyObject *obj) else { const char unicode_op = UNICODE; - encoded = raw_unicode_escape(PyUnicode_AS_UNICODE(obj), - PyUnicode_GET_SIZE(obj)); + encoded = raw_unicode_escape(obj); if (encoded == NULL) goto error; @@ -2544,7 +2526,9 @@ save_dict(PicklerObject *self, PyObject *obj) status = batch_dict_exact(self, obj); Py_LeaveRecursiveCall(); } else { - items = PyObject_CallMethod(obj, "items", "()"); + _Py_IDENTIFIER(items); + + items = _PyObject_CallMethodId(obj, &PyId_items, "()"); if (items == NULL) goto error; iter = PyObject_GetIter(items); @@ -2828,6 +2812,60 @@ save_global(PicklerObject *self, PyObject *obj, PyObject *name) } static int +save_ellipsis(PicklerObject *self, PyObject *obj) +{ + PyObject *str = PyUnicode_FromString("Ellipsis"); + int res; + if (str == NULL) + return -1; + res = save_global(self, Py_Ellipsis, str); + Py_DECREF(str); + return res; +} + +static int +save_notimplemented(PicklerObject *self, PyObject *obj) +{ + PyObject *str = PyUnicode_FromString("NotImplemented"); + int res; + if (str == NULL) + return -1; + res = save_global(self, Py_NotImplemented, str); + Py_DECREF(str); + return res; +} + +static int +save_singleton_type(PicklerObject *self, PyObject *obj, PyObject *singleton) +{ + PyObject *reduce_value; + int status; + + reduce_value = Py_BuildValue("O(O)", &PyType_Type, singleton); + if (reduce_value == NULL) { + return -1; + } + status = save_reduce(self, reduce_value, obj); + Py_DECREF(reduce_value); + return status; +} + +static int +save_type(PicklerObject *self, PyObject *obj) +{ + if (obj == (PyObject *)&_PyNone_Type) { + return save_singleton_type(self, obj, Py_None); + } + else if (obj == (PyObject *)&PyEllipsis_Type) { + return save_singleton_type(self, obj, Py_Ellipsis); + } + else if (obj == (PyObject *)&_PyNotImplemented_Type) { + return save_singleton_type(self, obj, Py_NotImplemented); + } + return save_global(self, obj, NULL); +} + +static int save_pers(PicklerObject *self, PyObject *obj, PyObject *func) { PyObject *pid = NULL; @@ -2881,6 +2919,28 @@ save_pers(PicklerObject *self, PyObject *obj, PyObject *func) return status; } +static PyObject * +get_class(PyObject *obj) +{ + PyObject *cls; + static PyObject *str_class; + + if (str_class == NULL) { + str_class = PyUnicode_InternFromString("__class__"); + if (str_class == NULL) + return NULL; + } + cls = PyObject_GetAttr(obj, str_class); + if (cls == NULL) { + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Clear(); + cls = (PyObject *) Py_TYPE(obj); + Py_INCREF(cls); + } + } + return cls; +} + /* We're saving obj, and args is the 2-thru-5 tuple returned by the * appropriate __reduce__ method for obj. */ @@ -2928,7 +2988,7 @@ save_reduce(PicklerObject *self, PyObject *args, PyObject *obj) if (listitems == Py_None) listitems = NULL; else if (!PyIter_Check(listitems)) { - PyErr_Format(PicklingError, "Fourth element of tuple" + PyErr_Format(PicklingError, "fourth element of the tuple " "returned by __reduce__ must be an iterator, not %s", Py_TYPE(listitems)->tp_name); return -1; @@ -2937,7 +2997,7 @@ save_reduce(PicklerObject *self, PyObject *args, PyObject *obj) if (dictitems == Py_None) dictitems = NULL; else if (!PyIter_Check(dictitems)) { - PyErr_Format(PicklingError, "Fifth element of tuple" + PyErr_Format(PicklingError, "fifth element of the tuple " "returned by __reduce__ must be an iterator, not %s", Py_TYPE(dictitems)->tp_name); return -1; @@ -2946,17 +3006,18 @@ save_reduce(PicklerObject *self, PyObject *args, PyObject *obj) /* Protocol 2 special case: if callable's name is __newobj__, use NEWOBJ. */ if (use_newobj) { - static PyObject *newobj_str = NULL; - PyObject *name_str; + static PyObject *newobj_str = NULL, *name_str = NULL; + PyObject *name; if (newobj_str == NULL) { newobj_str = PyUnicode_InternFromString("__newobj__"); - if (newobj_str == NULL) + name_str = PyUnicode_InternFromString("__name__"); + if (newobj_str == NULL || name_str == NULL) return -1; } - name_str = PyObject_GetAttrString(callable, "__name__"); - if (name_str == NULL) { + name = PyObject_GetAttr(callable, name_str); + if (name == NULL) { if (PyErr_ExceptionMatches(PyExc_AttributeError)) PyErr_Clear(); else @@ -2964,9 +3025,9 @@ save_reduce(PicklerObject *self, PyObject *args, PyObject *obj) use_newobj = 0; } else { - use_newobj = PyUnicode_Check(name_str) && - PyUnicode_Compare(name_str, newobj_str) == 0; - Py_DECREF(name_str); + use_newobj = PyUnicode_Check(name) && + PyUnicode_Compare(name, newobj_str) == 0; + Py_DECREF(name); } } if (use_newobj) { @@ -2982,20 +3043,14 @@ save_reduce(PicklerObject *self, PyObject *args, PyObject *obj) } cls = PyTuple_GET_ITEM(argtup, 0); - if (!PyObject_HasAttrString(cls, "__new__")) { + if (!PyType_Check(cls)) { PyErr_SetString(PicklingError, "args[0] from " - "__newobj__ args has no __new__"); + "__newobj__ args is not a type"); return -1; } if (obj != NULL) { - obj_class = PyObject_GetAttrString(obj, "__class__"); - if (obj_class == NULL) { - if (PyErr_ExceptionMatches(PyExc_AttributeError)) - PyErr_Clear(); - else - return -1; - } + obj_class = get_class(obj); p = obj_class != cls; /* true iff a problem */ Py_DECREF(obj_class); if (p) { @@ -3069,7 +3124,7 @@ save_reduce(PicklerObject *self, PyObject *args, PyObject *obj) return -1; if (state) { - if (save(self, state, 0) < 0 || + if (save(self, state, 0) < 0 || _Pickler_Write(self, &build_op, 1) < 0) return -1; } @@ -3113,6 +3168,14 @@ save(PicklerObject *self, PyObject *obj, int pers_save) status = save_none(self, obj); goto done; } + else if (obj == Py_Ellipsis) { + status = save_ellipsis(self, obj); + goto done; + } + else if (obj == Py_NotImplemented) { + status = save_notimplemented(self, obj); + goto done; + } else if (obj == Py_False || obj == Py_True) { status = save_bool(self, obj); goto done; @@ -3156,7 +3219,7 @@ save(PicklerObject *self, PyObject *obj, int pers_save) goto done; } else if (type == &PyType_Type) { - status = save_global(self, obj, NULL); + status = save_type(self, obj); goto done; } else if (type == &PyFunction_Type) { @@ -3177,17 +3240,24 @@ save(PicklerObject *self, PyObject *obj, int pers_save) /* XXX: This part needs some unit tests. */ /* Get a reduction callable, and call it. This may come from - * copyreg.dispatch_table, the object's __reduce_ex__ method, - * or the object's __reduce__ method. + * self.dispatch_table, copyreg.dispatch_table, the object's + * __reduce_ex__ method, or the object's __reduce__ method. */ - reduce_func = PyDict_GetItem(dispatch_table, (PyObject *)type); + if (self->dispatch_table == NULL) { + reduce_func = PyDict_GetItem(dispatch_table, (PyObject *)type); + /* PyDict_GetItem() unlike PyObject_GetItem() and + PyObject_GetAttr() returns a borrowed ref */ + Py_XINCREF(reduce_func); + } else { + reduce_func = PyObject_GetItem(self->dispatch_table, (PyObject *)type); + if (reduce_func == NULL) { + if (PyErr_ExceptionMatches(PyExc_KeyError)) + PyErr_Clear(); + else + goto error; + } + } if (reduce_func != NULL) { - /* Here, the reference count of the reduce_func object returned by - PyDict_GetItem needs to be increased to be consistent with the one - returned by PyObject_GetAttr. This is allow us to blindly DECREF - reduce_func at the end of the save() routine. - */ - Py_INCREF(reduce_func); Py_INCREF(obj); reduce_value = _Pickler_FastCall(self, reduce_func, obj); } @@ -3323,7 +3393,7 @@ Pickler_dump(PicklerObject *self, PyObject *args) Developers often forget to call __init__() in their subclasses, which would trigger a segfault without this check. */ if (self->write == NULL) { - PyErr_Format(PicklingError, + PyErr_Format(PicklingError, "Pickler.__init__() was not called by %s.__init__()", Py_TYPE(self)->tp_name); return NULL; @@ -3360,6 +3430,7 @@ Pickler_dealloc(PicklerObject *self) Py_XDECREF(self->output_buffer); Py_XDECREF(self->write); Py_XDECREF(self->pers_func); + Py_XDECREF(self->dispatch_table); Py_XDECREF(self->arg); Py_XDECREF(self->fast_memo); @@ -3373,6 +3444,7 @@ Pickler_traverse(PicklerObject *self, visitproc visit, void *arg) { Py_VISIT(self->write); Py_VISIT(self->pers_func); + Py_VISIT(self->dispatch_table); Py_VISIT(self->arg); Py_VISIT(self->fast_memo); return 0; @@ -3384,6 +3456,7 @@ Pickler_clear(PicklerObject *self) Py_CLEAR(self->output_buffer); Py_CLEAR(self->write); Py_CLEAR(self->pers_func); + Py_CLEAR(self->dispatch_table); Py_CLEAR(self->arg); Py_CLEAR(self->fast_memo); @@ -3427,6 +3500,8 @@ Pickler_init(PicklerObject *self, PyObject *args, PyObject *kwds) PyObject *file; PyObject *proto_obj = NULL; PyObject *fix_imports = Py_True; + _Py_IDENTIFIER(persistent_id); + _Py_IDENTIFIER(dispatch_table); if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|OO:Pickler", kwlist, &file, &proto_obj, &fix_imports)) @@ -3462,12 +3537,19 @@ Pickler_init(PicklerObject *self, PyObject *args, PyObject *kwds) self->fast_nesting = 0; self->fast_memo = NULL; self->pers_func = NULL; - if (PyObject_HasAttrString((PyObject *)self, "persistent_id")) { - self->pers_func = PyObject_GetAttrString((PyObject *)self, - "persistent_id"); + if (_PyObject_HasAttrId((PyObject *)self, &PyId_persistent_id)) { + self->pers_func = _PyObject_GetAttrId((PyObject *)self, + &PyId_persistent_id); if (self->pers_func == NULL) return -1; } + self->dispatch_table = NULL; + if (_PyObject_HasAttrId((PyObject *)self, &PyId_dispatch_table)) { + self->dispatch_table = _PyObject_GetAttrId((PyObject *)self, + &PyId_dispatch_table); + if (self->dispatch_table == NULL) + return -1; + } return 0; } @@ -3749,6 +3831,7 @@ Pickler_set_persid(PicklerObject *self, PyObject *value) static PyMemberDef Pickler_members[] = { {"bin", T_INT, offsetof(PicklerObject, bin)}, {"fast", T_INT, offsetof(PicklerObject, fast)}, + {"dispatch_table", T_OBJECT_EX, offsetof(PicklerObject, dispatch_table)}, {NULL} }; @@ -3803,7 +3886,7 @@ static PyTypeObject Pickler_Type = { 0, /*tp_is_gc*/ }; -/* Temporary helper for calling self.find_class(). +/* Temporary helper for calling self.find_class(). XXX: It would be nice to able to avoid Python function call overhead, by using directly the C version of find_class(), when find_class() is not @@ -3813,8 +3896,10 @@ static PyTypeObject Pickler_Type = { static PyObject * find_class(UnpicklerObject *self, PyObject *module_name, PyObject *global_name) { - return PyObject_CallMethod((PyObject *)self, "find_class", "OO", - module_name, global_name); + _Py_IDENTIFIER(find_class); + + return _PyObject_CallMethodId((PyObject *)self, &PyId_find_class, "OO", + module_name, global_name); } static Py_ssize_t @@ -3856,13 +3941,13 @@ load_int(UnpicklerObject *self) return bad_readline(); errno = 0; - /* XXX: Should the base argument of strtol() be explicitly set to 10? + /* XXX: Should the base argument of strtol() be explicitly set to 10? XXX(avassalotti): Should this uses PyOS_strtol()? */ x = strtol(s, &endptr, 0); if (errno || (*endptr != '\n' && *endptr != '\0')) { /* Hm, maybe we've got something long. Let's try reading - * it as a Python long object. */ + * it as a Python int object. */ errno = 0; /* XXX: Same thing about the base here. */ value = PyLong_FromString(s, NULL, 0); @@ -4116,7 +4201,7 @@ load_string(UnpicklerObject *self) if ((len = _Unpickler_Readline(self, &s)) < 0) return -1; - if (len < 3) + if (len < 2) return bad_readline(); if ((s = strdup(s)) == NULL) { PyErr_NoMemory(); @@ -4124,14 +4209,14 @@ load_string(UnpicklerObject *self) } /* Strip outermost quotes */ - while (s[len - 1] <= ' ') + while (len > 0 && s[len - 1] <= ' ') len--; - if (s[0] == '"' && s[len - 1] == '"') { + if (len > 1 && s[0] == '"' && s[len - 1] == '"') { s[len - 1] = '\0'; p = s + 1; len -= 2; } - else if (s[0] == '\'' && s[len - 1] == '\'') { + else if (len > 1 && s[0] == '\'' && s[len - 1] == '\'') { s[len - 1] = '\0'; p = s + 1; len -= 2; @@ -4221,7 +4306,7 @@ load_binstring(UnpicklerObject *self) x = calc_binint(s, 4); if (x < 0) { - PyErr_SetString(UnpicklingError, + PyErr_SetString(UnpicklingError, "BINSTRING pickle has negative byte count"); return -1; } @@ -4418,16 +4503,19 @@ static PyObject * instantiate(PyObject *cls, PyObject *args) { PyObject *result = NULL; + _Py_IDENTIFIER(__getinitargs__); /* Caller must assure args are a tuple. Normally, args come from Pdata_poptuple which packs objects from the top of the stack into a newly created tuple. */ assert(PyTuple_Check(args)); if (Py_SIZE(args) > 0 || !PyType_Check(cls) || - PyObject_HasAttrString(cls, "__getinitargs__")) { + _PyObject_HasAttrId(cls, &PyId___getinitargs__)) { result = PyObject_CallObject(cls, args); } else { - result = PyObject_CallMethod(cls, "__new__", "O", cls); + _Py_IDENTIFIER(__new__); + + result = _PyObject_CallMethodId(cls, &PyId___new__, "O", cls); } return result; } @@ -4607,7 +4695,7 @@ load_persid(UnpicklerObject *self) if (self->pers_func) { if ((len = _Unpickler_Readline(self, &s)) < 0) return -1; - if (len < 2) + if (len < 1) return bad_readline(); pid = PyBytes_FromStringAndSize(s, len - 1); @@ -4758,9 +4846,10 @@ load_binget(UnpicklerObject *self) value = _Unpickler_MemoGet(self, idx); if (value == NULL) { PyObject *key = PyLong_FromSsize_t(idx); - if (!PyErr_Occurred()) + if (key != NULL) { PyErr_SetObject(PyExc_KeyError, key); - Py_DECREF(key); + Py_DECREF(key); + } return -1; } @@ -4783,9 +4872,10 @@ load_long_binget(UnpicklerObject *self) value = _Unpickler_MemoGet(self, idx); if (value == NULL) { PyObject *key = PyLong_FromSsize_t(idx); - if (!PyErr_Occurred()) + if (key != NULL) { PyErr_SetObject(PyExc_KeyError, key); - Py_DECREF(key); + Py_DECREF(key); + } return -1; } @@ -4968,8 +5058,9 @@ do_append(UnpicklerObject *self, Py_ssize_t x) } else { PyObject *append_func; + _Py_IDENTIFIER(append); - append_func = PyObject_GetAttrString(list, "append"); + append_func = _PyObject_GetAttrId(list, &PyId_append); if (append_func == NULL) return -1; for (i = x; i < len; i++) { @@ -4980,11 +5071,13 @@ do_append(UnpicklerObject *self, Py_ssize_t x) if (result == NULL) { Pdata_clear(self->stack, i + 1); Py_SIZE(self->stack) = x; + Py_DECREF(append_func); return -1; } Py_DECREF(result); } Py_SIZE(self->stack) = x; + Py_DECREF(append_func); } return 0; @@ -5015,7 +5108,7 @@ do_setitems(UnpicklerObject *self, Py_ssize_t x) return stack_underflow(); if (len == x) /* nothing to do */ return 0; - if ((len - x) % 2 != 0) { + if ((len - x) % 2 != 0) { /* Currupt or hostile pickle -- we never write one like this. */ PyErr_SetString(UnpicklingError, "odd number of items for SETITEMS"); return -1; @@ -5056,6 +5149,7 @@ load_build(UnpicklerObject *self) PyObject *state, *inst, *slotstate; PyObject *setstate; int status = 0; + _Py_IDENTIFIER(__setstate__); /* Stack is ... instance, state. We want to leave instance at * the stack top, possibly mutated via instance.__setstate__(state). @@ -5069,7 +5163,7 @@ load_build(UnpicklerObject *self) inst = self->stack->data[Py_SIZE(self->stack) - 1]; - setstate = PyObject_GetAttrString(inst, "__setstate__"); + setstate = _PyObject_GetAttrId(inst, &PyId___setstate__); if (setstate == NULL) { if (PyErr_ExceptionMatches(PyExc_AttributeError)) PyErr_Clear(); @@ -5112,12 +5206,13 @@ load_build(UnpicklerObject *self) PyObject *dict; PyObject *d_key, *d_value; Py_ssize_t i; + _Py_IDENTIFIER(__dict__); if (!PyDict_Check(state)) { PyErr_SetString(UnpicklingError, "state is not a dictionary"); goto error; } - dict = PyObject_GetAttrString(inst, "__dict__"); + dict = _PyObject_GetAttrId(inst, &PyId___dict__); if (dict == NULL) goto error; @@ -5372,7 +5467,7 @@ Unpickler_load(UnpicklerObject *self) not call Unpickler.__init__(). Here, we simply ensure that self->read is not NULL. */ if (self->read == NULL) { - PyErr_Format(UnpicklingError, + PyErr_Format(UnpicklingError, "Unpickler.__init__() was not called by %s.__init__()", Py_TYPE(self)->tp_name); return NULL; @@ -5474,7 +5569,7 @@ Unpickler_find_class(UnpicklerObject *self, PyObject *args) global = PyObject_GetAttr(module, global_name); Py_DECREF(module); } - else { + else { global = PyObject_GetAttr(module, global_name); } return global; @@ -5582,6 +5677,7 @@ Unpickler_init(UnpicklerObject *self, PyObject *args, PyObject *kwds) PyObject *fix_imports = Py_True; char *encoding = NULL; char *errors = NULL; + _Py_IDENTIFIER(persistent_load); /* XXX: That is an horrible error message. But, I don't know how to do better... */ @@ -5616,9 +5712,9 @@ Unpickler_init(UnpicklerObject *self, PyObject *args, PyObject *kwds) if (self->fix_imports == -1) return -1; - if (PyObject_HasAttrString((PyObject *)self, "persistent_load")) { - self->pers_func = PyObject_GetAttrString((PyObject *)self, - "persistent_load"); + if (_PyObject_HasAttrId((PyObject *)self, &PyId_persistent_load)) { + self->pers_func = _PyObject_GetAttrId((PyObject *)self, + &PyId_persistent_load); if (self->pers_func == NULL) return -1; } @@ -5651,7 +5747,7 @@ Unpickler_init(UnpicklerObject *self, PyObject *args, PyObject *kwds) * intentional, as these should be treated as black-box implementation details. * * We do, however, have to implement pickling/unpickling support because of - * real-world code like cvs2svn. + * real-world code like cvs2svn. */ typedef struct { @@ -5867,6 +5963,11 @@ Unpickler_set_memo(UnpicklerObject *self, PyObject *obj) idx = PyLong_AsSsize_t(key); if (idx == -1 && PyErr_Occurred()) goto error; + if (idx < 0) { + PyErr_SetString(PyExc_ValueError, + "memo key must be positive integers."); + goto error; + } if (_Unpickler_MemoPut(self, idx, value) < 0) goto error; } |