diff options
Diffstat (limited to 'Modules/_io/fileio.c')
| -rw-r--r-- | Modules/_io/fileio.c | 198 |
1 files changed, 110 insertions, 88 deletions
diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index f3ce776993..27995e543b 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -52,7 +52,7 @@ typedef struct { unsigned int appending : 1; signed int seekable : 2; /* -1 means unknown */ unsigned int closefd : 1; - unsigned int deallocating: 1; + char finalizing; PyObject *weakreflist; PyObject *dict; } fileio; @@ -129,7 +129,7 @@ fileio_close(fileio *self) self->fd = -1; Py_RETURN_NONE; } - if (self->deallocating) { + if (self->finalizing) { PyObject *r = fileio_dealloc_warn(self, (PyObject *) self); if (r) Py_DECREF(r); @@ -204,6 +204,9 @@ check_fd(int fd) return 0; } +#ifdef O_CLOEXEC +extern int _Py_open_cloexec_works; +#endif static int fileio_init(PyObject *oself, PyObject *args, PyObject *kwds) @@ -223,6 +226,11 @@ fileio_init(PyObject *oself, PyObject *args, PyObject *kwds) int fd = -1; int closefd = 1; int fd_is_own = 0; +#ifdef O_CLOEXEC + int *atomic_flag_works = &_Py_open_cloexec_works; +#elif !defined(MS_WINDOWS) + int *atomic_flag_works = NULL; +#endif assert(PyFileIO_Check(oself)); if (self->fd >= 0) { @@ -343,6 +351,12 @@ fileio_init(PyObject *oself, PyObject *args, PyObject *kwds) flags |= O_BINARY; #endif +#ifdef MS_WINDOWS + flags |= O_NOINHERIT; +#elif defined(O_CLOEXEC) + flags |= O_CLOEXEC; +#endif + if (fd >= 0) { if (check_fd(fd)) goto error; @@ -366,10 +380,18 @@ fileio_init(PyObject *oself, PyObject *args, PyObject *kwds) else #endif self->fd = open(name, flags, 0666); + Py_END_ALLOW_THREADS - } else { - PyObject *fdobj = PyObject_CallFunction( - opener, "Oi", nameobj, flags); + } + else { + PyObject *fdobj; + +#ifndef MS_WINDOWS + /* the opener may clear the atomic flag */ + atomic_flag_works = NULL; +#endif + + fdobj = PyObject_CallFunction(opener, "Oi", nameobj, flags); if (fdobj == NULL) goto error; if (!PyLong_Check(fdobj)) { @@ -388,14 +410,14 @@ fileio_init(PyObject *oself, PyObject *args, PyObject *kwds) fd_is_own = 1; if (self->fd < 0) { -#ifdef MS_WINDOWS - if (widename != NULL) - PyErr_SetFromErrnoWithFilenameObject(PyExc_IOError, nameobj); - else -#endif - PyErr_SetFromErrnoWithFilename(PyExc_IOError, name); + PyErr_SetFromErrnoWithFilenameObject(PyExc_OSError, nameobj); goto error; } + +#ifndef MS_WINDOWS + if (_Py_set_inheritable(self->fd, 0, atomic_flag_works) < 0) + goto error; +#endif } if (dircheck(self, nameobj) < 0) goto error; @@ -449,7 +471,7 @@ fileio_clear(fileio *self) static void fileio_dealloc(fileio *self) { - self->deallocating = 1; + self->finalizing = 1; if (_PyIOBase_finalize((PyObject *) self) < 0) return; _PyObject_GC_UNTRACK(self); @@ -535,7 +557,7 @@ fileio_readinto(fileio *self, PyObject *args) len = pbuf.len; Py_BEGIN_ALLOW_THREADS errno = 0; -#if defined(MS_WIN64) || defined(MS_WINDOWS) +#ifdef MS_WINDOWS if (len > INT_MAX) len = INT_MAX; n = read(self->fd, pbuf.buf, (int)len); @@ -558,33 +580,27 @@ fileio_readinto(fileio *self, PyObject *args) return PyLong_FromSsize_t(n); } +#ifndef HAVE_FSTAT + +static PyObject * +fileio_readall(fileio *self) +{ + _Py_IDENTIFIER(readall); + return _PyObject_CallMethodId((PyObject*)&PyRawIOBase_Type, + &PyId_readall, "O", self); +} + +#else + static size_t -new_buffersize(fileio *self, size_t currentsize -#ifdef HAVE_FSTAT - , Py_off_t pos, Py_off_t end -#endif - ) +new_buffersize(fileio *self, size_t currentsize) { size_t addend; -#ifdef HAVE_FSTAT - if (end != (Py_off_t)-1) { - /* Files claiming a size smaller than SMALLCHUNK may - actually be streaming pseudo-files. In this case, we - apply the more aggressive algorithm below. - */ - if (end >= SMALLCHUNK && end >= pos && pos >= 0) { - /* Add 1 so if the file were to grow we'd notice. */ - Py_off_t bufsize = currentsize + end - pos + 1; - if (bufsize < PY_SSIZE_T_MAX) - return (size_t)bufsize; - else - return PY_SSIZE_T_MAX; - } - } -#endif + /* Expand the buffer by an amount proportional to the current size, giving us amortized linear-time behavior. For bigger sizes, use a less-than-double growth factor to avoid excessive allocation. */ + assert(currentsize <= PY_SSIZE_T_MAX); if (currentsize > 65536) addend = currentsize >> 3; else @@ -598,26 +614,19 @@ new_buffersize(fileio *self, size_t currentsize static PyObject * fileio_readall(fileio *self) { -#ifdef HAVE_FSTAT struct stat st; Py_off_t pos, end; -#endif PyObject *result; - Py_ssize_t total = 0; + Py_ssize_t bytes_read = 0; Py_ssize_t n; - size_t newsize; + size_t bufsize; if (self->fd < 0) return err_closed(); if (!_PyVerify_fd(self->fd)) return PyErr_SetFromErrno(PyExc_IOError); - result = PyBytes_FromStringAndSize(NULL, SMALLCHUNK); - if (result == NULL) - return NULL; - -#ifdef HAVE_FSTAT -#if defined(MS_WIN64) || defined(MS_WINDOWS) +#ifdef MS_WINDOWS pos = _lseeki64(self->fd, 0L, SEEK_CUR); #else pos = lseek(self->fd, 0L, SEEK_CUR); @@ -626,44 +635,46 @@ fileio_readall(fileio *self) end = st.st_size; else end = (Py_off_t)-1; -#endif + + if (end > 0 && end >= pos && pos >= 0 && end - pos < PY_SSIZE_T_MAX) { + /* This is probably a real file, so we try to allocate a + buffer one byte larger than the rest of the file. If the + calculation is right then we should get EOF without having + to enlarge the buffer. */ + bufsize = (size_t)(end - pos + 1); + } else { + bufsize = SMALLCHUNK; + } + + result = PyBytes_FromStringAndSize(NULL, bufsize); + if (result == NULL) + return NULL; + while (1) { -#ifdef HAVE_FSTAT - newsize = new_buffersize(self, total, pos, end); -#else - newsize = new_buffersize(self, total); -#endif - if (newsize > PY_SSIZE_T_MAX || newsize <= 0) { - PyErr_SetString(PyExc_OverflowError, - "unbounded read returned more bytes " - "than a Python string can hold "); - Py_DECREF(result); - return NULL; - } + if (bytes_read >= (Py_ssize_t)bufsize) { + bufsize = new_buffersize(self, bytes_read); + if (bufsize > PY_SSIZE_T_MAX || bufsize <= 0) { + PyErr_SetString(PyExc_OverflowError, + "unbounded read returned more bytes " + "than a Python string can hold"); + Py_DECREF(result); + return NULL; + } - if (PyBytes_GET_SIZE(result) < (Py_ssize_t)newsize) { - if (_PyBytes_Resize(&result, newsize) < 0) { - if (total == 0) { - Py_DECREF(result); + if (PyBytes_GET_SIZE(result) < (Py_ssize_t)bufsize) { + if (_PyBytes_Resize(&result, bufsize) < 0) return NULL; - } - PyErr_Clear(); - break; } } Py_BEGIN_ALLOW_THREADS errno = 0; - n = newsize - total; -#if defined(MS_WIN64) || defined(MS_WINDOWS) + n = bufsize - bytes_read; +#ifdef MS_WINDOWS if (n > INT_MAX) n = INT_MAX; - n = read(self->fd, - PyBytes_AS_STRING(result) + total, - (int)n); + n = read(self->fd, PyBytes_AS_STRING(result) + bytes_read, (int)n); #else - n = read(self->fd, - PyBytes_AS_STRING(result) + total, - n); + n = read(self->fd, PyBytes_AS_STRING(result) + bytes_read, n); #endif Py_END_ALLOW_THREADS if (n == 0) @@ -676,7 +687,7 @@ fileio_readall(fileio *self) } continue; } - if (total > 0) + if (bytes_read > 0) break; if (errno == EAGAIN) { Py_DECREF(result); @@ -686,22 +697,19 @@ fileio_readall(fileio *self) PyErr_SetFromErrno(PyExc_IOError); return NULL; } - total += n; -#ifdef HAVE_FSTAT + bytes_read += n; pos += n; -#endif } - if (PyBytes_GET_SIZE(result) > total) { - if (_PyBytes_Resize(&result, total) < 0) { - /* This should never happen, but just in case */ - Py_DECREF(result); + if (PyBytes_GET_SIZE(result) > bytes_read) { + if (_PyBytes_Resize(&result, bytes_read) < 0) return NULL; - } } return result; } +#endif /* HAVE_FSTAT */ + static PyObject * fileio_read(fileio *self, PyObject *args) { @@ -722,7 +730,7 @@ fileio_read(fileio *self, PyObject *args) return fileio_readall(self); } -#if defined(MS_WIN64) || defined(MS_WINDOWS) +#ifdef MS_WINDOWS if (size > INT_MAX) size = INT_MAX; #endif @@ -734,7 +742,7 @@ fileio_read(fileio *self, PyObject *args) if (_PyVerify_fd(self->fd)) { Py_BEGIN_ALLOW_THREADS errno = 0; -#if defined(MS_WIN64) || defined(MS_WINDOWS) +#ifdef MS_WINDOWS n = read(self->fd, ptr, (int)size); #else n = read(self->fd, ptr, size); @@ -755,7 +763,7 @@ fileio_read(fileio *self, PyObject *args) if (n != size) { if (_PyBytes_Resize(&bytes, n) < 0) { - Py_DECREF(bytes); + Py_CLEAR(bytes); return NULL; } } @@ -782,7 +790,7 @@ fileio_write(fileio *self, PyObject *args) Py_BEGIN_ALLOW_THREADS errno = 0; len = pbuf.len; -#if defined(MS_WIN64) || defined(MS_WINDOWS) +#ifdef MS_WINDOWS if (len > 32767 && isatty(self->fd)) { /* Issue #11395: the Windows console returns an error (12: not enough space error) on writing into stdout if stdout mode is @@ -855,7 +863,7 @@ portable_lseek(int fd, PyObject *posobj, int whence) if (_PyVerify_fd(fd)) { Py_BEGIN_ALLOW_THREADS -#if defined(MS_WIN64) || defined(MS_WINDOWS) +#ifdef MS_WINDOWS res = _lseeki64(fd, pos, whence); #else res = lseek(fd, pos, whence); @@ -1204,6 +1212,11 @@ static PyGetSetDef fileio_getsetlist[] = { {NULL}, }; +static PyMemberDef fileio_members[] = { + {"_finalizing", T_BOOL, offsetof(fileio, finalizing), 0}, + {NULL} +}; + PyTypeObject PyFileIO_Type = { PyVarObject_HEAD_INIT(NULL, 0) "_io.FileIO", @@ -1225,7 +1238,7 @@ PyTypeObject PyFileIO_Type = { 0, /* tp_setattro */ 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE - | Py_TPFLAGS_HAVE_GC, /* tp_flags */ + | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_FINALIZE, /* tp_flags */ fileio_doc, /* tp_doc */ (traverseproc)fileio_traverse, /* tp_traverse */ (inquiry)fileio_clear, /* tp_clear */ @@ -1234,7 +1247,7 @@ PyTypeObject PyFileIO_Type = { 0, /* tp_iter */ 0, /* tp_iternext */ fileio_methods, /* tp_methods */ - 0, /* tp_members */ + fileio_members, /* tp_members */ fileio_getsetlist, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ @@ -1245,4 +1258,13 @@ PyTypeObject PyFileIO_Type = { PyType_GenericAlloc, /* tp_alloc */ fileio_new, /* tp_new */ PyObject_GC_Del, /* tp_free */ + 0, /* tp_is_gc */ + 0, /* tp_bases */ + 0, /* tp_mro */ + 0, /* tp_cache */ + 0, /* tp_subclasses */ + 0, /* tp_weaklist */ + 0, /* tp_del */ + 0, /* tp_version_tag */ + 0, /* tp_finalize */ }; |