diff options
Diffstat (limited to 'Modules/_io/fileio.c')
| -rw-r--r-- | Modules/_io/fileio.c | 178 |
1 files changed, 131 insertions, 47 deletions
diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index ad4a7e6bcd..f3ce776993 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -46,8 +46,10 @@ typedef struct { PyObject_HEAD int fd; + unsigned int created : 1; unsigned int readable : 1; unsigned int writable : 1; + unsigned int appending : 1; signed int seekable : 2; /* -1 means unknown */ unsigned int closefd : 1; unsigned int deallocating: 1; @@ -122,6 +124,7 @@ internal_close(fileio *self) static PyObject * fileio_close(fileio *self) { + _Py_IDENTIFIER(close); if (!self->closefd) { self->fd = -1; Py_RETURN_NONE; @@ -137,8 +140,8 @@ fileio_close(fileio *self) if (errno < 0) return NULL; - return PyObject_CallMethod((PyObject*)&PyRawIOBase_Type, - "close", "O", self); + return _PyObject_CallMethodId((PyObject*)&PyRawIOBase_Type, + &PyId_close, "O", self); } static PyObject * @@ -151,8 +154,10 @@ fileio_new(PyTypeObject *type, PyObject *args, PyObject *kwds) self = (fileio *) type->tp_alloc(type, 0); if (self != NULL) { self->fd = -1; + self->created = 0; self->readable = 0; self->writable = 0; + self->appending = 0; self->seekable = -1; self->closefd = 1; self->weakreflist = NULL; @@ -168,7 +173,7 @@ fileio_new(PyTypeObject *type, PyObject *args, PyObject *kwds) static int dircheck(fileio* self, PyObject *nameobj) { -#if defined(HAVE_FSTAT) && defined(S_IFDIR) && defined(EISDIR) +#if defined(HAVE_FSTAT) && defined(S_ISDIR) && defined(EISDIR) struct stat buf; if (self->fd < 0) return 0; @@ -204,16 +209,16 @@ static int fileio_init(PyObject *oself, PyObject *args, PyObject *kwds) { fileio *self = (fileio *) oself; - static char *kwlist[] = {"file", "mode", "closefd", NULL}; + static char *kwlist[] = {"file", "mode", "closefd", "opener", NULL}; const char *name = NULL; - PyObject *nameobj, *stringobj = NULL; + PyObject *nameobj, *stringobj = NULL, *opener = Py_None; char *mode = "r"; char *s; #ifdef MS_WINDOWS Py_UNICODE *widename = NULL; #endif int ret = 0; - int rwa = 0, plus = 0, append = 0; + int rwa = 0, plus = 0; int flags = 0; int fd = -1; int closefd = 1; @@ -230,8 +235,9 @@ fileio_init(PyObject *oself, PyObject *args, PyObject *kwds) self->fd = -1; } - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|si:fileio", - kwlist, &nameobj, &mode, &closefd)) + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|siO:fileio", + kwlist, &nameobj, &mode, &closefd, + &opener)) return -1; if (PyFloat_Check(nameobj)) { @@ -258,9 +264,10 @@ fileio_init(PyObject *oself, PyObject *args, PyObject *kwds) PyErr_SetString(PyExc_TypeError, "embedded NUL character"); return -1; } - widename = PyUnicode_AS_UNICODE(nameobj); - } - if (widename == NULL) + widename = PyUnicode_AsUnicode(nameobj); + if (widename == NULL) + return -1; + } else #endif if (fd < 0) { @@ -273,15 +280,23 @@ fileio_init(PyObject *oself, PyObject *args, PyObject *kwds) s = mode; while (*s) { switch (*s++) { - case 'r': + case 'x': if (rwa) { bad_mode: PyErr_SetString(PyExc_ValueError, - "Must have exactly one of read/write/append " + "Must have exactly one of create/read/write/append " "mode and at most one plus"); goto error; } rwa = 1; + self->created = 1; + self->writable = 1; + flags |= O_EXCL | O_CREAT; + break; + case 'r': + if (rwa) + goto bad_mode; + rwa = 1; self->readable = 1; break; case 'w': @@ -296,8 +311,8 @@ fileio_init(PyObject *oself, PyObject *args, PyObject *kwds) goto bad_mode; rwa = 1; self->writable = 1; - flags |= O_CREAT; - append = 1; + self->appending = 1; + flags |= O_APPEND | O_CREAT; break; case 'b': break; @@ -328,11 +343,6 @@ fileio_init(PyObject *oself, PyObject *args, PyObject *kwds) flags |= O_BINARY; #endif -#ifdef O_APPEND - if (append) - flags |= O_APPEND; -#endif - if (fd >= 0) { if (check_fd(fd)) goto error; @@ -347,20 +357,40 @@ fileio_init(PyObject *oself, PyObject *args, PyObject *kwds) goto error; } - Py_BEGIN_ALLOW_THREADS errno = 0; + if (opener == Py_None) { + Py_BEGIN_ALLOW_THREADS #ifdef MS_WINDOWS - if (widename != NULL) - self->fd = _wopen(widename, flags, 0666); - else + if (widename != NULL) + self->fd = _wopen(widename, flags, 0666); + else #endif - self->fd = open(name, flags, 0666); - Py_END_ALLOW_THREADS + self->fd = open(name, flags, 0666); + Py_END_ALLOW_THREADS + } else { + PyObject *fdobj = PyObject_CallFunction( + opener, "Oi", nameobj, flags); + if (fdobj == NULL) + goto error; + if (!PyLong_Check(fdobj)) { + Py_DECREF(fdobj); + PyErr_SetString(PyExc_TypeError, + "expected integer from opener"); + goto error; + } + + self->fd = _PyLong_AsInt(fdobj); + Py_DECREF(fdobj); + if (self->fd == -1) { + goto error; + } + } + fd_is_own = 1; if (self->fd < 0) { #ifdef MS_WINDOWS if (widename != NULL) - PyErr_SetFromErrnoWithUnicodeFilename(PyExc_IOError, widename); + PyErr_SetFromErrnoWithFilenameObject(PyExc_IOError, nameobj); else #endif PyErr_SetFromErrnoWithFilename(PyExc_IOError, name); @@ -378,7 +408,7 @@ fileio_init(PyObject *oself, PyObject *args, PyObject *kwds) if (PyObject_SetAttrString((PyObject *)self, "name", nameobj) < 0) goto error; - if (append) { + if (self->appending) { /* For consistent behaviour, we explicitly seek to the end of file (otherwise, it might be done only on the first write()). */ @@ -529,36 +559,53 @@ fileio_readinto(fileio *self, PyObject *args) } static size_t -new_buffersize(fileio *self, size_t currentsize) +new_buffersize(fileio *self, size_t currentsize +#ifdef HAVE_FSTAT + , Py_off_t pos, Py_off_t end +#endif + ) { + size_t addend; #ifdef HAVE_FSTAT - off_t pos, end; - struct stat st; - if (fstat(self->fd, &st) == 0) { - end = st.st_size; - pos = lseek(self->fd, 0L, SEEK_CUR); + if (end != (Py_off_t)-1) { /* Files claiming a size smaller than SMALLCHUNK may actually be streaming pseudo-files. In this case, we apply the more aggressive algorithm below. */ if (end >= SMALLCHUNK && end >= pos && pos >= 0) { /* Add 1 so if the file were to grow we'd notice. */ - return currentsize + end - pos + 1; + Py_off_t bufsize = currentsize + end - pos + 1; + if (bufsize < PY_SSIZE_T_MAX) + return (size_t)bufsize; + else + return PY_SSIZE_T_MAX; } } #endif /* Expand the buffer by an amount proportional to the current size, - giving us amortized linear-time behavior. Use a less-than-double - growth factor to avoid excessive allocation. */ - return currentsize + (currentsize >> 3) + 6; + giving us amortized linear-time behavior. For bigger sizes, use a + less-than-double growth factor to avoid excessive allocation. */ + if (currentsize > 65536) + addend = currentsize >> 3; + else + addend = 256 + currentsize; + if (addend < SMALLCHUNK) + /* Avoid tiny read() calls. */ + addend = SMALLCHUNK; + return addend + currentsize; } static PyObject * fileio_readall(fileio *self) { +#ifdef HAVE_FSTAT + struct stat st; + Py_off_t pos, end; +#endif PyObject *result; Py_ssize_t total = 0; Py_ssize_t n; + size_t newsize; if (self->fd < 0) return err_closed(); @@ -569,8 +616,23 @@ fileio_readall(fileio *self) if (result == NULL) return NULL; +#ifdef HAVE_FSTAT +#if defined(MS_WIN64) || defined(MS_WINDOWS) + pos = _lseeki64(self->fd, 0L, SEEK_CUR); +#else + pos = lseek(self->fd, 0L, SEEK_CUR); +#endif + if (fstat(self->fd, &st) == 0) + end = st.st_size; + else + end = (Py_off_t)-1; +#endif while (1) { - size_t newsize = new_buffersize(self, total); +#ifdef HAVE_FSTAT + newsize = new_buffersize(self, total, pos, end); +#else + newsize = new_buffersize(self, total); +#endif if (newsize > PY_SSIZE_T_MAX || newsize <= 0) { PyErr_SetString(PyExc_OverflowError, "unbounded read returned more bytes " @@ -625,6 +687,9 @@ fileio_readall(fileio *self) return NULL; } total += n; +#ifdef HAVE_FSTAT + pos += n; +#endif } if (PyBytes_GET_SIZE(result) > total) { @@ -938,7 +1003,19 @@ fileio_truncate(fileio *self, PyObject *args) static char * mode_string(fileio *self) { - if (self->readable) { + if (self->created) { + if (self->readable) + return "xb+"; + else + return "xb"; + } + if (self->appending) { + if (self->readable) + return "ab+"; + else + return "ab"; + } + else if (self->readable) { if (self->writable) return "rb+"; else @@ -951,12 +1028,13 @@ mode_string(fileio *self) static PyObject * fileio_repr(fileio *self) { + _Py_IDENTIFIER(name); PyObject *nameobj, *res; if (self->fd < 0) return PyUnicode_FromFormat("<_io.FileIO [closed]>"); - nameobj = PyObject_GetAttrString((PyObject *) self, "name"); + nameobj = _PyObject_GetAttrId((PyObject *) self, &PyId_name); if (nameobj == NULL) { if (PyErr_ExceptionMatches(PyExc_AttributeError)) PyErr_Clear(); @@ -996,13 +1074,19 @@ fileio_getstate(fileio *self) PyDoc_STRVAR(fileio_doc, -"file(name: str[, mode: str]) -> file IO object\n" +"file(name: str[, mode: str][, opener: None]) -> file IO object\n" "\n" -"Open a file. The mode can be 'r', 'w' or 'a' for reading (default),\n" -"writing or appending. The file will be created if it doesn't exist\n" -"when opened for writing or appending; it will be truncated when\n" -"opened for writing. Add a '+' to the mode to allow simultaneous\n" -"reading and writing."); +"Open a file. The mode can be 'r', 'w', 'x' or 'a' for reading (default),\n" +"writing, exclusive creation or appending. The file will be created if it\n" +"doesn't exist when opened for writing or appending; it will be truncated\n" +"when opened for writing. A `FileExistsError` will be raised if it already\n" +"exists when opened for creating. Opening a file for creating implies\n" +"writing so this mode behaves in a similar way to 'w'.Add a '+' to the mode\n" +"to allow simultaneous reading and writing. A custom opener can be used by\n" +"passing a callable as *opener*. The underlying file descriptor for the file\n" +"object is then obtained by calling opener with (*name*, *flags*).\n" +"*opener* must return an open file descriptor (passing os.open as *opener*\n" +"results in functionality similar to passing None)."); PyDoc_STRVAR(read_doc, "read(size: int) -> bytes. read at most size bytes, returned as bytes.\n" |