From 41224435f0695630a8df433ed28a65d5bb74e0c3 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Mon, 9 May 2011 17:04:27 +0200 Subject: Issue #11743: Rewrite multiprocessing connection classes in pure Python. --- Modules/_multiprocessing/win32_functions.c | 166 +++++++++++++++++++++++++++++ 1 file changed, 166 insertions(+) (limited to 'Modules/_multiprocessing/win32_functions.c') diff --git a/Modules/_multiprocessing/win32_functions.c b/Modules/_multiprocessing/win32_functions.c index 452d608648..12dc0cd847 100644 --- a/Modules/_multiprocessing/win32_functions.c +++ b/Modules/_multiprocessing/win32_functions.c @@ -215,6 +215,164 @@ win32_WaitNamedPipe(PyObject *self, PyObject *args) Py_RETURN_NONE; } +static PyObject * +win32_closesocket(PyObject *self, PyObject *args) +{ + HANDLE handle; + int ret; + + if (!PyArg_ParseTuple(args, F_HANDLE ":closesocket" , &handle)) + return NULL; + + Py_BEGIN_ALLOW_THREADS + ret = closesocket((SOCKET) handle); + Py_END_ALLOW_THREADS + + if (ret) + return PyErr_SetExcFromWindowsErr(PyExc_IOError, WSAGetLastError()); + Py_RETURN_NONE; +} + +static PyObject * +win32_recv(PyObject *self, PyObject *args) +{ + HANDLE handle; + int size, nread; + PyObject *buf; + + if (!PyArg_ParseTuple(args, F_HANDLE "i:recv" , &handle, &size)) + return NULL; + + buf = PyBytes_FromStringAndSize(NULL, size); + if (!buf) + return NULL; + + Py_BEGIN_ALLOW_THREADS + nread = recv((SOCKET) handle, PyBytes_AS_STRING(buf), size, 0); + Py_END_ALLOW_THREADS + + if (nread < 0) { + Py_DECREF(buf); + return PyErr_SetExcFromWindowsErr(PyExc_IOError, WSAGetLastError()); + } + _PyBytes_Resize(&buf, nread); + return buf; +} + +static PyObject * +win32_send(PyObject *self, PyObject *args) +{ + HANDLE handle; + Py_buffer buf; + int ret; + + if (!PyArg_ParseTuple(args, F_HANDLE "y*:send" , &handle, &buf)) + return NULL; + + Py_BEGIN_ALLOW_THREADS + ret = send((SOCKET) handle, buf.buf, buf.len, 0); + Py_END_ALLOW_THREADS + + PyBuffer_Release(&buf); + if (ret < 0) + return PyErr_SetExcFromWindowsErr(PyExc_IOError, WSAGetLastError()); + return PyLong_FromLong(ret); +} + +static PyObject * +win32_WriteFile(PyObject *self, PyObject *args) +{ + HANDLE handle; + Py_buffer buf; + int written; + BOOL ret; + + if (!PyArg_ParseTuple(args, F_HANDLE "y*:WriteFile" , &handle, &buf)) + return NULL; + + Py_BEGIN_ALLOW_THREADS + ret = WriteFile(handle, buf.buf, buf.len, &written, NULL); + Py_END_ALLOW_THREADS + + PyBuffer_Release(&buf); + if (!ret) + return PyErr_SetExcFromWindowsErr(PyExc_IOError, 0); + return PyLong_FromLong(written); +} + +static PyObject * +win32_ReadFile(PyObject *self, PyObject *args) +{ + HANDLE handle; + int size; + DWORD nread; + PyObject *buf; + BOOL ret; + + if (!PyArg_ParseTuple(args, F_HANDLE "i:ReadFile" , &handle, &size)) + return NULL; + + buf = PyBytes_FromStringAndSize(NULL, size); + if (!buf) + return NULL; + + Py_BEGIN_ALLOW_THREADS + ret = ReadFile(handle, PyBytes_AS_STRING(buf), size, &nread, NULL); + Py_END_ALLOW_THREADS + + if (!ret && GetLastError() != ERROR_MORE_DATA) { + Py_DECREF(buf); + return PyErr_SetExcFromWindowsErr(PyExc_IOError, 0); + } + if (_PyBytes_Resize(&buf, nread)) + return NULL; + return Py_BuildValue("NN", buf, PyBool_FromLong(ret)); +} + +static PyObject * +win32_PeekNamedPipe(PyObject *self, PyObject *args) +{ + HANDLE handle; + int size = 0; + PyObject *buf = NULL; + DWORD nread, navail, nleft; + BOOL ret; + + if (!PyArg_ParseTuple(args, F_HANDLE "|i:PeekNamedPipe" , &handle, &size)) + return NULL; + + if (size < 0) { + PyErr_SetString(PyExc_ValueError, "negative size"); + return NULL; + } + + if (size) { + buf = PyBytes_FromStringAndSize(NULL, size); + if (!buf) + return NULL; + Py_BEGIN_ALLOW_THREADS + ret = PeekNamedPipe(handle, PyBytes_AS_STRING(buf), size, &nread, + &navail, &nleft); + Py_END_ALLOW_THREADS + if (!ret) { + Py_DECREF(buf); + return PyErr_SetExcFromWindowsErr(PyExc_IOError, 0); + } + if (_PyBytes_Resize(&buf, nread)) + return NULL; + return Py_BuildValue("Nii", buf, navail, nleft); + } + else { + Py_BEGIN_ALLOW_THREADS + ret = PeekNamedPipe(handle, NULL, 0, NULL, &navail, &nleft); + Py_END_ALLOW_THREADS + if (!ret) { + return PyErr_SetExcFromWindowsErr(PyExc_IOError, 0); + } + return Py_BuildValue("ii", navail, nleft); + } +} + static PyMethodDef win32_methods[] = { WIN32_FUNCTION(CloseHandle), WIN32_FUNCTION(GetLastError), @@ -223,8 +381,14 @@ static PyMethodDef win32_methods[] = { WIN32_FUNCTION(ConnectNamedPipe), WIN32_FUNCTION(CreateFile), WIN32_FUNCTION(CreateNamedPipe), + WIN32_FUNCTION(ReadFile), + WIN32_FUNCTION(PeekNamedPipe), WIN32_FUNCTION(SetNamedPipeHandleState), WIN32_FUNCTION(WaitNamedPipe), + WIN32_FUNCTION(WriteFile), + WIN32_FUNCTION(closesocket), + WIN32_FUNCTION(recv), + WIN32_FUNCTION(send), {NULL} }; @@ -244,6 +408,8 @@ create_win32_namespace(void) Py_INCREF(&Win32Type); WIN32_CONSTANT(F_DWORD, ERROR_ALREADY_EXISTS); + WIN32_CONSTANT(F_DWORD, ERROR_BROKEN_PIPE); + WIN32_CONSTANT(F_DWORD, ERROR_NO_SYSTEM_RESOURCES); WIN32_CONSTANT(F_DWORD, ERROR_PIPE_BUSY); WIN32_CONSTANT(F_DWORD, ERROR_PIPE_CONNECTED); WIN32_CONSTANT(F_DWORD, ERROR_SEM_TIMEOUT); -- cgit v1.2.1 From 8d62ed29484e5717297d02b7f861bc432423ed33 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Wed, 8 Jun 2011 17:21:55 +0200 Subject: Issue #9205: concurrent.futures.ProcessPoolExecutor now detects killed children and raises BrokenProcessPool in such a situation. Previously it would reliably freeze/deadlock. --- Modules/_multiprocessing/win32_functions.c | 390 +++++++++++++++++++++++++++-- 1 file changed, 374 insertions(+), 16 deletions(-) (limited to 'Modules/_multiprocessing/win32_functions.c') diff --git a/Modules/_multiprocessing/win32_functions.c b/Modules/_multiprocessing/win32_functions.c index 12dc0cd847..c017b2a6b7 100644 --- a/Modules/_multiprocessing/win32_functions.c +++ b/Modules/_multiprocessing/win32_functions.c @@ -12,10 +12,223 @@ #define WIN32_FUNCTION(func) \ {#func, (PyCFunction)win32_ ## func, METH_VARARGS | METH_STATIC, ""} +#define WIN32_KWARGS_FUNCTION(func) \ + {#func, (PyCFunction)win32_ ## func, METH_VARARGS | METH_KEYWORDS | METH_STATIC, ""} + #define WIN32_CONSTANT(fmt, con) \ PyDict_SetItemString(Win32Type.tp_dict, #con, Py_BuildValue(fmt, con)) +/* Grab CancelIoEx dynamically from kernel32 */ +static int has_CancelIoEx = -1; +static BOOL (CALLBACK *Py_CancelIoEx)(HANDLE, LPOVERLAPPED); + +static int +check_CancelIoEx() +{ + if (has_CancelIoEx == -1) + { + HINSTANCE hKernel32 = GetModuleHandle("KERNEL32"); + * (FARPROC *) &Py_CancelIoEx = GetProcAddress(hKernel32, + "CancelIoEx"); + has_CancelIoEx = (Py_CancelIoEx != NULL); + } + return has_CancelIoEx; +} + + +/* + * A Python object wrapping an OVERLAPPED structure and other useful data + * for overlapped I/O + */ + +typedef struct { + PyObject_HEAD + OVERLAPPED overlapped; + /* For convenience, we store the file handle too */ + HANDLE handle; + /* Whether there's I/O in flight */ + int pending; + /* Whether I/O completed successfully */ + int completed; + /* Buffer used for reading (optional) */ + PyObject *read_buffer; + /* Buffer used for writing (optional) */ + Py_buffer write_buffer; +} OverlappedObject; + +static void +overlapped_dealloc(OverlappedObject *self) +{ + int err = GetLastError(); + if (self->pending) { + if (check_CancelIoEx()) + Py_CancelIoEx(self->handle, &self->overlapped); + else { + PyErr_SetString(PyExc_RuntimeError, + "I/O operations still in flight while destroying " + "Overlapped object, the process may crash"); + PyErr_WriteUnraisable(NULL); + } + } + CloseHandle(self->overlapped.hEvent); + SetLastError(err); + if (self->write_buffer.obj) + PyBuffer_Release(&self->write_buffer); + Py_CLEAR(self->read_buffer); + PyObject_Del(self); +} + +static PyObject * +overlapped_GetOverlappedResult(OverlappedObject *self, PyObject *waitobj) +{ + int wait; + BOOL res; + DWORD transferred = 0; + + wait = PyObject_IsTrue(waitobj); + if (wait < 0) + return NULL; + Py_BEGIN_ALLOW_THREADS + res = GetOverlappedResult(self->handle, &self->overlapped, &transferred, + wait != 0); + Py_END_ALLOW_THREADS + + if (!res) { + int err = GetLastError(); + if (err == ERROR_IO_INCOMPLETE) + Py_RETURN_NONE; + if (err != ERROR_MORE_DATA) { + self->pending = 0; + return PyErr_SetExcFromWindowsErr(PyExc_IOError, err); + } + } + self->pending = 0; + self->completed = 1; + if (self->read_buffer) { + assert(PyBytes_CheckExact(self->read_buffer)); + if (_PyBytes_Resize(&self->read_buffer, transferred)) + return NULL; + } + return Py_BuildValue("lN", (long) transferred, PyBool_FromLong(res)); +} + +static PyObject * +overlapped_getbuffer(OverlappedObject *self) +{ + PyObject *res; + if (!self->completed) { + PyErr_SetString(PyExc_ValueError, + "can't get read buffer before GetOverlappedResult() " + "signals the operation completed"); + return NULL; + } + res = self->read_buffer ? self->read_buffer : Py_None; + Py_INCREF(res); + return res; +} + +static PyObject * +overlapped_cancel(OverlappedObject *self) +{ + BOOL res = TRUE; + + if (self->pending) { + Py_BEGIN_ALLOW_THREADS + if (check_CancelIoEx()) + res = Py_CancelIoEx(self->handle, &self->overlapped); + else + res = CancelIo(self->handle); + Py_END_ALLOW_THREADS + } + + /* CancelIoEx returns ERROR_NOT_FOUND if the I/O completed in-between */ + if (!res && GetLastError() != ERROR_NOT_FOUND) + return PyErr_SetExcFromWindowsErr(PyExc_IOError, 0); + self->pending = 0; + Py_RETURN_NONE; +} + +static PyMethodDef overlapped_methods[] = { + {"GetOverlappedResult", (PyCFunction) overlapped_GetOverlappedResult, + METH_O, NULL}, + {"getbuffer", (PyCFunction) overlapped_getbuffer, METH_NOARGS, NULL}, + {"cancel", (PyCFunction) overlapped_cancel, METH_NOARGS, NULL}, + {NULL} +}; + +static PyMemberDef overlapped_members[] = { + {"event", T_HANDLE, + offsetof(OverlappedObject, overlapped) + offsetof(OVERLAPPED, hEvent), + READONLY, "overlapped event handle"}, + {NULL} +}; + +PyTypeObject OverlappedType = { + PyVarObject_HEAD_INIT(NULL, 0) + /* tp_name */ "_multiprocessing.win32.Overlapped", + /* tp_basicsize */ sizeof(OverlappedObject), + /* tp_itemsize */ 0, + /* tp_dealloc */ (destructor) overlapped_dealloc, + /* tp_print */ 0, + /* tp_getattr */ 0, + /* tp_setattr */ 0, + /* tp_reserved */ 0, + /* tp_repr */ 0, + /* tp_as_number */ 0, + /* tp_as_sequence */ 0, + /* tp_as_mapping */ 0, + /* tp_hash */ 0, + /* tp_call */ 0, + /* tp_str */ 0, + /* tp_getattro */ 0, + /* tp_setattro */ 0, + /* tp_as_buffer */ 0, + /* tp_flags */ Py_TPFLAGS_DEFAULT, + /* tp_doc */ "OVERLAPPED structure wrapper", + /* tp_traverse */ 0, + /* tp_clear */ 0, + /* tp_richcompare */ 0, + /* tp_weaklistoffset */ 0, + /* tp_iter */ 0, + /* tp_iternext */ 0, + /* tp_methods */ overlapped_methods, + /* tp_members */ overlapped_members, + /* tp_getset */ 0, + /* tp_base */ 0, + /* tp_dict */ 0, + /* tp_descr_get */ 0, + /* tp_descr_set */ 0, + /* tp_dictoffset */ 0, + /* tp_init */ 0, + /* tp_alloc */ 0, + /* tp_new */ 0, +}; + +static OverlappedObject * +new_overlapped(HANDLE handle) +{ + OverlappedObject *self; + + self = PyObject_New(OverlappedObject, &OverlappedType); + if (!self) + return NULL; + self->handle = handle; + self->read_buffer = NULL; + self->pending = 0; + self->completed = 0; + memset(&self->overlapped, 0, sizeof(OVERLAPPED)); + memset(&self->write_buffer, 0, sizeof(Py_buffer)); + /* Manual reset, initially non-signalled */ + self->overlapped.hEvent = CreateEvent(NULL, TRUE, FALSE, NULL); + return self; +} + + +/* + * Module functions + */ + static PyObject * win32_CloseHandle(PyObject *self, PyObject *args) { @@ -36,20 +249,44 @@ win32_CloseHandle(PyObject *self, PyObject *args) } static PyObject * -win32_ConnectNamedPipe(PyObject *self, PyObject *args) +win32_ConnectNamedPipe(PyObject *self, PyObject *args, PyObject *kwds) { HANDLE hNamedPipe; - LPOVERLAPPED lpOverlapped; + int use_overlapped = 0; BOOL success; + OverlappedObject *overlapped = NULL; + static char *kwlist[] = {"handle", "overlapped", NULL}; - if (!PyArg_ParseTuple(args, F_HANDLE F_POINTER, - &hNamedPipe, &lpOverlapped)) + if (!PyArg_ParseTupleAndKeywords(args, kwds, + F_HANDLE "|i", kwlist, + &hNamedPipe, &use_overlapped)) return NULL; + if (use_overlapped) { + overlapped = new_overlapped(hNamedPipe); + if (!overlapped) + return NULL; + } + Py_BEGIN_ALLOW_THREADS - success = ConnectNamedPipe(hNamedPipe, lpOverlapped); + success = ConnectNamedPipe(hNamedPipe, + overlapped ? &overlapped->overlapped : NULL); Py_END_ALLOW_THREADS + if (overlapped) { + int err = GetLastError(); + /* Overlapped ConnectNamedPipe never returns a success code */ + assert(success == 0); + if (err == ERROR_IO_PENDING) + overlapped->pending = 1; + else if (err == ERROR_PIPE_CONNECTED) + SetEvent(overlapped->overlapped.hEvent); + else { + Py_DECREF(overlapped); + return PyErr_SetFromWindowsErr(err); + } + return (PyObject *) overlapped; + } if (!success) return PyErr_SetFromWindowsErr(0); @@ -280,46 +517,109 @@ win32_send(PyObject *self, PyObject *args) } static PyObject * -win32_WriteFile(PyObject *self, PyObject *args) +win32_WriteFile(PyObject *self, PyObject *args, PyObject *kwds) { HANDLE handle; - Py_buffer buf; + Py_buffer _buf, *buf; + PyObject *bufobj; int written; BOOL ret; + int use_overlapped = 0; + OverlappedObject *overlapped = NULL; + static char *kwlist[] = {"handle", "buffer", "overlapped", NULL}; + + /* First get handle and use_overlapped to know which Py_buffer to use */ + if (!PyArg_ParseTupleAndKeywords(args, kwds, + F_HANDLE "O|i:WriteFile", kwlist, + &handle, &bufobj, &use_overlapped)) + return NULL; + + if (use_overlapped) { + overlapped = new_overlapped(handle); + if (!overlapped) + return NULL; + buf = &overlapped->write_buffer; + } + else + buf = &_buf; - if (!PyArg_ParseTuple(args, F_HANDLE "y*:WriteFile" , &handle, &buf)) + if (!PyArg_Parse(bufobj, "y*", buf)) { + Py_XDECREF(overlapped); return NULL; + } Py_BEGIN_ALLOW_THREADS - ret = WriteFile(handle, buf.buf, buf.len, &written, NULL); + ret = WriteFile(handle, buf->buf, buf->len, &written, + overlapped ? &overlapped->overlapped : NULL); Py_END_ALLOW_THREADS - PyBuffer_Release(&buf); + if (overlapped) { + int err = GetLastError(); + if (!ret) { + if (err == ERROR_IO_PENDING) + overlapped->pending = 1; + else { + Py_DECREF(overlapped); + return PyErr_SetExcFromWindowsErr(PyExc_IOError, 0); + } + } + return (PyObject *) overlapped; + } + + PyBuffer_Release(buf); if (!ret) return PyErr_SetExcFromWindowsErr(PyExc_IOError, 0); return PyLong_FromLong(written); } static PyObject * -win32_ReadFile(PyObject *self, PyObject *args) +win32_ReadFile(PyObject *self, PyObject *args, PyObject *kwds) { HANDLE handle; int size; DWORD nread; PyObject *buf; BOOL ret; + int use_overlapped = 0; + OverlappedObject *overlapped = NULL; + static char *kwlist[] = {"handle", "size", "overlapped", NULL}; - if (!PyArg_ParseTuple(args, F_HANDLE "i:ReadFile" , &handle, &size)) + if (!PyArg_ParseTupleAndKeywords(args, kwds, + F_HANDLE "i|i:ReadFile", kwlist, + &handle, &size, &use_overlapped)) return NULL; buf = PyBytes_FromStringAndSize(NULL, size); if (!buf) return NULL; + if (use_overlapped) { + overlapped = new_overlapped(handle); + if (!overlapped) { + Py_DECREF(buf); + return NULL; + } + /* Steals reference to buf */ + overlapped->read_buffer = buf; + } Py_BEGIN_ALLOW_THREADS - ret = ReadFile(handle, PyBytes_AS_STRING(buf), size, &nread, NULL); + ret = ReadFile(handle, PyBytes_AS_STRING(buf), size, &nread, + overlapped ? &overlapped->overlapped : NULL); Py_END_ALLOW_THREADS + if (overlapped) { + int err = GetLastError(); + if (!ret) { + if (err == ERROR_IO_PENDING) + overlapped->pending = 1; + else if (err != ERROR_MORE_DATA) { + Py_DECREF(overlapped); + return PyErr_SetExcFromWindowsErr(PyExc_IOError, 0); + } + } + return (PyObject *) overlapped; + } + if (!ret && GetLastError() != ERROR_MORE_DATA) { Py_DECREF(buf); return PyErr_SetExcFromWindowsErr(PyExc_IOError, 0); @@ -373,19 +673,71 @@ win32_PeekNamedPipe(PyObject *self, PyObject *args) } } +static PyObject * +win32_WaitForMultipleObjects(PyObject* self, PyObject* args) +{ + DWORD result; + PyObject *handle_seq; + HANDLE handles[MAXIMUM_WAIT_OBJECTS]; + Py_ssize_t nhandles, i; + int wait_flag; + int milliseconds = INFINITE; + + if (!PyArg_ParseTuple(args, "Oi|i:WaitForMultipleObjects", + &handle_seq, &wait_flag, &milliseconds)) + return NULL; + + if (!PySequence_Check(handle_seq)) { + PyErr_Format(PyExc_TypeError, + "sequence type expected, got '%s'", + Py_TYPE(handle_seq)->tp_doc); + return NULL; + } + nhandles = PySequence_Length(handle_seq); + if (nhandles == -1) + return NULL; + if (nhandles < 0 || nhandles >= MAXIMUM_WAIT_OBJECTS) { + PyErr_Format(PyExc_ValueError, + "need at most %zd handles, got a sequence of length %zd", + MAXIMUM_WAIT_OBJECTS, nhandles); + return NULL; + } + for (i = 0; i < nhandles; i++) { + HANDLE h; + PyObject *v = PySequence_GetItem(handle_seq, i); + if (v == NULL) + return NULL; + if (!PyArg_Parse(v, F_HANDLE, &h)) + return NULL; + handles[i] = h; + } + + Py_BEGIN_ALLOW_THREADS + result = WaitForMultipleObjects((DWORD) nhandles, handles, + (BOOL) wait_flag, (DWORD) milliseconds); + Py_END_ALLOW_THREADS + + if (result == WAIT_FAILED) + return PyErr_SetExcFromWindowsErr(PyExc_IOError, 0); + + return PyLong_FromLong((int) result); +} + + static PyMethodDef win32_methods[] = { WIN32_FUNCTION(CloseHandle), WIN32_FUNCTION(GetLastError), WIN32_FUNCTION(OpenProcess), WIN32_FUNCTION(ExitProcess), - WIN32_FUNCTION(ConnectNamedPipe), + WIN32_KWARGS_FUNCTION(ConnectNamedPipe), WIN32_FUNCTION(CreateFile), WIN32_FUNCTION(CreateNamedPipe), - WIN32_FUNCTION(ReadFile), + WIN32_KWARGS_FUNCTION(ReadFile), WIN32_FUNCTION(PeekNamedPipe), WIN32_FUNCTION(SetNamedPipeHandleState), + WIN32_FUNCTION(WaitForMultipleObjects), WIN32_FUNCTION(WaitNamedPipe), - WIN32_FUNCTION(WriteFile), + WIN32_KWARGS_FUNCTION(WriteFile), WIN32_FUNCTION(closesocket), WIN32_FUNCTION(recv), WIN32_FUNCTION(send), @@ -407,12 +759,18 @@ create_win32_namespace(void) return NULL; Py_INCREF(&Win32Type); + if (PyType_Ready(&OverlappedType) < 0) + return NULL; + PyDict_SetItemString(Win32Type.tp_dict, "Overlapped", + (PyObject *) &OverlappedType); + WIN32_CONSTANT(F_DWORD, ERROR_ALREADY_EXISTS); WIN32_CONSTANT(F_DWORD, ERROR_BROKEN_PIPE); WIN32_CONSTANT(F_DWORD, ERROR_NO_SYSTEM_RESOURCES); WIN32_CONSTANT(F_DWORD, ERROR_PIPE_BUSY); WIN32_CONSTANT(F_DWORD, ERROR_PIPE_CONNECTED); WIN32_CONSTANT(F_DWORD, ERROR_SEM_TIMEOUT); + WIN32_CONSTANT(F_DWORD, FILE_FLAG_OVERLAPPED); WIN32_CONSTANT(F_DWORD, GENERIC_READ); WIN32_CONSTANT(F_DWORD, GENERIC_WRITE); WIN32_CONSTANT(F_DWORD, INFINITE); -- cgit v1.2.1 From 6aa9b246c7e6a5c63e62e8e0f05b14c1694644cc Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Mon, 21 Nov 2011 21:26:56 +0100 Subject: Issue #12328: Under Windows, refactor handling of Ctrl-C events and make _multiprocessing.win32.WaitForMultipleObjects interruptible when the wait_flag parameter is false. Patch by sbt. --- Modules/_multiprocessing/win32_functions.c | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) (limited to 'Modules/_multiprocessing/win32_functions.c') diff --git a/Modules/_multiprocessing/win32_functions.c b/Modules/_multiprocessing/win32_functions.c index c017b2a6b7..ddc496d941 100644 --- a/Modules/_multiprocessing/win32_functions.c +++ b/Modules/_multiprocessing/win32_functions.c @@ -679,6 +679,7 @@ win32_WaitForMultipleObjects(PyObject* self, PyObject* args) DWORD result; PyObject *handle_seq; HANDLE handles[MAXIMUM_WAIT_OBJECTS]; + HANDLE sigint_event = NULL; Py_ssize_t nhandles, i; int wait_flag; int milliseconds = INFINITE; @@ -696,10 +697,10 @@ win32_WaitForMultipleObjects(PyObject* self, PyObject* args) nhandles = PySequence_Length(handle_seq); if (nhandles == -1) return NULL; - if (nhandles < 0 || nhandles >= MAXIMUM_WAIT_OBJECTS) { + if (nhandles < 0 || nhandles >= MAXIMUM_WAIT_OBJECTS - 1) { PyErr_Format(PyExc_ValueError, "need at most %zd handles, got a sequence of length %zd", - MAXIMUM_WAIT_OBJECTS, nhandles); + MAXIMUM_WAIT_OBJECTS - 1, nhandles); return NULL; } for (i = 0; i < nhandles; i++) { @@ -711,14 +712,27 @@ win32_WaitForMultipleObjects(PyObject* self, PyObject* args) return NULL; handles[i] = h; } + /* If this is the main thread then make the wait interruptible + by Ctrl-C unless we are waiting for *all* handles */ + if (!wait_flag && _PyOS_IsMainThread()) { + sigint_event = _PyOS_SigintEvent(); + assert(sigint_event != NULL); + handles[nhandles++] = sigint_event; + } Py_BEGIN_ALLOW_THREADS + if (sigint_event != NULL) + ResetEvent(sigint_event); result = WaitForMultipleObjects((DWORD) nhandles, handles, (BOOL) wait_flag, (DWORD) milliseconds); Py_END_ALLOW_THREADS if (result == WAIT_FAILED) return PyErr_SetExcFromWindowsErr(PyExc_IOError, 0); + else if (sigint_event != NULL && result == WAIT_OBJECT_0 + nhandles - 1) { + errno = EINTR; + return PyErr_SetFromErrno(PyExc_IOError); + } return PyLong_FromLong((int) result); } -- cgit v1.2.1 From 818356540ccf1fb471e371ce68951e5638a9bdd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Charles-Fran=C3=A7ois=20Natali?= Date: Wed, 8 Feb 2012 21:15:58 +0100 Subject: Issue #8184: multiprocessing: On Windows, don't set SO_REUSEADDR on Connection sockets, and set FILE_FLAG_FIRST_PIPE_INSTANCE on named pipes, to make sure two listeners can't bind to the same socket/pipe (or any existing socket/pipe). --- Modules/_multiprocessing/win32_functions.c | 1 + 1 file changed, 1 insertion(+) (limited to 'Modules/_multiprocessing/win32_functions.c') diff --git a/Modules/_multiprocessing/win32_functions.c b/Modules/_multiprocessing/win32_functions.c index ddc496d941..e1759971cb 100644 --- a/Modules/_multiprocessing/win32_functions.c +++ b/Modules/_multiprocessing/win32_functions.c @@ -784,6 +784,7 @@ create_win32_namespace(void) WIN32_CONSTANT(F_DWORD, ERROR_PIPE_BUSY); WIN32_CONSTANT(F_DWORD, ERROR_PIPE_CONNECTED); WIN32_CONSTANT(F_DWORD, ERROR_SEM_TIMEOUT); + WIN32_CONSTANT(F_DWORD, FILE_FLAG_FIRST_PIPE_INSTANCE); WIN32_CONSTANT(F_DWORD, FILE_FLAG_OVERLAPPED); WIN32_CONSTANT(F_DWORD, GENERIC_READ); WIN32_CONSTANT(F_DWORD, GENERIC_WRITE); -- cgit v1.2.1 From 72418d6926d48a950c871138dcf278e80ad60abb Mon Sep 17 00:00:00 2001 From: Stefan Krah Date: Mon, 27 Feb 2012 13:51:02 +0100 Subject: Issue #14125: Fix multiprocessing refleak on Windows. Patch by sbt. --- Modules/_multiprocessing/win32_functions.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'Modules/_multiprocessing/win32_functions.c') diff --git a/Modules/_multiprocessing/win32_functions.c b/Modules/_multiprocessing/win32_functions.c index e1759971cb..15aeedae33 100644 --- a/Modules/_multiprocessing/win32_functions.c +++ b/Modules/_multiprocessing/win32_functions.c @@ -708,9 +708,12 @@ win32_WaitForMultipleObjects(PyObject* self, PyObject* args) PyObject *v = PySequence_GetItem(handle_seq, i); if (v == NULL) return NULL; - if (!PyArg_Parse(v, F_HANDLE, &h)) + if (!PyArg_Parse(v, F_HANDLE, &h)) { + Py_DECREF(v); return NULL; + } handles[i] = h; + Py_DECREF(v); } /* If this is the main thread then make the wait interruptible by Ctrl-C unless we are waiting for *all* handles */ -- cgit v1.2.1 From 0e0ccbf2431f5e426916ed2b9ae083c6244285a5 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Mon, 5 Mar 2012 19:28:37 +0100 Subject: Issue #12328: Fix multiprocessing's use of overlapped I/O on Windows. Also, add a multiprocessing.connection.wait(rlist, timeout=None) function for polling multiple objects at once. Patch by sbt. Complete changelist from sbt's patch: * Adds a wait(rlist, timeout=None) function for polling multiple objects at once. On Unix this is just a wrapper for select(rlist, [], [], timeout=None). * Removes use of the SentinelReady exception and the sentinels argument to certain methods. concurrent.futures.process has been changed to use wait() instead of SentinelReady. * Fixes bugs concerning PipeConnection.poll() and messages of zero length. * Fixes PipeListener.accept() to call ConnectNamedPipe() with overlapped=True. * Fixes Queue.empty() and SimpleQueue.empty() so that they are threadsafe on Windows. * Now PipeConnection.poll() and wait() will not modify the pipe except possibly by consuming a zero length message. (Previously poll() could consume a partial message.) * All of multiprocesing's pipe related blocking functions/methods are now interruptible by SIGINT on Windows. --- Modules/_multiprocessing/win32_functions.c | 69 ++++++++++++++++++------------ 1 file changed, 42 insertions(+), 27 deletions(-) (limited to 'Modules/_multiprocessing/win32_functions.c') diff --git a/Modules/_multiprocessing/win32_functions.c b/Modules/_multiprocessing/win32_functions.c index 15aeedae33..93c8fc9cde 100644 --- a/Modules/_multiprocessing/win32_functions.c +++ b/Modules/_multiprocessing/win32_functions.c @@ -60,16 +60,18 @@ typedef struct { static void overlapped_dealloc(OverlappedObject *self) { + DWORD bytes; int err = GetLastError(); if (self->pending) { - if (check_CancelIoEx()) - Py_CancelIoEx(self->handle, &self->overlapped); - else { - PyErr_SetString(PyExc_RuntimeError, - "I/O operations still in flight while destroying " - "Overlapped object, the process may crash"); - PyErr_WriteUnraisable(NULL); - } + /* make it a programming error to deallocate while operation + is pending, even if we can safely cancel it */ + if (check_CancelIoEx() && + Py_CancelIoEx(self->handle, &self->overlapped)) + GetOverlappedResult(self->handle, &self->overlapped, &bytes, TRUE); + PyErr_SetString(PyExc_RuntimeError, + "I/O operations still in flight while destroying " + "Overlapped object, the process may crash"); + PyErr_WriteUnraisable(NULL); } CloseHandle(self->overlapped.hEvent); SetLastError(err); @@ -85,6 +87,7 @@ overlapped_GetOverlappedResult(OverlappedObject *self, PyObject *waitobj) int wait; BOOL res; DWORD transferred = 0; + DWORD err; wait = PyObject_IsTrue(waitobj); if (wait < 0) @@ -94,23 +97,27 @@ overlapped_GetOverlappedResult(OverlappedObject *self, PyObject *waitobj) wait != 0); Py_END_ALLOW_THREADS - if (!res) { - int err = GetLastError(); - if (err == ERROR_IO_INCOMPLETE) - Py_RETURN_NONE; - if (err != ERROR_MORE_DATA) { + err = res ? ERROR_SUCCESS : GetLastError(); + switch (err) { + case ERROR_SUCCESS: + case ERROR_MORE_DATA: + case ERROR_OPERATION_ABORTED: + self->completed = 1; + self->pending = 0; + break; + case ERROR_IO_INCOMPLETE: + break; + default: self->pending = 0; return PyErr_SetExcFromWindowsErr(PyExc_IOError, err); - } } - self->pending = 0; - self->completed = 1; - if (self->read_buffer) { + if (self->completed && self->read_buffer != NULL) { assert(PyBytes_CheckExact(self->read_buffer)); - if (_PyBytes_Resize(&self->read_buffer, transferred)) + if (transferred != PyBytes_GET_SIZE(self->read_buffer) && + _PyBytes_Resize(&self->read_buffer, transferred)) return NULL; } - return Py_BuildValue("lN", (long) transferred, PyBool_FromLong(res)); + return Py_BuildValue("II", (unsigned) transferred, (unsigned) err); } static PyObject * @@ -522,9 +529,10 @@ win32_WriteFile(PyObject *self, PyObject *args, PyObject *kwds) HANDLE handle; Py_buffer _buf, *buf; PyObject *bufobj; - int written; + DWORD written; BOOL ret; int use_overlapped = 0; + DWORD err; OverlappedObject *overlapped = NULL; static char *kwlist[] = {"handle", "buffer", "overlapped", NULL}; @@ -553,8 +561,9 @@ win32_WriteFile(PyObject *self, PyObject *args, PyObject *kwds) overlapped ? &overlapped->overlapped : NULL); Py_END_ALLOW_THREADS + err = ret ? 0 : GetLastError(); + if (overlapped) { - int err = GetLastError(); if (!ret) { if (err == ERROR_IO_PENDING) overlapped->pending = 1; @@ -563,13 +572,13 @@ win32_WriteFile(PyObject *self, PyObject *args, PyObject *kwds) return PyErr_SetExcFromWindowsErr(PyExc_IOError, 0); } } - return (PyObject *) overlapped; + return Py_BuildValue("NI", (PyObject *) overlapped, err); } PyBuffer_Release(buf); if (!ret) return PyErr_SetExcFromWindowsErr(PyExc_IOError, 0); - return PyLong_FromLong(written); + return Py_BuildValue("II", written, err); } static PyObject * @@ -581,6 +590,7 @@ win32_ReadFile(PyObject *self, PyObject *args, PyObject *kwds) PyObject *buf; BOOL ret; int use_overlapped = 0; + DWORD err; OverlappedObject *overlapped = NULL; static char *kwlist[] = {"handle", "size", "overlapped", NULL}; @@ -607,8 +617,9 @@ win32_ReadFile(PyObject *self, PyObject *args, PyObject *kwds) overlapped ? &overlapped->overlapped : NULL); Py_END_ALLOW_THREADS + err = ret ? 0 : GetLastError(); + if (overlapped) { - int err = GetLastError(); if (!ret) { if (err == ERROR_IO_PENDING) overlapped->pending = 1; @@ -617,16 +628,16 @@ win32_ReadFile(PyObject *self, PyObject *args, PyObject *kwds) return PyErr_SetExcFromWindowsErr(PyExc_IOError, 0); } } - return (PyObject *) overlapped; + return Py_BuildValue("NI", (PyObject *) overlapped, err); } - if (!ret && GetLastError() != ERROR_MORE_DATA) { + if (!ret && err != ERROR_MORE_DATA) { Py_DECREF(buf); return PyErr_SetExcFromWindowsErr(PyExc_IOError, 0); } if (_PyBytes_Resize(&buf, nread)) return NULL; - return Py_BuildValue("NN", buf, PyBool_FromLong(ret)); + return Py_BuildValue("NI", buf, err); } static PyObject * @@ -783,7 +794,11 @@ create_win32_namespace(void) WIN32_CONSTANT(F_DWORD, ERROR_ALREADY_EXISTS); WIN32_CONSTANT(F_DWORD, ERROR_BROKEN_PIPE); + WIN32_CONSTANT(F_DWORD, ERROR_IO_PENDING); + WIN32_CONSTANT(F_DWORD, ERROR_MORE_DATA); + WIN32_CONSTANT(F_DWORD, ERROR_NETNAME_DELETED); WIN32_CONSTANT(F_DWORD, ERROR_NO_SYSTEM_RESOURCES); + WIN32_CONSTANT(F_DWORD, ERROR_OPERATION_ABORTED); WIN32_CONSTANT(F_DWORD, ERROR_PIPE_BUSY); WIN32_CONSTANT(F_DWORD, ERROR_PIPE_CONNECTED); WIN32_CONSTANT(F_DWORD, ERROR_SEM_TIMEOUT); -- cgit v1.2.1