Windows: Fix deadlock in backend when submitting transfers.

Without this change the Windows backend needed to call usbi_fd_notification()
from within the backend's submit_transfer. This can cause deadlock when
attempting to lock the event lock if another thread was processing events on
the just-submitted transfer.

The deadlock comes about as the thread calling libusb_submit_transfer acquires
the transfer mutex before trying to acquire the event lock; this is the other
order of lock acquisition from an event thread handling activity on the just
submitted transfer. This could lead to one of two deadlocks:

1) If the transfer completes while usbi_fd_notification() is waiting for
the event lock and the callback attempts to resubmit the transfer.

2) If the transfer timeout is hit while usbi_fd_notification() is waiting
for the event lock then the attempt to cancel the transfer will deadlock.

This patch fixes both of these deadlocks by having libusb_submit_transfer()
only call usbi_fd_notification() after having released the transfer mutex.

4 files changed, 11 insertions, 4 deletions

diff --git a/libusb/io.c b/libusb/io.c
index d7fae7e..ab32e70 100644
--- a/libusb/io.c
+++ b/libusb/io.c
@@ -1292,6 +1292,7 @@ int API_EXPORTED libusb_submit_transfer(struct libusb_transfer *transfer)
 		LIBUSB_TRANSFER_TO_USBI_TRANSFER(transfer);
 	int r;
 	int first;
+	int updated_fds;
 
 	usbi_mutex_lock(&itransfer->lock);
 	itransfer->transferred = 0;
@@ -1325,7 +1326,10 @@ int API_EXPORTED libusb_submit_transfer(struct libusb_transfer *transfer)
 #endif
 
 out:
+	updated_fds = (itransfer->flags & USBI_TRANSFER_UPDATED_FDS);
 	usbi_mutex_unlock(&itransfer->lock);
+	if (updated_fds)
+		usbi_fd_notification(ctx);
 	return r;
 }
 
diff --git a/libusb/libusbi.h b/libusb/libusbi.h
index c3d2158..0aa6bf9 100644
--- a/libusb/libusbi.h
+++ b/libusb/libusbi.h
@@ -347,6 +347,9 @@ enum usbi_transfer_flags {
 
 	/* Operation on the transfer failed because the device disappeared */
 	USBI_TRANSFER_DEVICE_DISAPPEARED = 1 << 3,
+
+	/* Set by backend submit_transfer() if the fds in use have been updated */
+	USBI_TRANSFER_UPDATED_FDS = 1 << 4,
 };
 
 #define USBI_TRANSFER_TO_LIBUSB_TRANSFER(transfer) \
diff --git a/libusb/os/windows_usb.c b/libusb/os/windows_usb.c
index 1957964..f29f84c 100644
--- a/libusb/os/windows_usb.c
+++ b/libusb/os/windows_usb.c
@@ -1770,7 +1770,7 @@ static int submit_bulk_transfer(struct usbi_transfer *itransfer)
 	usbi_add_pollfd(ctx, transfer_priv->pollable_fd.fd,
 		(short)(IS_XFERIN(transfer) ? POLLIN : POLLOUT));
 
-	usbi_fd_notification(ctx);
+	itransfer->flags |= USBI_TRANSFER_UPDATED_FDS;
 	return LIBUSB_SUCCESS;
 }
 
@@ -1790,7 +1790,7 @@ static int submit_iso_transfer(struct usbi_transfer *itransfer)
 	usbi_add_pollfd(ctx, transfer_priv->pollable_fd.fd,
 		(short)(IS_XFERIN(transfer) ? POLLIN : POLLOUT));
 
-	usbi_fd_notification(ctx);
+	itransfer->flags |= USBI_TRANSFER_UPDATED_FDS;
 	return LIBUSB_SUCCESS;
 }
 
@@ -1809,7 +1809,7 @@ static int submit_control_transfer(struct usbi_transfer *itransfer)
 
 	usbi_add_pollfd(ctx, transfer_priv->pollable_fd.fd, POLLIN);
 
-	usbi_fd_notification(ctx);
+	itransfer->flags |= USBI_TRANSFER_UPDATED_FDS;
 	return LIBUSB_SUCCESS;
 
 }
diff --git a/libusb/version.h b/libusb/version.h
index ef81eba..c0df768 100644
--- a/libusb/version.h
+++ b/libusb/version.h
@@ -9,7 +9,7 @@
 #define LIBUSB_MICRO 10
 #endif
 #ifndef LIBUSB_NANO
-#define LIBUSB_NANO 10488
+#define LIBUSB_NANO 10489
 #endif
 /* LIBUSB_RC is the release candidate suffix. Should normally be empty. */
 #ifndef LIBUSB_RC