Bug 1404061, Permanent version of TCP Fast Open, Cancel overlappedIO before closing it. r=mcmanus, r=:kaie:NSPR_4_18_BETA1

8 files changed, 171 insertions, 49 deletions

diff --git a/pr/include/private/pprio.h b/pr/include/private/pprio.h
index 83d3715b..26bcd0d0 100644
--- a/pr/include/private/pprio.h
+++ b/pr/include/private/pprio.h
@@ -237,20 +237,6 @@ NSPR_API(PRStatus) PR_NT_CancelIo(PRFileDesc *fd);
 
 #endif /* WIN32 */
 
-/* FUNCTION: PR_EXPERIMENTAL_ONLY_IN_4_17_GetOverlappedIOHandle
-** DESCRIPTION:
-** This function will be available only in nspr version 4.17
-** This functionality is only available on windows. Some windows operation use
-** asynchronous (call overlapped) io. One of them is ConnectEx. NSPR uses
-** ConnectEx for enabling TCP Fast Open.
-** This function returns an OVERLAPPED structure associated with ConnectEx call.
-** If ConnectEx has not been called or the io has already finished, the
-** function will return PR_INVALID_ARGUMENT_ERROR.
-** PRFileDesc continues to be owner of the structure and the structure must not
-** be destroyed.
-*/
-NSPR_API(PRStatus) PR_EXPERIMENTAL_ONLY_IN_4_17_GetOverlappedIOHandle(PRFileDesc *fd, void **ol);
-
 PR_END_EXTERN_C
 
 #endif /* pprio_h___ */
diff --git a/pr/include/private/primpl.h b/pr/include/private/primpl.h
index dc24a257..e43416c5 100644
--- a/pr/include/private/primpl.h
+++ b/pr/include/private/primpl.h
@@ -2165,6 +2165,18 @@ extern PRUint32 connectCount;
 
 #endif /* XP_BEOS */
 
+#if defined(_WIN64) && defined(WIN95)
+typedef struct _PRFileDescList {
+  PRFileDesc *fd;
+  struct _PRFileDescList *next;
+} PRFileDescList;
+
+extern PRLock *_fd_waiting_for_overlapped_done_lock;
+extern PRFileDescList *_fd_waiting_for_overlapped_done;
+extern void CheckOverlappedPendingSocketsAreDone();
+#endif
+
+
 PR_END_EXTERN_C
 
 #endif /* primpl_h___ */
diff --git a/pr/src/io/prio.c b/pr/src/io/prio.c
index bf9763a2..10ae5e09 100644
--- a/pr/src/io/prio.c
+++ b/pr/src/io/prio.c
@@ -137,11 +137,82 @@ PR_IMPLEMENT(void) PR_FreeFileDesc(PRFileDesc *fd)
     _PR_Putfd(fd);
 }
 
+#if defined(_WIN64) && defined(WIN95)
+
+PRFileDescList *_fd_waiting_for_overlapped_done = NULL;
+PRLock *_fd_waiting_for_overlapped_done_lock = NULL;
+
+void CheckOverlappedPendingSocketsAreDone()
+{
+  if (!_fd_waiting_for_overlapped_done_lock ||
+      !_fd_waiting_for_overlapped_done) {
+    return;
+  }
+
+  PR_Lock(_fd_waiting_for_overlapped_done_lock);
+
+  PRFileDescList *cur = _fd_waiting_for_overlapped_done;
+  PRFileDescList *previous = NULL;
+  while (cur) {
+    PR_ASSERT(cur->fd->secret->overlappedActive);
+    PRFileDesc *fd = cur->fd;
+    DWORD rvSent;
+    if (GetOverlappedResult((HANDLE)fd->secret->md.osfd, &fd->secret->ol, &rvSent, FALSE) == TRUE) {
+      fd->secret->overlappedActive = PR_FALSE;
+      PR_LOG(_pr_io_lm, PR_LOG_MIN,
+             ("CheckOverlappedPendingSocketsAreDone GetOverlappedResult succeeded\n"));
+    } else {
+      DWORD err = WSAGetLastError();
+      PR_LOG(_pr_io_lm, PR_LOG_MIN,
+             ("CheckOverlappedPendingSocketsAreDone GetOverlappedResult failed %d\n", err));
+      if (err != ERROR_IO_INCOMPLETE) {
+        fd->secret->overlappedActive = PR_FALSE;
+      }
+    }
+
+    if (!fd->secret->overlappedActive) {
+
+      _PR_MD_CLOSE_SOCKET(fd->secret->md.osfd);
+      fd->secret->state = _PR_FILEDESC_CLOSED;
+#ifdef _PR_HAVE_PEEK_BUFFER
+      if (fd->secret->peekBuffer) {
+        PR_ASSERT(fd->secret->peekBufSize > 0);
+        PR_DELETE(fd->secret->peekBuffer);
+        fd->secret->peekBufSize = 0;
+        fd->secret->peekBytes = 0;
+      }
+#endif
+
+      PR_FreeFileDesc(fd);
+
+      if (previous) {
+        previous->next = cur->next;
+      } else {
+        _fd_waiting_for_overlapped_done = cur->next;
+      }
+      PRFileDescList *del = cur;
+      cur = cur->next;
+      PR_Free(del);
+    } else {
+      previous = cur;
+      cur = cur->next;
+    }
+  }
+
+  PR_Unlock(_fd_waiting_for_overlapped_done_lock);
+}
+#endif
+
 /*
 ** Wait for some i/o to finish on one or more more poll descriptors.
 */
 PR_IMPLEMENT(PRInt32) PR_Poll(PRPollDesc *pds, PRIntn npds, PRIntervalTime timeout)
 {
+#if defined(_WIN64) && defined(WIN95)
+  // For each iteration check if TFO overlapped IOs are down.
+  CheckOverlappedPendingSocketsAreDone();
+#endif
+
 	return(_PR_MD_PR_POLL(pds, npds, timeout));
 }
 
diff --git a/pr/src/io/prsocket.c b/pr/src/io/prsocket.c
index b42f0568..26f7a245 100644
--- a/pr/src/io/prsocket.c
+++ b/pr/src/io/prsocket.c
@@ -737,6 +737,56 @@ static PRStatus PR_CALLBACK SocketClose(PRFileDesc *fd)
 	}
 
 	if (fd->secret->state == _PR_FILEDESC_OPEN) {
+#if defined(_WIN64) && defined(WIN95)
+    /* TCP Fast Open on Windows must use ConnectEx, which uses overlapped
+     * input/output. Before closing such a socket we must cancelIO.
+     */
+    if (fd->secret->overlappedActive) {
+      PR_ASSERT(fd->secret->nonblocking);
+      if (CancelIo((HANDLE) fd->secret->md.osfd) == TRUE) {
+        PR_LOG(_pr_io_lm, PR_LOG_MIN,
+               ("SocketClose - CancelIo succeeded\n"));
+      } else {
+        DWORD err = WSAGetLastError();
+        PR_LOG(_pr_io_lm, PR_LOG_MIN,
+               ("SocketClose - CancelIo failed err=%x\n", err));
+      }
+
+      DWORD rvSent;
+      if (GetOverlappedResult((HANDLE)fd->secret->md.osfd, &fd->secret->ol, &rvSent, FALSE) == TRUE) {
+        fd->secret->overlappedActive = PR_FALSE;
+        PR_LOG(_pr_io_lm, PR_LOG_MIN,
+               ("SocketClose GetOverlappedResult succeeded\n"));
+      } else {
+        DWORD err = WSAGetLastError();
+        PR_LOG(_pr_io_lm, PR_LOG_MIN,
+               ("SocketClose GetOverlappedResult failed %d\n", err));
+        if (err != ERROR_IO_INCOMPLETE) {
+          _PR_MD_MAP_CONNECT_ERROR(err);
+          fd->secret->overlappedActive = PR_FALSE;
+        }
+      }
+    }
+
+    if (fd->secret->overlappedActive &&
+        _fd_waiting_for_overlapped_done_lock) {
+      // Put osfd into the list to be checked later.
+      PRFileDescList *forWaiting = PR_NEW(PRFileDescList);
+      if (!forWaiting) {
+        PR_SetError(PR_OUT_OF_MEMORY_ERROR, 0);
+        return PR_FAILURE;
+      }
+      forWaiting->fd = fd;
+
+      PR_Lock(_fd_waiting_for_overlapped_done_lock);
+      forWaiting->next = _fd_waiting_for_overlapped_done;
+      _fd_waiting_for_overlapped_done = forWaiting;
+      PR_Unlock(_fd_waiting_for_overlapped_done_lock);
+
+      return PR_SUCCESS;
+    }
+#endif
+
 		if (_PR_MD_CLOSE_SOCKET(fd->secret->md.osfd) < 0) {
 			return PR_FAILURE;
 		}
@@ -1684,33 +1734,6 @@ PR_ChangeFileDescNativeHandle(PRFileDesc *fd, PROsfd handle)
 		fd->secret->md.osfd = handle;
 }
 
-/* Expose OVERLAPPED if present. OVERLAPPED is implemented only on WIN95. */
-PR_IMPLEMENT(PRStatus)
-PR_EXPERIMENTAL_ONLY_IN_4_17_GetOverlappedIOHandle(PRFileDesc *fd, void **ol)
-{
-#if defined(_WIN64) && defined(WIN95)
-    *ol = NULL;
-    if (fd) {
-        fd = PR_GetIdentitiesLayer(fd, PR_NSPR_IO_LAYER);
-    }
-    if (!fd) {
-        PR_SetError(PR_INVALID_ARGUMENT_ERROR, 0);
-        return PR_FAILURE;
-    }
-
-    if (!fd->secret->overlappedActive) {
-        PR_SetError(PR_INVALID_ARGUMENT_ERROR, 0);
-        return PR_FAILURE;
-    }
-
-    *ol = &fd->secret->ol;
-    return PR_SUCCESS;
-#else
-    PR_SetError(PR_NOT_IMPLEMENTED_ERROR, 0);
-    return PR_FAILURE;
-#endif
-}
-
 /*
 ** Select compatibility
 **
diff --git a/pr/src/md/windows/w95sock.c b/pr/src/md/windows/w95sock.c
index 0429c655..c6a3ec11 100644
--- a/pr/src/md/windows/w95sock.c
+++ b/pr/src/md/windows/w95sock.c
@@ -382,6 +382,11 @@ PRInt32
 _PR_MD_TCPSENDTO(PRFileDesc *fd, const void *buf, PRInt32 amount, PRIntn flags,
                  const PRNetAddr *addr, PRUint32 addrlen, PRIntervalTime timeout)
 {
+    if (!_fd_waiting_for_overlapped_done_lock) {
+        PR_SetError(PR_NOT_IMPLEMENTED_ERROR, 0);
+        return PR_FAILURE;
+    }
+
     if (PR_CallOnce(&_pr_has_connectex_once, _pr_set_connectex) != PR_SUCCESS) {
         PR_SetError(PR_NOT_IMPLEMENTED_ERROR, 0);
         return PR_FAILURE;
diff --git a/pr/src/md/windows/w95thred.c b/pr/src/md/windows/w95thred.c
index c27d982a..3499f9f2 100644
--- a/pr/src/md/windows/w95thred.c
+++ b/pr/src/md/windows/w95thred.c
@@ -35,6 +35,10 @@ _PR_MD_EARLY_INIT()
     _pr_lastThreadIndex = TlsAlloc();
     _pr_currentCPUIndex = TlsAlloc();
 #endif
+
+#if defined(_WIN64) && defined(WIN95)
+    _fd_waiting_for_overlapped_done_lock = PR_NewLock();
+#endif
 }
 
 void _PR_MD_CLEANUP_BEFORE_EXIT(void)
@@ -50,6 +54,29 @@ void _PR_MD_CLEANUP_BEFORE_EXIT(void)
     TlsFree(_pr_lastThreadIndex);
     TlsFree(_pr_currentCPUIndex);
 #endif
+
+#if defined(_WIN64) && defined(WIN95)
+    // For each iteration check if TFO overlapped IOs are down.
+    if (_fd_waiting_for_overlapped_done_lock) {
+        PRIntervalTime delay = PR_MillisecondsToInterval(1000);
+        PRFileDescList *cur;
+        do {
+            CheckOverlappedPendingSocketsAreDone();
+
+            PR_Lock(_fd_waiting_for_overlapped_done_lock);
+            cur = _fd_waiting_for_overlapped_done;
+            PR_Unlock(_fd_waiting_for_overlapped_done_lock);
+#if define DO_NOT_WAIT_FOR_CONNECT_OVERLAPPED_OPERATIONS
+            cur = NULL;
+#endif
+            if (cur) {
+                PR_Sleep(delay); // wait another 1s.
+            }
+        } while (cur);
+
+        PR_DestroyLock(_fd_waiting_for_overlapped_done_lock);
+    }
+#endif
 }
 
 PRStatus
diff --git a/pr/src/pthreads/ptio.c b/pr/src/pthreads/ptio.c
index ebe6d8d8..9dde0319 100644
--- a/pr/src/pthreads/ptio.c
+++ b/pr/src/pthreads/ptio.c
@@ -4744,14 +4744,6 @@ PR_IMPLEMENT(PRInt32) PR_FileDesc2NativeHandle(PRFileDesc *bottom)
     return osfd;
 }  /* PR_FileDesc2NativeHandle */
 
-/* Expose OVERLAPPED if present. OVERLAPPED is implemented only on WIN95. */
-PR_IMPLEMENT(PRStatus)
-PR_EXPERIMENTAL_ONLY_IN_4_17_GetOverlappedIOHandle(PRFileDesc *fd, void **ol)
-{
-    PR_SetError(PR_NOT_IMPLEMENTED_ERROR, 0);
-    return PR_FAILURE;
-}
-
 PR_IMPLEMENT(void) PR_ChangeFileDescNativeHandle(PRFileDesc *fd,
     PRInt32 handle)
 {
diff --git a/pr/src/pthreads/ptthread.c b/pr/src/pthreads/ptthread.c
index 9e12606e..e57abd61 100644
--- a/pr/src/pthreads/ptthread.c
+++ b/pr/src/pthreads/ptthread.c
@@ -1064,6 +1064,11 @@ void PR_HPUX10xInit(shl_t handle, int loading)
 
 void _PR_Fini(void)
 {
+    /* We disable the cleanup code on Mac OSX, see bug 1399746.
+     * The .dylib containing NSPR can get unloaded, and _PR_Fini called,
+     * and other code calling NSPR functions can get executed afterwards.
+    */
+#ifndef DARWIN
     void *thred;
     int rv;
 
@@ -1095,6 +1100,7 @@ void _PR_Fini(void)
     pt_book.keyCreated = PR_FALSE;
     /* TODO: free other resources used by NSPR */
     /* _pr_initialized = PR_FALSE; */
+#endif
 }  /* _PR_Fini */
 
 PR_IMPLEMENT(PRStatus) PR_Cleanup(void)