[core] _WIN32 alternative fdarray for Windows

_WIN32 SOCKET (long long unsigned) handles are assigned differently
from how POSIX allocates file descriptors (lowest number available).

On _WIN32, the SOCKET descriptor should not be used to index an array
of (fdnode *), so this commit provides an alternative method to store
(fdnode *) for use by select() and by WSAPoll().

select(): commonly used unix select() idioms may be incorrect on _WIN32
https://devblogs.microsoft.com/oldnewthing/20221102-00/?p=107343
https://devblogs.microsoft.com/oldnewthing/20161221-00/?p=94985

WSAPoll():
https://learn.microsoft.com/en-us/windows/win32/api/winsock2/nf-winsock2-wsapoll
As of Windows 10 version 2004, when a TCP socket fails to connect,
(POLLHUP | POLLERR | POLLWRNORM) is indicated.
(note: this was broken in WSAPoll() in all earlier Windows versions)

6 files changed, 131 insertions, 10 deletions

diff --git a/src/fdevent.h b/src/fdevent.h
index 8c9950c2..8ee741b3 100644
--- a/src/fdevent.h
+++ b/src/fdevent.h
@@ -15,10 +15,22 @@ struct fdnode_st {
     int fd;
     int events;
     int fde_ndx;
+  #ifdef _WIN32
+    int fda_ndx;
+  #endif
 };
 
 /* These must match POLL* values from operating system headers */
 
+#ifdef _WIN32 /* MS is different; definitely not better */
+#define FDEVENT_IN     (0x0100 | 0x0200)
+#define FDEVENT_PRI    (0x0400)
+#define FDEVENT_OUT    (0x0010)
+#define FDEVENT_ERR    (0x0001)
+#define FDEVENT_HUP    (0x0002)
+#define FDEVENT_NVAL   (0x0004)
+#define FDEVENT_RDHUP  0x2000
+#else
 #define FDEVENT_IN     0x0001
 #define FDEVENT_PRI    0x0002
 #define FDEVENT_OUT    0x0004
@@ -30,6 +42,7 @@ struct fdnode_st {
 #else
 #define FDEVENT_RDHUP  0x2000
 #endif
+#endif
 
 #define FDEVENT_STREAM_REQUEST                  BV(0)
 #define FDEVENT_STREAM_REQUEST_BUFMIN           BV(1)
diff --git a/src/fdevent_fdnode.c b/src/fdevent_fdnode.c
index 4dbb3878..f498c57b 100644
--- a/src/fdevent_fdnode.c
+++ b/src/fdevent_fdnode.c
@@ -25,7 +25,12 @@ fdnode_free (fdnode *fdn)
 fdnode *
 fdevent_register (fdevents *ev, int fd, fdevent_handler handler, void *ctx)
 {
+  #ifdef _WIN32
+    fdnode *fdn  = fdnode_init();
+    ev->fdarray[(fdn->fda_ndx = ev->count++)] = fdn;
+  #else
     fdnode *fdn  = ev->fdarray[fd] = fdnode_init();
+  #endif
     fdn->handler = handler;
     fdn->fd      = fd;
     fdn->ctx     = ctx;
@@ -34,11 +39,26 @@ fdevent_register (fdevents *ev, int fd, fdevent_handler handler, void *ctx)
     return fdn;
 }
 
+#ifdef _WIN32
+#define fdevent_fdarray_slot(ev,fdn) &(ev)->fdarray[(fdn)->fda_ndx]
+#else
+#define fdevent_fdarray_slot(ev,fdn) &(ev)->fdarray[(fdn)->fd]
+#endif
+
 void
 fdevent_unregister (fdevents *ev, fdnode *fdn)
 {
-    fdnode **fdn_slot = &ev->fdarray[fdn->fd];
+    fdnode **fdn_slot = fdevent_fdarray_slot(ev, fdn);
     if ((uintptr_t)*fdn_slot & 0x3) return; /*(should not happen)*/
+  #ifdef _WIN32
+    if (--ev->count != fdn->fda_ndx) {
+        /* compact fdarray; move element in last slot */
+        fdnode **fdn_last = &ev->fdarray[ev->count];
+        *fdn_slot = *fdn_last;
+        ((fdnode *)((uintptr_t)*fdn_slot & ~0x3))->fda_ndx = fdn->fda_ndx;
+        fdn_slot = fdn_last;
+    }
+  #endif
     *fdn_slot = NULL;
     fdnode_free(fdn);
 }
@@ -46,7 +66,7 @@ fdevent_unregister (fdevents *ev, fdnode *fdn)
 void
 fdevent_sched_close (fdevents *ev, fdnode *fdn)
 {
-    fdnode **fdn_slot = &ev->fdarray[fdn->fd];
+    fdnode **fdn_slot = fdevent_fdarray_slot(ev, fdn);
     if ((uintptr_t)*fdn_slot & 0x3) return;
     *fdn_slot = (fdnode *)((uintptr_t)fdn | 0x3);
     fdn->handler = (fdevent_handler)NULL;
diff --git a/src/fdevent_impl.c b/src/fdevent_impl.c
index 973e7990..78576ab2 100644
--- a/src/fdevent_impl.c
+++ b/src/fdevent_impl.c
@@ -12,7 +12,10 @@
 #include <string.h>
 
 #ifdef _WIN32
-#include <winsock2.h>   /* closesocket */
+#ifdef _MSC_VER
+#pragma comment(lib, "ws2_32.lib")
+#endif
+#include <winsock2.h>   /* closesocket(), WSAPoll() */
 #endif
 
 #ifdef FDEVENT_USE_LINUX_EPOLL
@@ -290,9 +293,15 @@ fdevent_sched_run (fdevents * const ev)
 
         fdnode * const fdn_tmp = fdn;
         fdn = (fdnode *)fdn->ctx; /* next */
+      #ifdef _WIN32
+        /*(remove 0x3 flags from fdarray fdn before call to unregister below)*/
+        ev->fdarray[fdn_tmp->fda_ndx] = fdn_tmp;
+        fdevent_unregister(ev, fdn_tmp);
+      #else
         /*(fdevent_unregister)*/
         ev->fdarray[fd] = NULL;
         free(fdn_tmp); /*fdnode_free(fdn_tmp);*/
+      #endif
     }
     ev->pendclose = NULL;
 }
@@ -723,6 +732,8 @@ fdevent_solaris_devpoll_init (fdevents *ev)
 
 #ifdef HAVE_POLL_H
 #include <poll.h>
+#elif defined(_WIN32)
+#define poll(fdArray,fds,timeout) WSAPoll((fdArray),(fds),(timeout))
 #else
 #include <sys/poll.h>
 #endif
@@ -732,7 +743,7 @@ fdevent_poll_event_del (fdevents *ev, fdnode *fdn)
 {
     int fd = fdn->fd;
     int k = fdn->fde_ndx;
-    if ((uint32_t)k >= ev->used || ev->pollfds[k].fd != fd)
+    if ((uint32_t)k >= ev->used || (int)ev->pollfds[k].fd != fd)
         return (errno = EINVAL, -1);
 
     ev->pollfds[k].fd = -1;
@@ -761,7 +772,7 @@ fdevent_poll_event_set (fdevents *ev, fdnode *fdn, int events)
   #endif
 
     if (k >= 0) {
-        if ((uint32_t)k >= ev->used || ev->pollfds[k].fd != fd)
+        if ((uint32_t)k >= ev->used || (int)ev->pollfds[k].fd != fd)
             return (errno = EINVAL, -1);
         ev->pollfds[k].events = events;
         return 0;
@@ -792,6 +803,26 @@ fdevent_poll_poll (fdevents *ev, int timeout_ms)
 {
     const int n = poll(ev->pollfds, ev->used, timeout_ms);
     fdnode ** const fdarray = ev->fdarray;
+  #ifdef _WIN32
+    /* XXX: O(m x n) search through fdarray; improve later for many fds
+     * Since number of ready events is typically going to be small,
+     * walk fdarray and for each fdn, scan ready pollfds (rather than
+     * walking ready pollfds and then scanning fdarray for matching fd) */
+    const int nfds = (int)ev->used;
+    const int count = ev->count;
+    for (int m = 0, a = 0; m < n && a < count; ++a) {
+        struct pollfd * const restrict pfds = ev->pollfds;
+        const fdnode * const fdn = fdarray[a];
+        SOCKET fd = (SOCKET)((fdnode *)((uintptr_t)fdn & ~0x3))->fd;
+        for (int i = 0; i < nfds; ++i) {
+            if (0 == pfds[i].revents || fd != pfds[i].fd) continue;
+            if (0 == ((uintptr_t)fdn & 0x3))
+                (*fdn->handler)(fdn->ctx, pfds[i].revents);
+            ++m;
+            break;
+        }
+    }
+  #else
     for (int i = 0, m = 0; m < n; ++i, ++m) {
         struct pollfd * const restrict pfds = ev->pollfds;
         while (0 == pfds[i].revents) ++i;
@@ -799,6 +830,7 @@ fdevent_poll_poll (fdevents *ev, int timeout_ms)
         if (0 == ((uintptr_t)fdn & 0x3))
             (*fdn->handler)(fdn->ctx, pfds[i].revents);
     }
+  #endif
     return n;
 }
 
@@ -846,7 +878,9 @@ fdevent_select_reset (fdevents *ev)
     FD_ZERO(&(ev->select_set_read));
     FD_ZERO(&(ev->select_set_write));
     FD_ZERO(&(ev->select_set_error));
+  #ifndef _WIN32
     ev->select_max_fd = -1;
+  #endif
     return 0;
 }
 
@@ -866,7 +900,12 @@ fdevent_select_event_set (fdevents *ev, fdnode *fdn, int events)
     int fd = fdn->fde_ndx = fdn->fd;
 
     /* we should be protected by max-fds, but you never know */
+  #ifdef _WIN32
+    force_assert(ev->count < ((int)FD_SETSIZE));
+  #else
     force_assert(fd < ((int)FD_SETSIZE));
+    if (fd > ev->select_max_fd) ev->select_max_fd = fd;
+  #endif
 
     if (events & FDEVENT_IN)
         FD_SET(fd, &(ev->select_set_read));
@@ -880,8 +919,6 @@ fdevent_select_event_set (fdevents *ev, fdnode *fdn, int events)
 
     FD_SET(fd, &(ev->select_set_error));
 
-    if (fd > ev->select_max_fd) ev->select_max_fd = fd;
-
     return 0;
 }
 
@@ -896,6 +933,30 @@ fdevent_select_poll (fdevents *ev, int timeout_ms)
     ev->select_write = ev->select_set_write;
     ev->select_error = ev->select_set_error;
 
+  #ifdef _WIN32
+
+    const int nfds = ev->count;
+    const int n =
+      select(nfds, &ev->select_read, &ev->select_write, &ev->select_error, &tv);
+    if (n <= 0) return n;
+    fdnode **fda = ev->fdarray;
+    for (int ndx = -1, i = n; ++ndx < nfds; ) {
+        const fdnode *fdn = *fda++;
+        int fd = ((fdnode *)((uintptr_t)fdn & ~0x3))->fd;
+        int revents = 0;
+        if (FD_ISSET(fd, &ev->select_read))  revents |= FDEVENT_IN;
+        if (FD_ISSET(fd, &ev->select_write)) revents |= FDEVENT_OUT;
+        if (FD_ISSET(fd, &ev->select_error)) revents |= FDEVENT_ERR;
+        if (revents) {
+            if (0 == ((uintptr_t)fdn & 0x3))
+                (*fdn->handler)(fdn->ctx, revents);
+            if (0 == --i)
+                break;
+        }
+    }
+
+  #else
+
     const int nfds = ev->select_max_fd + 1;
     const int n =
       select(nfds, &ev->select_read, &ev->select_write, &ev->select_error, &tv);
@@ -913,6 +974,9 @@ fdevent_select_poll (fdevents *ev, int timeout_ms)
                 break;
         }
     }
+
+  #endif
+
     return n;
 }
 
diff --git a/src/fdevent_impl.h b/src/fdevent_impl.h
index e2ebbc14..d8eb0311 100644
--- a/src/fdevent_impl.h
+++ b/src/fdevent_impl.h
@@ -16,6 +16,13 @@ struct epoll_event;     /* declaration */
 struct pollfd;          /* declaration */
 #endif
 
+#if 0 /*(disabled; needs further development for stability of results)*/
+#ifdef _WIN32
+# define FDEVENT_USE_POLL
+struct pollfd;          /* declaration */
+#endif
+#endif
+
 #ifndef FDEVENT_USE_POLL
 #if defined HAVE_SELECT
 # ifdef __WIN32
@@ -83,6 +90,9 @@ struct fdevents {
     log_error_st *errh;
     int *cur_fds;
     uint32_t maxfds;
+  #ifdef _WIN32
+    int count;
+  #endif
   #ifdef FDEVENT_USE_LINUX_EPOLL
     int epoll_fd;
     struct epoll_event *epoll_events;
@@ -108,6 +118,9 @@ struct fdevents {
     buffer_int unused;
   #endif
   #ifdef FDEVENT_USE_SELECT
+   #ifndef _WIN32
+    int select_max_fd;
+   #endif
     fd_set select_read;
     fd_set select_write;
     fd_set select_error;
@@ -115,8 +128,6 @@ struct fdevents {
     fd_set select_set_read;
     fd_set select_set_write;
     fd_set select_set_error;
-
-    int select_max_fd;
   #endif
 
     int (*reset)(struct fdevents *ev);
diff --git a/src/network.c b/src/network.c
index 887ed140..dc32e6f8 100644
--- a/src/network.c
+++ b/src/network.c
@@ -556,8 +556,11 @@ static int network_server_init(server *srv, const network_socket_config *s, buff
 #endif
 	}
 
-	/* */
+  #ifdef _WIN32
+	++srv->cur_fds;
+  #else
 	srv->cur_fds = srv_socket->fd;
+  #endif
 
 	if (fdevent_set_so_reuseaddr(srv_socket->fd, 1) < 0) {
 		log_perror(srv->errh, __FILE__, __LINE__, "setsockopt(SO_REUSEADDR)");
diff --git a/src/server.c b/src/server.c
index 35ef51cb..e81b7c2f 100644
--- a/src/server.c
+++ b/src/server.c
@@ -1660,6 +1660,12 @@ static int server_main_setup (server * const srv, int argc, char **argv) {
 			rlim.rlim_cur = rlim.rlim_max;
 			setrlimit(RLIMIT_CORE, &rlim);
 		}
+#else
+	  #ifdef _WIN32
+		/*(default upper limit of 4k if server.max-fds not specified)*/
+		if (0 == srv->srvconf.max_fds)
+			srv->srvconf.max_fds = 4096;
+	  #endif
 #endif
 	}
 
@@ -1920,6 +1926,9 @@ static int server_main_setup (server * const srv, int argc, char **argv) {
 #endif
 
 	/* get the current number of FDs */
+  #ifdef _WIN32
+	srv->cur_fds = 3; /*(estimate on _WIN32)*/
+  #else
 	{
 		int fd = fdevent_open_devnull();
 		if (fd >= 0) {
@@ -1927,6 +1936,7 @@ static int server_main_setup (server * const srv, int argc, char **argv) {
 			close(fd);
 		}
 	}
+  #endif
 
 	if (0 != server_sockets_set_nb_cloexec(srv)) {
 		return -1;