[core] connect, write, read timeouts on backends (fixes #3086)

implement connect(), write(), read() timeouts on backends

"connect-timeout"
"write-timeout"
"read-timeout"

x-ref:
  "sockets disabled, out-of-fds with proxy module"
  https://redmine.lighttpd.net/issues/3086

2 files changed, 138 insertions, 11 deletions

diff --git a/src/gw_backend.c b/src/gw_backend.c
index e3067aae..12837756 100644
--- a/src/gw_backend.c
+++ b/src/gw_backend.c
@@ -1289,6 +1289,15 @@ int gw_set_defaults_backend(server *srv, gw_plugin_data *p, const array *a, gw_p
      ,{ CONST_STR_LEN("tcp-fin-propagate"),
         T_CONFIG_BOOL,
         T_CONFIG_SCOPE_CONNECTION }
+     ,{ CONST_STR_LEN("connect-timeout"),
+        T_CONFIG_INT,
+        T_CONFIG_SCOPE_CONNECTION }
+     ,{ CONST_STR_LEN("write-timeout"),
+        T_CONFIG_INT,
+        T_CONFIG_SCOPE_CONNECTION }
+     ,{ CONST_STR_LEN("read-timeout"),
+        T_CONFIG_INT,
+        T_CONFIG_SCOPE_CONNECTION }
      ,{ NULL, 0,
         T_CONFIG_UNSET,
         T_CONFIG_SCOPE_UNSET }
@@ -1471,6 +1480,15 @@ int gw_set_defaults_backend(server *srv, gw_plugin_data *p, const array *a, gw_p
                   case 22:/* tcp-fin-propagate */
                     host->tcp_fin_propagate = (0 != cpv->v.u);
                     break;
+                  case 23:/* connect-timeout */
+                    host->connect_timeout = cpv->v.u;
+                    break;
+                  case 24:/* write-timeout */
+                    host->write_timeout = cpv->v.u;
+                    break;
+                  case 25:/* read-timeout */
+                    host->read_timeout = cpv->v.u;
+                    break;
                   default:
                     break;
                 }
@@ -1763,6 +1781,34 @@ void gw_set_transparent(gw_handler_ctx *hctx) {
 }
 
 
+static void gw_host_hctx_enq(gw_handler_ctx * const hctx) {
+    gw_host * const host = hctx->host;
+    /*if (__builtin_expect( (host == NULL), 0)) return;*/
+
+    hctx->prev = NULL;
+    hctx->next = host->hctxs;
+    if (hctx->next)
+        hctx->next->prev = hctx;
+    host->hctxs = hctx;
+}
+
+
+static void gw_host_hctx_deq(gw_handler_ctx * const hctx) {
+    /*if (__builtin_expect( (hctx->host == NULL), 0)) return;*/
+
+    if (hctx->prev)
+        hctx->prev->next = hctx->next;
+    else
+        hctx->host->hctxs= hctx->next;
+
+    if (hctx->next)
+        hctx->next->prev = hctx->prev;
+
+    hctx->next = NULL;
+    hctx->prev = NULL;
+}
+
+
 static void gw_backend_close(gw_handler_ctx * const hctx, request_st * const r) {
     if (hctx->fd >= 0) {
         fdevent_fdnode_event_del(hctx->ev, hctx->fdn);
@@ -1770,6 +1816,7 @@ static void gw_backend_close(gw_handler_ctx * const hctx, request_st * const r)
         fdevent_sched_close(hctx->ev, hctx->fd, 1);
         hctx->fdn = NULL;
         hctx->fd = -1;
+        gw_host_hctx_deq(hctx);
     }
 
     if (hctx->host) {
@@ -1885,6 +1932,7 @@ static handler_t gw_write_request(gw_handler_ctx * const hctx, request_st * cons
         }
 
         hctx->write_ts = log_monotonic_secs;
+        gw_host_hctx_enq(hctx);
         switch (gw_establish_connection(r, hctx->host, hctx->proc, hctx->pid,
                                         hctx->fd, hctx->conf.debug)) {
         case 1: /* connection is in progress */
@@ -2595,17 +2643,86 @@ handler_t gw_check_extension(request_st * const r, gw_plugin_data * const p, int
     return HANDLER_GO_ON;
 }
 
+__attribute_cold__
+__attribute_noinline__
+static void gw_handle_trigger_hctx_timeout(gw_handler_ctx * const hctx, const char * const msg) {
+
+    request_st * const r = hctx->r;
+    joblist_append(r->con);
+
+    if (*msg == 'c') { /* "connect" */
+        /* temporarily disable backend proc */
+        gw_proc_connect_error(r, hctx->host, hctx->proc, hctx->pid,
+                              ETIMEDOUT, hctx->conf.debug);
+        /* cleanup this request and let request handler start request again */
+        /* retry only once since request already waited write_timeout secs */
+        if (hctx->reconnects++ < 1) {
+            gw_reconnect(hctx, r);
+            return;
+        }
+        r->http_status = 503; /* Service Unavailable */
+    }
+    else { /* "read" or "write" */
+        /* blocked waiting to send (more) data to or to receive response
+         * (neither are a definite indication that the proc is no longer
+         *  responsive on other socket connections; not marking proc overloaded)
+         * (If connect() to backend succeeded, then we began sending
+         *  request and filled kernel socket buffers, so request is
+         *  in progress and it is not safe or possible to retry) */
+        /*if (hctx->conf.debug)*/
+            log_error(r->conf.errh, __FILE__, __LINE__,
+              "%s timeout on socket: %s (fd: %d)",
+              msg, hctx->proc->connection_name->ptr, hctx->fd);
+
+        if (*msg == 'w') { /* "write" */
+            gw_write_error(hctx, r); /*(calls gw_backend_error())*/
+            if (r->http_status == 503) r->http_status = 504; /*Gateway Timeout*/
+            return;
+        } /* else "read" */
+    }
+    gw_backend_error(hctx, r);
+    if (r->http_status == 500 && !r->resp_body_started && !r->handler_module)
+        r->http_status = 504; /*Gateway Timeout*/
+}
+
+__attribute_noinline__
+static void gw_handle_trigger_host_timeouts(gw_host * const host) {
+
+    if (NULL == host->hctxs) return;
+    const unix_time64_t rsecs = (unix_time64_t)host->read_timeout;
+    const unix_time64_t wsecs = (unix_time64_t)host->write_timeout;
+    const unix_time64_t csecs = (unix_time64_t)host->connect_timeout;
+    if (!rsecs && !wsecs && !csecs)
+        return; /*(no timeout policy (default))*/
+
+    const unix_time64_t mono = log_monotonic_secs; /*(could have callers pass)*/
+    for (gw_handler_ctx *hctx = host->hctxs, *next; hctx; hctx = next) {
+        /* if timeout occurs, hctx might be invalidated and removed from list,
+         * so next element must be store before checking for timeout */
+        next = hctx->next;
+
+        if (hctx->state == GW_STATE_CONNECT_DELAYED) {
+            if (mono - hctx->write_ts > csecs && csecs) /*(waiting for write)*/
+                gw_handle_trigger_hctx_timeout(hctx, "connect");
+            continue; /*(do not apply wsecs below to GW_STATE_CONNECT_DELAYED)*/
+        }
+
+        const int events = fdevent_fdnode_interest(hctx->fdn);
+        if ((events & FDEVENT_IN) && mono - hctx->read_ts > rsecs && rsecs) {
+            gw_handle_trigger_hctx_timeout(hctx, "read");
+            continue;
+        }
+        if ((events & FDEVENT_OUT) && mono - hctx->write_ts > wsecs && wsecs) {
+            gw_handle_trigger_hctx_timeout(hctx, "write");
+            continue;
+        }
+    }
+}
+
 static void gw_handle_trigger_host(gw_host * const host, log_error_st * const errh, const int debug) {
-    /*
-     * TODO:
-     *
-     * - add timeout for a connect to a non-gw process
-     *   (use state_timestamp + state)
-     *
-     * perhaps we should kill a connect attempt after 10-15 seconds
-     *
-     * currently we wait for the TCP timeout which is 180 seconds on Linux
-     */
+
+    /* check for socket timeouts on active requests to backend host */
+    gw_handle_trigger_host_timeouts(host);
 
     /* check each child proc to detect if proc exited */
 
@@ -2679,6 +2796,7 @@ static void gw_handle_trigger_exts_wkr(gw_exts *exts, log_error_st *errh) {
         gw_extension * const ex = exts->exts+j;
         for (uint32_t n = 0; n < ex->used; ++n) {
             gw_host * const host = ex->hosts[n];
+            gw_handle_trigger_host_timeouts(host);
             for (gw_proc *proc = host->first; proc; proc = proc->next) {
                 if (proc->state == PROC_STATE_OVERLOADED)
                     gw_proc_check_enable(host, proc, errh);
diff --git a/src/gw_backend.h b/src/gw_backend.h
index a948fc88..dda7d11f 100644
--- a/src/gw_backend.h
+++ b/src/gw_backend.h
@@ -44,6 +44,8 @@ typedef struct gw_proc {
     unsigned short port;  /* config.port + pno */
 } gw_proc;
 
+struct gw_handler_ctx;  /* declaration */
+
 typedef struct {
     /* list of processes handling this extension
      * sorted by lowest load
@@ -113,13 +115,18 @@ typedef struct {
 
     unsigned short disable_time;
 
+    unsigned short read_timeout;
+    unsigned short write_timeout;
+    unsigned short connect_timeout;
+    struct gw_handler_ctx *hctxs;
+
     /*
      * some gw processes get a little bit larger
      * than wanted. max_requests_per_proc kills a
      * process after a number of handled requests.
      *
      */
-    uint32_t max_requests_per_proc;
+    /*uint32_t max_requests_per_proc;*//* not implemented */
 
 
     /* config */
@@ -327,6 +334,8 @@ typedef struct gw_handler_ctx {
     unix_time64_t write_ts;
     handler_t(*stdin_append)(struct gw_handler_ctx *hctx);
     handler_t(*create_env)(struct gw_handler_ctx *hctx);
+    struct gw_handler_ctx *prev;
+    struct gw_handler_ctx *next;
     void(*backend_error)(struct gw_handler_ctx *hctx);
     void(*handler_ctx_free)(void *hctx);
 } gw_handler_ctx;