experimental maxconns_fast option

Also fixes -c option to allow reducing the maximum connection limit.

This gives a new option "-o maxconns_fast", which changes how memcached
handles hitting the maximum connection limit. By default, it disables the
accept listener and new connections will wait in the listen queue. With
maxconns_fast enabled, new connections over the limited have an error written
to them and are immediately closed by the listener thread.

This is currently experimental, as we aren't sure how clients will handle the
change. It may become the default in the future.

8 files changed, 82 insertions, 11 deletions

diff --git a/doc/memcached.1 b/doc/memcached.1
index deff4fb..b6f633d 100644
--- a/doc/memcached.1
+++ b/doc/memcached.1
@@ -130,6 +130,10 @@ Override the default size of each slab page. Default is 1mb. Default is 1m,
 minimum is 1k, max is 128m. Adjusting this value changes the item size limit.
 Beware that this also increases the number of slabs (use -v to view), and the
 overal memory usage of memcached.
+.TP
+.B \-o <options>
+Comma separated list of extended or experimental options. See -h or wiki for
+up to date list.
 .br
 .SH LICENSE
 The memcached daemon is copyright Danga Interactive and is distributed under
diff --git a/doc/protocol.txt b/doc/protocol.txt
index e2b6b86..de21476 100644
--- a/doc/protocol.txt
+++ b/doc/protocol.txt
@@ -405,6 +405,7 @@ integers separated by a colon (treat this as a floating point number).
 |                       |         | the server started running                |
 | connection_structures | 32u     | Number of connection structures allocated |
 |                       |         | by the server                             |
+| reserved_fds          | 32u     | Number of misc fds used internally        |
 | cmd_get               | 64u     | Cumulative number of retrieval reqs       |
 | cmd_set               | 64u     | Cumulative number of storage reqs         |
 | cmd_flush             | 64u     | Cumulative number of flush reqs           |
diff --git a/memcached.c b/memcached.c
index f8316c7..335fa3b 100644
--- a/memcached.c
+++ b/memcached.c
@@ -166,7 +166,7 @@ static rel_time_t realtime(const time_t exptime) {
 static void stats_init(void) {
     stats.curr_items = stats.total_items = stats.curr_conns = stats.total_conns = stats.conn_structs = 0;
     stats.get_cmds = stats.set_cmds = stats.get_hits = stats.get_misses = stats.evictions = stats.reclaimed = 0;
-    stats.touch_cmds = stats.touch_misses = 0;
+    stats.touch_cmds = stats.touch_misses = stats.touch_hits = stats.rejected_conns = 0;
     stats.curr_bytes = stats.listen_disabled_num = 0;
     stats.accepting_conns = true; /* assuming we start in this state. */
 
@@ -181,6 +181,7 @@ static void stats_init(void) {
 static void stats_reset(void) {
     STATS_LOCK();
     stats.total_items = stats.total_conns = 0;
+    stats.rejected_conns = 0;
     stats.evictions = 0;
     stats.reclaimed = 0;
     stats.listen_disabled_num = 0;
@@ -213,6 +214,7 @@ static void settings_init(void) {
     settings.backlog = 1024;
     settings.binding_protocol = negotiating_prot;
     settings.item_size_max = 1024 * 1024; /* The famous 1MB upper limit. */
+    settings.maxconns_fast = false;
 }
 
 /*
@@ -2519,7 +2521,11 @@ static void server_stats(ADD_STAT add_stats, conn *c) {
 
     APPEND_STAT("curr_connections", "%u", stats.curr_conns - 1);
     APPEND_STAT("total_connections", "%u", stats.total_conns);
+    if (settings.maxconns_fast) {
+        APPEND_STAT("rejected_connections", "%llu", (unsigned long long)stats.rejected_conns);
+    }
     APPEND_STAT("connection_structures", "%u", stats.conn_structs);
+    APPEND_STAT("reserved_fds", "%u", stats.reserved_fds);
     APPEND_STAT("cmd_get", "%llu", (unsigned long long)thread_stats.get_cmds);
     APPEND_STAT("cmd_set", "%llu", (unsigned long long)slab_stats.set_cmds);
     APPEND_STAT("cmd_flush", "%llu", (unsigned long long)thread_stats.flush_cmds);
@@ -2576,6 +2582,7 @@ static void process_stat_settings(ADD_STAT add_stats, void *c) {
                 prot_text(settings.binding_protocol));
     APPEND_STAT("auth_enabled_sasl", "%s", settings.sasl ? "yes" : "no");
     APPEND_STAT("item_size_max", "%d", settings.item_size_max);
+    APPEND_STAT("maxconns_fast", "%s", settings.maxconns_fast ? "yes" : "no");
 }
 
 static void process_stat(conn *c, token_t *tokens, const size_t ntokens) {
@@ -3625,6 +3632,7 @@ static void drive_machine(conn *c) {
     struct sockaddr_storage addr;
     int nreqs = settings.reqs_per_event;
     int res;
+    const char *str;
 
     assert(c != NULL);
 
@@ -3655,8 +3663,19 @@ static void drive_machine(conn *c) {
                 break;
             }
 
-            dispatch_conn_new(sfd, conn_new_cmd, EV_READ | EV_PERSIST,
+            if (settings.maxconns_fast &&
+                stats.curr_conns + stats.reserved_fds >= settings.maxconns - 1) {
+                str = "ERROR Too many open connections\r\n";
+                res = write(sfd, str, strlen(str));
+                close(sfd);
+                STATS_LOCK();
+                stats.rejected_conns++;
+                STATS_UNLOCK();
+            } else {
+                dispatch_conn_new(sfd, conn_new_cmd, EV_READ | EV_PERSIST,
                                      DATA_BUFFER_SIZE, tcp_transport);
+            }
+
             stop = true;
             break;
 
@@ -4036,6 +4055,11 @@ static int server_socket(const char *interface,
             /* getaddrinfo can return "junk" addresses,
              * we make sure at least one works before erroring.
              */
+            if (errno == EMFILE) {
+                /* ...unless we're out of fds */
+                perror("server_socket");
+                exit(EX_OSERR);
+            }
             continue;
         }
 
@@ -4360,6 +4384,9 @@ static void usage(void) {
 #ifdef ENABLE_SASL
     printf("-S            Turn on Sasl authentication\n");
 #endif
+    printf("-o            Comma separated list of extended or experimental options\n"
+           "              - (EXPERIMENTAL) maxconns_fast: immediately close new\n"
+           "                connections if over maxconns limit\n");
     return;
 }
 
@@ -4573,6 +4600,16 @@ int main (int argc, char **argv) {
     bool tcp_specified = false;
     bool udp_specified = false;
 
+    char *subopts;
+    char *subopts_value;
+    enum {
+        MAXCONNS_FAST = 0
+    };
+    char *const subopts_tokens[] = {
+        [MAXCONNS_FAST] = "maxconns_fast",
+        NULL
+    };
+
     if (!sanitycheck()) {
         return EX_OSERR;
     }
@@ -4614,6 +4651,7 @@ int main (int argc, char **argv) {
           "B:"  /* Binding protocol */
           "I:"  /* Max item size */
           "S"   /* Sasl ON */
+          "o:"  /* Extended generic options */
         ))) {
         switch (c) {
         case 'a':
@@ -4788,6 +4826,22 @@ int main (int argc, char **argv) {
 #endif
             settings.sasl = true;
             break;
+        case 'o': /* It's sub-opts time! */
+            subopts = optarg;
+
+            while (*subopts != '\0') {
+
+            switch (getsubopt(&subopts, subopts_tokens, &subopts_value)) {
+            case MAXCONNS_FAST:
+                settings.maxconns_fast = true;
+                break;
+            default:
+                printf("Illegal suboption \"%s\"\n", subopts_value);
+                return 1;
+            }
+
+            }
+            break;
         default:
             fprintf(stderr, "Illegal argument \"%c\"\n", c);
             return 1;
@@ -4856,13 +4910,9 @@ int main (int argc, char **argv) {
         fprintf(stderr, "failed to getrlimit number of files\n");
         exit(EX_OSERR);
     } else {
-        int maxfiles = settings.maxconns;
-        if (rlim.rlim_cur < maxfiles)
-            rlim.rlim_cur = maxfiles;
-        if (rlim.rlim_max < rlim.rlim_cur)
-            rlim.rlim_max = rlim.rlim_cur;
+        rlim.rlim_cur = settings.maxconns;
         if (setrlimit(RLIMIT_NOFILE, &rlim) != 0) {
-            fprintf(stderr, "failed to set rlimit for open files. Try running as root or requesting smaller maxconns value.\n");
+            fprintf(stderr, "failed to set rlimit for open files. Try starting as root or requesting smaller maxconns value.\n");
             exit(EX_OSERR);
         }
     }
@@ -4995,6 +5045,15 @@ int main (int argc, char **argv) {
         }
     }
 
+    /* Give the sockets a moment to open. I know this is dumb, but the error
+     * is only an advisory.
+     */
+    usleep(1000);
+    if (stats.curr_conns + stats.reserved_fds >= settings.maxconns - 1) {
+        fprintf(stderr, "Maxconns setting is too low, use -c to increase.\n");
+        exit(EXIT_FAILURE);
+    }
+
     if (pid_file != NULL) {
         save_pid(pid_file);
     }
diff --git a/memcached.h b/memcached.h
index 6aa0317..79b80d9 100644
--- a/memcached.h
+++ b/memcached.h
@@ -242,6 +242,8 @@ struct stats {
     uint64_t      curr_bytes;
     unsigned int  curr_conns;
     unsigned int  total_conns;
+    uint64_t      rejected_conns;
+    unsigned int  reserved_fds;
     unsigned int  conn_structs;
     uint64_t      get_cmds;
     uint64_t      set_cmds;
@@ -287,6 +289,7 @@ struct settings {
     int backlog;
     int item_size_max;        /* Maximum item size, and upper end for slabs */
     bool sasl;              /* SASL on/off */
+    bool maxconns_fast;     /* Whether or not to early close connections */
 };
 
 extern struct stats stats;
diff --git a/t/binary.t b/t/binary.t
index 838e948..8fa27d1 100755
--- a/t/binary.t
+++ b/t/binary.t
@@ -2,7 +2,7 @@
 
 use strict;
 use warnings;
-use Test::More tests => 3435;
+use Test::More tests => 3450;
 use FindBin qw($Bin);
 use lib "$Bin/lib";
 use MemcachedTest;
diff --git a/t/maxconns.t b/t/maxconns.t
index d30966a..14b5eae 100755
--- a/t/maxconns.t
+++ b/t/maxconns.t
@@ -1,4 +1,6 @@
 #!/usr/bin/perl
+# NOTE: This test never worked. Memcached would ignore maxconns requests lower
+# than the current ulimit. Test needs to be updated.
 
 use strict;
 use warnings;
@@ -11,7 +13,7 @@ use MemcachedTest;
 
 
 # start up a server with 10 maximum connections
-my $server = new_memcached('-c 10');
+my $server = new_memcached('-c 100');
 my $sock = $server->sock;
 my @sockets;
 
diff --git a/t/stats.t b/t/stats.t
index f47d6a4..7c78617 100755
--- a/t/stats.t
+++ b/t/stats.t
@@ -58,7 +58,7 @@ my $sock = $server->sock;
 my $stats = mem_stats($sock);
 
 # Test number of keys
-is(scalar(keys(%$stats)), 42, "42 stats values");
+is(scalar(keys(%$stats)), 43, "43 stats values");
 
 # Test initial state
 foreach my $key (qw(curr_items total_items bytes cmd_get cmd_set get_hits evictions get_misses
diff --git a/thread.c b/thread.c
index 904368f..8fe797f 100644
--- a/thread.c
+++ b/thread.c
@@ -624,6 +624,8 @@ void thread_init(int nthreads, struct event_base *main_base) {
         threads[i].notify_send_fd = fds[1];
 
         setup_thread(&threads[i]);
+        /* Reserve three fds for the libevent base, and two for the pipe */
+        stats.reserved_fds += 5;
     }
 
     /* Create threads after we've done all the libevent setup. */