Track store errors in thread stats

Add two new stat keys, `store_too_large` and `store_no_memory`, to track
occurrences of storage request rejections due to writing too large of a
value and writing beyond available provisioned memory, respectively.

6 files changed, 44 insertions, 7 deletions

diff --git a/doc/protocol.txt b/doc/protocol.txt
index af97270..0c640e0 100644
--- a/doc/protocol.txt
+++ b/doc/protocol.txt
@@ -1267,6 +1267,12 @@ integers separated by a colon (treat this as a floating point number).
 |                       |         | with a new expiration time                |
 | touch_misses          | 64u     | Number of items that have been touched    |
 |                       |         | and not found                             |
+| store_too_large       | 64u     | Number of rejected storage requests       |
+|                       |         | caused by attempting to write a value     |
+|                       |         | larger than the -I limit                  |
+| store_no_memory       | 64u     | Number of rejected storage requests       |
+|                       |         | caused by exhaustion of the -m memory     |
+|                       |         | limit (relevant when -M is used)          |
 | auth_cmds             | 64u     | Number of authentication commands         |
 |                       |         | handled, success or failure.              |
 | auth_errors           | 64u     | Number of failed authentications.         |
diff --git a/memcached.c b/memcached.c
index a63797a..e61e560 100644
--- a/memcached.c
+++ b/memcached.c
@@ -1824,6 +1824,8 @@ void server_stats(ADD_STAT add_stats, conn *c) {
     APPEND_STAT("cas_badval", "%llu", (unsigned long long)slab_stats.cas_badval);
     APPEND_STAT("touch_hits", "%llu", (unsigned long long)slab_stats.touch_hits);
     APPEND_STAT("touch_misses", "%llu", (unsigned long long)thread_stats.touch_misses);
+    APPEND_STAT("store_too_large", "%llu", (unsigned long long)thread_stats.store_too_large);
+    APPEND_STAT("store_no_memory", "%llu", (unsigned long long)thread_stats.store_no_memory);
     APPEND_STAT("auth_cmds", "%llu", (unsigned long long)thread_stats.auth_cmds);
     APPEND_STAT("auth_errors", "%llu", (unsigned long long)thread_stats.auth_errors);
     if (settings.idle_timeout) {
diff --git a/memcached.h b/memcached.h
index b7d4ad5..b0e10f7 100644
--- a/memcached.h
+++ b/memcached.h
@@ -320,7 +320,9 @@ struct slab_stats {
     X(response_obj_oom) \
     X(response_obj_count) \
     X(response_obj_bytes) \
-    X(read_buf_oom)
+    X(read_buf_oom) \
+    X(store_too_large) \
+    X(store_no_memory)
 
 #ifdef EXTSTORE
 #define EXTSTORE_THREAD_STATS_FIELDS \
diff --git a/proto_text.c b/proto_text.c
index 6cc1f66..e33c204 100644
--- a/proto_text.c
+++ b/proto_text.c
@@ -1480,9 +1480,15 @@ static void process_mset_command(conn *c, token_t *tokens, const size_t ntokens)
         if (! item_size_ok(nkey, of.client_flags, vlen)) {
             errstr = "SERVER_ERROR object too large for cache";
             status = TOO_LARGE;
+            pthread_mutex_lock(&c->thread->stats.mutex);
+            c->thread->stats.store_too_large++;
+            pthread_mutex_unlock(&c->thread->stats.mutex);
         } else {
             errstr = "SERVER_ERROR out of memory storing object";
             status = NO_MEMORY;
+            pthread_mutex_lock(&c->thread->stats.mutex);
+            c->thread->stats.store_no_memory++;
+            pthread_mutex_unlock(&c->thread->stats.mutex);
         }
         // FIXME: LOGGER_LOG specific to mset, include options.
         LOGGER_LOG(c->thread->l, LOG_MUTATIONS, LOGGER_ITEM_STORE,
@@ -1962,9 +1968,15 @@ static void process_update_command(conn *c, token_t *tokens, const size_t ntoken
         if (! item_size_ok(nkey, flags, vlen)) {
             out_string(c, "SERVER_ERROR object too large for cache");
             status = TOO_LARGE;
+            pthread_mutex_lock(&c->thread->stats.mutex);
+            c->thread->stats.store_too_large++;
+            pthread_mutex_unlock(&c->thread->stats.mutex);
         } else {
             out_of_memory(c, "SERVER_ERROR out of memory storing object");
             status = NO_MEMORY;
+            pthread_mutex_lock(&c->thread->stats.mutex);
+            c->thread->stats.store_no_memory++;
+            pthread_mutex_unlock(&c->thread->stats.mutex);
         }
         LOGGER_LOG(c->thread->l, LOG_MUTATIONS, LOGGER_ITEM_STORE,
                 NULL, status, comm, key, nkey, 0, 0, c->sfd);
diff --git a/t/dash-M.t b/t/dash-M.t
index df55f62..7e01b23 100644
--- a/t/dash-M.t
+++ b/t/dash-M.t
@@ -23,12 +23,16 @@ while($resp eq "STORED\r\n") {
 
 my $max_stored = $key - 1;
 
-plan tests => $max_stored + 1;
+plan tests => $max_stored + 2;
 
 print $sock "set dash$key 0 0 $vallen\r\n$value\r\n";
 is(scalar <$sock>, "SERVER_ERROR out of memory storing object\r\n",
    "failed to add another one.");
 
+my $stats = mem_stats($sock);
+is($stats->{"store_no_memory"}, 2,
+    "recorded store failures due to no memory");
+
 for($key = 0; $key < $max_stored; $key++) {
     mem_get_is $sock, "dash$key", $value, "Failed at dash$key";
 }
diff --git a/t/stats.t b/t/stats.t
index 106b242..69872ba 100755
--- a/t/stats.t
+++ b/t/stats.t
@@ -1,12 +1,12 @@
 #!/usr/bin/perl
 
 use strict;
-use Test::More tests => 109;
+use Test::More tests => 113;
 use FindBin qw($Bin);
 use lib "$Bin/lib";
 use MemcachedTest;
 
-my $server = new_memcached("-o no_lru_crawler,no_lru_maintainer");
+my $server = new_memcached("-I 1024 -o slab_chunk_max=1024,no_lru_crawler,no_lru_maintainer");
 my $sock = $server->sock;
 
 
@@ -28,15 +28,16 @@ if (MemcachedTest::enabled_tls_testing()) {
     # when TLS is enabled, stats contains additional keys:
     #   - ssl_handshake_errors
     #   - time_since_server_cert_refresh
-    is(scalar(keys(%$stats)), 83, "expected count of stats values");
+    is(scalar(keys(%$stats)), 85, "expected count of stats values");
 } else {
-    is(scalar(keys(%$stats)), 81, "expected count of stats values");
+    is(scalar(keys(%$stats)), 83, "expected count of stats values");
 }
 
 # Test initial state
 foreach my $key (qw(curr_items total_items bytes cmd_get cmd_set get_hits evictions get_misses get_expired
                  bytes_written delete_hits delete_misses incr_hits incr_misses decr_hits get_flushed
-                 decr_misses listen_disabled_num lrutail_reflocked time_in_listen_disabled_us)) {
+                 decr_misses listen_disabled_num lrutail_reflocked time_in_listen_disabled_us
+                 store_too_large store_no_memory)) {
     is($stats->{$key}, 0, "initial $key is zero");
 }
 is($stats->{accepting_conns}, 1, "initial accepting_conns is one");
@@ -190,3 +191,13 @@ is(scalar <$sock>, "END\r\n", "flushed item not returned");
 my $stats = mem_stats($sock);
 is($stats->{cmd_flush}, 1, "after one flush cmd_flush is 1");
 is($stats->{get_flushed}, 1, "after flush and a get, get_flushed is 1");
+
+# item too large
+my $large = "B" x 2048;
+my $largelen = length($large);
+print $sock "set too_large 0 0 $largelen\r\n$large\r\n";
+is(scalar <$sock>, "SERVER_ERROR object too large for cache\r\n",
+    "set rejected due to value too large");
+$stats = mem_stats($sock);
+is($stats->{'store_too_large'}, 1,
+    "recorded store failure due to value too large")