From de021a9c40eb8a6f57d1525a6c4ca88cc42d8ba2 Mon Sep 17 00:00:00 2001 From: Trond Norbye Date: Wed, 17 Oct 2012 11:06:38 +0200 Subject: Add statistics for allocation failures This patch adds a new stat "malloc_fails" that is a counter of how many times malloc/realloc/calloc returned NULL when we _needed_ it to return something else (resulting in closing the connection or something like that). Conditions where we could live without malloc returning a new chunk of memory is not tracked with this counter. --- memcached.c | 65 ++++++++++++++++++++++++++++++++++++++++++++++++++----------- memcached.h | 1 + thread.c | 6 +++++- 3 files changed, 60 insertions(+), 12 deletions(-) diff --git a/memcached.c b/memcached.c index 06c2efc..9ec1597 100644 --- a/memcached.c +++ b/memcached.c @@ -170,6 +170,7 @@ static void stats_init(void) { stats.curr_items = stats.total_items = stats.curr_conns = stats.total_conns = stats.conn_structs = 0; stats.get_cmds = stats.set_cmds = stats.get_hits = stats.get_misses = stats.evictions = stats.reclaimed = 0; stats.touch_cmds = stats.touch_misses = stats.touch_hits = stats.rejected_conns = 0; + stats.malloc_fails = 0; stats.curr_bytes = stats.listen_disabled_num = 0; stats.hash_power_level = stats.hash_bytes = stats.hash_is_expanding = 0; stats.expired_unfetched = stats.evicted_unfetched = 0; @@ -189,6 +190,7 @@ static void stats_reset(void) { STATS_LOCK(); stats.total_items = stats.total_conns = 0; stats.rejected_conns = 0; + stats.malloc_fails = 0; stats.evictions = 0; stats.reclaimed = 0; stats.listen_disabled_num = 0; @@ -242,8 +244,12 @@ static int add_msghdr(conn *c) if (c->msgsize == c->msgused) { msg = realloc(c->msglist, c->msgsize * 2 * sizeof(struct msghdr)); - if (! msg) + if (! msg) { + STATS_LOCK(); + stats.malloc_fails++; + STATS_UNLOCK(); return -1; + } c->msglist = msg; c->msgsize *= 2; } @@ -358,7 +364,10 @@ conn *conn_new(const int sfd, enum conn_states init_state, if (NULL == c) { if (!(c = (conn *)calloc(1, sizeof(conn)))) { - fprintf(stderr, "calloc()\n"); + STATS_LOCK(); + stats.malloc_fails++; + STATS_UNLOCK(); + fprintf(stderr, "Failed to allocate connection object\n"); return NULL; } MEMCACHED_CONN_CREATE(c); @@ -388,7 +397,10 @@ conn *conn_new(const int sfd, enum conn_states init_state, if (c->rbuf == 0 || c->wbuf == 0 || c->ilist == 0 || c->iov == 0 || c->msglist == 0 || c->suffixlist == 0) { conn_free(c); - fprintf(stderr, "malloc()\n"); + STATS_LOCK(); + stats.malloc_fails++; + STATS_UNLOCK(); + fprintf(stderr, "Failed to allocate buffers for connection\n"); return NULL; } @@ -672,8 +684,12 @@ static int ensure_iov_space(conn *c) { int i, iovnum; struct iovec *new_iov = (struct iovec *)realloc(c->iov, (c->iovsize * 2) * sizeof(struct iovec)); - if (! new_iov) + if (! new_iov) { + STATS_LOCK(); + stats.malloc_fails++; + STATS_UNLOCK(); return -1; + } c->iov = new_iov; c->iovsize *= 2; @@ -756,12 +772,18 @@ static int build_udp_headers(conn *c) { if (c->msgused > c->hdrsize) { void *new_hdrbuf; - if (c->hdrbuf) + if (c->hdrbuf) { new_hdrbuf = realloc(c->hdrbuf, c->msgused * 2 * UDP_HEADER_SIZE); - else + } else { new_hdrbuf = malloc(c->msgused * 2 * UDP_HEADER_SIZE); - if (! new_hdrbuf) + } + + if (! new_hdrbuf) { + STATS_LOCK(); + stats.malloc_fails++; + STATS_UNLOCK(); return -1; + } c->hdrbuf = (unsigned char *)new_hdrbuf; c->hdrsize = c->msgused * 2; } @@ -1438,6 +1460,9 @@ static bool grow_stats_buf(conn *c, size_t needed) { c->stats.buffer = ptr; c->stats.size = nsize; } else { + STATS_LOCK(); + stats.malloc_fails++; + STATS_UNLOCK(); rv = false; } } @@ -1561,6 +1586,9 @@ static void bin_read_key(conn *c, enum bin_substates next_substate, int extra) { } char *newm = realloc(c->rbuf, nsize); if (newm == NULL) { + STATS_LOCK(); + stats.malloc_fails++; + STATS_UNLOCK(); if (settings.verbose) { fprintf(stderr, "%d: Failed to grow buffer.. closing connection\n", c->sfd); @@ -2591,6 +2619,8 @@ static void server_stats(ADD_STAT add_stats, conn *c) { APPEND_STAT("slab_reassign_running", "%u", stats.slab_reassign_running); APPEND_STAT("slabs_moved", "%llu", stats.slabs_moved); } + APPEND_STAT("malloc_fails", "%llu", + (unsigned long long)stats.malloc_fails); STATS_UNLOCK(); } @@ -2736,6 +2766,9 @@ static inline void process_get_command(conn *c, token_t *tokens, size_t ntokens, c->isize *= 2; c->ilist = new_list; } else { + STATS_LOCK(); + stats.malloc_fails++; + STATS_UNLOCK(); item_remove(it); break; } @@ -2761,6 +2794,9 @@ static inline void process_get_command(conn *c, token_t *tokens, size_t ntokens, c->suffixsize *= 2; c->suffixlist = new_suffix_list; } else { + STATS_LOCK(); + stats.malloc_fails++; + STATS_UNLOCK(); item_remove(it); break; } @@ -2768,9 +2804,12 @@ static inline void process_get_command(conn *c, token_t *tokens, size_t ntokens, suffix = cache_alloc(c->thread->suffix_cache); if (suffix == NULL) { - out_string(c, "SERVER_ERROR out of memory making CAS suffix"); - item_remove(it); - return; + STATS_LOCK(); + stats.malloc_fails++; + STATS_UNLOCK(); + out_string(c, "SERVER_ERROR out of memory making CAS suffix"); + item_remove(it); + return; } *(c->suffixlist + i) = suffix; int suffix_len = snprintf(suffix, SUFFIX_SIZE, @@ -3593,8 +3632,12 @@ static enum try_read_result try_read_network(conn *c) { ++num_allocs; char *new_rbuf = realloc(c->rbuf, c->rsize * 2); if (!new_rbuf) { - if (settings.verbose > 0) + STATS_LOCK(); + stats.malloc_fails++; + STATS_UNLOCK(); + if (settings.verbose > 0) { fprintf(stderr, "Couldn't realloc input buffer\n"); + } c->rbytes = 0; /* ignore what we read */ out_string(c, "SERVER_ERROR out of memory reading request"); c->write_and_go = conn_closing; diff --git a/memcached.h b/memcached.h index 2bb9100..8c335c1 100644 --- a/memcached.h +++ b/memcached.h @@ -250,6 +250,7 @@ struct stats { unsigned int curr_conns; unsigned int total_conns; uint64_t rejected_conns; + uint64_t malloc_fails; unsigned int reserved_fds; unsigned int conn_structs; uint64_t get_cmds; diff --git a/thread.c b/thread.c index 2295752..80cbac3 100644 --- a/thread.c +++ b/thread.c @@ -260,8 +260,12 @@ static CQ_ITEM *cqi_new(void) { /* Allocate a bunch of items at once to reduce fragmentation */ item = malloc(sizeof(CQ_ITEM) * ITEMS_PER_ALLOC); - if (NULL == item) + if (NULL == item) { + STATS_LOCK(); + stats.malloc_fails++; + STATS_UNLOCK(); return NULL; + } /* * Link together all the new items except the first one -- cgit v1.2.1