From 2379e5db6207eac93e7d68435ddc6cbb0fbe1618 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Tue, 6 Mar 2018 08:24:20 +0200 Subject: memory reporting of clients argv track and report memory used by clients argv. this is very usaful in case clients started sending a command and didn't complete it. in which case the first args of the command are already trimmed from the query buffer. --- src/aof.c | 2 ++ src/networking.c | 43 ++++++++++++++++++++++++++++++++++++++----- src/object.c | 6 ++++++ src/server.h | 1 + tests/unit/introspection.tcl | 2 +- 5 files changed, 48 insertions(+), 6 deletions(-) diff --git a/src/aof.c b/src/aof.c index f8f26bdfe..bd8ccd64d 100644 --- a/src/aof.c +++ b/src/aof.c @@ -634,6 +634,7 @@ struct client *createFakeClient(void) { c->querybuf_peak = 0; c->argc = 0; c->argv = NULL; + c->argv_bytes = 0; c->bufpos = 0; c->flags = 0; c->btype = BLOCKED_NONE; @@ -657,6 +658,7 @@ void freeFakeClientArgv(struct client *c) { for (j = 0; j < c->argc; j++) decrRefCount(c->argv[j]); zfree(c->argv); + c->argv_bytes = 0; } void freeFakeClient(struct client *c) { diff --git a/src/networking.c b/src/networking.c index 06715ee41..3a46e308b 100644 --- a/src/networking.c +++ b/src/networking.c @@ -44,7 +44,7 @@ size_t sdsZmallocSize(sds s) { } /* Return the amount of memory used by the sds string at object->ptr - * for a string object. */ + * for a string object. This includes internal fragmentation. */ size_t getStringObjectSdsUsedMemory(robj *o) { serverAssertWithInfo(NULL,o,o->type == OBJ_STRING); switch(o->encoding) { @@ -54,6 +54,17 @@ size_t getStringObjectSdsUsedMemory(robj *o) { } } +/* Return approximate memory used by the sds string at object->ptr + * for a string object. This method does not include internal fragmentation. */ +size_t getStringObjectSize(robj *o) { + serverAssertWithInfo(NULL,o,o->type == OBJ_STRING); + switch(o->encoding) { + case OBJ_ENCODING_RAW: return sdsAllocSize(o->ptr); + case OBJ_ENCODING_EMBSTR: return sdsAllocSize(o->ptr); + default: return 0; /* Just integer encoding for now. */ + } +} + /* Client.reply list dup and free methods. */ void *dupClientReplyValue(void *o) { clientReplyBlock *old = o; @@ -117,6 +128,7 @@ client *createClient(int fd) { c->reqtype = 0; c->argc = 0; c->argv = NULL; + c->argv_bytes = 0; c->cmd = c->lastcmd = NULL; c->multibulklen = 0; c->bulklen = -1; @@ -757,6 +769,7 @@ static void freeClientArgv(client *c) { decrRefCount(c->argv[j]); c->argc = 0; c->cmd = NULL; + c->argv_bytes = 0; } /* Close all the slaves connections. This is useful in chained replication @@ -903,6 +916,7 @@ void freeClient(client *c) { * and finally release the client structure itself. */ if (c->name) decrRefCount(c->name); zfree(c->argv); + c->argv_bytes = 0; freeClientMultiState(c); sdsfree(c->peerid); zfree(c); @@ -1158,12 +1172,14 @@ int processInlineBuffer(client *c) { if (argc) { if (c->argv) zfree(c->argv); c->argv = zmalloc(sizeof(robj*)*argc); + c->argv_bytes = 0; } /* Create redis objects for all arguments. */ for (c->argc = 0, j = 0; j < argc; j++) { if (sdslen(argv[j])) { c->argv[c->argc] = createObject(OBJ_STRING,argv[j]); + c->argv_bytes += getStringObjectSize(c->argv[c->argc]); c->argc++; } else { sdsfree(argv[j]); @@ -1256,6 +1272,7 @@ int processMultibulkBuffer(client *c) { /* Setup argv array on client structure */ if (c->argv) zfree(c->argv); c->argv = zmalloc(sizeof(robj*)*c->multibulklen); + c->argv_bytes = 0; } serverAssertWithInfo(c,NULL,c->multibulklen > 0); @@ -1326,15 +1343,19 @@ int processMultibulkBuffer(client *c) { c->bulklen >= PROTO_MBULK_BIG_ARG && sdslen(c->querybuf) == (size_t)(c->bulklen+2)) { - c->argv[c->argc++] = createObject(OBJ_STRING,c->querybuf); + c->argv[c->argc] = createObject(OBJ_STRING,c->querybuf); + c->argv_bytes += getStringObjectSize(c->argv[c->argc]); + c->argc++; sdsIncrLen(c->querybuf,-2); /* remove CRLF */ /* Assume that if we saw a fat argument we'll see another one * likely... */ c->querybuf = sdsnewlen(SDS_NOINIT,c->bulklen+2); sdsclear(c->querybuf); } else { - c->argv[c->argc++] = + c->argv[c->argc] = createStringObject(c->querybuf+c->qb_pos,c->bulklen); + c->argv_bytes += getStringObjectSize(c->argv[c->argc]); + c->argc++; c->qb_pos += c->bulklen+2; } c->bulklen = -1; @@ -1605,7 +1626,7 @@ sds catClientInfoString(sds s, client *client) { if (emask & AE_WRITABLE) *p++ = 'w'; *p = '\0'; return sdscatfmt(s, - "id=%U addr=%s fd=%i name=%s age=%I idle=%I flags=%s db=%i sub=%i psub=%i multi=%i qbuf=%U qbuf-free=%U obl=%U oll=%U omem=%U events=%s cmd=%s", + "id=%U addr=%s fd=%i name=%s age=%I idle=%I flags=%s db=%i sub=%i psub=%i multi=%i qbuf=%U qbuf-free=%U argv=%U obl=%U oll=%U omem=%U events=%s cmd=%s", (unsigned long long) client->id, getClientPeerId(client), client->fd, @@ -1619,9 +1640,10 @@ sds catClientInfoString(sds s, client *client) { (client->flags & CLIENT_MULTI) ? client->mstate.count : -1, (unsigned long long) sdslen(client->querybuf), (unsigned long long) sdsavail(client->querybuf), + (unsigned long long) zmalloc_size(client->argv) + client->argv_bytes, (unsigned long long) client->bufpos, (unsigned long long) listLength(client->reply), - (unsigned long long) getClientOutputBufferMemoryUsage(client), + (unsigned long long) getClientOutputBufferMemoryUsage(client) + sizeof(client->buf), events, client->lastcmd ? client->lastcmd->name : "NULL"); } @@ -1907,6 +1929,10 @@ void rewriteClientCommandVector(client *c, int argc, ...) { /* Replace argv and argc with our new versions. */ c->argv = argv; c->argc = argc; + c->argv_bytes = 0; + for (j = 0; j < c->argc; j++) + if (c->argv[j]) + c->argv_bytes += getStringObjectSize(c->argv[j]); c->cmd = lookupCommandOrOriginal(c->argv[0]->ptr); serverAssertWithInfo(c,NULL,c->cmd != NULL); va_end(ap); @@ -1914,10 +1940,15 @@ void rewriteClientCommandVector(client *c, int argc, ...) { /* Completely replace the client command vector with the provided one. */ void replaceClientCommandVector(client *c, int argc, robj **argv) { + int j; freeClientArgv(c); zfree(c->argv); c->argv = argv; c->argc = argc; + c->argv_bytes = 0; + for (j = 0; j < c->argc; j++) + if (c->argv[j]) + c->argv_bytes += getStringObjectSize(c->argv[j]); c->cmd = lookupCommandOrOriginal(c->argv[0]->ptr); serverAssertWithInfo(c,NULL,c->cmd != NULL); } @@ -1942,6 +1973,8 @@ void rewriteClientCommandArgument(client *c, int i, robj *newval) { c->argv[i] = NULL; } oldval = c->argv[i]; + if (oldval) c->argv_bytes -= getStringObjectSize(oldval); + if (newval) c->argv_bytes += getStringObjectSize(newval); c->argv[i] = newval; incrRefCount(newval); if (oldval) decrRefCount(oldval); diff --git a/src/object.c b/src/object.c index d8a56dd26..61674329f 100644 --- a/src/object.c +++ b/src/object.c @@ -986,6 +986,9 @@ struct redisMemOverhead *getMemoryOverheadData(void) { mem += getClientOutputBufferMemoryUsage(c); mem += sdsAllocSize(c->querybuf); mem += sizeof(client); + mem += c->argv_bytes; + if (c->argv) + mem += zmalloc_size(c->argv); } } mh->clients_slaves = mem; @@ -1004,6 +1007,9 @@ struct redisMemOverhead *getMemoryOverheadData(void) { mem += getClientOutputBufferMemoryUsage(c); mem += sdsAllocSize(c->querybuf); mem += sizeof(client); + mem += c->argv_bytes; + if (c->argv) + mem += zmalloc_size(c->argv); } } mh->clients_normal = mem; diff --git a/src/server.h b/src/server.h index 4c4c0ce55..51547b912 100644 --- a/src/server.h +++ b/src/server.h @@ -719,6 +719,7 @@ typedef struct client { size_t querybuf_peak; /* Recent (100ms or more) peak of querybuf size. */ int argc; /* Num of arguments of current command. */ robj **argv; /* Arguments of current command. */ + unsigned long long argv_bytes; /* Tot bytes of objects in argv list. */ struct redisCommand *cmd, *lastcmd; /* Last command executed. */ int reqtype; /* Request protocol type: PROTO_REQ_* */ int multibulklen; /* Number of multi bulk arguments left to read. */ diff --git a/tests/unit/introspection.tcl b/tests/unit/introspection.tcl index 2581eb83d..68dfede58 100644 --- a/tests/unit/introspection.tcl +++ b/tests/unit/introspection.tcl @@ -1,7 +1,7 @@ start_server {tags {"introspection"}} { test {CLIENT LIST} { r client list - } {*addr=*:* fd=* age=* idle=* flags=N db=9 sub=0 psub=0 multi=-1 qbuf=26 qbuf-free=* obl=0 oll=0 omem=0 events=r cmd=client*} + } {*addr=*:* fd=* age=* idle=* flags=N db=9 sub=0 psub=0 multi=-1 qbuf=26 qbuf-free=* argv=* obl=0 oll=0 omem=0 events=r cmd=client*} test {MONITOR can log executed commands} { set rd [redis_deferring_client] -- cgit v1.2.1