From 919e8c3021259a75161d20d68b695e153766e693 Mon Sep 17 00:00:00 2001 From: Matt Stancliff Date: Tue, 16 Dec 2014 00:49:14 -0500 Subject: Config: Add quicklist, remove old list options This removes: - list-max-ziplist-entries - list-max-ziplist-value This adds: - list-max-ziplist-size - list-compress-depth Also updates config file with new sections and updates tests to use quicklist settings instead of old list settings. --- redis.conf | 35 ++++++++++++++++++++++++++++++----- src/config.c | 28 ++++++++++++++++------------ src/rdb.c | 8 +++++--- src/redis.c | 4 ++-- src/redis.h | 12 ++++++++---- src/t_list.c | 14 +++++++------- tests/unit/sort.tcl | 3 +-- tests/unit/type/list-2.tcl | 3 +-- tests/unit/type/list-3.tcl | 3 +-- tests/unit/type/list.tcl | 3 +-- 10 files changed, 72 insertions(+), 41 deletions(-) diff --git a/redis.conf b/redis.conf index 7bb94fbe9..781e726bc 100644 --- a/redis.conf +++ b/redis.conf @@ -818,11 +818,36 @@ notify-keyspace-events "" hash-max-ziplist-entries 512 hash-max-ziplist-value 64 -# Similarly to hashes, small lists are also encoded in a special way in order -# to save a lot of space. The special representation is only used when -# you are under the following limits: -list-max-ziplist-entries 512 -list-max-ziplist-value 64 +# Lists are also encoded in a special way to save a lot of space. +# The number of entries allowed per internal list node can be specified +# as a fixed maximum size or a maximum number of elements. +# For a fixed maximum size, use -5 through -1, meaning: +# -5: max size: 64 Kb <-- not recommended for normal workloads +# -4: max size: 32 Kb <-- not recommended +# -3: max size: 16 Kb <-- probably not recommended +# -2: max size: 8 Kb <-- good +# -1: max size: 4 Kb <-- good +# Positive numbers mean store up to _exactly_ that number of elements +# per list node. +# The highest performing option is usually -2 (8 Kb size) or -1 (4 Kb size), +# but if your use case is unique, adjust the settings as necessary. +list-max-ziplist-size -2 + +# Lists may also be compressed. +# Compress depth is the number of quicklist ziplist nodes from *each* side of +# the list to *exclude* from compression. The head and tail of the list +# are always uncompressed for fast push/pop operations. Settings are: +# 0: disable all list compression +# 1: depth 1 means "don't start compressing until after 1 node into the list, +# going from either the head or tail" +# So: [head]->node->node->...->node->[tail] +# [head], [tail] will always be uncompressed; inner nodes will compress. +# 2: [head]->[next]->node->node->...->node->[prev]->[tail] +# 2 here means: don't compress head or head->next or tail->prev or tail, +# but compress all nodes between them. +# 3: [head]->[next]->[next]->node->node->...->node->[prev]->[prev]->[tail] +# etc. +list-compress-depth 0 # Sets have a special encoding in just one case: when a set is composed # of just strings that happen to be integers in radix 10 in the range diff --git a/src/config.c b/src/config.c index 2140b769f..b0245954c 100644 --- a/src/config.c +++ b/src/config.c @@ -397,9 +397,13 @@ void loadServerConfigFromString(char *config) { } else if (!strcasecmp(argv[0],"hash-max-ziplist-value") && argc == 2) { server.hash_max_ziplist_value = memtoll(argv[1], NULL); } else if (!strcasecmp(argv[0],"list-max-ziplist-entries") && argc == 2){ - server.list_max_ziplist_entries = memtoll(argv[1], NULL); + /* DEAD OPTION */ } else if (!strcasecmp(argv[0],"list-max-ziplist-value") && argc == 2) { - server.list_max_ziplist_value = memtoll(argv[1], NULL); + /* DEAD OPTION */ + } else if (!strcasecmp(argv[0],"list-max-ziplist-size") && argc == 2) { + server.list_max_ziplist_size = atoi(argv[1]); + } else if (!strcasecmp(argv[0],"list-compress-depth") && argc == 2) { + server.list_compress_depth = atoi(argv[1]); } else if (!strcasecmp(argv[0],"set-max-intset-entries") && argc == 2) { server.set_max_intset_entries = memtoll(argv[1], NULL); } else if (!strcasecmp(argv[0],"zset-max-ziplist-entries") && argc == 2) { @@ -795,12 +799,12 @@ void configSetCommand(redisClient *c) { } else if (!strcasecmp(c->argv[2]->ptr,"hash-max-ziplist-value")) { if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll < 0) goto badfmt; server.hash_max_ziplist_value = ll; - } else if (!strcasecmp(c->argv[2]->ptr,"list-max-ziplist-entries")) { + } else if (!strcasecmp(c->argv[2]->ptr,"list-max-ziplist-size")) { if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll < 0) goto badfmt; - server.list_max_ziplist_entries = ll; - } else if (!strcasecmp(c->argv[2]->ptr,"list-max-ziplist-value")) { + server.list_max_ziplist_size = ll; + } else if (!strcasecmp(c->argv[2]->ptr,"list-compress-depth")) { if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll < 0) goto badfmt; - server.list_max_ziplist_value = ll; + server.list_compress_depth = ll; } else if (!strcasecmp(c->argv[2]->ptr,"set-max-intset-entries")) { if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll < 0) goto badfmt; server.set_max_intset_entries = ll; @@ -1047,10 +1051,10 @@ void configGetCommand(redisClient *c) { server.hash_max_ziplist_entries); config_get_numerical_field("hash-max-ziplist-value", server.hash_max_ziplist_value); - config_get_numerical_field("list-max-ziplist-entries", - server.list_max_ziplist_entries); - config_get_numerical_field("list-max-ziplist-value", - server.list_max_ziplist_value); + config_get_numerical_field("list-max-ziplist-size", + server.list_max_ziplist_size); + config_get_numerical_field("list-compress-depth", + server.list_compress_depth); config_get_numerical_field("set-max-intset-entries", server.set_max_intset_entries); config_get_numerical_field("zset-max-ziplist-entries", @@ -1857,8 +1861,8 @@ int rewriteConfig(char *path) { rewriteConfigNotifykeyspaceeventsOption(state); rewriteConfigNumericalOption(state,"hash-max-ziplist-entries",server.hash_max_ziplist_entries,REDIS_HASH_MAX_ZIPLIST_ENTRIES); rewriteConfigNumericalOption(state,"hash-max-ziplist-value",server.hash_max_ziplist_value,REDIS_HASH_MAX_ZIPLIST_VALUE); - rewriteConfigNumericalOption(state,"list-max-ziplist-entries",server.list_max_ziplist_entries,REDIS_LIST_MAX_ZIPLIST_ENTRIES); - rewriteConfigNumericalOption(state,"list-max-ziplist-value",server.list_max_ziplist_value,REDIS_LIST_MAX_ZIPLIST_VALUE); + rewriteConfigNumericalOption(state,"list-max-ziplist-size",server.list_max_ziplist_size,REDIS_LIST_MAX_ZIPLIST_SIZE); + rewriteConfigNumericalOption(state,"list-compress-depth",server.list_compress_depth,REDIS_LIST_COMPRESS_DEPTH); rewriteConfigNumericalOption(state,"set-max-intset-entries",server.set_max_intset_entries,REDIS_SET_MAX_INTSET_ENTRIES); rewriteConfigNumericalOption(state,"zset-max-ziplist-entries",server.zset_max_ziplist_entries,REDIS_ZSET_MAX_ZIPLIST_ENTRIES); rewriteConfigNumericalOption(state,"zset-max-ziplist-value",server.zset_max_ziplist_value,REDIS_ZSET_MAX_ZIPLIST_VALUE); diff --git a/src/rdb.c b/src/rdb.c index 209a0da11..53b47e4f0 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -835,15 +835,15 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) { if ((len = rdbLoadLen(rdb,NULL)) == REDIS_RDB_LENERR) return NULL; o = createQuicklistObject(); - quicklistSetFill(o->ptr, server.list_max_ziplist_entries); - quicklistSetCompress(o->ptr, 0 /*FIXME*/); + quicklistSetOptions(o->ptr, server.list_max_ziplist_size, + server.list_compress_depth); /* Load every single element of the list */ while(len--) { if ((ele = rdbLoadEncodedStringObject(rdb)) == NULL) return NULL; dec = getDecodedObject(ele); size_t len = sdslen(dec->ptr); - o->ptr = quicklistPushTail(o->ptr, dec->ptr, len); + quicklistPushTail(o->ptr, dec->ptr, len); decrRefCount(dec); decrRefCount(ele); } @@ -985,6 +985,8 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) { } else if (rdbtype == REDIS_RDB_TYPE_LIST_QUICKLIST) { if ((len = rdbLoadLen(rdb,NULL)) == REDIS_RDB_LENERR) return NULL; o = createQuicklistObject(); + quicklistSetOptions(o->ptr, server.list_max_ziplist_size, + server.list_compress_depth); while (len--) { if ((ele = rdbLoadStringObject(rdb)) == NULL) return NULL; diff --git a/src/redis.c b/src/redis.c index ee88eddb5..a4d9e562c 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1449,8 +1449,8 @@ void initServerConfig(void) { server.maxmemory_samples = REDIS_DEFAULT_MAXMEMORY_SAMPLES; server.hash_max_ziplist_entries = REDIS_HASH_MAX_ZIPLIST_ENTRIES; server.hash_max_ziplist_value = REDIS_HASH_MAX_ZIPLIST_VALUE; - server.list_max_ziplist_entries = REDIS_LIST_MAX_ZIPLIST_ENTRIES; - server.list_max_ziplist_value = REDIS_LIST_MAX_ZIPLIST_VALUE; + server.list_max_ziplist_size = REDIS_LIST_MAX_ZIPLIST_SIZE; + server.list_compress_depth = REDIS_LIST_COMPRESS_DEPTH; server.set_max_intset_entries = REDIS_SET_MAX_INTSET_ENTRIES; server.zset_max_ziplist_entries = REDIS_ZSET_MAX_ZIPLIST_ENTRIES; server.zset_max_ziplist_value = REDIS_ZSET_MAX_ZIPLIST_VALUE; diff --git a/src/redis.h b/src/redis.h index 38c072478..ab02275fb 100644 --- a/src/redis.h +++ b/src/redis.h @@ -331,12 +331,14 @@ typedef long long mstime_t; /* millisecond time type. */ /* Zip structure related defaults */ #define REDIS_HASH_MAX_ZIPLIST_ENTRIES 512 #define REDIS_HASH_MAX_ZIPLIST_VALUE 64 -#define REDIS_LIST_MAX_ZIPLIST_ENTRIES 512 -#define REDIS_LIST_MAX_ZIPLIST_VALUE 64 #define REDIS_SET_MAX_INTSET_ENTRIES 512 #define REDIS_ZSET_MAX_ZIPLIST_ENTRIES 128 #define REDIS_ZSET_MAX_ZIPLIST_VALUE 64 +/* List defaults */ +#define REDIS_LIST_MAX_ZIPLIST_SIZE -2 +#define REDIS_LIST_COMPRESS_DEPTH 0 + /* HyperLogLog defines */ #define REDIS_DEFAULT_HLL_SPARSE_MAX_BYTES 3000 @@ -871,12 +873,14 @@ struct redisServer { /* Zip structure config, see redis.conf for more information */ size_t hash_max_ziplist_entries; size_t hash_max_ziplist_value; - size_t list_max_ziplist_entries; - size_t list_max_ziplist_value; size_t set_max_intset_entries; size_t zset_max_ziplist_entries; size_t zset_max_ziplist_value; size_t hll_sparse_max_bytes; + /* List parameters */ + int list_max_ziplist_size; + int list_compress_depth; + /* time cache */ time_t unixtime; /* Unix time sampled every cron cycle. */ long long mstime; /* Like 'unixtime' but with milliseconds resolution. */ /* Pubsub */ diff --git a/src/t_list.c b/src/t_list.c index 61fdf2ad8..232cb5c52 100644 --- a/src/t_list.c +++ b/src/t_list.c @@ -181,10 +181,10 @@ void listTypeConvert(robj *subject, int enc) { redisAssertWithInfo(NULL,subject,subject->encoding==REDIS_ENCODING_ZIPLIST); if (enc == REDIS_ENCODING_QUICKLIST) { - size_t zlen = server.list_max_ziplist_entries; - + size_t zlen = server.list_max_ziplist_size; + int depth = server.list_compress_depth; + subject->ptr = quicklistCreateFromZiplist(zlen, depth, subject->ptr); subject->encoding = REDIS_ENCODING_QUICKLIST; - subject->ptr = quicklistCreateFromZiplist(zlen, 0 /*FIXME*/, subject->ptr); } else { redisPanic("Unsupported list conversion"); } @@ -207,8 +207,8 @@ void pushGenericCommand(redisClient *c, int where) { c->argv[j] = tryObjectEncoding(c->argv[j]); if (!lobj) { lobj = createQuicklistObject(); - quicklistSetFill(lobj->ptr, server.list_max_ziplist_entries); - quicklistSetCompress(lobj->ptr, 0 /*FIXME*/); + quicklistSetOptions(lobj->ptr, server.list_max_ziplist_size, + server.list_compress_depth); dbAdd(c->db,c->argv[1],lobj); } listTypePush(lobj,c->argv[j],where); @@ -537,8 +537,8 @@ void rpoplpushHandlePush(redisClient *c, robj *dstkey, robj *dstobj, robj *value /* Create the list if the key does not exist */ if (!dstobj) { dstobj = createQuicklistObject(); - quicklistSetFill(dstobj->ptr, server.list_max_ziplist_entries); - quicklistSetCompress(dstobj->ptr, 0 /*FIXME*/); + quicklistSetOptions(dstobj->ptr, server.list_max_ziplist_size, + server.list_compress_depth); dbAdd(c->db,dstkey,dstobj); } signalModifiedKey(c->db,dstkey); diff --git a/tests/unit/sort.tcl b/tests/unit/sort.tcl index 5ae48d450..083c4540d 100644 --- a/tests/unit/sort.tcl +++ b/tests/unit/sort.tcl @@ -1,8 +1,7 @@ start_server { tags {"sort"} overrides { - "list-max-ziplist-value" 16 - "list-max-ziplist-entries" 32 + "list-max-ziplist-size" 32 "set-max-intset-entries" 32 } } { diff --git a/tests/unit/type/list-2.tcl b/tests/unit/type/list-2.tcl index d25873912..4c7d6d91c 100644 --- a/tests/unit/type/list-2.tcl +++ b/tests/unit/type/list-2.tcl @@ -1,8 +1,7 @@ start_server { tags {"list"} overrides { - "list-max-ziplist-value" 16 - "list-max-ziplist-entries" 4 + "list-max-ziplist-size" 4 } } { source "tests/unit/type/list-common.tcl" diff --git a/tests/unit/type/list-3.tcl b/tests/unit/type/list-3.tcl index 81ef9b3e5..ece6ea2d5 100644 --- a/tests/unit/type/list-3.tcl +++ b/tests/unit/type/list-3.tcl @@ -1,8 +1,7 @@ start_server { tags {list ziplist} overrides { - "list-max-ziplist-value" 200000 - "list-max-ziplist-entries" 16 + "list-max-ziplist-size" 16 } } { test {Explicit regression for a list bug} { diff --git a/tests/unit/type/list.tcl b/tests/unit/type/list.tcl index 0c0c07761..e4d568cf1 100644 --- a/tests/unit/type/list.tcl +++ b/tests/unit/type/list.tcl @@ -1,8 +1,7 @@ start_server { tags {"list"} overrides { - "list-max-ziplist-value" 16 - "list-max-ziplist-entries" 5 + "list-max-ziplist-size" 5 } } { source "tests/unit/type/list-common.tcl" -- cgit v1.2.1