diff options
Diffstat (limited to 'src')
65 files changed, 4250 insertions, 1492 deletions
@@ -1531,18 +1531,15 @@ static int ACLSelectorCheckKey(aclSelector *selector, const char *key, int keyle return ACL_DENIED_KEY; } -/* Returns if a given command may possibly access channels. For this context, - * the unsubscribe commands do not have channels. */ -static int ACLDoesCommandHaveChannels(struct redisCommand *cmd) { - return (cmd->proc == publishCommand - || cmd->proc == subscribeCommand - || cmd->proc == psubscribeCommand - || cmd->proc == spublishCommand - || cmd->proc == ssubscribeCommand); -} - -/* Checks a channel against a provide list of channels. */ -static int ACLCheckChannelAgainstList(list *reference, const char *channel, int channellen, int literal) { +/* Checks a channel against a provided list of channels. The is_pattern + * argument should only be used when subscribing (not when publishing) + * and controls whether the input channel is evaluated as a channel pattern + * (like in PSUBSCRIBE) or a plain channel name (like in SUBSCRIBE). + * + * Note that a plain channel name like in PUBLISH or SUBSCRIBE can be + * matched against ACL channel patterns, but the pattern provided in PSUBSCRIBE + * can only be matched as a literal against an ACL pattern (using plain string compare). */ +static int ACLCheckChannelAgainstList(list *reference, const char *channel, int channellen, int is_pattern) { listIter li; listNode *ln; @@ -1550,8 +1547,10 @@ static int ACLCheckChannelAgainstList(list *reference, const char *channel, int while((ln = listNext(&li))) { sds pattern = listNodeValue(ln); size_t plen = sdslen(pattern); - if ((literal && !strcmp(pattern,channel)) || - (!literal && stringmatchlen(pattern,plen,channel,channellen,0))) + /* Channel patterns are matched literally against the channels in + * the list. Regular channels perform pattern matching. */ + if ((is_pattern && !strcmp(pattern,channel)) || + (!is_pattern && stringmatchlen(pattern,plen,channel,channellen,0))) { return ACL_OK; } @@ -1559,28 +1558,6 @@ static int ACLCheckChannelAgainstList(list *reference, const char *channel, int return ACL_DENIED_CHANNEL; } -/* Check if the pub/sub channels of the command can be executed - * according to the ACL channels associated with the specified selector. - * - * idx and count are the index and count of channel arguments from the - * command. The literal argument controls whether the selector's ACL channels are - * evaluated as literal values or matched as glob-like patterns. - * - * If the selector can execute the command ACL_OK is returned, otherwise - * ACL_DENIED_CHANNEL. */ -static int ACLSelectorCheckPubsubArguments(aclSelector *s, robj **argv, int idx, int count, int literal, int *idxptr) { - for (int j = idx; j < idx+count; j++) { - if (ACLCheckChannelAgainstList(s->channels, argv[j]->ptr, sdslen(argv[j]->ptr), literal != ACL_OK)) { - if (idxptr) *idxptr = j; - return ACL_DENIED_CHANNEL; - } - } - - /* If we survived all the above checks, the selector can execute the - * command. */ - return ACL_OK; -} - /* To prevent duplicate calls to getKeysResult, a cache is maintained * in between calls to the various selectors. */ typedef struct { @@ -1645,7 +1622,7 @@ static int ACLSelectorCheckCmd(aclSelector *selector, struct redisCommand *cmd, int idx = resultidx[j].pos; ret = ACLSelectorCheckKey(selector, argv[idx]->ptr, sdslen(argv[idx]->ptr), resultidx[j].flags); if (ret != ACL_OK) { - if (resultidx) *keyidxptr = resultidx[j].pos; + if (keyidxptr) *keyidxptr = resultidx[j].pos; return ret; } } @@ -1653,26 +1630,30 @@ static int ACLSelectorCheckCmd(aclSelector *selector, struct redisCommand *cmd, /* Check if the user can execute commands explicitly touching the channels * mentioned in the command arguments */ - if (!(selector->flags & SELECTOR_FLAG_ALLCHANNELS) && ACLDoesCommandHaveChannels(cmd)) { - if (cmd->proc == publishCommand || cmd->proc == spublishCommand) { - ret = ACLSelectorCheckPubsubArguments(selector,argv, 1, 1, 0, keyidxptr); - } else if (cmd->proc == subscribeCommand || cmd->proc == ssubscribeCommand) { - ret = ACLSelectorCheckPubsubArguments(selector, argv, 1, argc-1, 0, keyidxptr); - } else if (cmd->proc == psubscribeCommand) { - ret = ACLSelectorCheckPubsubArguments(selector, argv, 1, argc-1, 1, keyidxptr); - } else { - serverPanic("Encountered a command declared with channels but not handled"); - } - if (ret != ACL_OK) { - /* keyidxptr is set by ACLSelectorCheckPubsubArguments */ - return ret; + const int channel_flags = CMD_CHANNEL_PUBLISH | CMD_CHANNEL_SUBSCRIBE; + if (!(selector->flags & SELECTOR_FLAG_ALLCHANNELS) && doesCommandHaveChannelsWithFlags(cmd, channel_flags)) { + getKeysResult channels = (getKeysResult) GETKEYS_RESULT_INIT; + getChannelsFromCommand(cmd, argv, argc, &channels); + keyReference *channelref = channels.keys; + for (int j = 0; j < channels.numkeys; j++) { + int idx = channelref[j].pos; + if (!(channelref[j].flags & channel_flags)) continue; + int is_pattern = channelref[j].flags & CMD_CHANNEL_PATTERN; + int ret = ACLCheckChannelAgainstList(selector->channels, argv[idx]->ptr, sdslen(argv[idx]->ptr), is_pattern); + if (ret != ACL_OK) { + if (keyidxptr) *keyidxptr = channelref[j].pos; + getKeysFreeResult(&channels); + return ret; + } } + getKeysFreeResult(&channels); } return ACL_OK; } /* Check if the key can be accessed by the client according to - * the ACLs associated with the specified user. + * the ACLs associated with the specified user according to the + * keyspec access flags. * * If the user can access the key, ACL_OK is returned, otherwise * ACL_DENIED_KEY is returned. */ @@ -1699,7 +1680,7 @@ int ACLUserCheckKeyPerm(user *u, const char *key, int keylen, int flags) { * * If the user can access the key, ACL_OK is returned, otherwise * ACL_DENIED_CHANNEL is returned. */ -int ACLUserCheckChannelPerm(user *u, sds channel, int literal) { +int ACLUserCheckChannelPerm(user *u, sds channel, int is_pattern) { listIter li; listNode *ln; @@ -1714,7 +1695,7 @@ int ACLUserCheckChannelPerm(user *u, sds channel, int literal) { if (s->flags & SELECTOR_FLAG_ALLCHANNELS) return ACL_OK; /* Otherwise, loop over the selectors list and check each channel */ - if (ACLCheckChannelAgainstList(s->channels, channel, sdslen(channel), literal) == ACL_OK) { + if (ACLCheckChannelAgainstList(s->channels, channel, sdslen(channel), is_pattern) == ACL_OK) { return ACL_OK; } } @@ -42,11 +42,13 @@ #include <sys/param.h> void freeClientArgv(client *c); -off_t getAppendOnlyFileSize(sds filename); -off_t getBaseAndIncrAppendOnlyFilesSize(aofManifest *am); +off_t getAppendOnlyFileSize(sds filename, int *status); +off_t getBaseAndIncrAppendOnlyFilesSize(aofManifest *am, int *status); int getBaseAndIncrAppendOnlyFilesNum(aofManifest *am); int aofFileExist(char *filename); int rewriteAppendOnlyFile(char *filename); +aofManifest *aofLoadManifestFromFile(sds am_filepath); +void aofManifestFreeAndUpdate(aofManifest *am); /* ---------------------------------------------------------------------------- * AOF Manifest file implementation. @@ -226,13 +228,8 @@ sds getAofManifestAsString(aofManifest *am) { * in order to support seamless upgrades from previous versions which did not * use them. */ -#define MANIFEST_MAX_LINE 1024 void aofLoadManifestFromDisk(void) { - const char *err = NULL; - long long maxseq = 0; - server.aof_manifest = aofManifestCreate(); - if (!dirExists(server.aof_dirname)) { serverLog(LL_NOTICE, "The AOF directory %s doesn't exist", server.aof_dirname); return; @@ -247,16 +244,26 @@ void aofLoadManifestFromDisk(void) { return; } + aofManifest *am = aofLoadManifestFromFile(am_filepath); + if (am) aofManifestFreeAndUpdate(am); + sdsfree(am_name); + sdsfree(am_filepath); +} + +/* Generic manifest loading function, used in `aofLoadManifestFromDisk` and redis-check-aof tool. */ +#define MANIFEST_MAX_LINE 1024 +aofManifest *aofLoadManifestFromFile(sds am_filepath) { + const char *err = NULL; + long long maxseq = 0; + + aofManifest *am = aofManifestCreate(); FILE *fp = fopen(am_filepath, "r"); if (fp == NULL) { serverLog(LL_WARNING, "Fatal error: can't open the AOF manifest " - "file %s for reading: %s", am_name, strerror(errno)); + "file %s for reading: %s", am_filepath, strerror(errno)); exit(1); } - sdsfree(am_name); - sdsfree(am_filepath); - char buf[MANIFEST_MAX_LINE+1]; sds *argv = NULL; int argc; @@ -292,14 +299,14 @@ void aofLoadManifestFromDisk(void) { line = sdstrim(sdsnew(buf), " \t\r\n"); if (!sdslen(line)) { - err = "The AOF manifest file is invalid format"; + err = "Invalid AOF manifest file format"; goto loaderr; } argv = sdssplitargs(line, &argc); /* 'argc < 6' was done for forward compatibility. */ if (argv == NULL || argc < 6 || (argc % 2)) { - err = "The AOF manifest file is invalid format"; + err = "Invalid AOF manifest file format"; goto loaderr; } @@ -321,7 +328,7 @@ void aofLoadManifestFromDisk(void) { /* We have to make sure we load all the information. */ if (!ai->file_name || !ai->file_seq || !ai->file_type) { - err = "The AOF manifest file is invalid format"; + err = "Invalid AOF manifest file format"; goto loaderr; } @@ -329,21 +336,21 @@ void aofLoadManifestFromDisk(void) { argv = NULL; if (ai->file_type == AOF_FILE_TYPE_BASE) { - if (server.aof_manifest->base_aof_info) { + if (am->base_aof_info) { err = "Found duplicate base file information"; goto loaderr; } - server.aof_manifest->base_aof_info = ai; - server.aof_manifest->curr_base_file_seq = ai->file_seq; + am->base_aof_info = ai; + am->curr_base_file_seq = ai->file_seq; } else if (ai->file_type == AOF_FILE_TYPE_HIST) { - listAddNodeTail(server.aof_manifest->history_aof_list, ai); + listAddNodeTail(am->history_aof_list, ai); } else if (ai->file_type == AOF_FILE_TYPE_INCR) { if (ai->file_seq <= maxseq) { err = "Found a non-monotonic sequence number"; goto loaderr; } - listAddNodeTail(server.aof_manifest->incr_aof_list, ai); - server.aof_manifest->curr_incr_file_seq = ai->file_seq; + listAddNodeTail(am->incr_aof_list, ai); + am->curr_incr_file_seq = ai->file_seq; maxseq = ai->file_seq; } else { err = "Unknown AOF file type"; @@ -356,7 +363,7 @@ void aofLoadManifestFromDisk(void) { } fclose(fp); - return; + return am; loaderr: /* Sanitizer suppression: may report a false positive if we goto loaderr @@ -627,7 +634,7 @@ void aofUpgradePrepare(aofManifest *am) { server.aof_dirname, strerror(errno)); sdsfree(aof_filepath); - exit(1);; + exit(1); } sdsfree(aof_filepath); @@ -721,7 +728,7 @@ void aofOpenIfNeededOnServerStart(void) { exit(1); } - server.aof_last_incr_size = getAppendOnlyFileSize(aof_name); + server.aof_last_incr_size = getAppendOnlyFileSize(aof_name, NULL); } int aofFileExist(char *filename) { @@ -1338,26 +1345,35 @@ int loadSingleAppendOnlyFile(char *filename) { client *old_client = server.current_client; fakeClient = server.current_client = createAOFClient(); - /* Check if this AOF file has an RDB preamble. In that case we need to - * load the RDB file and later continue loading the AOF tail. */ + /* Check if the AOF file is in RDB format (it may be RDB encoded base AOF + * or old style RDB-preamble AOF). In that case we need to load the RDB file + * and later continue loading the AOF tail if it is an old style RDB-preamble AOF. */ char sig[5]; /* "REDIS" */ if (fread(sig,1,5,fp) != 5 || memcmp(sig,"REDIS",5) != 0) { - /* No RDB preamble, seek back at 0 offset. */ + /* Not in RDB format, seek back at 0 offset. */ if (fseek(fp,0,SEEK_SET) == -1) goto readerr; } else { - /* RDB preamble. Pass loading the RDB functions. */ + /* RDB format. Pass loading the RDB functions. */ rio rdb; + int old_style = !strcmp(filename, server.aof_filename); + if (old_style) + serverLog(LL_NOTICE, "Reading RDB preamble from AOF file..."); + else + serverLog(LL_NOTICE, "Reading RDB base file on AOF loading..."); - serverLog(LL_NOTICE,"Reading RDB preamble from AOF file..."); if (fseek(fp,0,SEEK_SET) == -1) goto readerr; rioInitWithFile(&rdb,fp); if (rdbLoadRio(&rdb,RDBFLAGS_AOF_PREAMBLE,NULL) != C_OK) { - serverLog(LL_WARNING,"Error reading the RDB preamble of the AOF file %s, AOF loading aborted", filename); + if (old_style) + serverLog(LL_WARNING, "Error reading the RDB preamble of the AOF file %s, AOF loading aborted", filename); + else + serverLog(LL_WARNING, "Error reading the RDB base file %s, AOF loading aborted", filename); + goto readerr; } else { loadingAbsProgress(ftello(fp)); last_progress_report_size = ftello(fp); - serverLog(LL_NOTICE,"Reading the remaining AOF tail..."); + if (old_style) serverLog(LL_NOTICE, "Reading the remaining AOF tail..."); } } @@ -1517,15 +1533,15 @@ uxeof: /* Unexpected AOF end of file. */ } } } - serverLog(LL_WARNING,"Unexpected end of file reading the append only file %s. You can: \ - 1) Make a backup of your AOF file, then use ./redis-check-aof --fix <filename>. \ - 2) Alternatively you can set the 'aof-load-truncated' configuration option to yes and restart the server.", filename); + serverLog(LL_WARNING, "Unexpected end of file reading the append only file %s. You can: " + "1) Make a backup of your AOF file, then use ./redis-check-aof --fix <filename.manifest>. " + "2) Alternatively you can set the 'aof-load-truncated' configuration option to yes and restart the server.", filename); ret = AOF_FAILED; goto cleanup; fmterr: /* Format error. */ - serverLog(LL_WARNING,"Bad file format reading the append only file %s: \ - make a backup of your AOF file, then use ./redis-check-aof --fix <filename>", filename); + serverLog(LL_WARNING, "Bad file format reading the append only file %s: " + "make a backup of your AOF file, then use ./redis-check-aof --fix <filename.manifest>", filename); ret = AOF_FAILED; /* fall through to cleanup. */ @@ -1540,7 +1556,7 @@ cleanup: /* Load the AOF files according the aofManifest pointed by am. */ int loadAppendOnlyFiles(aofManifest *am) { serverAssert(am != NULL); - int ret = C_OK; + int status, ret = C_OK; long long start; off_t total_size = 0; sds aof_name; @@ -1574,7 +1590,16 @@ int loadAppendOnlyFiles(aofManifest *am) { /* Here we calculate the total size of all BASE and INCR files in * advance, it will be set to `server.loading_total_bytes`. */ - total_size = getBaseAndIncrAppendOnlyFilesSize(am); + total_size = getBaseAndIncrAppendOnlyFilesSize(am, &status); + if (status != AOF_OK) { + /* If an AOF exists in the manifest but not on the disk, we consider this to be a fatal error. */ + if (status == AOF_NOT_EXIST) status = AOF_FAILED; + + return status; + } else if (total_size == 0) { + return AOF_EMPTY; + } + startLoading(total_size, RDBFLAGS_AOF_PREAMBLE, 0); /* Load BASE AOF if needed. */ @@ -1590,9 +1615,8 @@ int loadAppendOnlyFiles(aofManifest *am) { aof_name, (float)(ustime()-start)/1000000); } - /* If an AOF exists in the manifest but not on the disk, Or the truncated - * file is not the last file, we consider this to be a fatal error. */ - if (ret == AOF_NOT_EXIST || (ret == AOF_TRUNCATED && !last_file)) { + /* If the truncated file is not the last file, we consider this to be a fatal error. */ + if (ret == AOF_TRUNCATED && !last_file) { ret = AOF_FAILED; } @@ -1620,7 +1644,11 @@ int loadAppendOnlyFiles(aofManifest *am) { aof_name, (float)(ustime()-start)/1000000); } - if (ret == AOF_NOT_EXIST || (ret == AOF_TRUNCATED && !last_file)) { + /* We know that (at least) one of the AOF files has data (total_size > 0), + * so empty incr AOF file doesn't count as a AOF_EMPTY result */ + if (ret == AOF_EMPTY) ret = AOF_OK; + + if (ret == AOF_TRUNCATED && !last_file) { ret = AOF_FAILED; } @@ -1635,7 +1663,7 @@ int loadAppendOnlyFiles(aofManifest *am) { server.aof_fsync_offset = server.aof_current_size; cleanup: - stopLoading(ret == AOF_OK); + stopLoading(ret == AOF_OK || ret == AOF_TRUNCATED); return ret; } @@ -2007,10 +2035,14 @@ int rewriteStreamObject(rio *r, robj *key, robj *o) { /* Append XSETID after XADD, make sure lastid is correct, * in case of XDEL lastid. */ - if (!rioWriteBulkCount(r,'*',3) || + if (!rioWriteBulkCount(r,'*',7) || !rioWriteBulkString(r,"XSETID",6) || !rioWriteBulkObject(r,key) || - !rioWriteBulkStreamID(r,&s->last_id)) + !rioWriteBulkStreamID(r,&s->last_id) || + !rioWriteBulkString(r,"ENTRIESADDED",12) || + !rioWriteBulkLongLong(r,s->entries_added) || + !rioWriteBulkString(r,"MAXDELETEDID",12) || + !rioWriteBulkStreamID(r,&s->max_deleted_entry_id)) { streamIteratorStop(&si); return 0; @@ -2025,12 +2057,14 @@ int rewriteStreamObject(rio *r, robj *key, robj *o) { while(raxNext(&ri)) { streamCG *group = ri.data; /* Emit the XGROUP CREATE in order to create the group. */ - if (!rioWriteBulkCount(r,'*',5) || + if (!rioWriteBulkCount(r,'*',7) || !rioWriteBulkString(r,"XGROUP",6) || !rioWriteBulkString(r,"CREATE",6) || !rioWriteBulkObject(r,key) || !rioWriteBulkString(r,(char*)ri.key,ri.key_len) || - !rioWriteBulkStreamID(r,&group->last_id)) + !rioWriteBulkStreamID(r,&group->last_id) || + !rioWriteBulkString(r,"ENTRIESREAD",11) || + !rioWriteBulkLongLong(r,group->entries_read)) { raxStop(&ri); streamIteratorStop(&si); @@ -2332,7 +2366,7 @@ int rewriteAppendOnlyFileBackground(void) { server.aof_selected_db = -1; flushAppendOnlyFile(1); if (openNewIncrAofForAppend() != C_OK) return C_ERR; - + server.stat_aof_rewrites++; if ((childpid = redisFork(CHILD_TYPE_AOF)) == 0) { char tmpfile[256]; @@ -2388,7 +2422,10 @@ void aofRemoveTempFile(pid_t childpid) { bg_unlink(tmpfile); } -off_t getAppendOnlyFileSize(sds filename) { +/* Get size of an AOF file. + * The status argument is an optional output argument to be filled with + * one of the AOF_ status values. */ +off_t getAppendOnlyFileSize(sds filename, int *status) { struct redis_stat sb; off_t size; mstime_t latency; @@ -2396,10 +2433,12 @@ off_t getAppendOnlyFileSize(sds filename) { sds aof_filepath = makePath(server.aof_dirname, filename); latencyStartMonitor(latency); if (redis_stat(aof_filepath, &sb) == -1) { + if (status) *status = errno == ENOENT ? AOF_NOT_EXIST : AOF_OPEN_ERR; serverLog(LL_WARNING, "Unable to obtain the AOF file %s length. stat: %s", filename, strerror(errno)); size = 0; } else { + if (status) *status = AOF_OK; size = sb.st_size; } latencyEndMonitor(latency); @@ -2408,22 +2447,27 @@ off_t getAppendOnlyFileSize(sds filename) { return size; } -off_t getBaseAndIncrAppendOnlyFilesSize(aofManifest *am) { +/* Get size of all AOF files referred by the manifest (excluding history). + * The status argument is an output argument to be filled with + * one of the AOF_ status values. */ +off_t getBaseAndIncrAppendOnlyFilesSize(aofManifest *am, int *status) { off_t size = 0; - listNode *ln; listIter li; if (am->base_aof_info) { serverAssert(am->base_aof_info->file_type == AOF_FILE_TYPE_BASE); - size += getAppendOnlyFileSize(am->base_aof_info->file_name); + + size += getAppendOnlyFileSize(am->base_aof_info->file_name, status); + if (*status != AOF_OK) return 0; } listRewind(am->incr_aof_list, &li); while ((ln = listNext(&li)) != NULL) { aofInfo *ai = (aofInfo*)ln->value; serverAssert(ai->file_type == AOF_FILE_TYPE_INCR); - size += getAppendOnlyFileSize(ai->file_name); + size += getAppendOnlyFileSize(ai->file_name, status); + if (*status != AOF_OK) return 0; } return size; @@ -2497,7 +2541,7 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) { if (server.aof_fd != -1) { /* AOF enabled. */ server.aof_selected_db = -1; /* Make sure SELECT is re-issued */ - server.aof_current_size = getAppendOnlyFileSize(new_base_filename) + server.aof_last_incr_size; + server.aof_current_size = getAppendOnlyFileSize(new_base_filename, NULL) + server.aof_last_incr_size; server.aof_rewrite_base_size = server.aof_current_size; server.aof_fsync_offset = server.aof_current_size; server.aof_last_fsync = server.unixtime; diff --git a/src/blocked.c b/src/blocked.c index 0feab4a69..aa298cffb 100644 --- a/src/blocked.c +++ b/src/blocked.c @@ -108,9 +108,11 @@ void blockClient(client *c, int btype) { /* This function is called after a client has finished a blocking operation * in order to update the total command duration, log the command into * the Slow log if needed, and log the reply duration event if needed. */ -void updateStatsOnUnblock(client *c, long blocked_us, long reply_us){ +void updateStatsOnUnblock(client *c, long blocked_us, long reply_us, int had_errors){ const ustime_t total_cmd_duration = c->duration + blocked_us + reply_us; c->lastcmd->microseconds += total_cmd_duration; + if (had_errors) + c->lastcmd->failed_calls++; if (server.latency_tracking_enabled) updateCommandLatencyHistogram(&(c->lastcmd->latency_histogram), total_cmd_duration*1000); /* Log the command into the Slow log if needed. */ @@ -314,6 +316,7 @@ void serveClientsBlockedOnListKey(robj *o, readyList *rl) { * call. */ if (dstkey) incrRefCount(dstkey); + long long prev_error_replies = server.stat_total_error_replies; client *old_client = server.current_client; server.current_client = receiver; monotime replyTimer; @@ -322,7 +325,7 @@ void serveClientsBlockedOnListKey(robj *o, readyList *rl) { rl->key, dstkey, rl->db, wherefrom, whereto, &deleted); - updateStatsOnUnblock(receiver, 0, elapsedUs(replyTimer)); + updateStatsOnUnblock(receiver, 0, elapsedUs(replyTimer), server.stat_total_error_replies != prev_error_replies); unblockClient(receiver); afterCommand(receiver); server.current_client = old_client; @@ -366,6 +369,7 @@ void serveClientsBlockedOnSortedSetKey(robj *o, readyList *rl) { ? 1 : 0; int reply_nil_when_empty = use_nested_array; + long long prev_error_replies = server.stat_total_error_replies; client *old_client = server.current_client; server.current_client = receiver; monotime replyTimer; @@ -388,7 +392,7 @@ void serveClientsBlockedOnSortedSetKey(robj *o, readyList *rl) { decrRefCount(argv[1]); if (count != -1) decrRefCount(argv[2]); - updateStatsOnUnblock(receiver, 0, elapsedUs(replyTimer)); + updateStatsOnUnblock(receiver, 0, elapsedUs(replyTimer), server.stat_total_error_replies != prev_error_replies); unblockClient(receiver); afterCommand(receiver); server.current_client = old_client; @@ -421,6 +425,12 @@ void serveClientsBlockedOnStreamKey(robj *o, readyList *rl) { bkinfo *bki = dictFetchValue(receiver->bpop.keys,rl->key); streamID *gt = &bki->stream_id; + long long prev_error_replies = server.stat_total_error_replies; + client *old_client = server.current_client; + server.current_client = receiver; + monotime replyTimer; + elapsedStart(&replyTimer); + /* If we blocked in the context of a consumer * group, we need to resolve the group and update the * last ID the client is blocked for: this is needed @@ -440,8 +450,7 @@ void serveClientsBlockedOnStreamKey(robj *o, readyList *rl) { addReplyError(receiver, "-NOGROUP the consumer group this client " "was blocked on no longer exists"); - unblockClient(receiver); - continue; + goto unblock_receiver; } else { *gt = group->last_id; } @@ -470,10 +479,6 @@ void serveClientsBlockedOnStreamKey(robj *o, readyList *rl) { } } - client *old_client = server.current_client; - server.current_client = receiver; - monotime replyTimer; - elapsedStart(&replyTimer); /* Emit the two elements sub-array consisting of * the name of the stream and the data we * extracted from it. Wrapped in a single-item @@ -493,11 +498,13 @@ void serveClientsBlockedOnStreamKey(robj *o, readyList *rl) { streamReplyWithRange(receiver,s,&start,NULL, receiver->bpop.xread_count, 0, group, consumer, noack, &pi); - updateStatsOnUnblock(receiver, 0, elapsedUs(replyTimer)); /* Note that after we unblock the client, 'gt' * and other receiver->bpop stuff are no longer * valid, so we must do the setup above before - * this call. */ + * the unblockClient call. */ + +unblock_receiver: + updateStatsOnUnblock(receiver, 0, elapsedUs(replyTimer), server.stat_total_error_replies != prev_error_replies); unblockClient(receiver); afterCommand(receiver); server.current_client = old_client; @@ -545,12 +552,13 @@ void serveClientsBlockedOnKeyByModule(readyList *rl) { * different modules with different triggers to consider if a key * is ready or not. This means we can't exit the loop but need * to continue after the first failure. */ + long long prev_error_replies = server.stat_total_error_replies; client *old_client = server.current_client; server.current_client = receiver; monotime replyTimer; elapsedStart(&replyTimer); if (!moduleTryServeClientBlockedOnKey(receiver, rl->key)) continue; - updateStatsOnUnblock(receiver, 0, elapsedUs(replyTimer)); + updateStatsOnUnblock(receiver, 0, elapsedUs(replyTimer), server.stat_total_error_replies != prev_error_replies); moduleUnblockClient(receiver); afterCommand(receiver); server.current_client = old_client; diff --git a/src/call_reply.c b/src/call_reply.c index 7aa79d089..3694db55e 100644 --- a/src/call_reply.c +++ b/src/call_reply.c @@ -60,7 +60,7 @@ struct CallReply { double d; /* Reply value for double reply. */ struct CallReply *array; /* Array of sub-reply elements. used for set, array, map, and attribute */ } val; - + list *deferred_error_list; /* list of errors in sds form or NULL */ struct CallReply *attribute; /* attribute reply, NULL if not exists */ }; @@ -237,6 +237,8 @@ void freeCallReply(CallReply *rep) { freeCallReplyInternal(rep); } sdsfree(rep->original_proto); + if (rep->deferred_error_list) + listRelease(rep->deferred_error_list); zfree(rep); } @@ -488,6 +490,11 @@ int callReplyIsResp3(CallReply *rep) { return rep->flags & REPLY_FLAG_RESP3; } +/* Returns a list of errors in sds form, or NULL. */ +list *callReplyDeferredErrorList(CallReply *rep) { + return rep->deferred_error_list; +} + /* Create a new CallReply struct from the reply blob. * * The function will own the reply blob, so it must not be used or freed by @@ -496,6 +503,9 @@ int callReplyIsResp3(CallReply *rep) { * The reply blob will be freed when the returned CallReply struct is later * freed using freeCallReply(). * + * The deferred_error_list is an optional list of errors that are present + * in the reply blob, if given, this function will take ownership on it. + * * The private_data is optional and can later be accessed using * callReplyGetPrivateData(). * @@ -504,7 +514,7 @@ int callReplyIsResp3(CallReply *rep) { * DESIGNED TO HANDLE USER INPUT and using it to parse invalid replies is * unsafe. */ -CallReply *callReplyCreate(sds reply, void *private_data) { +CallReply *callReplyCreate(sds reply, list *deferred_error_list, void *private_data) { CallReply *res = zmalloc(sizeof(*res)); res->flags = REPLY_FLAG_ROOT; res->original_proto = reply; @@ -512,5 +522,6 @@ CallReply *callReplyCreate(sds reply, void *private_data) { res->proto_len = sdslen(reply); res->private_data = private_data; res->attribute = NULL; + res->deferred_error_list = deferred_error_list; return res; } diff --git a/src/call_reply.h b/src/call_reply.h index 5b07dc437..ff98f7f5a 100644 --- a/src/call_reply.h +++ b/src/call_reply.h @@ -34,7 +34,7 @@ typedef struct CallReply CallReply; -CallReply *callReplyCreate(sds reply, void *private_data); +CallReply *callReplyCreate(sds reply, list *deferred_error_list, void *private_data); int callReplyType(CallReply *rep); const char *callReplyGetString(CallReply *rep, size_t *len); long long callReplyGetLongLong(CallReply *rep); @@ -51,6 +51,7 @@ const char *callReplyGetVerbatim(CallReply *rep, size_t *len, const char **forma const char *callReplyGetProto(CallReply *rep, size_t *len); void *callReplyGetPrivateData(CallReply *rep); int callReplyIsResp3(CallReply *rep); +list *callReplyDeferredErrorList(CallReply *rep); void freeCallReply(CallReply *rep); #endif /* SRC_CALL_REPLY_H_ */ diff --git a/src/cluster.c b/src/cluster.c index d2d179e6f..32335bbf9 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -106,7 +106,7 @@ dictType clusterNodesDictType = { }; /* Cluster re-addition blacklist. This maps node IDs to the time - * we can re-add this node. The goal is to avoid readding a removed + * we can re-add this node. The goal is to avoid reading a removed * node for some time. */ dictType clusterNodesBlackListDictType = { dictSdsCaseHash, /* hash function */ @@ -243,10 +243,9 @@ int clusterLoadConfig(char *filename) { if (hostname) { *hostname = '\0'; hostname++; - zfree(n->hostname); - n->hostname = zstrdup(hostname); - } else { - n->hostname = NULL; + n->hostname = sdscpy(n->hostname, hostname); + } else if (sdslen(n->hostname) != 0) { + sdsclear(n->hostname); } /* The plaintext port for client in a TLS cluster (n->pport) is not @@ -570,20 +569,15 @@ void clusterUpdateMyselfIp(void) { /* Update the hostname for the specified node with the provided C string. */ static void updateAnnouncedHostname(clusterNode *node, char *new) { - if (!node->hostname && !new) { - return; - } - /* Previous and new hostname are the same, no need to update. */ - if (new && node->hostname && !strcmp(new, node->hostname)) { + if (new && !strcmp(new, node->hostname)) { return; } - if (node->hostname) zfree(node->hostname); if (new) { - node->hostname = zstrdup(new); - } else { - node->hostname = NULL; + node->hostname = sdscpy(node->hostname, new); + } else if (sdslen(node->hostname) != 0) { + sdsclear(node->hostname); } } @@ -959,7 +953,7 @@ clusterNode *createClusterNode(char *nodename, int flags) { node->link = NULL; node->inbound_link = NULL; memset(node->ip,0,sizeof(node->ip)); - node->hostname = NULL; + node->hostname = sdsempty(); node->port = 0; node->cport = 0; node->pport = 0; @@ -1125,7 +1119,7 @@ void freeClusterNode(clusterNode *n) { nodename = sdsnewlen(n->name, CLUSTER_NAMELEN); serverAssert(dictDelete(server.cluster->nodes,nodename) == DICT_OK); sdsfree(nodename); - zfree(n->hostname); + sdsfree(n->hostname); /* Release links and associated data structures. */ if (n->link) freeClusterLink(n->link); @@ -1947,9 +1941,9 @@ static clusterMsgPingExt *getNextPingExt(clusterMsgPingExt *ext) { * will be 8 byte padded. */ int getHostnamePingExtSize() { /* If hostname is not set, we don't send this extension */ - if (!myself->hostname) return 0; + if (sdslen(myself->hostname) == 0) return 0; - int totlen = sizeof(clusterMsgPingExt) + EIGHT_BYTE_ALIGN(strlen(myself->hostname) + 1); + int totlen = sizeof(clusterMsgPingExt) + EIGHT_BYTE_ALIGN(sdslen(myself->hostname) + 1); return totlen; } @@ -1958,19 +1952,18 @@ int getHostnamePingExtSize() { * will return the amount of bytes written. */ int writeHostnamePingExt(clusterMsgPingExt **cursor) { /* If hostname is not set, we don't send this extension */ - if (!myself->hostname) return 0; + if (sdslen(myself->hostname) == 0) return 0; /* Add the hostname information at the extension cursor */ clusterMsgPingExtHostname *ext = &(*cursor)->ext[0].hostname; - size_t hostname_len = strlen(myself->hostname); - memcpy(ext->hostname, myself->hostname, hostname_len); + memcpy(ext->hostname, myself->hostname, sdslen(myself->hostname)); uint32_t extension_size = getHostnamePingExtSize(); /* Move the write cursor */ (*cursor)->type = CLUSTERMSG_EXT_TYPE_HOSTNAME; (*cursor)->length = htonl(extension_size); /* Make sure the string is NULL terminated by adding 1 */ - *cursor = (clusterMsgPingExt *) (ext->hostname + EIGHT_BYTE_ALIGN(strlen(myself->hostname) + 1)); + *cursor = (clusterMsgPingExt *) (ext->hostname + EIGHT_BYTE_ALIGN(sdslen(myself->hostname) + 1)); return extension_size; } @@ -2975,7 +2968,7 @@ void clusterSendPing(clusterLink *link, int type) { /* Set the initial extension position */ clusterMsgPingExt *cursor = getInitialPingExt(hdr, gossipcount); /* Add in the extensions */ - if (myself->hostname) { + if (sdslen(myself->hostname) != 0) { hdr->mflags[0] |= CLUSTERMSG_FLAG0_EXT_DATA; totlen += writeHostnamePingExt(&cursor); extensions++; @@ -3959,7 +3952,8 @@ void clusterCron(void) { iteration++; /* Number of times this function was called so far. */ - updateAnnouncedHostname(myself, server.cluster_announce_hostname); + clusterUpdateMyselfHostname(); + /* The handshake timeout is the time after which a handshake node that was * not turned into a normal node is removed from the nodes. Usually it is * just the NODE_TIMEOUT value, but when NODE_TIMEOUT is too small we use @@ -4578,7 +4572,7 @@ sds clusterGenNodeDescription(clusterNode *node, int use_pport) { /* Node coordinates */ ci = sdscatlen(sdsempty(),node->name,CLUSTER_NAMELEN); - if (node->hostname) { + if (sdslen(node->hostname) != 0) { ci = sdscatfmt(ci," %s:%i@%i,%s ", node->ip, port, @@ -4804,7 +4798,7 @@ void addReplyClusterLinksDescription(client *c) { const char *getPreferredEndpoint(clusterNode *n) { switch(server.cluster_preferred_endpoint_type) { case CLUSTER_ENDPOINT_TYPE_IP: return n->ip; - case CLUSTER_ENDPOINT_TYPE_HOSTNAME: return n->hostname ? n->hostname : "?"; + case CLUSTER_ENDPOINT_TYPE_HOSTNAME: return (sdslen(n->hostname) != 0) ? n->hostname : "?"; case CLUSTER_ENDPOINT_TYPE_UNKNOWN_ENDPOINT: return ""; } return "unknown"; @@ -4898,7 +4892,7 @@ void addNodeToNodeReply(client *c, clusterNode *node) { if (server.cluster_preferred_endpoint_type == CLUSTER_ENDPOINT_TYPE_IP) { addReplyBulkCString(c, node->ip); } else if (server.cluster_preferred_endpoint_type == CLUSTER_ENDPOINT_TYPE_HOSTNAME) { - addReplyBulkCString(c, node->hostname ? node->hostname : "?"); + addReplyBulkCString(c, sdslen(node->hostname) != 0 ? node->hostname : "?"); } else if (server.cluster_preferred_endpoint_type == CLUSTER_ENDPOINT_TYPE_UNKNOWN_ENDPOINT) { addReplyNull(c); } else { @@ -4921,7 +4915,7 @@ void addNodeToNodeReply(client *c, clusterNode *node) { length++; } if (server.cluster_preferred_endpoint_type != CLUSTER_ENDPOINT_TYPE_HOSTNAME - && node->hostname) + && sdslen(node->hostname) != 0) { addReplyBulkCString(c, "hostname"); addReplyBulkCString(c, node->hostname); @@ -5032,7 +5026,7 @@ void clusterCommand(client *c) { " Reset current node (default: soft).", "SET-CONFIG-EPOCH <epoch>", " Set config epoch of current node.", -"SETSLOT <slot> (IMPORTING|MIGRATING|STABLE|NODE <node-id>)", +"SETSLOT <slot> (IMPORTING <node-id>|MIGRATING <node-id>|STABLE|NODE <node-id>)", " Set slot state.", "REPLICAS <node-id>", " Return <node-id> replicas.", @@ -5226,6 +5220,10 @@ NULL (char*)c->argv[4]->ptr); return; } + if (nodeIsSlave(n)) { + addReplyError(c,"Target node is not a master"); + return; + } /* If this hash slot was served by 'myself' before to switch * make sure there are no longer local keys for this hash slot. */ if (server.cluster->slots[slot] == myself && n != myself) { @@ -5285,7 +5283,7 @@ NULL addReplySds(c,reply); } else if (!strcasecmp(c->argv[1]->ptr,"info") && c->argc == 2) { /* CLUSTER INFO */ - char *statestr[] = {"ok","fail","needhelp"}; + char *statestr[] = {"ok","fail"}; int slots_assigned = 0, slots_ok = 0, slots_pfail = 0, slots_fail = 0; uint64_t myepoch; int j; @@ -5703,7 +5701,7 @@ int verifyDumpPayload(unsigned char *p, size_t len, uint16_t *rdbver_ptr) { if (len < 10) return C_ERR; footer = p+(len-10); - /* Verify RDB version */ + /* Set and verify RDB version. */ rdbver = (footer[1] << 8) | footer[0]; if (rdbver_ptr) { *rdbver_ptr = rdbver; diff --git a/src/cluster.h b/src/cluster.h index 465654205..314b747be 100644 --- a/src/cluster.h +++ b/src/cluster.h @@ -96,8 +96,8 @@ typedef struct clusterLink { #define CLUSTERMSG_TYPE_UPDATE 7 /* Another node slots configuration */ #define CLUSTERMSG_TYPE_MFSTART 8 /* Pause clients for manual failover */ #define CLUSTERMSG_TYPE_MODULE 9 /* Module cluster API message. */ -#define CLUSTERMSG_TYPE_COUNT 10 /* Total number of message types. */ -#define CLUSTERMSG_TYPE_PUBLISHSHARD 11 /* Pub/Sub Publish shard propagation */ +#define CLUSTERMSG_TYPE_PUBLISHSHARD 10 /* Pub/Sub Publish shard propagation */ +#define CLUSTERMSG_TYPE_COUNT 11 /* Total number of message types. */ /* Flags that a module can set in order to prevent certain Redis Cluster * features to be enabled. Useful when implementing a different distributed @@ -134,8 +134,8 @@ typedef struct clusterNode { mstime_t repl_offset_time; /* Unix time we received offset for this node */ mstime_t orphaned_time; /* Starting time of orphaned master condition */ long long repl_offset; /* Last known repl offset for this node. */ - char ip[NET_IP_STR_LEN]; /* Latest known IP address of this node */ - char *hostname; /* The known hostname for this node */ + char ip[NET_IP_STR_LEN]; /* Latest known IP address of this node */ + sds hostname; /* The known hostname for this node */ int port; /* Latest known clients port (TLS or plain). */ int pport; /* Latest known clients plaintext port. Only used if the main clients port is for TLS. */ @@ -339,8 +339,6 @@ typedef struct { * changes in clusterMsg be caught at compile time. */ -/* Avoid static_assert on non-C11 compilers. */ -#if __STDC_VERSION__ >= 201112L static_assert(offsetof(clusterMsg, sig) == 0, "unexpected field offset"); static_assert(offsetof(clusterMsg, totlen) == 4, "unexpected field offset"); static_assert(offsetof(clusterMsg, ver) == 8, "unexpected field offset"); @@ -362,7 +360,6 @@ static_assert(offsetof(clusterMsg, flags) == 2250, "unexpected field offset"); static_assert(offsetof(clusterMsg, state) == 2252, "unexpected field offset"); static_assert(offsetof(clusterMsg, mflags) == 2253, "unexpected field offset"); static_assert(offsetof(clusterMsg, data) == 2256, "unexpected field offset"); -#endif #define CLUSTERMSG_MIN_LEN (sizeof(clusterMsg)-sizeof(union clusterMsgData)) diff --git a/src/commands.c b/src/commands.c index 62291f33f..d3b2e45b3 100644 --- a/src/commands.c +++ b/src/commands.c @@ -588,9 +588,9 @@ NULL /* CLUSTER SETSLOT subcommand argument table */ struct redisCommandArg CLUSTER_SETSLOT_subcommand_Subargs[] = { -{"node-id",ARG_TYPE_INTEGER,-1,"IMPORTING",NULL,NULL,CMD_ARG_NONE}, -{"node-id",ARG_TYPE_INTEGER,-1,"MIGRATING",NULL,NULL,CMD_ARG_NONE}, -{"node-id",ARG_TYPE_INTEGER,-1,"NODE",NULL,NULL,CMD_ARG_NONE}, +{"node-id",ARG_TYPE_STRING,-1,"IMPORTING",NULL,NULL,CMD_ARG_NONE}, +{"node-id",ARG_TYPE_STRING,-1,"MIGRATING",NULL,NULL,CMD_ARG_NONE}, +{"node-id",ARG_TYPE_STRING,-1,"NODE",NULL,NULL,CMD_ARG_NONE}, {"stable",ARG_TYPE_PURE_TOKEN,-1,"STABLE",NULL,NULL,CMD_ARG_NONE}, {0} }; @@ -3092,6 +3092,12 @@ struct redisCommandArg PUBSUB_SHARDCHANNELS_Args[] = { /* PUBSUB SHARDNUMSUB tips */ #define PUBSUB_SHARDNUMSUB_tips NULL +/* PUBSUB SHARDNUMSUB argument table */ +struct redisCommandArg PUBSUB_SHARDNUMSUB_Args[] = { +{"channel",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_OPTIONAL|CMD_ARG_MULTIPLE}, +{0} +}; + /* PUBSUB command table */ struct redisCommand PUBSUB_Subcommands[] = { {"channels","List active channels","O(N) where N is the number of active channels, and assuming constant time pattern matching (relatively short channels and patterns)","2.8.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_PUBSUB,PUBSUB_CHANNELS_History,PUBSUB_CHANNELS_tips,pubsubCommand,-2,CMD_PUBSUB|CMD_LOADING|CMD_STALE,0,.args=PUBSUB_CHANNELS_Args}, @@ -3099,7 +3105,7 @@ struct redisCommand PUBSUB_Subcommands[] = { {"numpat","Get the count of unique patterns pattern subscriptions","O(1)","2.8.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_PUBSUB,PUBSUB_NUMPAT_History,PUBSUB_NUMPAT_tips,pubsubCommand,2,CMD_PUBSUB|CMD_LOADING|CMD_STALE,0}, {"numsub","Get the count of subscribers for channels","O(N) for the NUMSUB subcommand, where N is the number of requested channels","2.8.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_PUBSUB,PUBSUB_NUMSUB_History,PUBSUB_NUMSUB_tips,pubsubCommand,-2,CMD_PUBSUB|CMD_LOADING|CMD_STALE,0,.args=PUBSUB_NUMSUB_Args}, {"shardchannels","List active shard channels","O(N) where N is the number of active shard channels, and assuming constant time pattern matching (relatively short channels).","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_PUBSUB,PUBSUB_SHARDCHANNELS_History,PUBSUB_SHARDCHANNELS_tips,pubsubCommand,-2,CMD_PUBSUB|CMD_LOADING|CMD_STALE,0,.args=PUBSUB_SHARDCHANNELS_Args}, -{"shardnumsub","Get the count of subscribers for shard channels","O(N) for the SHARDNUMSUB subcommand, where N is the number of requested channels","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_PUBSUB,PUBSUB_SHARDNUMSUB_History,PUBSUB_SHARDNUMSUB_tips,pubsubCommand,-2,CMD_PUBSUB|CMD_LOADING|CMD_STALE,0}, +{"shardnumsub","Get the count of subscribers for shard channels","O(N) for the SHARDNUMSUB subcommand, where N is the number of requested channels","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_PUBSUB,PUBSUB_SHARDNUMSUB_History,PUBSUB_SHARDNUMSUB_tips,pubsubCommand,-2,CMD_PUBSUB|CMD_LOADING|CMD_STALE,0,.args=PUBSUB_SHARDNUMSUB_Args}, {0} }; @@ -3315,7 +3321,7 @@ NULL /* FUNCTION DELETE argument table */ struct redisCommandArg FUNCTION_DELETE_Args[] = { -{"function-name",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_NONE}, +{"library-name",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_NONE}, {0} }; @@ -3404,7 +3410,7 @@ struct redisCommandArg FUNCTION_LOAD_Args[] = { {"engine-name",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_NONE}, {"library-name",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_NONE}, {"replace",ARG_TYPE_PURE_TOKEN,-1,"REPLACE",NULL,NULL,CMD_ARG_OPTIONAL}, -{"library-description",ARG_TYPE_STRING,-1,"DESC",NULL,NULL,CMD_ARG_OPTIONAL}, +{"library-description",ARG_TYPE_STRING,-1,"DESCRIPTION",NULL,NULL,CMD_ARG_OPTIONAL}, {"function-code",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_NONE}, {0} }; @@ -3759,7 +3765,7 @@ struct redisCommand SCRIPT_Subcommands[] = { struct redisCommand SENTINEL_Subcommands[] = { {"ckquorum","Check for a Sentinel quorum",NULL,"2.8.4",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SENTINEL,SENTINEL_CKQUORUM_History,SENTINEL_CKQUORUM_tips,sentinelCommand,3,CMD_ADMIN|CMD_SENTINEL|CMD_ONLY_SENTINEL,0}, {"config","Configure Sentinel","O(1)","6.2.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SENTINEL,SENTINEL_CONFIG_History,SENTINEL_CONFIG_tips,sentinelCommand,-3,CMD_ADMIN|CMD_SENTINEL|CMD_ONLY_SENTINEL,0}, -{"debug",NULL,NULL,"7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SENTINEL,SENTINEL_DEBUG_History,SENTINEL_DEBUG_tips,sentinelCommand,-2,CMD_ADMIN|CMD_SENTINEL|CMD_ONLY_SENTINEL,0}, +{"debug","List or update the current configurable parameters","O(N) where N is the number of configurable parameters","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SENTINEL,SENTINEL_DEBUG_History,SENTINEL_DEBUG_tips,sentinelCommand,-2,CMD_ADMIN|CMD_SENTINEL|CMD_ONLY_SENTINEL,0}, {"failover","Force a failover",NULL,"2.8.4",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SENTINEL,SENTINEL_FAILOVER_History,SENTINEL_FAILOVER_tips,sentinelCommand,3,CMD_ADMIN|CMD_SENTINEL|CMD_ONLY_SENTINEL,0}, {"flushconfig","Rewrite configuration file","O(1)","2.8.4",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SENTINEL,SENTINEL_FLUSHCONFIG_History,SENTINEL_FLUSHCONFIG_tips,sentinelCommand,2,CMD_ADMIN|CMD_SENTINEL|CMD_ONLY_SENTINEL,0}, {"get-master-addr-by-name","Get port and address of a master","O(1)","2.8.4",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SENTINEL,SENTINEL_GET_MASTER_ADDR_BY_NAME_History,SENTINEL_GET_MASTER_ADDR_BY_NAME_tips,sentinelCommand,3,CMD_ADMIN|CMD_SENTINEL|CMD_ONLY_SENTINEL,0}, @@ -4033,6 +4039,14 @@ struct redisCommandArg COMMAND_DOCS_Args[] = { /* COMMAND GETKEYS tips */ #define COMMAND_GETKEYS_tips NULL +/********** COMMAND GETKEYSANDFLAGS ********************/ + +/* COMMAND GETKEYSANDFLAGS history */ +#define COMMAND_GETKEYSANDFLAGS_History NULL + +/* COMMAND GETKEYSANDFLAGS tips */ +#define COMMAND_GETKEYSANDFLAGS_tips NULL + /********** COMMAND HELP ********************/ /* COMMAND HELP history */ @@ -4085,6 +4099,7 @@ struct redisCommand COMMAND_Subcommands[] = { {"count","Get total number of Redis commands","O(1)","2.8.13",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SERVER,COMMAND_COUNT_History,COMMAND_COUNT_tips,commandCountCommand,2,CMD_LOADING|CMD_STALE,ACL_CATEGORY_CONNECTION}, {"docs","Get array of specific Redis command documentation","O(N) where N is the number of commands to look up","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SERVER,COMMAND_DOCS_History,COMMAND_DOCS_tips,commandDocsCommand,-2,CMD_LOADING|CMD_STALE,ACL_CATEGORY_CONNECTION,.args=COMMAND_DOCS_Args}, {"getkeys","Extract keys given a full Redis command","O(N) where N is the number of arguments to the command","2.8.13",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SERVER,COMMAND_GETKEYS_History,COMMAND_GETKEYS_tips,commandGetKeysCommand,-4,CMD_LOADING|CMD_STALE,ACL_CATEGORY_CONNECTION}, +{"getkeysandflags","Extract keys given a full Redis command","O(N) where N is the number of arguments to the command","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SERVER,COMMAND_GETKEYSANDFLAGS_History,COMMAND_GETKEYSANDFLAGS_tips,commandGetKeysAndFlagsCommand,-4,CMD_LOADING|CMD_STALE,ACL_CATEGORY_CONNECTION}, {"help","Show helpful text about the different subcommands","O(1)","5.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SERVER,COMMAND_HELP_History,COMMAND_HELP_tips,commandHelpCommand,2,CMD_LOADING|CMD_STALE,ACL_CATEGORY_CONNECTION}, {"info","Get array of specific Redis command details, or all when no argument is given.","O(N) where N is the number of commands to look up","2.8.13",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SERVER,COMMAND_INFO_History,COMMAND_INFO_tips,commandInfoCommand,-2,CMD_LOADING|CMD_STALE,ACL_CATEGORY_CONNECTION,.args=COMMAND_INFO_Args}, {"list","Get an array of Redis command names","O(N) where N is the total number of Redis commands","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SERVER,COMMAND_LIST_History,COMMAND_LIST_tips,commandListCommand,-2,CMD_LOADING|CMD_STALE,ACL_CATEGORY_CONNECTION,.args=COMMAND_LIST_Args}, @@ -4300,7 +4315,10 @@ struct redisCommandArg FLUSHDB_Args[] = { /********** INFO ********************/ /* INFO history */ -#define INFO_History NULL +commandHistory INFO_History[] = { +{"7.0.0","Added support for taking multiple section arguments."}, +{0} +}; /* INFO tips */ const char *INFO_tips[] = { @@ -4312,7 +4330,7 @@ NULL /* INFO argument table */ struct redisCommandArg INFO_Args[] = { -{"section",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_OPTIONAL}, +{"section",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_OPTIONAL|CMD_ARG_MULTIPLE}, {0} }; @@ -5913,7 +5931,10 @@ struct redisCommandArg XADD_Args[] = { /********** XAUTOCLAIM ********************/ /* XAUTOCLAIM history */ -#define XAUTOCLAIM_History NULL +commandHistory XAUTOCLAIM_History[] = { +{"7.0.0","Added an element to the reply array, containing deleted entries the command cleared from the PEL"}, +{0} +}; /* XAUTOCLAIM tips */ const char *XAUTOCLAIM_tips[] = { @@ -5977,7 +5998,10 @@ struct redisCommandArg XDEL_Args[] = { /********** XGROUP CREATE ********************/ /* XGROUP CREATE history */ -#define XGROUP_CREATE_History NULL +commandHistory XGROUP_CREATE_History[] = { +{"7.0.0","Added the `entries_read` named argument."}, +{0} +}; /* XGROUP CREATE tips */ #define XGROUP_CREATE_tips NULL @@ -5995,6 +6019,7 @@ struct redisCommandArg XGROUP_CREATE_Args[] = { {"groupname",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_NONE}, {"id",ARG_TYPE_ONEOF,-1,NULL,NULL,NULL,CMD_ARG_NONE,.subargs=XGROUP_CREATE_id_Subargs}, {"mkstream",ARG_TYPE_PURE_TOKEN,-1,"MKSTREAM",NULL,NULL,CMD_ARG_OPTIONAL}, +{"entries_read",ARG_TYPE_INTEGER,-1,"ENTRIESREAD",NULL,NULL,CMD_ARG_OPTIONAL}, {0} }; @@ -6056,7 +6081,10 @@ struct redisCommandArg XGROUP_DESTROY_Args[] = { /********** XGROUP SETID ********************/ /* XGROUP SETID history */ -#define XGROUP_SETID_History NULL +commandHistory XGROUP_SETID_History[] = { +{"7.0.0","Added the optional `entries_read` argument."}, +{0} +}; /* XGROUP SETID tips */ #define XGROUP_SETID_tips NULL @@ -6073,6 +6101,7 @@ struct redisCommandArg XGROUP_SETID_Args[] = { {"key",ARG_TYPE_KEY,0,NULL,NULL,NULL,CMD_ARG_NONE}, {"groupname",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_NONE}, {"id",ARG_TYPE_ONEOF,-1,NULL,NULL,NULL,CMD_ARG_NONE,.subargs=XGROUP_SETID_id_Subargs}, +{"entries_read",ARG_TYPE_INTEGER,-1,"ENTRIESREAD",NULL,NULL,CMD_ARG_OPTIONAL}, {0} }; @@ -6083,7 +6112,7 @@ struct redisCommand XGROUP_Subcommands[] = { {"delconsumer","Delete a consumer from a consumer group.","O(1)","5.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_STREAM,XGROUP_DELCONSUMER_History,XGROUP_DELCONSUMER_tips,xgroupCommand,5,CMD_WRITE,ACL_CATEGORY_STREAM,{{NULL,CMD_KEY_RW|CMD_KEY_DELETE,KSPEC_BS_INDEX,.bs.index={2},KSPEC_FK_RANGE,.fk.range={0,1,0}}},.args=XGROUP_DELCONSUMER_Args}, {"destroy","Destroy a consumer group.","O(N) where N is the number of entries in the group's pending entries list (PEL).","5.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_STREAM,XGROUP_DESTROY_History,XGROUP_DESTROY_tips,xgroupCommand,4,CMD_WRITE,ACL_CATEGORY_STREAM,{{NULL,CMD_KEY_RW|CMD_KEY_DELETE,KSPEC_BS_INDEX,.bs.index={2},KSPEC_FK_RANGE,.fk.range={0,1,0}}},.args=XGROUP_DESTROY_Args}, {"help","Show helpful text about the different subcommands","O(1)","5.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_STREAM,XGROUP_HELP_History,XGROUP_HELP_tips,xgroupCommand,2,CMD_LOADING|CMD_STALE,ACL_CATEGORY_STREAM}, -{"setid","Set a consumer group to an arbitrary last delivered ID value.","O(1)","5.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_STREAM,XGROUP_SETID_History,XGROUP_SETID_tips,xgroupCommand,5,CMD_WRITE,ACL_CATEGORY_STREAM,{{NULL,CMD_KEY_RW|CMD_KEY_UPDATE,KSPEC_BS_INDEX,.bs.index={2},KSPEC_FK_RANGE,.fk.range={0,1,0}}},.args=XGROUP_SETID_Args}, +{"setid","Set a consumer group to an arbitrary last delivered ID value.","O(1)","5.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_STREAM,XGROUP_SETID_History,XGROUP_SETID_tips,xgroupCommand,-5,CMD_WRITE,ACL_CATEGORY_STREAM,{{NULL,CMD_KEY_RW|CMD_KEY_UPDATE,KSPEC_BS_INDEX,.bs.index={2},KSPEC_FK_RANGE,.fk.range={0,1,0}}},.args=XGROUP_SETID_Args}, {0} }; @@ -6116,7 +6145,10 @@ struct redisCommandArg XINFO_CONSUMERS_Args[] = { /********** XINFO GROUPS ********************/ /* XINFO GROUPS history */ -#define XINFO_GROUPS_History NULL +commandHistory XINFO_GROUPS_History[] = { +{"7.0.0","Added the `entries-read` and `lag` fields"}, +{0} +}; /* XINFO GROUPS tips */ #define XINFO_GROUPS_tips NULL @@ -6138,7 +6170,10 @@ struct redisCommandArg XINFO_GROUPS_Args[] = { /********** XINFO STREAM ********************/ /* XINFO STREAM history */ -#define XINFO_STREAM_History NULL +commandHistory XINFO_STREAM_History[] = { +{"7.0.0","Added the `max-deleted-entry-id`, `entries-added`, `recorded-first-entry-id`, `entries-read` and `lag` fields"}, +{0} +}; /* XINFO STREAM tips */ #define XINFO_STREAM_tips NULL @@ -6317,7 +6352,10 @@ struct redisCommandArg XREVRANGE_Args[] = { /********** XSETID ********************/ /* XSETID history */ -#define XSETID_History NULL +commandHistory XSETID_History[] = { +{"7.0.0","Added the `entries_added` and `max_deleted_entry_id` arguments."}, +{0} +}; /* XSETID tips */ #define XSETID_tips NULL @@ -6326,6 +6364,8 @@ struct redisCommandArg XREVRANGE_Args[] = { struct redisCommandArg XSETID_Args[] = { {"key",ARG_TYPE_KEY,0,NULL,NULL,NULL,CMD_ARG_NONE}, {"last-id",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_NONE}, +{"entries_added",ARG_TYPE_INTEGER,-1,"ENTRIESADDED",NULL,NULL,CMD_ARG_OPTIONAL}, +{"max_deleted_entry_id",ARG_TYPE_STRING,-1,"MAXDELETEDID",NULL,NULL,CMD_ARG_OPTIONAL}, {0} }; @@ -6922,18 +6962,18 @@ struct redisCommand redisCommandTable[] = { {"publish","Post a message to a channel","O(N+M) where N is the number of clients subscribed to the receiving channel and M is the total number of subscribed patterns (by any client).","2.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_PUBSUB,PUBLISH_History,PUBLISH_tips,publishCommand,3,CMD_PUBSUB|CMD_LOADING|CMD_STALE|CMD_FAST|CMD_MAY_REPLICATE|CMD_SENTINEL,0,.args=PUBLISH_Args}, {"pubsub","A container for Pub/Sub commands","Depends on subcommand.","2.8.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_PUBSUB,PUBSUB_History,PUBSUB_tips,NULL,-2,0,0,.subcommands=PUBSUB_Subcommands}, {"punsubscribe","Stop listening for messages posted to channels matching the given patterns","O(N+M) where N is the number of patterns the client is already subscribed and M is the number of total patterns subscribed in the system (by any client).","2.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_PUBSUB,PUNSUBSCRIBE_History,PUNSUBSCRIBE_tips,punsubscribeCommand,-1,CMD_PUBSUB|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_SENTINEL,0,.args=PUNSUBSCRIBE_Args}, -{"spublish","Post a message to a shard channel","O(N) where N is the number of clients subscribed to the receiving shard channel.","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_PUBSUB,SPUBLISH_History,SPUBLISH_tips,spublishCommand,3,CMD_PUBSUB|CMD_LOADING|CMD_STALE|CMD_FAST|CMD_MAY_REPLICATE,0,{{NULL,CMD_KEY_CHANNEL,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}}},.args=SPUBLISH_Args}, -{"ssubscribe","Listen for messages published to the given shard channels","O(N) where N is the number of shard channels to subscribe to.","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_PUBSUB,SSUBSCRIBE_History,SSUBSCRIBE_tips,ssubscribeCommand,-2,CMD_PUBSUB|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE,0,{{NULL,CMD_KEY_CHANNEL,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={-1,1,0}}},.args=SSUBSCRIBE_Args}, +{"spublish","Post a message to a shard channel","O(N) where N is the number of clients subscribed to the receiving shard channel.","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_PUBSUB,SPUBLISH_History,SPUBLISH_tips,spublishCommand,3,CMD_PUBSUB|CMD_LOADING|CMD_STALE|CMD_FAST|CMD_MAY_REPLICATE,0,{{NULL,CMD_KEY_NOT_KEY,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}}},.args=SPUBLISH_Args}, +{"ssubscribe","Listen for messages published to the given shard channels","O(N) where N is the number of shard channels to subscribe to.","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_PUBSUB,SSUBSCRIBE_History,SSUBSCRIBE_tips,ssubscribeCommand,-2,CMD_PUBSUB|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE,0,{{NULL,CMD_KEY_NOT_KEY,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={-1,1,0}}},.args=SSUBSCRIBE_Args}, {"subscribe","Listen for messages published to the given channels","O(N) where N is the number of channels to subscribe to.","2.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_PUBSUB,SUBSCRIBE_History,SUBSCRIBE_tips,subscribeCommand,-2,CMD_PUBSUB|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_SENTINEL,0,.args=SUBSCRIBE_Args}, -{"sunsubscribe","Stop listening for messages posted to the given shard channels","O(N) where N is the number of clients already subscribed to a channel.","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_PUBSUB,SUNSUBSCRIBE_History,SUNSUBSCRIBE_tips,sunsubscribeCommand,-1,CMD_PUBSUB|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE,0,{{NULL,CMD_KEY_CHANNEL,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={-1,1,0}}},.args=SUNSUBSCRIBE_Args}, +{"sunsubscribe","Stop listening for messages posted to the given shard channels","O(N) where N is the number of clients already subscribed to a channel.","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_PUBSUB,SUNSUBSCRIBE_History,SUNSUBSCRIBE_tips,sunsubscribeCommand,-1,CMD_PUBSUB|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE,0,{{NULL,CMD_KEY_NOT_KEY,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={-1,1,0}}},.args=SUNSUBSCRIBE_Args}, {"unsubscribe","Stop listening for messages posted to the given channels","O(N) where N is the number of clients already subscribed to a channel.","2.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_PUBSUB,UNSUBSCRIBE_History,UNSUBSCRIBE_tips,unsubscribeCommand,-1,CMD_PUBSUB|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_SENTINEL,0,.args=UNSUBSCRIBE_Args}, /* scripting */ {"eval","Execute a Lua script server side","Depends on the script that is executed.","2.6.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SCRIPTING,EVAL_History,EVAL_tips,evalCommand,-3,CMD_NOSCRIPT|CMD_SKIP_MONITOR|CMD_MAY_REPLICATE|CMD_NO_MANDATORY_KEYS|CMD_STALE,ACL_CATEGORY_SCRIPTING,{{"We cannot tell how the keys will be used so we assume the worst, RW and UPDATE",CMD_KEY_RW|CMD_KEY_ACCESS|CMD_KEY_UPDATE,KSPEC_BS_INDEX,.bs.index={2},KSPEC_FK_KEYNUM,.fk.keynum={0,1,1}}},evalGetKeys,.args=EVAL_Args}, {"evalsha","Execute a Lua script server side","Depends on the script that is executed.","2.6.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SCRIPTING,EVALSHA_History,EVALSHA_tips,evalShaCommand,-3,CMD_NOSCRIPT|CMD_SKIP_MONITOR|CMD_MAY_REPLICATE|CMD_NO_MANDATORY_KEYS|CMD_STALE,ACL_CATEGORY_SCRIPTING,{{NULL,CMD_KEY_RW|CMD_KEY_ACCESS|CMD_KEY_UPDATE,KSPEC_BS_INDEX,.bs.index={2},KSPEC_FK_KEYNUM,.fk.keynum={0,1,1}}},evalGetKeys,.args=EVALSHA_Args}, {"evalsha_ro","Execute a read-only Lua script server side","Depends on the script that is executed.","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SCRIPTING,EVALSHA_RO_History,EVALSHA_RO_tips,evalShaRoCommand,-3,CMD_NOSCRIPT|CMD_SKIP_MONITOR|CMD_NO_MANDATORY_KEYS|CMD_STALE,ACL_CATEGORY_SCRIPTING,{{NULL,CMD_KEY_RO|CMD_KEY_ACCESS,KSPEC_BS_INDEX,.bs.index={2},KSPEC_FK_KEYNUM,.fk.keynum={0,1,1}}},evalGetKeys,.args=EVALSHA_RO_Args}, {"eval_ro","Execute a read-only Lua script server side","Depends on the script that is executed.","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SCRIPTING,EVAL_RO_History,EVAL_RO_tips,evalRoCommand,-3,CMD_NOSCRIPT|CMD_SKIP_MONITOR|CMD_NO_MANDATORY_KEYS|CMD_STALE,ACL_CATEGORY_SCRIPTING,{{"We cannot tell how the keys will be used so we assume the worst, RO and ACCESS",CMD_KEY_RO|CMD_KEY_ACCESS,KSPEC_BS_INDEX,.bs.index={2},KSPEC_FK_KEYNUM,.fk.keynum={0,1,1}}},evalGetKeys,.args=EVAL_RO_Args}, -{"fcall","PATCH__TBD__38__","PATCH__TBD__37__","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SCRIPTING,FCALL_History,FCALL_tips,fcallCommand,-3,CMD_NOSCRIPT|CMD_SKIP_MONITOR|CMD_MAY_REPLICATE|CMD_NO_MANDATORY_KEYS|CMD_STALE,ACL_CATEGORY_SCRIPTING,{{"We cannot tell how the keys will be used so we assume the worst, RW and UPDATE",CMD_KEY_RW|CMD_KEY_ACCESS|CMD_KEY_UPDATE,KSPEC_BS_INDEX,.bs.index={2},KSPEC_FK_KEYNUM,.fk.keynum={0,1,1}}},functionGetKeys,.args=FCALL_Args}, -{"fcall_ro","PATCH__TBD__7__","PATCH__TBD__6__","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SCRIPTING,FCALL_RO_History,FCALL_RO_tips,fcallroCommand,-3,CMD_NOSCRIPT|CMD_SKIP_MONITOR|CMD_NO_MANDATORY_KEYS|CMD_STALE,ACL_CATEGORY_SCRIPTING,{{"We cannot tell how the keys will be used so we assume the worst, RO and ACCESS",CMD_KEY_RO|CMD_KEY_ACCESS,KSPEC_BS_INDEX,.bs.index={2},KSPEC_FK_KEYNUM,.fk.keynum={0,1,1}}},functionGetKeys,.args=FCALL_RO_Args}, +{"fcall","Invoke a function","Depends on the function that is executed.","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SCRIPTING,FCALL_History,FCALL_tips,fcallCommand,-3,CMD_NOSCRIPT|CMD_SKIP_MONITOR|CMD_MAY_REPLICATE|CMD_NO_MANDATORY_KEYS|CMD_STALE,ACL_CATEGORY_SCRIPTING,{{"We cannot tell how the keys will be used so we assume the worst, RW and UPDATE",CMD_KEY_RW|CMD_KEY_ACCESS|CMD_KEY_UPDATE,KSPEC_BS_INDEX,.bs.index={2},KSPEC_FK_KEYNUM,.fk.keynum={0,1,1}}},functionGetKeys,.args=FCALL_Args}, +{"fcall_ro","Invoke a read-only function","Depends on the function that is executed.","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SCRIPTING,FCALL_RO_History,FCALL_RO_tips,fcallroCommand,-3,CMD_NOSCRIPT|CMD_SKIP_MONITOR|CMD_NO_MANDATORY_KEYS|CMD_STALE,ACL_CATEGORY_SCRIPTING,{{"We cannot tell how the keys will be used so we assume the worst, RO and ACCESS",CMD_KEY_RO|CMD_KEY_ACCESS,KSPEC_BS_INDEX,.bs.index={2},KSPEC_FK_KEYNUM,.fk.keynum={0,1,1}}},functionGetKeys,.args=FCALL_RO_Args}, {"function","A container for function commands","Depends on subcommand.","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SCRIPTING,FUNCTION_History,FUNCTION_tips,NULL,-2,0,0,.subcommands=FUNCTION_Subcommands}, {"script","A container for Lua scripts management commands","Depends on subcommand.","2.6.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SCRIPTING,SCRIPT_History,SCRIPT_tips,NULL,-2,0,0,.subcommands=SCRIPT_Subcommands}, /* sentinel */ @@ -6957,13 +6997,13 @@ struct redisCommand redisCommandTable[] = { {"module","A container for module commands","Depends on subcommand.","4.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SERVER,MODULE_History,MODULE_tips,NULL,-2,0,0,.subcommands=MODULE_Subcommands}, {"monitor","Listen for all requests received by the server in real time",NULL,"1.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SERVER,MONITOR_History,MONITOR_tips,monitorCommand,1,CMD_ADMIN|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE,0}, {"psync","Internal command used for replication",NULL,"2.8.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SERVER,PSYNC_History,PSYNC_tips,syncCommand,-3,CMD_NO_ASYNC_LOADING|CMD_ADMIN|CMD_NO_MULTI|CMD_NOSCRIPT,0,.args=PSYNC_Args}, -{"replconf","An internal command for configuring the replication stream","O(1)","3.0.0",CMD_DOC_SYSCMD,NULL,NULL,COMMAND_GROUP_SERVER,REPLCONF_History,REPLCONF_tips,replconfCommand,-1,CMD_ADMIN|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE,0}, -{"replicaof","Make the server a replica of another instance, or promote it as master.","O(1)","5.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SERVER,REPLICAOF_History,REPLICAOF_tips,replicaofCommand,3,CMD_NO_ASYNC_LOADING|CMD_ADMIN|CMD_ALLOW_BUSY|CMD_NOSCRIPT|CMD_STALE,0,.args=REPLICAOF_Args}, +{"replconf","An internal command for configuring the replication stream","O(1)","3.0.0",CMD_DOC_SYSCMD,NULL,NULL,COMMAND_GROUP_SERVER,REPLCONF_History,REPLCONF_tips,replconfCommand,-1,CMD_ADMIN|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_ALLOW_BUSY,0}, +{"replicaof","Make the server a replica of another instance, or promote it as master.","O(1)","5.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SERVER,REPLICAOF_History,REPLICAOF_tips,replicaofCommand,3,CMD_NO_ASYNC_LOADING|CMD_ADMIN|CMD_NOSCRIPT|CMD_STALE,0,.args=REPLICAOF_Args}, {"restore-asking","An internal command for migrating keys in a cluster","O(1) to create the new key and additional O(N*M) to reconstruct the serialized value, where N is the number of Redis objects composing the value and M their average size. For small string values the time complexity is thus O(1)+O(1*M) where M is small, so simply O(1). However for sorted set values the complexity is O(N*M*log(N)) because inserting values into sorted sets is O(log(N)).","3.0.0",CMD_DOC_SYSCMD,NULL,NULL,COMMAND_GROUP_SERVER,RESTORE_ASKING_History,RESTORE_ASKING_tips,restoreCommand,-4,CMD_WRITE|CMD_DENYOOM|CMD_ASKING,ACL_CATEGORY_KEYSPACE|ACL_CATEGORY_DANGEROUS,{{NULL,CMD_KEY_OW|CMD_KEY_UPDATE,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}}}}, {"role","Return the role of the instance in the context of replication","O(1)","2.8.12",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SERVER,ROLE_History,ROLE_tips,roleCommand,1,CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_FAST|CMD_SENTINEL,ACL_CATEGORY_ADMIN|ACL_CATEGORY_DANGEROUS}, {"save","Synchronously save the dataset to disk","O(N) where N is the total number of keys in all databases","1.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SERVER,SAVE_History,SAVE_tips,saveCommand,1,CMD_NO_ASYNC_LOADING|CMD_ADMIN|CMD_NOSCRIPT|CMD_NO_MULTI,0}, {"shutdown","Synchronously save the dataset to disk and then shut down the server","O(N) when saving, where N is the total number of keys in all databases when saving data, otherwise O(1)","1.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SERVER,SHUTDOWN_History,SHUTDOWN_tips,shutdownCommand,-1,CMD_ADMIN|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_NO_MULTI|CMD_SENTINEL|CMD_ALLOW_BUSY,0,.args=SHUTDOWN_Args}, -{"slaveof","Make the server a replica of another instance, or promote it as master. Deprecated starting with Redis 5. Use REPLICAOF instead.","O(1)","1.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SERVER,SLAVEOF_History,SLAVEOF_tips,replicaofCommand,3,CMD_NO_ASYNC_LOADING|CMD_ADMIN|CMD_NOSCRIPT|CMD_STALE,0,.args=SLAVEOF_Args}, +{"slaveof","Make the server a replica of another instance, or promote it as master.","O(1)","1.0.0",CMD_DOC_DEPRECATED,"`REPLICAOF`","5.0.0",COMMAND_GROUP_SERVER,SLAVEOF_History,SLAVEOF_tips,replicaofCommand,3,CMD_NO_ASYNC_LOADING|CMD_ADMIN|CMD_NOSCRIPT|CMD_STALE,0,.args=SLAVEOF_Args}, {"slowlog","A container for slow log commands","Depends on subcommand.","2.2.12",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SERVER,SLOWLOG_History,SLOWLOG_tips,NULL,-2,0,0,.subcommands=SLOWLOG_Subcommands}, {"swapdb","Swaps two Redis databases","O(N) where N is the count of clients watching or blocking on keys from both databases.","4.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SERVER,SWAPDB_History,SWAPDB_tips,swapdbCommand,3,CMD_WRITE|CMD_FAST,ACL_CATEGORY_KEYSPACE|ACL_CATEGORY_DANGEROUS,.args=SWAPDB_Args}, {"sync","Internal command used for replication",NULL,"1.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SERVER,SYNC_History,SYNC_tips,syncCommand,1,CMD_NO_ASYNC_LOADING|CMD_ADMIN|CMD_NO_MULTI|CMD_NOSCRIPT,0}, @@ -7036,7 +7076,7 @@ struct redisCommand redisCommandTable[] = { {"xread","Return never seen elements in multiple streams, with IDs greater than the ones reported by the caller for each stream. Can block.","For each stream mentioned: O(N) with N being the number of elements being returned, it means that XREAD-ing with a fixed COUNT is O(1). Note that when the BLOCK option is used, XADD will pay O(M) time in order to serve the M clients blocked on the stream getting new data.","5.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_STREAM,XREAD_History,XREAD_tips,xreadCommand,-4,CMD_BLOCKING|CMD_READONLY|CMD_BLOCKING,ACL_CATEGORY_STREAM,{{NULL,CMD_KEY_RO|CMD_KEY_ACCESS,KSPEC_BS_KEYWORD,.bs.keyword={"STREAMS",1},KSPEC_FK_RANGE,.fk.range={-1,1,2}}},xreadGetKeys,.args=XREAD_Args}, {"xreadgroup","Return new entries from a stream using a consumer group, or access the history of the pending entries for a given consumer. Can block.","For each stream mentioned: O(M) with M being the number of elements returned. If M is constant (e.g. always asking for the first 10 elements with COUNT), you can consider it O(1). On the other side when XREADGROUP blocks, XADD will pay the O(N) time in order to serve the N clients blocked on the stream getting new data.","5.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_STREAM,XREADGROUP_History,XREADGROUP_tips,xreadCommand,-7,CMD_BLOCKING|CMD_WRITE,ACL_CATEGORY_STREAM,{{NULL,CMD_KEY_RO|CMD_KEY_ACCESS,KSPEC_BS_KEYWORD,.bs.keyword={"STREAMS",4},KSPEC_FK_RANGE,.fk.range={-1,1,2}}},xreadGetKeys,.args=XREADGROUP_Args}, {"xrevrange","Return a range of elements in a stream, with IDs matching the specified IDs interval, in reverse order (from greater to smaller IDs) compared to XRANGE","O(N) with N being the number of elements returned. If N is constant (e.g. always asking for the first 10 elements with COUNT), you can consider it O(1).","5.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_STREAM,XREVRANGE_History,XREVRANGE_tips,xrevrangeCommand,-4,CMD_READONLY,ACL_CATEGORY_STREAM,{{NULL,CMD_KEY_RO|CMD_KEY_ACCESS,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}}},.args=XREVRANGE_Args}, -{"xsetid","An internal command for replicating stream values","O(1)","5.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_STREAM,XSETID_History,XSETID_tips,xsetidCommand,3,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_STREAM,{{NULL,CMD_KEY_RW|CMD_KEY_UPDATE,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}}},.args=XSETID_Args}, +{"xsetid","An internal command for replicating stream values","O(1)","5.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_STREAM,XSETID_History,XSETID_tips,xsetidCommand,-3,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_STREAM,{{NULL,CMD_KEY_RW|CMD_KEY_UPDATE,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}}},.args=XSETID_Args}, {"xtrim","Trims the stream to (approximately if '~' is passed) a certain size","O(N), with N being the number of evicted entries. Constant times are very small however, since entries are organized in macro nodes containing multiple entries that can be released with a single deallocation.","5.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_STREAM,XTRIM_History,XTRIM_tips,xtrimCommand,-4,CMD_WRITE,ACL_CATEGORY_STREAM,{{NULL,CMD_KEY_RW|CMD_KEY_DELETE,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}}},.args=XTRIM_Args}, /* string */ {"append","Append a value to a key","O(1). The amortized time complexity is O(1) assuming the appended value is small and the already present value is of any size, since the dynamic string library used by Redis will double the free space available on every reallocation.","2.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_STRING,APPEND_History,APPEND_tips,appendCommand,3,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_STRING,{{NULL,CMD_KEY_RW|CMD_KEY_INSERT,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}}},.args=APPEND_Args}, diff --git a/src/commands/cluster-setslot.json b/src/commands/cluster-setslot.json index dc9af6138..5d1aa45fc 100644 --- a/src/commands/cluster-setslot.json +++ b/src/commands/cluster-setslot.json @@ -26,17 +26,17 @@ "arguments": [ { "name": "node-id", - "type": "integer", + "type": "string", "token": "IMPORTING" }, { "name": "node-id", - "type": "integer", + "type": "string", "token": "MIGRATING" }, { "name": "node-id", - "type": "integer", + "type": "string", "token": "NODE" }, { diff --git a/src/commands/command-getkeysandflags.json b/src/commands/command-getkeysandflags.json new file mode 100644 index 000000000..1ac8e990d --- /dev/null +++ b/src/commands/command-getkeysandflags.json @@ -0,0 +1,18 @@ +{ + "GETKEYSANDFLAGS": { + "summary": "Extract keys given a full Redis command", + "complexity": "O(N) where N is the number of arguments to the command", + "group": "server", + "since": "7.0.0", + "arity": -4, + "container": "COMMAND", + "function": "commandGetKeysAndFlagsCommand", + "command_flags": [ + "LOADING", + "STALE" + ], + "acl_categories": [ + "CONNECTION" + ] + } +} diff --git a/src/commands/fcall.json b/src/commands/fcall.json index 45fad3c9b..27b7b4e35 100644 --- a/src/commands/fcall.json +++ b/src/commands/fcall.json @@ -1,7 +1,7 @@ { "FCALL": { - "summary": "PATCH__TBD__38__", - "complexity": "PATCH__TBD__37__", + "summary": "Invoke a function", + "complexity": "Depends on the function that is executed.", "group": "scripting", "since": "7.0.0", "arity": -3, diff --git a/src/commands/fcall_ro.json b/src/commands/fcall_ro.json index d1caf5f65..092915700 100644 --- a/src/commands/fcall_ro.json +++ b/src/commands/fcall_ro.json @@ -1,7 +1,7 @@ { "FCALL_RO": { - "summary": "PATCH__TBD__7__", - "complexity": "PATCH__TBD__6__", + "summary": "Invoke a read-only function", + "complexity": "Depends on the function that is executed.", "group": "scripting", "since": "7.0.0", "arity": -3, diff --git a/src/commands/function-delete.json b/src/commands/function-delete.json index 2cfb4068c..01dc78ba4 100644 --- a/src/commands/function-delete.json +++ b/src/commands/function-delete.json @@ -20,7 +20,7 @@ ], "arguments": [ { - "name": "function-name", + "name": "library-name", "type": "string" } ] diff --git a/src/commands/function-load.json b/src/commands/function-load.json index df756ceab..0a363e328 100644 --- a/src/commands/function-load.json +++ b/src/commands/function-load.json @@ -37,7 +37,7 @@ { "name": "library-description", "type": "string", - "token": "DESC", + "token": "DESCRIPTION", "optional": true }, { diff --git a/src/commands/info.json b/src/commands/info.json index 48720f9c3..612294d34 100644 --- a/src/commands/info.json +++ b/src/commands/info.json @@ -6,6 +6,12 @@ "since": "1.0.0", "arity": -1, "function": "infoCommand", + "history": [ + [ + "7.0.0", + "Added support for taking multiple section arguments." + ] + ], "command_flags": [ "LOADING", "STALE", @@ -23,6 +29,7 @@ { "name": "section", "type": "string", + "multiple": true, "optional": true } ] diff --git a/src/commands/pubsub-shardnumsub.json b/src/commands/pubsub-shardnumsub.json index 167132b31..55f5101ac 100644 --- a/src/commands/pubsub-shardnumsub.json +++ b/src/commands/pubsub-shardnumsub.json @@ -11,6 +11,14 @@ "PUBSUB", "LOADING", "STALE" + ], + "arguments": [ + { + "name": "channel", + "type": "string", + "optional": true, + "multiple": true + } ] } } diff --git a/src/commands/replconf.json b/src/commands/replconf.json index e880aa929..630b62136 100644 --- a/src/commands/replconf.json +++ b/src/commands/replconf.json @@ -13,7 +13,8 @@ "ADMIN", "NOSCRIPT", "LOADING", - "STALE" + "STALE", + "ALLOW_BUSY" ] } } diff --git a/src/commands/replicaof.json b/src/commands/replicaof.json index 90ec59fed..805b81e4c 100644 --- a/src/commands/replicaof.json +++ b/src/commands/replicaof.json @@ -9,7 +9,6 @@ "command_flags": [ "NO_ASYNC_LOADING", "ADMIN", - "ALLOW_BUSY", "NOSCRIPT", "STALE" ], diff --git a/src/commands/sentinel-debug.json b/src/commands/sentinel-debug.json index ed771c4f7..44c6bec9b 100644 --- a/src/commands/sentinel-debug.json +++ b/src/commands/sentinel-debug.json @@ -1,5 +1,7 @@ { "DEBUG": { + "summary": "List or update the current configurable parameters", + "complexity": "O(N) where N is the number of configurable parameters", "group": "sentinel", "since": "7.0.0", "arity": -2, diff --git a/src/commands/slaveof.json b/src/commands/slaveof.json index f9db6afbb..271eb2d1b 100644 --- a/src/commands/slaveof.json +++ b/src/commands/slaveof.json @@ -1,11 +1,16 @@ { "SLAVEOF": { - "summary": "Make the server a replica of another instance, or promote it as master. Deprecated starting with Redis 5. Use REPLICAOF instead.", + "summary": "Make the server a replica of another instance, or promote it as master.", "complexity": "O(1)", "group": "server", "since": "1.0.0", "arity": 3, "function": "replicaofCommand", + "deprecated_since": "5.0.0", + "replaced_by": "`REPLICAOF`", + "doc_flags": [ + "DEPRECATED" + ], "command_flags": [ "NO_ASYNC_LOADING", "ADMIN", diff --git a/src/commands/spublish.json b/src/commands/spublish.json index e6c4fb062..2cbcdc19a 100644 --- a/src/commands/spublish.json +++ b/src/commands/spublish.json @@ -26,7 +26,7 @@ "key_specs": [ { "flags": [ - "CHANNEL" + "NOT_KEY" ], "begin_search": { "index": { diff --git a/src/commands/ssubscribe.json b/src/commands/ssubscribe.json index dd1070046..eb570ea53 100644 --- a/src/commands/ssubscribe.json +++ b/src/commands/ssubscribe.json @@ -22,7 +22,7 @@ "key_specs": [ { "flags": [ - "CHANNEL" + "NOT_KEY" ], "begin_search": { "index": { diff --git a/src/commands/sunsubscribe.json b/src/commands/sunsubscribe.json index f7271825e..481415490 100644 --- a/src/commands/sunsubscribe.json +++ b/src/commands/sunsubscribe.json @@ -23,7 +23,7 @@ "key_specs": [ { "flags": [ - "CHANNEL" + "NOT_KEY" ], "begin_search": { "index": { diff --git a/src/commands/xautoclaim.json b/src/commands/xautoclaim.json index b951eac29..726bf38fe 100644 --- a/src/commands/xautoclaim.json +++ b/src/commands/xautoclaim.json @@ -6,6 +6,12 @@ "since": "6.2.0", "arity": -6, "function": "xautoclaimCommand", + "history": [ + [ + "7.0.0", + "Added an element to the reply array, containing deleted entries the command cleared from the PEL" + ] + ], "command_flags": [ "WRITE", "FAST" diff --git a/src/commands/xgroup-create.json b/src/commands/xgroup-create.json index a14c0e577..2b1ee03b4 100644 --- a/src/commands/xgroup-create.json +++ b/src/commands/xgroup-create.json @@ -7,6 +7,12 @@ "arity": -5, "container": "XGROUP", "function": "xgroupCommand", + "history": [ + [ + "7.0.0", + "Added the `entries_read` named argument." + ] + ], "command_flags": [ "WRITE", "DENYOOM" @@ -64,6 +70,12 @@ "name": "mkstream", "type": "pure-token", "optional": true + }, + { + "token": "ENTRIESREAD", + "name": "entries_read", + "type": "integer", + "optional": true } ] } diff --git a/src/commands/xgroup-setid.json b/src/commands/xgroup-setid.json index 5c2ffcf19..af4b83c19 100644 --- a/src/commands/xgroup-setid.json +++ b/src/commands/xgroup-setid.json @@ -4,9 +4,15 @@ "complexity": "O(1)", "group": "stream", "since": "5.0.0", - "arity": 5, + "arity": -5, "container": "XGROUP", "function": "xgroupCommand", + "history": [ + [ + "7.0.0", + "Added the optional `entries_read` argument." + ] + ], "command_flags": [ "WRITE" ], @@ -57,6 +63,12 @@ "token": "$" } ] + }, + { + "name": "entries_read", + "token": "ENTRIESREAD", + "type": "integer", + "optional": true } ] } diff --git a/src/commands/xinfo-groups.json b/src/commands/xinfo-groups.json index 546d2030e..e9b61ba06 100644 --- a/src/commands/xinfo-groups.json +++ b/src/commands/xinfo-groups.json @@ -6,6 +6,12 @@ "since": "5.0.0", "arity": 3, "container": "XINFO", + "history": [ + [ + "7.0.0", + "Added the `entries-read` and `lag` fields" + ] + ], "function": "xinfoCommand", "command_flags": [ "READONLY" diff --git a/src/commands/xinfo-stream.json b/src/commands/xinfo-stream.json index 43ae9bc8b..5b7d9ad57 100644 --- a/src/commands/xinfo-stream.json +++ b/src/commands/xinfo-stream.json @@ -6,6 +6,12 @@ "since": "5.0.0", "arity": -3, "container": "XINFO", + "history": [ + [ + "7.0.0", + "Added the `max-deleted-entry-id`, `entries-added`, `recorded-first-entry-id`, `entries-read` and `lag` fields" + ] + ], "function": "xinfoCommand", "command_flags": [ "READONLY" diff --git a/src/commands/xsetid.json b/src/commands/xsetid.json index 8faa2c7e9..7654784e1 100644 --- a/src/commands/xsetid.json +++ b/src/commands/xsetid.json @@ -4,8 +4,14 @@ "complexity": "O(1)", "group": "stream", "since": "5.0.0", - "arity": 3, + "arity": -3, "function": "xsetidCommand", + "history": [ + [ + "7.0.0", + "Added the `entries_added` and `max_deleted_entry_id` arguments." + ] + ], "command_flags": [ "WRITE", "DENYOOM", @@ -43,6 +49,18 @@ { "name": "last-id", "type": "string" + }, + { + "name": "entries_added", + "token": "ENTRIESADDED", + "type": "integer", + "optional": true + }, + { + "name": "max_deleted_entry_id", + "token": "MAXDELETEDID", + "type": "string", + "optional": true } ] } diff --git a/src/connection.c b/src/connection.c index 3a17d983d..11fc4ba28 100644 --- a/src/connection.c +++ b/src/connection.c @@ -178,6 +178,21 @@ static int connSocketWrite(connection *conn, const void *data, size_t data_len) return ret; } +static int connSocketWritev(connection *conn, const struct iovec *iov, int iovcnt) { + int ret = writev(conn->fd, iov, iovcnt); + if (ret < 0 && errno != EAGAIN) { + conn->last_errno = errno; + + /* Don't overwrite the state of a connection that is not already + * connected, not to mess with handler callbacks. + */ + if (errno != EINTR && conn->state == CONN_STATE_CONNECTED) + conn->state = CONN_STATE_ERROR; + } + + return ret; +} + static int connSocketRead(connection *conn, void *buf, size_t buf_len) { int ret = read(conn->fd, buf, buf_len); if (!ret) { @@ -349,6 +364,7 @@ ConnectionType CT_Socket = { .ae_handler = connSocketEventHandler, .close = connSocketClose, .write = connSocketWrite, + .writev = connSocketWritev, .read = connSocketRead, .accept = connSocketAccept, .connect = connSocketConnect, diff --git a/src/connection.h b/src/connection.h index 07c1d4dd8..dad2e2fd6 100644 --- a/src/connection.h +++ b/src/connection.h @@ -32,6 +32,7 @@ #define __REDIS_CONNECTION_H #include <errno.h> +#include <sys/uio.h> #define CONN_INFO_LEN 32 @@ -59,6 +60,7 @@ typedef struct ConnectionType { void (*ae_handler)(struct aeEventLoop *el, int fd, void *clientData, int mask); int (*connect)(struct connection *conn, const char *addr, int port, const char *source_addr, ConnectionCallbackFunc connect_handler); int (*write)(struct connection *conn, const void *data, size_t data_len); + int (*writev)(struct connection *conn, const struct iovec *iov, int iovcnt); int (*read)(struct connection *conn, void *buf, size_t buf_len); void (*close)(struct connection *conn); int (*accept)(struct connection *conn, ConnectionCallbackFunc accept_handler); @@ -142,6 +144,18 @@ static inline int connWrite(connection *conn, const void *data, size_t data_len) return conn->type->write(conn, data, data_len); } +/* Gather output data from the iovcnt buffers specified by the members of the iov + * array: iov[0], iov[1], ..., iov[iovcnt-1] and write to connection, behaves the same as writev(3). + * + * Like writev(3), a short write is possible. A -1 return indicates an error. + * + * The caller should NOT rely on errno. Testing for an EAGAIN-like condition, use + * connGetState() to see if the connection state is still CONN_STATE_CONNECTED. + */ +static inline int connWritev(connection *conn, const struct iovec *iov, int iovcnt) { + return conn->type->writev(conn, iov, iovcnt); +} + /* Read from the connection, behaves the same as read(2). * * Like read(2), a short read is possible. A return value of 0 will indicate the @@ -83,16 +83,16 @@ robj *lookupKey(redisDb *db, robj *key, int flags) { robj *val = NULL; if (de) { val = dictGetVal(de); - int force_delete_expired = flags & LOOKUP_WRITE; - if (force_delete_expired) { - /* Forcing deletion of expired keys on a replica makes the replica - * inconsistent with the master. The reason it's allowed for write - * commands is to make writable replicas behave consistently. It - * shall not be used in readonly commands. Modules are accepted so - * that we don't break old modules. */ - client *c = server.in_script ? scriptGetClient() : server.current_client; - serverAssert(!c || !c->cmd || (c->cmd->flags & (CMD_WRITE|CMD_MODULE))); - } + /* Forcing deletion of expired keys on a replica makes the replica + * inconsistent with the master. We forbid it on readonly replicas, but + * we have to allow it on writable replicas to make write commands + * behave consistently. + * + * It's possible that the WRITE flag is set even during a readonly + * command, since the command may trigger events that cause modules to + * perform additional writes. */ + int is_ro_replica = server.masterhost && server.repl_slave_ro; + int force_delete_expired = flags & LOOKUP_WRITE && !is_ro_replica; if (expireIfNeeded(db, key, force_delete_expired)) { /* The key is no longer valid. */ val = NULL; @@ -1340,6 +1340,11 @@ int dbSwapDatabases(int id1, int id2) { redisDb aux = server.db[id1]; redisDb *db1 = &server.db[id1], *db2 = &server.db[id2]; + /* Swapdb should make transaction fail if there is any + * client watching keys */ + touchAllWatchedKeysInDb(db1, db2); + touchAllWatchedKeysInDb(db2, db1); + /* Swap hash tables. Note that we don't swap blocking_keys, * ready_keys and watched_keys, since we want clients to * remain in the same DB they were. */ @@ -1361,14 +1366,9 @@ int dbSwapDatabases(int id1, int id2) { * However normally we only do this check for efficiency reasons * in dbAdd() when a list is created. So here we need to rescan * the list of clients blocked on lists and signal lists as ready - * if needed. - * - * Also the swapdb should make transaction fail if there is any - * client watching keys */ + * if needed. */ scanDatabaseForReadyLists(db1); - touchAllWatchedKeysInDb(db1, db2); scanDatabaseForReadyLists(db2); - touchAllWatchedKeysInDb(db2, db1); return C_OK; } @@ -1387,6 +1387,10 @@ void swapMainDbWithTempDb(redisDb *tempDb) { redisDb aux = server.db[i]; redisDb *activedb = &server.db[i], *newdb = &tempDb[i]; + /* Swapping databases should make transaction fail if there is any + * client watching keys. */ + touchAllWatchedKeysInDb(activedb, newdb); + /* Swap hash tables. Note that we don't swap blocking_keys, * ready_keys and watched_keys, since clients * remain in the same DB they were. */ @@ -1408,12 +1412,8 @@ void swapMainDbWithTempDb(redisDb *tempDb) { * However normally we only do this check for efficiency reasons * in dbAdd() when a list is created. So here we need to rescan * the list of clients blocked on lists and signal lists as ready - * if needed. - * - * Also the swapdb should make transaction fail if there is any - * client watching keys. */ + * if needed. */ scanDatabaseForReadyLists(activedb); - touchAllWatchedKeysInDb(activedb, newdb); } trackingInvalidateKeysOnFlush(1); @@ -1692,7 +1692,7 @@ int64_t getAllKeySpecsFlags(struct redisCommand *cmd, int inv) { /* Fetch the keys based of the provided key specs. Returns the number of keys found, or -1 on error. * There are several flags that can be used to modify how this function finds keys in a command. * - * GET_KEYSPEC_INCLUDE_CHANNELS: Return channels as if they were keys. + * GET_KEYSPEC_INCLUDE_NOT_KEYS: Return 'fake' keys as if they were keys. * GET_KEYSPEC_RETURN_PARTIAL: Skips invalid and incomplete keyspecs but returns the keys * found in other valid keyspecs. */ @@ -1703,8 +1703,8 @@ int getKeysUsingKeySpecs(struct redisCommand *cmd, robj **argv, int argc, int se for (j = 0; j < cmd->key_specs_num; j++) { keySpec *spec = cmd->key_specs + j; serverAssert(spec->begin_search_type != KSPEC_BS_INVALID); - /* Skip specs that represent channels instead of keys */ - if ((spec->flags & CMD_KEY_CHANNEL) && !(search_flags & GET_KEYSPEC_INCLUDE_CHANNELS)) { + /* Skip specs that represent 'fake' keys */ + if ((spec->flags & CMD_KEY_NOT_KEY) && !(search_flags & GET_KEYSPEC_INCLUDE_NOT_KEYS)) { continue; } @@ -1821,31 +1821,123 @@ invalid_spec: * associated with how Redis will access the key. * * 'cmd' must be point to the corresponding entry into the redisCommand - * table, according to the command name in argv[0]. - * - * This function uses the command's key specs, which contain the key-spec flags, - * (e.g. RO / RW) and only resorts to the command-specific helper function if - * any of the keys-specs are marked as INCOMPLETE. */ + * table, according to the command name in argv[0]. */ int getKeysFromCommandWithSpecs(struct redisCommand *cmd, robj **argv, int argc, int search_flags, getKeysResult *result) { - if (cmd->flags & CMD_MODULE_GETKEYS) { + /* The command has at least one key-spec not marked as NOT_KEY */ + int has_keyspec = (getAllKeySpecsFlags(cmd, 1) & CMD_KEY_NOT_KEY); + /* The command has at least one key-spec marked as VARIABLE_FLAGS */ + int has_varflags = (getAllKeySpecsFlags(cmd, 0) & CMD_KEY_VARIABLE_FLAGS); + + /* Flags indicating that we have a getkeys callback */ + int has_module_getkeys = cmd->flags & CMD_MODULE_GETKEYS; + int has_native_getkeys = !(cmd->flags & CMD_MODULE) && cmd->getkeys_proc; + + /* The key-spec that's auto generated by RM_CreateCommand sets VARIABLE_FLAGS since no flags are given. + * If the module provides getkeys callback, we'll prefer it, but if it didn't, we'll use key-spec anyway. */ + if ((cmd->flags & CMD_MODULE) && has_varflags && !has_module_getkeys) + has_varflags = 0; + + /* We prefer key-specs if there are any, and their flags are reliable. */ + if (has_keyspec && !has_varflags) { + int ret = getKeysUsingKeySpecs(cmd,argv,argc,search_flags,result); + if (ret >= 0) + return ret; + /* If the specs returned with an error (probably an INVALID or INCOMPLETE spec), + * fallback to the callback method. */ + } + + /* Resort to getkeys callback methods. */ + if (has_module_getkeys) return moduleGetCommandKeysViaAPI(cmd,argv,argc,result); - } else { - if (!(getAllKeySpecsFlags(cmd, 0) & CMD_KEY_VARIABLE_FLAGS)) { - int ret = getKeysUsingKeySpecs(cmd,argv,argc,search_flags,result); - if (ret >= 0) - return ret; - } - if (!(cmd->flags & CMD_MODULE) && cmd->getkeys_proc) - return cmd->getkeys_proc(cmd,argv,argc,result); - return 0; - } + + /* We use native getkeys as a last resort, since not all these native getkeys provide + * flags properly (only the ones that correspond to INVALID, INCOMPLETE or VARIABLE_FLAGS do.*/ + if (has_native_getkeys) + return cmd->getkeys_proc(cmd,argv,argc,result); + return 0; } /* This function returns a sanity check if the command may have keys. */ int doesCommandHaveKeys(struct redisCommand *cmd) { return (!(cmd->flags & CMD_MODULE) && cmd->getkeys_proc) || /* has getkeys_proc (non modules) */ (cmd->flags & CMD_MODULE_GETKEYS) || /* module with GETKEYS */ - (getAllKeySpecsFlags(cmd, 1) & CMD_KEY_CHANNEL); /* has at least one key-spec not marked as CHANNEL */ + (getAllKeySpecsFlags(cmd, 1) & CMD_KEY_NOT_KEY); /* has at least one key-spec not marked as NOT_KEY */ +} + +/* A simplified channel spec table that contains all of the redis commands + * and which channels they have and how they are accessed. */ +typedef struct ChannelSpecs { + redisCommandProc *proc; /* Command procedure to match against */ + uint64_t flags; /* CMD_CHANNEL_* flags for this command */ + int start; /* The initial position of the first channel */ + int count; /* The number of channels, or -1 if all remaining + * arguments are channels. */ +} ChannelSpecs; + +ChannelSpecs commands_with_channels[] = { + {subscribeCommand, CMD_CHANNEL_SUBSCRIBE, 1, -1}, + {ssubscribeCommand, CMD_CHANNEL_SUBSCRIBE, 1, -1}, + {unsubscribeCommand, CMD_CHANNEL_UNSUBSCRIBE, 1, -1}, + {sunsubscribeCommand, CMD_CHANNEL_UNSUBSCRIBE, 1, -1}, + {psubscribeCommand, CMD_CHANNEL_PATTERN | CMD_CHANNEL_SUBSCRIBE, 1, -1}, + {punsubscribeCommand, CMD_CHANNEL_PATTERN | CMD_CHANNEL_UNSUBSCRIBE, 1, -1}, + {publishCommand, CMD_CHANNEL_PUBLISH, 1, 1}, + {spublishCommand, CMD_CHANNEL_PUBLISH, 1, 1}, + {NULL,0} /* Terminator. */ +}; + +/* Returns 1 if the command may access any channels matched by the flags + * argument. */ +int doesCommandHaveChannelsWithFlags(struct redisCommand *cmd, int flags) { + /* If a module declares get channels, we are just going to assume + * has channels. This API is allowed to return false positives. */ + if (cmd->flags & CMD_MODULE_GETCHANNELS) { + return 1; + } + for (ChannelSpecs *spec = commands_with_channels; spec->proc != NULL; spec += 1) { + if (cmd->proc == spec->proc) { + return !!(spec->flags & flags); + } + } + return 0; +} + +/* Return all the arguments that are channels in the command passed via argc / argv. + * This function behaves similar to getKeysFromCommandWithSpecs, but with channels + * instead of keys. + * + * The command returns the positions of all the channel arguments inside the array, + * so the actual return value is a heap allocated array of integers. The + * length of the array is returned by reference into *numkeys. + * + * Along with the position, this command also returns the flags that are + * associated with how Redis will access the channel. + * + * 'cmd' must be point to the corresponding entry into the redisCommand + * table, according to the command name in argv[0]. */ +int getChannelsFromCommand(struct redisCommand *cmd, robj **argv, int argc, getKeysResult *result) { + keyReference *keys; + /* If a module declares get channels, use that. */ + if (cmd->flags & CMD_MODULE_GETCHANNELS) { + return moduleGetCommandChannelsViaAPI(cmd, argv, argc, result); + } + /* Otherwise check the channel spec table */ + for (ChannelSpecs *spec = commands_with_channels; spec != NULL; spec += 1) { + if (cmd->proc == spec->proc) { + int start = spec->start; + int stop = (spec->count == -1) ? argc : start + spec->count; + if (stop > argc) stop = argc; + int count = 0; + keys = getKeysPrepareResult(result, stop - start); + for (int i = start; i < stop; i++ ) { + keys[count].pos = i; + keys[count++].flags = spec->flags; + } + result->numkeys = count; + return count; + } + } + return 0; } /* The base case is to use the keys position as given in the command table diff --git a/src/debug.c b/src/debug.c index 2da2c5d50..b95daaa36 100644 --- a/src/debug.c +++ b/src/debug.c @@ -482,6 +482,10 @@ void debugCommand(client *c) { " Show low level client eviction pools info (maxmemory-clients).", "PAUSE-CRON <0|1>", " Stop periodic cron job processing.", +"REPLYBUFFER-PEAK-RESET-TIME <NEVER||RESET|time>", +" Sets the time (in milliseconds) to wait between client reply buffer peak resets.", +" In case NEVER is provided the last observed peak will never be reset", +" In case RESET is provided the peak reset time will be restored to the default value", NULL }; addReplyHelp(c, help); @@ -825,7 +829,7 @@ NULL int memerr; unsigned long long sz = memtoull((const char *)c->argv[2]->ptr, &memerr); if (memerr || !quicklistisSetPackedThreshold(sz)) { - addReplyError(c, "argument must be a memory value bigger then 1 and smaller than 4gb"); + addReplyError(c, "argument must be a memory value bigger than 1 and smaller than 4gb"); } else { addReply(c,shared.ok); } @@ -921,12 +925,12 @@ NULL addReplyStatus(c,"Apparently Redis did not crash: test passed"); } else if (!strcasecmp(c->argv[1]->ptr,"set-disable-deny-scripts") && c->argc == 3) { - server.script_disable_deny_script = atoi(c->argv[2]->ptr);; + server.script_disable_deny_script = atoi(c->argv[2]->ptr); addReply(c,shared.ok); } else if (!strcasecmp(c->argv[1]->ptr,"config-rewrite-force-all") && c->argc == 2) { if (rewriteConfig(server.configfile, 1) == -1) - addReplyError(c, "CONFIG-REWRITE-FORCE-ALL failed"); + addReplyErrorFormat(c, "CONFIG-REWRITE-FORCE-ALL failed: %s", strerror(errno)); else addReply(c, shared.ok); } else if(!strcasecmp(c->argv[1]->ptr,"client-eviction") && c->argc == 2) { @@ -958,6 +962,16 @@ NULL { server.pause_cron = atoi(c->argv[2]->ptr); addReply(c,shared.ok); + } else if (!strcasecmp(c->argv[1]->ptr,"replybuffer-peak-reset-time") && c->argc == 3 ) { + if (!strcasecmp(c->argv[2]->ptr, "never")) { + server.reply_buffer_peak_reset_time = -1; + } else if(!strcasecmp(c->argv[2]->ptr, "reset")) { + server.reply_buffer_peak_reset_time = REPLY_BUFFER_DEFAULT_PEAK_RESET_TIME; + } else { + if (getLongFromObjectOrReply(c, c->argv[2], &server.reply_buffer_peak_reset_time, NULL) != C_OK) + return; + } + addReply(c, shared.ok); } else { addReplySubcommandSyntaxError(c); return; @@ -1681,13 +1695,19 @@ void logStackTrace(void *eip, int uplevel) { void logServerInfo(void) { sds infostring, clients; serverLogRaw(LL_WARNING|LL_RAW, "\n------ INFO OUTPUT ------\n"); - infostring = genRedisInfoString("all"); + int all = 0, everything = 0; + robj *argv[1]; + argv[0] = createStringObject("all", strlen("all")); + dict *section_dict = genInfoSectionDict(argv, 1, NULL, &all, &everything); + infostring = genRedisInfoString(section_dict, all, everything); serverLogRaw(LL_WARNING|LL_RAW, infostring); serverLogRaw(LL_WARNING|LL_RAW, "\n------ CLIENT LIST OUTPUT ------\n"); clients = getAllClientsInfoString(-1); serverLogRaw(LL_WARNING|LL_RAW, clients); sdsfree(infostring); sdsfree(clients); + releaseInfoSectionDict(section_dict); + decrRefCount(argv[0]); } /* Log certain config values, which can be used for debuggin */ @@ -1723,10 +1743,10 @@ void logCurrentClient(void) { sdsfree(client); for (j = 0; j < cc->argc; j++) { robj *decoded; - decoded = getDecodedObject(cc->argv[j]); - serverLog(LL_WARNING|LL_RAW,"argv[%d]: '%s'\n", j, - (char*)decoded->ptr); + sds repr = sdscatrepr(sdsempty(),decoded->ptr, min(sdslen(decoded->ptr), 128)); + serverLog(LL_WARNING|LL_RAW,"argv[%d]: '%s'\n", j, (char*)repr); + sdsfree(repr); decrRefCount(decoded); } /* Check if the first argument, usually a key, is found inside the @@ -1764,7 +1784,10 @@ int memtest_test_linux_anonymous_maps(void) { if (!fd) return 0; fp = fopen("/proc/self/maps","r"); - if (!fp) return 0; + if (!fp) { + closeDirectLogFiledes(fd); + return 0; + } while(fgets(line,sizeof(line),fp) != NULL) { char *start, *end, *p = line; diff --git a/src/debugmacro.h b/src/debugmacro.h index 58e6577e5..dcd79a33f 100644 --- a/src/debugmacro.h +++ b/src/debugmacro.h @@ -30,6 +30,9 @@ * POSSIBILITY OF SUCH DAMAGE. */ +#ifndef _REDIS_DEBUGMACRO_H_ +#define _REDIS_DEBUGMACRO_H_ + #include <stdio.h> #define D(...) \ do { \ @@ -39,3 +42,5 @@ fprintf(fp,"\n"); \ fclose(fp); \ } while (0) + +#endif /* _REDIS_DEBUGMACRO_H_ */ diff --git a/src/defrag.c b/src/defrag.c index 16b1a4d2d..d4983c6d5 100644 --- a/src/defrag.c +++ b/src/defrag.c @@ -128,6 +128,27 @@ robj *activeDefragStringOb(robj* ob, long *defragged) { return ret; } +/* Defrag helper for lua scripts + * + * returns NULL in case the allocation wasn't moved. + * when it returns a non-null value, the old pointer was already released + * and should NOT be accessed. */ +luaScript *activeDefragLuaScript(luaScript *script, long *defragged) { + luaScript *ret = NULL; + + /* try to defrag script struct */ + if ((ret = activeDefragAlloc(script))) { + script = ret; + (*defragged)++; + } + + /* try to defrag actual script object */ + robj *ob = activeDefragStringOb(script->body, defragged); + if (ob) script->body = ob; + + return ret; +} + /* Defrag helper for dictEntries to be used during dict iteration (called on * each step). Returns a stat of how many pointers were moved. */ long dictIterDefragEntry(dictIterator *iter) { @@ -256,6 +277,7 @@ long activeDefragZsetEntry(zset *zs, dictEntry *de) { #define DEFRAG_SDS_DICT_VAL_IS_SDS 1 #define DEFRAG_SDS_DICT_VAL_IS_STROB 2 #define DEFRAG_SDS_DICT_VAL_VOID_PTR 3 +#define DEFRAG_SDS_DICT_VAL_LUA_SCRIPT 4 /* Defrag a dict with sds key and optional value (either ptr, sds or robj string) */ long activeDefragSdsDict(dict* d, int val_type) { @@ -280,6 +302,10 @@ long activeDefragSdsDict(dict* d, int val_type) { void *newptr, *ptr = dictGetVal(de); if ((newptr = activeDefragAlloc(ptr))) de->v.val = newptr, defragged++; + } else if (val_type == DEFRAG_SDS_DICT_VAL_LUA_SCRIPT) { + void *newptr, *ptr = dictGetVal(de); + if ((newptr = activeDefragLuaScript(ptr, &defragged))) + de->v.val = newptr; } defragged += dictIterDefragEntry(di); } @@ -939,7 +965,7 @@ long defragOtherGlobals() { /* there are many more pointers to defrag (e.g. client argv, output / aof buffers, etc. * but we assume most of these are short lived, we only need to defrag allocations * that remain static for a long time */ - defragged += activeDefragSdsDict(evalScriptsDict(), DEFRAG_SDS_DICT_VAL_IS_STROB); + defragged += activeDefragSdsDict(evalScriptsDict(), DEFRAG_SDS_DICT_VAL_LUA_SCRIPT); defragged += moduleDefragGlobals(); return defragged; } @@ -1130,7 +1156,7 @@ void activeDefragCycle(void) { /* Move on to next database, and stop if we reached the last one. */ if (++current_db >= server.dbnum) { /* defrag other items not part of the db / keys */ - defragOtherGlobals(); + server.stat_active_defrag_hits += defragOtherGlobals(); long long now = ustime(); size_t frag_bytes; diff --git a/src/eval.c b/src/eval.c index 41dc3b611..1a9437a09 100644 --- a/src/eval.c +++ b/src/eval.c @@ -47,11 +47,6 @@ void ldbEnable(client *c); void evalGenericCommandWithDebugging(client *c, int evalsha); sds ldbCatStackValue(sds s, lua_State *lua, int idx); -typedef struct luaScript { - uint64_t flags; - robj *body; -} luaScript; - static void dictLuaScriptDestructor(dict *d, void *val) { UNUSED(d); if (val == NULL) return; /* Lazy freeing will set value to NULL. */ @@ -63,7 +58,7 @@ static uint64_t dictStrCaseHash(const void *key) { return dictGenCaseHashFunction((unsigned char*)key, strlen((char*)key)); } -/* server.lua_scripts sha (as sds string) -> scripts (as robj) cache. */ +/* server.lua_scripts sha (as sds string) -> scripts (as luaScript) cache. */ dictType shaScriptObjectDictType = { dictStrCaseHash, /* hash function */ NULL, /* key dup */ @@ -246,11 +241,14 @@ void scriptingInit(int setup) { " if i and i.what == 'C' then\n" " i = dbg.getinfo(3,'nSl')\n" " end\n" + " if type(err) ~= 'table' then\n" + " err = {err='ERR' .. tostring(err)}" + " end" " if i then\n" - " return i.source .. ':' .. i.currentline .. ': ' .. err\n" - " else\n" - " return err\n" - " end\n" + " err['source'] = i.source\n" + " err['line'] = i.currentline\n" + " end" + " return err\n" "end\n"; luaL_loadbuffer(lua,errh_func,strlen(errh_func),"@err_handler_def"); lua_pcall(lua,0,0,0); @@ -392,7 +390,7 @@ sds luaCreateFunction(client *c, robj *body) { if (luaL_loadbuffer(lctx.lua,funcdef,sdslen(funcdef),"@user_script")) { if (c != NULL) { addReplyErrorFormat(c, - "Error compiling script (new function): %s\n", + "Error compiling script (new function): %s", lua_tostring(lctx.lua,-1)); } lua_pop(lctx.lua,1); @@ -403,7 +401,7 @@ sds luaCreateFunction(client *c, robj *body) { if (lua_pcall(lctx.lua,0,0,0)) { if (c != NULL) { - addReplyErrorFormat(c,"Error running script (new function): %s\n", + addReplyErrorFormat(c,"Error running script (new function): %s", lua_tostring(lctx.lua,-1)); } lua_pop(lctx.lua,1); @@ -1479,8 +1477,8 @@ int ldbRepl(lua_State *lua) { while((argv = ldbReplParseCommand(&argc, &err)) == NULL) { char buf[1024]; if (err) { - lua_pushstring(lua, err); - lua_error(lua); + luaPushError(lua, err); + luaError(lua); } int nread = connRead(ldb.conn,buf,sizeof(buf)); if (nread <= 0) { @@ -1497,8 +1495,8 @@ int ldbRepl(lua_State *lua) { if (sdslen(ldb.cbuf) > 1<<20) { sdsfree(ldb.cbuf); ldb.cbuf = sdsempty(); - lua_pushstring(lua, "max client buffer reached"); - lua_error(lua); + luaPushError(lua, "max client buffer reached"); + luaError(lua); } } @@ -1558,8 +1556,8 @@ ldbLog(sdsnew(" next line of code.")); ldbEval(lua,argv,argc); ldbSendLogs(); } else if (!strcasecmp(argv[0],"a") || !strcasecmp(argv[0],"abort")) { - lua_pushstring(lua, "script aborted for user request"); - lua_error(lua); + luaPushError(lua, "script aborted for user request"); + luaError(lua); } else if (argc > 1 && (!strcasecmp(argv[0],"r") || !strcasecmp(argv[0],"redis"))) { ldbRedis(lua,argv,argc); @@ -1640,8 +1638,8 @@ void luaLdbLineHook(lua_State *lua, lua_Debug *ar) { /* If the client closed the connection and we have a timeout * connection, let's kill the script otherwise the process * will remain blocked indefinitely. */ - lua_pushstring(lua, "timeout during Lua debugging with client closing connection"); - lua_error(lua); + luaPushError(lua, "timeout during Lua debugging with client closing connection"); + luaError(lua); } rctx->start_time = getMonotonicUs(); rctx->snapshot_time = mstime(); diff --git a/src/function_lua.c b/src/function_lua.c index 3dbc8419e..8f21a1721 100644 --- a/src/function_lua.c +++ b/src/function_lua.c @@ -86,8 +86,8 @@ static void luaEngineLoadHook(lua_State *lua, lua_Debug *ar) { if (duration > LOAD_TIMEOUT_MS) { lua_sethook(lua, luaEngineLoadHook, LUA_MASKLINE, 0); - lua_pushstring(lua,"FUNCTION LOAD timeout"); - lua_error(lua); + luaPushError(lua,"FUNCTION LOAD timeout"); + luaError(lua); } } @@ -151,10 +151,13 @@ static int luaEngineCreate(void *engine_ctx, functionLibInfo *li, sds blob, sds lua_sethook(lua,luaEngineLoadHook,LUA_MASKCOUNT,100000); /* Run the compiled code to allow it to register functions */ if (lua_pcall(lua,0,0,0)) { - *err = sdscatprintf(sdsempty(), "Error registering functions: %s", lua_tostring(lua, -1)); + errorInfo err_info = {0}; + luaExtractErrorInformation(lua, &err_info); + *err = sdscatprintf(sdsempty(), "Error registering functions: %s", err_info.msg); lua_pop(lua, 2); /* pops the error and globals table */ lua_sethook(lua,NULL,0,0); /* Disable hook */ luaSaveOnRegistry(lua, REGISTRY_LOAD_CTX_NAME, NULL); + luaErrorInformationDiscard(&err_info); return C_ERR; } lua_sethook(lua,NULL,0,0); /* Disable hook */ @@ -429,11 +432,11 @@ static int luaRegisterFunction(lua_State *lua) { loadCtx *load_ctx = luaGetFromRegistry(lua, REGISTRY_LOAD_CTX_NAME); if (!load_ctx) { luaPushError(lua, "redis.register_function can only be called on FUNCTION LOAD command"); - return luaRaiseError(lua); + return luaError(lua); } if (luaRegisterFunctionReadArgs(lua, ®ister_f_args) != C_OK) { - return luaRaiseError(lua); + return luaError(lua); } sds err = NULL; @@ -441,7 +444,7 @@ static int luaRegisterFunction(lua_State *lua) { luaRegisterFunctionArgsDispose(lua, ®ister_f_args); luaPushError(lua, err); sdsfree(err); - return luaRaiseError(lua); + return luaError(lua); } return 0; @@ -475,11 +478,14 @@ int luaEngineInitEngine() { " if i and i.what == 'C' then\n" " i = dbg.getinfo(3,'nSl')\n" " end\n" + " if type(err) ~= 'table' then\n" + " err = {err='ERR' .. tostring(err)}" + " end" " if i then\n" - " return i.source .. ':' .. i.currentline .. ': ' .. err\n" - " else\n" - " return err\n" - " end\n" + " err['source'] = i.source\n" + " err['line'] = i.currentline\n" + " end" + " return err\n" "end\n" "return error_handler"; luaL_loadbuffer(lua_engine_ctx->lua, errh_func, strlen(errh_func), "@err_handler_def"); diff --git a/src/functions.c b/src/functions.c index 57fb7ec24..739d178aa 100644 --- a/src/functions.c +++ b/src/functions.c @@ -808,7 +808,7 @@ void functionFlushCommand(client *c) { void functionHelpCommand(client *c) { const char *help[] = { -"LOAD <ENGINE NAME> <LIBRARY NAME> [REPLACE] [DESC <LIBRARY DESCRIPTION>] <LIBRARY CODE>", +"LOAD <ENGINE NAME> <LIBRARY NAME> [REPLACE] [DESCRIPTION <LIBRARY DESCRIPTION>] <LIBRARY CODE>", " Create a new library with the given library name and code.", "DELETE <LIBRARY NAME>", " Delete the given library.", diff --git a/src/geohash.h b/src/geohash.h index 8fa324afd..4befb9303 100644 --- a/src/geohash.h +++ b/src/geohash.h @@ -34,7 +34,6 @@ #include <stddef.h> #include <stdint.h> -#include <stdint.h> #if defined(__cplusplus) extern "C" { diff --git a/src/geohash_helper.c b/src/geohash_helper.c index ec4dbd23a..a9224a1dd 100644 --- a/src/geohash_helper.c +++ b/src/geohash_helper.c @@ -91,8 +91,8 @@ uint8_t geohashEstimateStepsByRadius(double range_meters, double lat) { * \-----------------/ -------- \-----------------/ * \ / / \ \ / * \ (long,lat) / / (long,lat) \ \ (long,lat) / - * \ / / \ / \ - * --------- /----------------\ /--------------\ + * \ / / \ / \ + * --------- /----------------\ /---------------\ * Northern Hemisphere Southern Hemisphere Around the equator */ int geohashBoundingBox(GeoShape *shape, double *bounds) { @@ -164,14 +164,14 @@ GeoHashRadius geohashCalculateAreasByShapeWGS84(GeoShape *shape) { geohashDecode(long_range, lat_range, neighbors.east, &east); geohashDecode(long_range, lat_range, neighbors.west, &west); - if (geohashGetDistance(longitude,latitude,longitude,north.latitude.max) - < radius_meters) decrease_step = 1; - if (geohashGetDistance(longitude,latitude,longitude,south.latitude.min) - < radius_meters) decrease_step = 1; - if (geohashGetDistance(longitude,latitude,east.longitude.max,latitude) - < radius_meters) decrease_step = 1; - if (geohashGetDistance(longitude,latitude,west.longitude.min,latitude) - < radius_meters) decrease_step = 1; + if (north.latitude.max < max_lat) + decrease_step = 1; + if (south.latitude.min > min_lat) + decrease_step = 1; + if (east.longitude.max < max_lon) + decrease_step = 1; + if (west.longitude.min > min_lon) + decrease_step = 1; } if (steps > 1 && decrease_step) { diff --git a/src/help.h b/src/help.h index 2bceea2a4..edeb9b26e 100644 --- a/src/help.h +++ b/src/help.h @@ -1,4 +1,4 @@ -/* Automatically generated by ./utils/generate-command-help.rb, do not edit. */ +/* Automatically generated by utils/generate-command-help.rb, do not edit. */ #ifndef __REDIS_HELP_H #define __REDIS_HELP_H @@ -429,6 +429,11 @@ struct commandHelp { "Extract keys given a full Redis command", 9, "2.8.13" }, + { "COMMAND GETKEYSANDFLAGS", + "", + "Extract keys given a full Redis command", + 9, + "7.0.0" }, { "COMMAND HELP", "", "Show helpful text about the different subcommands", @@ -571,12 +576,12 @@ struct commandHelp { "6.2.0" }, { "FCALL", "function numkeys key [key ...] arg [arg ...]", - "PATCH__TBD__38__", + "Invoke a function", 10, "7.0.0" }, { "FCALL_RO", "function numkeys key [key ...] arg [arg ...]", - "PATCH__TBD__7__", + "Invoke a read-only function", 10, "7.0.0" }, { "FLUSHALL", @@ -595,7 +600,7 @@ struct commandHelp { 10, "7.0.0" }, { "FUNCTION DELETE", - "function-name", + "library-name", "Delete a function by name", 10, "7.0.0" }, @@ -625,7 +630,7 @@ struct commandHelp { 10, "7.0.0" }, { "FUNCTION LOAD", - "engine-name library-name [REPLACE] [DESC library-description] function-code", + "engine-name library-name [REPLACE] [DESCRIPTION library-description] function-code", "Create a function with the given arguments (name, code, description)", 10, "7.0.0" }, @@ -820,7 +825,7 @@ struct commandHelp { 1, "2.6.0" }, { "INFO", - "[section]", + "[section [section ...]]", "Get information and statistics about the server", 9, "1.0.0" }, @@ -1180,7 +1185,7 @@ struct commandHelp { 6, "7.0.0" }, { "PUBSUB SHARDNUMSUB", - "", + "[channel [channel ...]]", "Get the count of subscribers for shard channels", 6, "7.0.0" }, @@ -1391,7 +1396,7 @@ struct commandHelp { "1.0.0" }, { "SLAVEOF", "host port", - "Make the server a replica of another instance, or promote it as master. Deprecated starting with Redis 5. Use REPLICAOF instead.", + "Make the server a replica of another instance, or promote it as master.", 9, "1.0.0" }, { "SLOWLOG", @@ -1590,7 +1595,7 @@ struct commandHelp { 14, "5.0.0" }, { "XGROUP CREATE", - "key groupname id|$ [MKSTREAM]", + "key groupname id|$ [MKSTREAM] [ENTRIESREAD entries_read]", "Create a consumer group.", 14, "5.0.0" }, @@ -1615,7 +1620,7 @@ struct commandHelp { 14, "5.0.0" }, { "XGROUP SETID", - "key groupname id|$", + "key groupname id|$ [ENTRIESREAD entries_read]", "Set a consumer group to an arbitrary last delivered ID value.", 14, "5.0.0" }, @@ -1675,7 +1680,7 @@ struct commandHelp { 14, "5.0.0" }, { "XSETID", - "key last-id", + "key last-id [ENTRIESADDED entries_added] [MAXDELETEDID max_deleted_entry_id]", "An internal command for replicating stream values", 14, "5.0.0" }, diff --git a/src/module.c b/src/module.c index 8c44422c6..7130139a6 100644 --- a/src/module.c +++ b/src/module.c @@ -70,7 +70,7 @@ typedef struct RedisModuleInfoCtx { struct RedisModule *module; - const char *requested_section; + dict *requested_sections; sds info; /* info string we collected so far */ int sections; /* number of sections we collected so far */ int in_section; /* indication if we're in an active section or not */ @@ -154,7 +154,8 @@ struct RedisModuleCtx { gets called for clients blocked on keys. */ - /* Used if there is the REDISMODULE_CTX_KEYS_POS_REQUEST flag set. */ + /* Used if there is the REDISMODULE_CTX_KEYS_POS_REQUEST or + * REDISMODULE_CTX_CHANNEL_POS_REQUEST flag set. */ getKeysResult *keys_result; struct RedisModulePoolAllocBlock *pa_head; @@ -173,6 +174,7 @@ typedef struct RedisModuleCtx RedisModuleCtx; when the context is destroyed */ #define REDISMODULE_CTX_NEW_CLIENT (1<<7) /* Free client object when the context is destroyed */ +#define REDISMODULE_CTX_CHANNELS_POS_REQUEST (1<<8) /* This represents a Redis key opened with RM_OpenKey(). */ struct RedisModuleKey { @@ -390,6 +392,7 @@ typedef struct RedisModuleKeyOptCtx { In most cases, only 'from_dbid' is valid, but in callbacks such as `copy2`, 'from_dbid' and 'to_dbid' are both valid. */ } RedisModuleKeyOptCtx; + /* -------------------------------------------------------------------------- * Prototypes * -------------------------------------------------------------------------- */ @@ -404,6 +407,16 @@ static void moduleInitKeyTypeSpecific(RedisModuleKey *key); void RM_FreeDict(RedisModuleCtx *ctx, RedisModuleDict *d); void RM_FreeServerInfo(RedisModuleCtx *ctx, RedisModuleServerInfoData *data); +/* Helpers for RM_SetCommandInfo. */ +static int moduleValidateCommandInfo(const RedisModuleCommandInfo *info); +static int64_t moduleConvertKeySpecsFlags(int64_t flags, int from_api); +static int moduleValidateCommandArgs(RedisModuleCommandArg *args, + const RedisModuleCommandInfoVersion *version); +static struct redisCommandArg *moduleCopyCommandArgs(RedisModuleCommandArg *args, + const RedisModuleCommandInfoVersion *version); +static redisCommandArgType moduleConvertArgType(RedisModuleCommandArgType type, int *error); +static int moduleConvertArgFlags(int flags); + /* -------------------------------------------------------------------------- * ## Heap allocation raw functions * @@ -770,6 +783,25 @@ int moduleGetCommandKeysViaAPI(struct redisCommand *cmd, robj **argv, int argc, return result->numkeys; } +/* This function returns the list of channels, with the same interface as + * moduleGetCommandKeysViaAPI, for modules that declare "getchannels-api" + * during registration. Unlike keys, this is the only way to declare channels. */ +int moduleGetCommandChannelsViaAPI(struct redisCommand *cmd, robj **argv, int argc, getKeysResult *result) { + RedisModuleCommand *cp = (void*)(unsigned long)cmd->getkeys_proc; + RedisModuleCtx ctx; + moduleCreateContext(&ctx, cp->module, REDISMODULE_CTX_CHANNELS_POS_REQUEST); + + /* Initialize getKeysResult */ + getKeysPrepareResult(result, MAX_KEYS_BUFFER); + ctx.keys_result = result; + + cp->func(&ctx,(void**)argv,argc); + /* We currently always use the array allocated by RM_RM_ChannelAtPosWithFlags() and don't try + * to optimize for the pre-allocated buffer. */ + moduleFreeContext(&ctx); + return result->numkeys; +} + /* -------------------------------------------------------------------------- * ## Commands API * @@ -789,17 +821,23 @@ int RM_IsKeysPositionRequest(RedisModuleCtx *ctx) { * keys, since it was flagged as "getkeys-api" during the registration, * the command implementation checks for this special call using the * RedisModule_IsKeysPositionRequest() API and uses this function in - * order to report keys, like in the following example: + * order to report keys. + * + * The supported flags are the ones used by RM_SetCommandInfo, see REDISMODULE_CMD_KEY_*. + * + * + * The following is an example of how it could be used: * * if (RedisModule_IsKeysPositionRequest(ctx)) { - * RedisModule_KeyAtPos(ctx,1); - * RedisModule_KeyAtPos(ctx,2); + * RedisModule_KeyAtPosWithFlags(ctx, 2, REDISMODULE_CMD_KEY_RO | REDISMODULE_CMD_KEY_ACCESS); + * RedisModule_KeyAtPosWithFlags(ctx, 1, REDISMODULE_CMD_KEY_RW | REDISMODULE_CMD_KEY_UPDATE | REDISMODULE_CMD_KEY_ACCESS); * } * - * Note: in the example below the get keys API would not be needed since - * keys are at fixed positions. This interface is only used for commands - * with a more complex structure. */ -void RM_KeyAtPos(RedisModuleCtx *ctx, int pos) { + * Note: in the example above the get keys API could have been handled by key-specs (preferred). + * Implementing the getkeys-api is required only when is it not possible to declare key-specs that cover all keys. + * + */ +void RM_KeyAtPosWithFlags(RedisModuleCtx *ctx, int pos, int flags) { if (!(ctx->flags & REDISMODULE_CTX_KEYS_POS_REQUEST) || !ctx->keys_result) return; if (pos <= 0) return; @@ -811,7 +849,74 @@ void RM_KeyAtPos(RedisModuleCtx *ctx, int pos) { getKeysPrepareResult(res, newsize); } - res->keys[res->numkeys++].pos = pos; + res->keys[res->numkeys].pos = pos; + res->keys[res->numkeys].flags = moduleConvertKeySpecsFlags(flags, 1); + res->numkeys++; +} + +/* This API existed before RM_KeyAtPosWithFlags was added, now deprecated and + * can be used for compatibility with older versions, before key-specs and flags + * were introduced. */ +void RM_KeyAtPos(RedisModuleCtx *ctx, int pos) { + /* Default flags require full access */ + int flags = moduleConvertKeySpecsFlags(CMD_KEY_FULL_ACCESS, 0); + RM_KeyAtPosWithFlags(ctx, pos, flags); +} + +/* Return non-zero if a module command, that was declared with the + * flag "getchannels-api", is called in a special way to get the channel positions + * and not to get executed. Otherwise zero is returned. */ +int RM_IsChannelsPositionRequest(RedisModuleCtx *ctx) { + return (ctx->flags & REDISMODULE_CTX_CHANNELS_POS_REQUEST) != 0; +} + +/* When a module command is called in order to obtain the position of + * channels, since it was flagged as "getchannels-api" during the + * registration, the command implementation checks for this special call + * using the RedisModule_IsChannelsPositionRequest() API and uses this + * function in order to report the channels. + * + * The supported flags are: + * * REDISMODULE_CMD_CHANNEL_SUBSCRIBE: This command will subscribe to the channel. + * * REDISMODULE_CMD_CHANNEL_UNSUBSCRIBE: This command will unsubscribe from this channel. + * * REDISMODULE_CMD_CHANNEL_PUBLISH: This command will publish to this channel. + * * REDISMODULE_CMD_CHANNEL_PATTERN: Instead of acting on a specific channel, will act on any + * channel specified by the pattern. This is the same access + * used by the PSUBSCRIBE and PUNSUBSCRIBE commands available + * in Redis. Not intended to be used with PUBLISH permissions. + * + * The following is an example of how it could be used: + * + * if (RedisModule_IsChannelsPositionRequest(ctx)) { + * RedisModule_ChannelAtPosWithFlags(ctx, 1, REDISMODULE_CMD_CHANNEL_SUBSCRIBE | REDISMODULE_CMD_CHANNEL_PATTERN); + * RedisModule_ChannelAtPosWithFlags(ctx, 1, REDISMODULE_CMD_CHANNEL_PUBLISH); + * } + * + * Note: One usage of declaring channels is for evaluating ACL permissions. In this context, + * unsubscribing is always allowed, so commands will only be checked against subscribe and + * publish permissions. This is preferred over using RM_ACLCheckChannelPermissions, since + * it allows the ACLs to be checked before the command is executed. */ +void RM_ChannelAtPosWithFlags(RedisModuleCtx *ctx, int pos, int flags) { + if (!(ctx->flags & REDISMODULE_CTX_CHANNELS_POS_REQUEST) || !ctx->keys_result) return; + if (pos <= 0) return; + + getKeysResult *res = ctx->keys_result; + + /* Check overflow */ + if (res->numkeys == res->size) { + int newsize = res->size + (res->size > 8192 ? 8192 : res->size); + getKeysPrepareResult(res, newsize); + } + + int new_flags = 0; + if (flags & REDISMODULE_CMD_CHANNEL_SUBSCRIBE) new_flags |= CMD_CHANNEL_SUBSCRIBE; + if (flags & REDISMODULE_CMD_CHANNEL_UNSUBSCRIBE) new_flags |= CMD_CHANNEL_UNSUBSCRIBE; + if (flags & REDISMODULE_CMD_CHANNEL_PUBLISH) new_flags |= CMD_CHANNEL_PUBLISH; + if (flags & REDISMODULE_CMD_CHANNEL_PATTERN) new_flags |= CMD_CHANNEL_PATTERN; + + res->keys[res->numkeys].pos = pos; + res->keys[res->numkeys].flags = new_flags; + res->numkeys++; } /* Helper for RM_CreateCommand(). Turns a string representing command @@ -840,6 +945,7 @@ int64_t commandFlagsFromString(char *s) { else if (!strcasecmp(t,"no-auth")) flags |= CMD_NO_AUTH; else if (!strcasecmp(t,"may-replicate")) flags |= CMD_MAY_REPLICATE; else if (!strcasecmp(t,"getkeys-api")) flags |= CMD_MODULE_GETKEYS; + else if (!strcasecmp(t,"getchannels-api")) flags |= CMD_MODULE_GETCHANNELS; else if (!strcasecmp(t,"no-cluster")) flags |= CMD_MODULE_NO_CLUSTER; else if (!strcasecmp(t,"no-mandatory-keys")) flags |= CMD_NO_MANDATORY_KEYS; else if (!strcasecmp(t,"allow-busy")) flags |= CMD_ALLOW_BUSY; @@ -850,33 +956,6 @@ int64_t commandFlagsFromString(char *s) { return flags; } -/* Helper for RM_CreateCommand(). Turns a string representing keys spec - * flags into the keys spec flags used by the Redis core. - * - * It returns the set of flags, or -1 if unknown flags are found. */ -int64_t commandKeySpecsFlagsFromString(const char *s) { - int count, j; - int64_t flags = 0; - sds *tokens = sdssplitlen(s,strlen(s)," ",1,&count); - for (j = 0; j < count; j++) { - char *t = tokens[j]; - if (!strcasecmp(t,"RO")) flags |= CMD_KEY_RO; - else if (!strcasecmp(t,"RW")) flags |= CMD_KEY_RW; - else if (!strcasecmp(t,"OW")) flags |= CMD_KEY_OW; - else if (!strcasecmp(t,"RM")) flags |= CMD_KEY_RM; - else if (!strcasecmp(t,"access")) flags |= CMD_KEY_ACCESS; - else if (!strcasecmp(t,"insert")) flags |= CMD_KEY_INSERT; - else if (!strcasecmp(t,"update")) flags |= CMD_KEY_UPDATE; - else if (!strcasecmp(t,"delete")) flags |= CMD_KEY_DELETE; - else if (!strcasecmp(t,"channel")) flags |= CMD_KEY_CHANNEL; - else if (!strcasecmp(t,"incomplete")) flags |= CMD_KEY_INCOMPLETE; - else break; - } - sdsfreesplitres(tokens,count); - if (j != count) return -1; /* Some token not processed correctly. */ - return flags; -} - RedisModuleCommand *moduleCreateCommandProxy(struct RedisModule *module, sds declared_name, sds fullname, RedisModuleCmdFunc cmdfunc, int64_t flags, int firstkey, int lastkey, int keystep); /* Register a new command in the Redis server, that will be handled by @@ -946,6 +1025,8 @@ RedisModuleCommand *moduleCreateCommandProxy(struct RedisModule *module, sds dec * * **"allow-busy"**: Permit the command while the server is blocked either by * a script or by a slow module command, see * RM_Yield. + * * **"getchannels-api"**: The command implements the interface to return + * the arguments that are channels. * * The last three parameters specify which arguments of the new command are * Redis keys. See https://redis.io/commands/command for more information. @@ -965,9 +1046,7 @@ RedisModuleCommand *moduleCreateCommandProxy(struct RedisModule *module, sds dec * NOTE: The scheme described above serves a limited purpose and can * only be used to find keys that exist at constant indices. * For non-trivial key arguments, you may pass 0,0,0 and use - * RedisModule_AddCommandKeySpec (see documentation). - * - */ + * RedisModule_SetCommandInfo to set key specs using a more advanced scheme. */ int RM_CreateCommand(RedisModuleCtx *ctx, const char *name, RedisModuleCmdFunc cmdfunc, const char *strflags, int firstkey, int lastkey, int keystep) { int64_t flags = strflags ? commandFlagsFromString((char*)strflags) : 0; if (flags == -1) return REDISMODULE_ERR; @@ -1020,7 +1099,7 @@ RedisModuleCommand *moduleCreateCommandProxy(struct RedisModule *module, sds dec cp->rediscmd->key_specs = cp->rediscmd->key_specs_static; if (firstkey != 0) { cp->rediscmd->key_specs_num = 1; - cp->rediscmd->key_specs[0].flags = 0; + cp->rediscmd->key_specs[0].flags = CMD_KEY_FULL_ACCESS | CMD_KEY_VARIABLE_FLAGS; cp->rediscmd->key_specs[0].begin_search_type = KSPEC_BS_INDEX; cp->rediscmd->key_specs[0].bs.index.pos = firstkey; cp->rediscmd->key_specs[0].find_keys_type = KSPEC_FK_RANGE; @@ -1121,216 +1200,735 @@ int RM_CreateSubcommand(RedisModuleCommand *parent, const char *name, RedisModul return REDISMODULE_OK; } -/* Return `struct RedisModule *` as `void *` to avoid exposing it outside of module.c. */ -void *moduleGetHandleByName(char *modulename) { - return dictFetchValue(modules,modulename); -} +/* Accessors of array elements of structs where the element size is stored + * separately in the version struct. */ +static RedisModuleCommandHistoryEntry * +moduleCmdHistoryEntryAt(const RedisModuleCommandInfoVersion *version, + RedisModuleCommandHistoryEntry *entries, int index) { + off_t offset = index * version->sizeof_historyentry; + return (RedisModuleCommandHistoryEntry *)((char *)(entries) + offset); +} +static RedisModuleCommandKeySpec * +moduleCmdKeySpecAt(const RedisModuleCommandInfoVersion *version, + RedisModuleCommandKeySpec *keyspecs, int index) { + off_t offset = index * version->sizeof_keyspec; + return (RedisModuleCommandKeySpec *)((char *)(keyspecs) + offset); +} +static RedisModuleCommandArg * +moduleCmdArgAt(const RedisModuleCommandInfoVersion *version, + const RedisModuleCommandArg *args, int index) { + off_t offset = index * version->sizeof_arg; + return (RedisModuleCommandArg *)((char *)(args) + offset); +} + +/* Set additional command information. + * + * Affects the output of `COMMAND`, `COMMAND INFO` and `COMMAND DOCS`, Cluster, + * ACL and is used to filter commands with the wrong number of arguments before + * the call reaches the module code. + * + * This function can be called after creating a command using RM_CreateCommand + * and fetching the command pointer using RM_GetCommand. The information can + * only be set once for each command and has the following structure: + * + * typedef struct RedisModuleCommandInfo { + * const RedisModuleCommandInfoVersion *version; + * const char *summary; + * const char *complexity; + * const char *since; + * RedisModuleCommandHistoryEntry *history; + * const char *tips; + * int arity; + * RedisModuleCommandKeySpec *key_specs; + * RedisModuleCommandArg *args; + * } RedisModuleCommandInfo; + * + * All fields except `version` are optional. Explanation of the fields: + * + * - `version`: This field enables compatibility with different Redis versions. + * Always set this field to REDISMODULE_COMMAND_INFO_VERSION. + * + * - `summary`: A short description of the command (optional). + * + * - `complexity`: Complexity description (optional). + * + * - `since`: The version where the command was introduced (optional). + * Note: The version specified should be the module's, not Redis version. + * + * - `history`: An array of RedisModuleCommandHistoryEntry (optional), which is + * a struct with the following fields: + * + * const char *since; + * const char *changes; + * + * `since` is a version string and `changes` is a string describing the + * changes. The array is terminated by a zeroed entry, i.e. an entry with + * both strings set to NULL. + * + * - `tips`: A string of space-separated tips regarding this command, meant for + * clients and proxies. See https://redis.io/topics/command-tips. + * + * - `arity`: Number of arguments, including the command name itself. A positive + * number specifies an exact number of arguments and a negative number + * specifies a minimum number of arguments, so use -N to say >= N. Redis + * validates a call before passing it to a module, so this can replace an + * arity check inside the module command implementation. A value of 0 (or an + * omitted arity field) is equivalent to -2 if the command has sub commands + * and -1 otherwise. + * + * - `key_specs`: An array of RedisModuleCommandKeySpec, terminated by an + * element memset to zero. This is a scheme that tries to describe the + * positions of key arguments better than the old RM_CreateCommand arguments + * `firstkey`, `lastkey`, `keystep` and is needed if those three are not + * enough to describe the key positions. There are two steps to retrieve key + * positions: *begin search* (BS) in which index should find the first key and + * *find keys* (FK) which, relative to the output of BS, describes how can we + * will which arguments are keys. Additionally, there are key specific flags. + * + * Key-specs cause the triplet (firstkey, lastkey, keystep) given in + * RM_CreateCommand to be recomputed, but it is still useful to provide + * these three parameters in RM_CreateCommand, to better support old Redis + * versions where RM_SetCommandInfo is not available. + * + * Note that key-specs don't fully replace the "getkeys-api" (see + * RM_CreateCommand, RM_IsKeysPositionRequest and RM_KeyAtPosWithFlags) so + * it may be a good idea to supply both key-specs and implement the + * getkeys-api. + * + * A key-spec has the following structure: + * + * typedef struct RedisModuleCommandKeySpec { + * const char *notes; + * uint64_t flags; + * RedisModuleKeySpecBeginSearchType begin_search_type; + * union { + * struct { + * int pos; + * } index; + * struct { + * const char *keyword; + * int startfrom; + * } keyword; + * } bs; + * RedisModuleKeySpecFindKeysType find_keys_type; + * union { + * struct { + * int lastkey; + * int keystep; + * int limit; + * } range; + * struct { + * int keynumidx; + * int firstkey; + * int keystep; + * } keynum; + * } fk; + * } RedisModuleCommandKeySpec; + * + * Explanation of the fields of RedisModuleCommandKeySpec: + * + * * `notes`: Optional notes or clarifications about this key spec. + * + * * `flags`: A bitwise or of key-spec flags described below. + * + * * `begin_search_type`: This describes how the first key is discovered. + * There are two ways to determine the first key: + * + * * `REDISMODULE_KSPEC_BS_UNKNOWN`: There is no way to tell where the + * key args start. + * * `REDISMODULE_KSPEC_BS_INDEX`: Key args start at a constant index. + * * `REDISMODULE_KSPEC_BS_KEYWORD`: Key args start just after a + * specific keyword. + * + * * `bs`: This is a union in which the `index` or `keyword` branch is used + * depending on the value of the `begin_search_type` field. + * + * * `bs.index.pos`: The index from which we start the search for keys. + * (`REDISMODULE_KSPEC_BS_INDEX` only.) + * + * * `bs.keyword.keyword`: The keyword (string) that indicates the + * beginning of key arguments. (`REDISMODULE_KSPEC_BS_KEYWORD` only.) + * + * * `bs.keyword.startfrom`: An index in argv from which to start + * searching. Can be negative, which means start search from the end, + * in reverse. Example: -2 means to start in reverse from the + * penultimate argument. (`REDISMODULE_KSPEC_BS_KEYWORD` only.) + * + * * `find_keys_type`: After the "begin search", this describes which + * arguments are keys. The strategies are: + * + * * `REDISMODULE_KSPEC_BS_UNKNOWN`: There is no way to tell where the + * key args are located. + * * `REDISMODULE_KSPEC_FK_RANGE`: Keys end at a specific index (or + * relative to the last argument). + * * `REDISMODULE_KSPEC_FK_KEYNUM`: There's an argument that contains + * the number of key args somewhere before the keys themselves. + * + * `find_keys_type` and `fk` can be omitted if this keyspec describes + * exactly one key. + * + * * `fk`: This is a union in which the `range` or `keynum` branch is used + * depending on the value of the `find_keys_type` field. + * + * * `fk.range` (for `REDISMODULE_KSPEC_FK_RANGE`): A struct with the + * following fields: + * + * * `lastkey`: Index of the last key relative to the result of the + * begin search step. Can be negative, in which case it's not + * relative. -1 indicates the last argument, -2 one before the + * last and so on. + * + * * `keystep`: How many arguments should we skip after finding a + * key, in order to find the next one? + * + * * `limit`: If `lastkey` is -1, we use `limit` to stop the search + * by a factor. 0 and 1 mean no limit. 2 means 1/2 of the + * remaining args, 3 means 1/3, and so on. + * + * * `fk.keynum` (for `REDISMODULE_KSPEC_FK_KEYNUM`): A struct with the + * following fields: + * + * * `keynumidx`: Index of the argument containing the number of + * keys to come, relative to the result of the begin search step. + * + * * `firstkey`: Index of the fist key relative to the result of the + * begin search step. (Usually it's just after `keynumidx`, in + * which case it should be set to `keynumidx + 1`.) + * + * * `keystep`: How many argumentss should we skip after finding a + * key, in order to find the next one? + * + * Key-spec flags: + * + * The first four refer to what the command actually does with the *value or + * metadata of the key*, and not necessarily the user data or how it affects + * it. Each key-spec may must have exactly one of these. Any operation + * that's not distinctly deletion, overwrite or read-only would be marked as + * RW. + * + * * `REDISMODULE_CMD_KEY_RO`: Read-Only. Reads the value of the key, but + * doesn't necessarily return it. + * + * * `REDISMODULE_CMD_KEY_RW`: Read-Write. Modifies the data stored in the + * value of the key or its metadata. + * + * * `REDISMODULE_CMD_KEY_OW`: Overwrite. Overwrites the data stored in the + * value of the key. + * + * * `REDISMODULE_CMD_KEY_RM`: Deletes the key. + * + * The next four refer to *user data inside the value of the key*, not the + * metadata like LRU, type, cardinality. It refers to the logical operation + * on the user's data (actual input strings or TTL), being + * used/returned/copied/changed. It doesn't refer to modification or + * returning of metadata (like type, count, presence of data). ACCESS can be + * combined with one of the write operations INSERT, DELETE or UPDATE. Any + * write that's not an INSERT or a DELETE would be UPDATE. + * + * * `REDISMODULE_CMD_KEY_ACCESS`: Returns, copies or uses the user data + * from the value of the key. + * + * * `REDISMODULE_CMD_KEY_UPDATE`: Updates data to the value, new value may + * depend on the old value. + * + * * `REDISMODULE_CMD_KEY_INSERT`: Adds data to the value with no chance of + * modification or deletion of existing data. + * + * * `REDISMODULE_CMD_KEY_DELETE`: Explicitly deletes some content from the + * value of the key. + * + * Other flags: + * + * * `REDISMODULE_CMD_KEY_NOT_KEY`: The key is not actually a key, but + * should be routed in cluster mode as if it was a key. + * + * * `REDISMODULE_CMD_KEY_INCOMPLETE`: The keyspec might not point out all + * the keys it should cover. + * + * * `REDISMODULE_CMD_KEY_VARIABLE_FLAGS`: Some keys might have different + * flags depending on arguments. + * + * - `args`: An array of RedisModuleCommandArg, terminated by an element memset + * to zero. RedisModuleCommandArg is a structure with at the fields described + * below. + * + * typedef struct RedisModuleCommandArg { + * const char *name; + * RedisModuleCommandArgType type; + * int key_spec_index; + * const char *token; + * const char *summary; + * const char *since; + * int flags; + * struct RedisModuleCommandArg *subargs; + * } RedisModuleCommandArg; + * + * Explanation of the fields: + * + * * `name`: Name of the argument. + * + * * `type`: The type of the argument. See below for details. The types + * `REDISMODULE_ARG_TYPE_ONEOF` and `REDISMODULE_ARG_TYPE_BLOCK` require + * an argument to have sub-arguments, i.e. `subargs`. + * + * * `key_spec_index`: If the `type` is `REDISMODULE_ARG_TYPE_KEY` you must + * provide the index of the key-spec associated with this argument. See + * `key_specs` above. If the argument is not a key, you may specify -1. + * + * * `token`: The token preceding the argument (optional). Example: the + * argument `seconds` in `SET` has a token `EX`. If the argument consists + * of only a token (for example `NX` in `SET`) the type should be + * `REDISMODULE_ARG_TYPE_PURE_TOKEN` and `value` should be NULL. + * + * * `summary`: A short description of the argument (optional). + * + * * `since`: The first version which included this argument (optional). + * + * * `flags`: A bitwise or of the macros `REDISMODULE_CMD_ARG_*`. See below. + * + * * `value`: The display-value of the argument. This string is what should + * be displayed when creating the command syntax from the output of + * `COMMAND`. If `token` is not NULL, it should also be displayed. + * + * Explanation of `RedisModuleCommandArgType`: + * + * * `REDISMODULE_ARG_TYPE_STRING`: String argument. + * * `REDISMODULE_ARG_TYPE_INTEGER`: Integer argument. + * * `REDISMODULE_ARG_TYPE_DOUBLE`: Double-precision float argument. + * * `REDISMODULE_ARG_TYPE_KEY`: String argument representing a keyname. + * * `REDISMODULE_ARG_TYPE_PATTERN`: String, but regex pattern. + * * `REDISMODULE_ARG_TYPE_UNIX_TIME`: Integer, but Unix timestamp. + * * `REDISMODULE_ARG_TYPE_PURE_TOKEN`: Argument doesn't have a placeholder. + * It's just a token without a value. Example: the `KEEPTTL` option of the + * `SET` command. + * * `REDISMODULE_ARG_TYPE_ONEOF`: Used when the user can choose only one of + * a few sub-arguments. Requires `subargs`. Example: the `NX` and `XX` + * options of `SET`. + * * `REDISMODULE_ARG_TYPE_BLOCK`: Used when one wants to group together + * several sub-arguments, usually to apply something on all of them, like + * making the entire group "optional". Requires `subargs`. Example: the + * `LIMIT offset count` parameters in `ZRANGE`. + * + * Explanation of the command argument flags: + * + * * `REDISMODULE_CMD_ARG_OPTIONAL`: The argument is optional (like GET in + * the SET command). + * * `REDISMODULE_CMD_ARG_MULTIPLE`: The argument may repeat itself (like + * key in DEL). + * * `REDISMODULE_CMD_ARG_MULTIPLE_TOKEN`: The argument may repeat itself, + * and so does its token (like `GET pattern` in SORT). + * + * On success REDISMODULE_OK is returned. On error REDISMODULE_ERR is returned + * and `errno` is set to EINVAL if invalid info was provided or EEXIST if info + * has already been set. If the info is invalid, a warning is logged explaining + * which part of the info is invalid and why. */ +int RM_SetCommandInfo(RedisModuleCommand *command, const RedisModuleCommandInfo *info) { + if (!moduleValidateCommandInfo(info)) { + errno = EINVAL; + return REDISMODULE_ERR; + } -/* Returns 1 if `cmd` is a command of the module `modulename`. 0 otherwise. */ -int moduleIsModuleCommand(void *module_handle, struct redisCommand *cmd) { - if (cmd->proc != RedisModuleCommandDispatcher) - return 0; - if (module_handle == NULL) - return 0; - RedisModuleCommand *cp = (void*)(unsigned long)cmd->getkeys_proc; - return (cp->module == module_handle); -} + struct redisCommand *cmd = command->rediscmd; -void extendKeySpecsIfNeeded(struct redisCommand *cmd) { - /* We extend even if key_specs_num == key_specs_max because - * this function is called prior to adding a new spec */ - if (cmd->key_specs_num < cmd->key_specs_max) - return; + /* Check if any info has already been set. Overwriting info involves freeing + * the old info, which is not implemented. */ + if (cmd->summary || cmd->complexity || cmd->since || cmd->history || + cmd->tips || cmd->args || + !(cmd->key_specs_num == 0 || + /* Allow key spec populated from legacy (first,last,step) to exist. */ + (cmd->key_specs_num == 1 && cmd->key_specs == cmd->key_specs_static && + cmd->key_specs[0].begin_search_type == KSPEC_BS_INDEX && + cmd->key_specs[0].find_keys_type == KSPEC_FK_RANGE))) { + errno = EEXIST; + return REDISMODULE_ERR; + } - cmd->key_specs_max++; + if (info->summary) cmd->summary = zstrdup(info->summary); + if (info->complexity) cmd->complexity = zstrdup(info->complexity); + if (info->since) cmd->since = zstrdup(info->since); - if (cmd->key_specs == cmd->key_specs_static) { - cmd->key_specs = zmalloc(sizeof(keySpec) * cmd->key_specs_max); - memcpy(cmd->key_specs, cmd->key_specs_static, sizeof(keySpec) * cmd->key_specs_num); - } else { - cmd->key_specs = zrealloc(cmd->key_specs, sizeof(keySpec) * cmd->key_specs_max); + const RedisModuleCommandInfoVersion *version = info->version; + if (info->history) { + size_t count = 0; + while (moduleCmdHistoryEntryAt(version, info->history, count)->since) + count++; + serverAssert(count < SIZE_MAX / sizeof(commandHistory)); + cmd->history = zmalloc(sizeof(commandHistory) * (count + 1)); + for (size_t j = 0; j < count; j++) { + RedisModuleCommandHistoryEntry *entry = + moduleCmdHistoryEntryAt(version, info->history, j); + cmd->history[j].since = zstrdup(entry->since); + cmd->history[j].changes = zstrdup(entry->changes); + } + cmd->history[count].since = NULL; + cmd->history[count].changes = NULL; + cmd->num_history = count; + } + + if (info->tips) { + int count; + sds *tokens = sdssplitlen(info->tips, strlen(info->tips), " ", 1, &count); + if (tokens) { + cmd->tips = zmalloc(sizeof(char *) * (count + 1)); + for (int j = 0; j < count; j++) { + cmd->tips[j] = zstrdup(tokens[j]); + } + cmd->tips[count] = NULL; + cmd->num_tips = count; + sdsfreesplitres(tokens, count); + } } -} -int moduleAddCommandKeySpec(RedisModuleCommand *command, const char *specflags, int *index) { - int64_t flags = specflags ? commandKeySpecsFlagsFromString(specflags) : 0; - if (flags == -1) - return REDISMODULE_ERR; + if (info->arity) cmd->arity = info->arity; - struct redisCommand *cmd = command->rediscmd; + if (info->key_specs) { + /* Count and allocate the key specs. */ + size_t count = 0; + while (moduleCmdKeySpecAt(version, info->key_specs, count)->begin_search_type) + count++; + serverAssert(count < INT_MAX); + if (count <= STATIC_KEY_SPECS_NUM) { + cmd->key_specs_max = STATIC_KEY_SPECS_NUM; + cmd->key_specs = cmd->key_specs_static; + } else { + cmd->key_specs_max = count; + cmd->key_specs = zmalloc(sizeof(keySpec) * count); + } - extendKeySpecsIfNeeded(cmd); + /* Copy the contents of the RedisModuleCommandKeySpec array. */ + cmd->key_specs_num = count; + for (size_t j = 0; j < count; j++) { + RedisModuleCommandKeySpec *spec = + moduleCmdKeySpecAt(version, info->key_specs, j); + cmd->key_specs[j].notes = spec->notes ? zstrdup(spec->notes) : NULL; + cmd->key_specs[j].flags = moduleConvertKeySpecsFlags(spec->flags, 1); + switch (spec->begin_search_type) { + case REDISMODULE_KSPEC_BS_UNKNOWN: + cmd->key_specs[j].begin_search_type = KSPEC_BS_UNKNOWN; + break; + case REDISMODULE_KSPEC_BS_INDEX: + cmd->key_specs[j].begin_search_type = KSPEC_BS_INDEX; + cmd->key_specs[j].bs.index.pos = spec->bs.index.pos; + break; + case REDISMODULE_KSPEC_BS_KEYWORD: + cmd->key_specs[j].begin_search_type = KSPEC_BS_KEYWORD; + cmd->key_specs[j].bs.keyword.keyword = zstrdup(spec->bs.keyword.keyword); + cmd->key_specs[j].bs.keyword.startfrom = spec->bs.keyword.startfrom; + break; + default: + /* Can't happen; stopped in moduleValidateCommandInfo(). */ + serverPanic("Unknown begin_search_type"); + } - *index = cmd->key_specs_num; - cmd->key_specs[cmd->key_specs_num].begin_search_type = KSPEC_BS_INVALID; - cmd->key_specs[cmd->key_specs_num].find_keys_type = KSPEC_FK_INVALID; - cmd->key_specs[cmd->key_specs_num].flags = flags; - cmd->key_specs_num++; - return REDISMODULE_OK; -} + switch (spec->find_keys_type) { + case REDISMODULE_KSPEC_FK_OMITTED: + /* Omitted field is shorthand to say that it's a single key. */ + cmd->key_specs[j].find_keys_type = KSPEC_FK_RANGE; + cmd->key_specs[j].fk.range.lastkey = 0; + cmd->key_specs[j].fk.range.keystep = 1; + cmd->key_specs[j].fk.range.limit = 0; + break; + case REDISMODULE_KSPEC_FK_UNKNOWN: + cmd->key_specs[j].find_keys_type = KSPEC_FK_UNKNOWN; + break; + case REDISMODULE_KSPEC_FK_RANGE: + cmd->key_specs[j].find_keys_type = KSPEC_FK_RANGE; + cmd->key_specs[j].fk.range.lastkey = spec->fk.range.lastkey; + cmd->key_specs[j].fk.range.keystep = spec->fk.range.keystep; + cmd->key_specs[j].fk.range.limit = spec->fk.range.limit; + break; + case REDISMODULE_KSPEC_FK_KEYNUM: + cmd->key_specs[j].find_keys_type = KSPEC_FK_KEYNUM; + cmd->key_specs[j].fk.keynum.keynumidx = spec->fk.keynum.keynumidx; + cmd->key_specs[j].fk.keynum.firstkey = spec->fk.keynum.firstkey; + cmd->key_specs[j].fk.keynum.keystep = spec->fk.keynum.keystep; + break; + default: + /* Can't happen; stopped in moduleValidateCommandInfo(). */ + serverPanic("Unknown find_keys_type"); + } + } -int moduleSetCommandKeySpecBeginSearch(RedisModuleCommand *command, int index, keySpec *spec) { - struct redisCommand *cmd = command->rediscmd; + /* Update the legacy (first,last,step) spec used by the COMMAND command, + * by trying to "glue" consecutive range key specs. */ + populateCommandLegacyRangeSpec(cmd); + populateCommandMovableKeys(cmd); + } - if (index >= cmd->key_specs_num) - return REDISMODULE_ERR; + if (info->args) { + cmd->args = moduleCopyCommandArgs(info->args, version); + /* Populate arg.num_args with the number of subargs, recursively */ + cmd->num_args = populateArgsStructure(cmd->args); + } - cmd->key_specs[index].begin_search_type = spec->begin_search_type; - cmd->key_specs[index].bs = spec->bs; + /* Fields added in future versions to be added here, under conditions like + * `if (info->version >= 2) { access version 2 fields here }` */ return REDISMODULE_OK; } -int moduleSetCommandKeySpecFindKeys(RedisModuleCommand *command, int index, keySpec *spec) { - struct redisCommand *cmd = command->rediscmd; +/* Returns 1 if v is a power of two, 0 otherwise. */ +static inline int isPowerOfTwo(uint64_t v) { + return v && !(v & (v - 1)); +} - if (index >= cmd->key_specs_num) - return REDISMODULE_ERR; +/* Returns 1 if the command info is valid and 0 otherwise. */ +static int moduleValidateCommandInfo(const RedisModuleCommandInfo *info) { + const RedisModuleCommandInfoVersion *version = info->version; + if (!version) { + serverLog(LL_WARNING, "Invalid command info: version missing"); + return 0; + } + + /* No validation for the fields summary, complexity, since, tips (strings or + * NULL) and arity (any integer). */ + + /* History: If since is set, changes must also be set. */ + if (info->history) { + for (size_t j = 0; + moduleCmdHistoryEntryAt(version, info->history, j)->since; + j++) + { + if (!moduleCmdHistoryEntryAt(version, info->history, j)->changes) { + serverLog(LL_WARNING, "Invalid command info: history[%zd].changes missing", j); + return 0; + } + } + } - cmd->key_specs[index].find_keys_type = spec->find_keys_type; - cmd->key_specs[index].fk = spec->fk; + /* Key specs. */ + if (info->key_specs) { + for (size_t j = 0; + moduleCmdKeySpecAt(version, info->key_specs, j)->begin_search_type; + j++) + { + RedisModuleCommandKeySpec *spec = + moduleCmdKeySpecAt(version, info->key_specs, j); + if (j >= INT_MAX) { + serverLog(LL_WARNING, "Invalid command info: Too many key specs"); + return 0; /* redisCommand.key_specs_num is an int. */ + } - /* Refresh legacy range */ - populateCommandLegacyRangeSpec(cmd); - /* Refresh movablekeys flag */ - populateCommandMovableKeys(cmd); + /* Flags. Exactly one flag in a group is set if and only if the + * masked bits is a power of two. */ + uint64_t key_flags = + REDISMODULE_CMD_KEY_RO | REDISMODULE_CMD_KEY_RW | + REDISMODULE_CMD_KEY_OW | REDISMODULE_CMD_KEY_RM; + uint64_t write_flags = + REDISMODULE_CMD_KEY_INSERT | REDISMODULE_CMD_KEY_DELETE | + REDISMODULE_CMD_KEY_UPDATE; + if (!isPowerOfTwo(spec->flags & key_flags)) { + serverLog(LL_WARNING, + "Invalid command info: key_specs[%zd].flags: " + "Exactly one of the flags RO, RW, OW, RM reqired", j); + return 0; + } + if ((spec->flags & write_flags) != 0 && + !isPowerOfTwo(spec->flags & write_flags)) + { + serverLog(LL_WARNING, + "Invalid command info: key_specs[%zd].flags: " + "INSERT, DELETE and UPDATE are mutually exclusive", j); + return 0; + } - return REDISMODULE_OK; -} + switch (spec->begin_search_type) { + case REDISMODULE_KSPEC_BS_UNKNOWN: break; + case REDISMODULE_KSPEC_BS_INDEX: break; + case REDISMODULE_KSPEC_BS_KEYWORD: + if (spec->bs.keyword.keyword == NULL) { + serverLog(LL_WARNING, + "Invalid command info: key_specs[%zd].bs.keyword.keyword " + "required when begin_search_type is KEYWORD", j); + return 0; + } + break; + default: + serverLog(LL_WARNING, + "Invalid command info: key_specs[%zd].begin_search_type: " + "Invalid value %d", j, spec->begin_search_type); + return 0; + } -/* **The key spec API is not officially released and it is going to be changed - * in Redis 7.0. It has been disabled temporarily.** - * - * Key specs is a scheme that tries to describe the location - * of key arguments better than the old [first,last,step] scheme - * which is limited and doesn't fit many commands. - * - * This information is used by ACL, Cluster and the `COMMAND` command. - * - * There are two steps to retrieve the key arguments: - * - * - `begin_search` (BS): in which index should we start seacrhing for keys? - * - `find_keys` (FK): relative to the output of BS, how can we will which args are keys? - * - * There are two types of BS: - * - * - `index`: key args start at a constant index - * - `keyword`: key args start just after a specific keyword - * - * There are two kinds of FK: - * - * - `range`: keys end at a specific index (or relative to the last argument) - * - `keynum`: there's an arg that contains the number of key args somewhere before the keys themselves - * - * This function adds a new key spec to a command, returning a unique id in `spec_id`. - * The caller must then call one of the RedisModule_SetCommandKeySpecBeginSearch* APIs - * followed by one of the RedisModule_SetCommandKeySpecFindKeys* APIs. - * - * It should be called just after RedisModule_CreateCommand. - * - * Example: - * - * if (RedisModule_CreateCommand(ctx,"kspec.smove",kspec_legacy,"",0,0,0) == REDISMODULE_ERR) - * return REDISMODULE_ERR; - * - * if (RedisModule_AddCommandKeySpec(ctx,"kspec.smove","RW access delete",&spec_id) == REDISMODULE_ERR) - * return REDISMODULE_ERR; - * if (RedisModule_SetCommandKeySpecBeginSearchIndex(ctx,"kspec.smove",spec_id,1) == REDISMODULE_ERR) - * return REDISMODULE_ERR; - * if (RedisModule_SetCommandKeySpecFindKeysRange(ctx,"kspec.smove",spec_id,0,1,0) == REDISMODULE_ERR) - * return REDISMODULE_ERR; - * - * if (RedisModule_AddCommandKeySpec(ctx,"kspec.smove","RW insert",&spec_id) == REDISMODULE_ERR) - * return REDISMODULE_ERR; - * if (RedisModule_SetCommandKeySpecBeginSearchIndex(ctx,"kspec.smove",spec_id,2) == REDISMODULE_ERR) - * return REDISMODULE_ERR; - * if (RedisModule_SetCommandKeySpecFindKeysRange(ctx,"kspec.smove",spec_id,0,1,0) == REDISMODULE_ERR) - * return REDISMODULE_ERR; - * - * It is also possible to use this API on subcommands (See RedisModule_CreateSubcommand). - * The name of the subcommand should be the name of the parent command + "|" + name of subcommand. - * - * Example: - * - * RedisModule_AddCommandKeySpec(ctx,"module.object|encoding","RO",&spec_id) - * - * Returns REDISMODULE_OK on success - */ -int RM_AddCommandKeySpec(RedisModuleCommand *command, const char *specflags, int *spec_id) { - return moduleAddCommandKeySpec(command, specflags, spec_id); -} + /* Validate find_keys_type. */ + switch (spec->find_keys_type) { + case REDISMODULE_KSPEC_FK_OMITTED: break; /* short for RANGE {0,1,0} */ + case REDISMODULE_KSPEC_FK_UNKNOWN: break; + case REDISMODULE_KSPEC_FK_RANGE: break; + case REDISMODULE_KSPEC_FK_KEYNUM: break; + default: + serverLog(LL_WARNING, + "Invalid command info: key_specs[%zd].find_keys_type: " + "Invalid value %d", j, spec->find_keys_type); + return 0; + } + } + } -/* Set a "index" key arguments spec to a command (begin_search step). - * See RedisModule_AddCommandKeySpec's doc. - * - * - `index`: The index from which we start the search for keys - * - * Returns REDISMODULE_OK */ -int RM_SetCommandKeySpecBeginSearchIndex(RedisModuleCommand *command, int spec_id, int index) { - keySpec spec; - spec.begin_search_type = KSPEC_BS_INDEX; - spec.bs.index.pos = index; + /* Args, subargs (recursive) */ + return moduleValidateCommandArgs(info->args, version); +} + +/* When from_api is true, converts from REDISMODULE_CMD_KEY_* flags to CMD_KEY_* flags. + * When from_api is false, converts from CMD_KEY_* flags to REDISMODULE_CMD_KEY_* flags. */ +static int64_t moduleConvertKeySpecsFlags(int64_t flags, int from_api) { + int64_t out = 0; + int64_t map[][2] = { + {REDISMODULE_CMD_KEY_RO, CMD_KEY_RO}, + {REDISMODULE_CMD_KEY_RW, CMD_KEY_RW}, + {REDISMODULE_CMD_KEY_OW, CMD_KEY_OW}, + {REDISMODULE_CMD_KEY_RM, CMD_KEY_RM}, + {REDISMODULE_CMD_KEY_ACCESS, CMD_KEY_ACCESS}, + {REDISMODULE_CMD_KEY_INSERT, CMD_KEY_INSERT}, + {REDISMODULE_CMD_KEY_UPDATE, CMD_KEY_UPDATE}, + {REDISMODULE_CMD_KEY_DELETE, CMD_KEY_DELETE}, + {REDISMODULE_CMD_KEY_NOT_KEY, CMD_KEY_NOT_KEY}, + {REDISMODULE_CMD_KEY_INCOMPLETE, CMD_KEY_INCOMPLETE}, + {REDISMODULE_CMD_KEY_VARIABLE_FLAGS, CMD_KEY_VARIABLE_FLAGS}, + {0,0}}; + + int from_idx = from_api ? 0 : 1, to_idx = !from_idx; + for (int i=0; map[i][0]; i++) + if (flags & map[i][from_idx]) out |= map[i][to_idx]; + return out; +} + +/* Validates an array of RedisModuleCommandArg. Returns 1 if it's valid and 0 if + * it's invalid. */ +static int moduleValidateCommandArgs(RedisModuleCommandArg *args, + const RedisModuleCommandInfoVersion *version) { + if (args == NULL) return 1; /* Missing args is OK. */ + for (size_t j = 0; moduleCmdArgAt(version, args, j)->name != NULL; j++) { + RedisModuleCommandArg *arg = moduleCmdArgAt(version, args, j); + int arg_type_error = 0; + moduleConvertArgType(arg->type, &arg_type_error); + if (arg_type_error) { + serverLog(LL_WARNING, + "Invalid command info: Argument \"%s\": Undefined type %d", + arg->name, arg->type); + return 0; + } + if (arg->type == REDISMODULE_ARG_TYPE_PURE_TOKEN && !arg->token) { + serverLog(LL_WARNING, + "Invalid command info: Argument \"%s\": " + "token required when type is PURE_TOKEN", args[j].name); + return 0; + } - return moduleSetCommandKeySpecBeginSearch(command, spec_id, &spec); -} + if (arg->type == REDISMODULE_ARG_TYPE_KEY) { + if (arg->key_spec_index < 0) { + serverLog(LL_WARNING, + "Invalid command info: Argument \"%s\": " + "key_spec_index required when type is KEY", + arg->name); + return 0; + } + } else if (arg->key_spec_index != -1 && arg->key_spec_index != 0) { + /* 0 is allowed for convenience, to allow it to be omitted in + * compound struct literals on the form `.field = value`. */ + serverLog(LL_WARNING, + "Invalid command info: Argument \"%s\": " + "key_spec_index specified but type isn't KEY", + arg->name); + return 0; + } -/* Set a "keyword" key arguments spec to a command (begin_search step). - * See RedisModule_AddCommandKeySpec's doc. - * - * - `keyword`: The keyword that indicates the beginning of key args - * - `startfrom`: An index in argv from which to start searching. - * Can be negative, which means start search from the end, in reverse - * (Example: -2 means to start in reverse from the panultimate arg) - * - * Returns REDISMODULE_OK */ -int RM_SetCommandKeySpecBeginSearchKeyword(RedisModuleCommand *command, int spec_id, const char *keyword, int startfrom) { - keySpec spec; - spec.begin_search_type = KSPEC_BS_KEYWORD; - spec.bs.keyword.keyword = keyword; - spec.bs.keyword.startfrom = startfrom; + if (arg->flags & ~(_REDISMODULE_CMD_ARG_NEXT - 1)) { + serverLog(LL_WARNING, + "Invalid command info: Argument \"%s\": Invalid flags", + arg->name); + return 0; + } - return moduleSetCommandKeySpecBeginSearch(command, spec_id, &spec); + if (arg->type == REDISMODULE_ARG_TYPE_ONEOF || + arg->type == REDISMODULE_ARG_TYPE_BLOCK) + { + if (arg->subargs == NULL) { + serverLog(LL_WARNING, + "Invalid command info: Argument \"%s\": " + "subargs required when type is ONEOF or BLOCK", + arg->name); + return 0; + } + if (!moduleValidateCommandArgs(arg->subargs, version)) return 0; + } else { + if (arg->subargs != NULL) { + serverLog(LL_WARNING, + "Invalid command info: Argument \"%s\": " + "subargs specified but type isn't ONEOF nor BLOCK", + arg->name); + return 0; + } + } + } + return 1; } -/* Set a "range" key arguments spec to a command (find_keys step). - * See RedisModule_AddCommandKeySpec's doc. - * - * - `lastkey`: Relative index (to the result of the begin_search step) where the last key is. - * Can be negative, in which case it's not relative. -1 indicating till the last argument, - * -2 one before the last and so on. - * - `keystep`: How many args should we skip after finding a key, in order to find the next one. - * - `limit`: If lastkey is -1, we use limit to stop the search by a factor. 0 and 1 mean no limit. - * 2 means 1/2 of the remaining args, 3 means 1/3, and so on. - * - * Returns REDISMODULE_OK */ -int RM_SetCommandKeySpecFindKeysRange(RedisModuleCommand *command, int spec_id, int lastkey, int keystep, int limit) { - keySpec spec; - spec.find_keys_type = KSPEC_FK_RANGE; - spec.fk.range.lastkey = lastkey; - spec.fk.range.keystep = keystep; - spec.fk.range.limit = limit; +/* Converts an array of RedisModuleCommandArg into a freshly allocated array of + * struct redisCommandArg. */ +static struct redisCommandArg *moduleCopyCommandArgs(RedisModuleCommandArg *args, + const RedisModuleCommandInfoVersion *version) { + size_t count = 0; + while (moduleCmdArgAt(version, args, count)->name) count++; + serverAssert(count < SIZE_MAX / sizeof(struct redisCommandArg)); + struct redisCommandArg *realargs = zcalloc((count+1) * sizeof(redisCommandArg)); + + for (size_t j = 0; j < count; j++) { + RedisModuleCommandArg *arg = moduleCmdArgAt(version, args, j); + realargs[j].name = zstrdup(arg->name); + realargs[j].type = moduleConvertArgType(arg->type, NULL); + if (arg->type == REDISMODULE_ARG_TYPE_KEY) + realargs[j].key_spec_index = arg->key_spec_index; + else + realargs[j].key_spec_index = -1; + if (arg->token) realargs[j].token = zstrdup(arg->token); + if (arg->summary) realargs[j].summary = zstrdup(arg->summary); + if (arg->since) realargs[j].since = zstrdup(arg->since); + realargs[j].flags = moduleConvertArgFlags(arg->flags); + if (arg->subargs) realargs[j].subargs = moduleCopyCommandArgs(arg->subargs, version); + } + return realargs; +} + +static redisCommandArgType moduleConvertArgType(RedisModuleCommandArgType type, int *error) { + if (error) *error = 0; + switch (type) { + case REDISMODULE_ARG_TYPE_STRING: return ARG_TYPE_STRING; + case REDISMODULE_ARG_TYPE_INTEGER: return ARG_TYPE_INTEGER; + case REDISMODULE_ARG_TYPE_DOUBLE: return ARG_TYPE_DOUBLE; + case REDISMODULE_ARG_TYPE_KEY: return ARG_TYPE_KEY; + case REDISMODULE_ARG_TYPE_PATTERN: return ARG_TYPE_PATTERN; + case REDISMODULE_ARG_TYPE_UNIX_TIME: return ARG_TYPE_UNIX_TIME; + case REDISMODULE_ARG_TYPE_PURE_TOKEN: return ARG_TYPE_PURE_TOKEN; + case REDISMODULE_ARG_TYPE_ONEOF: return ARG_TYPE_ONEOF; + case REDISMODULE_ARG_TYPE_BLOCK: return ARG_TYPE_BLOCK; + default: + if (error) *error = 1; + return -1; + } +} - return moduleSetCommandKeySpecFindKeys(command, spec_id, &spec); +static int moduleConvertArgFlags(int flags) { + int realflags = 0; + if (flags & REDISMODULE_CMD_ARG_OPTIONAL) realflags |= CMD_ARG_OPTIONAL; + if (flags & REDISMODULE_CMD_ARG_MULTIPLE) realflags |= CMD_ARG_MULTIPLE; + if (flags & REDISMODULE_CMD_ARG_MULTIPLE_TOKEN) realflags |= CMD_ARG_MULTIPLE_TOKEN; + return realflags; } -/* Set a "keynum" key arguments spec to a command (find_keys step). - * See RedisModule_AddCommandKeySpec's doc. - * - * - `keynumidx`: Relative index (to the result of the begin_search step) where the arguments that - * contains the number of keys is. - * - `firstkey`: Relative index (to the result of the begin_search step) where the first key is - * found (Usually it's just after keynumidx, so it should be keynumidx+1) - * - `keystep`: How many args should we skip after finding a key, in order to find the next one. - * - * Returns REDISMODULE_OK */ -int RM_SetCommandKeySpecFindKeysKeynum(RedisModuleCommand *command, int spec_id, int keynumidx, int firstkey, int keystep) { - keySpec spec; - spec.find_keys_type = KSPEC_FK_KEYNUM; - spec.fk.keynum.keynumidx = keynumidx; - spec.fk.keynum.firstkey = firstkey; - spec.fk.keynum.keystep = keystep; +/* Return `struct RedisModule *` as `void *` to avoid exposing it outside of module.c. */ +void *moduleGetHandleByName(char *modulename) { + return dictFetchValue(modules,modulename); +} - return moduleSetCommandKeySpecFindKeys(command, spec_id, &spec); +/* Returns 1 if `cmd` is a command of the module `modulename`. 0 otherwise. */ +int moduleIsModuleCommand(void *module_handle, struct redisCommand *cmd) { + if (cmd->proc != RedisModuleCommandDispatcher) + return 0; + if (module_handle == NULL) + return 0; + RedisModuleCommand *cp = (void*)(unsigned long)cmd->getkeys_proc; + return (cp->module == module_handle); } /* -------------------------------------------------------------------------- @@ -2399,6 +2997,15 @@ int RM_ReplyWithCallReply(RedisModuleCtx *ctx, RedisModuleCallReply *reply) { size_t proto_len; const char *proto = callReplyGetProto(reply, &proto_len); addReplyProto(c, proto, proto_len); + /* Propagate the error list from that reply to the other client, to do some + * post error reply handling, like statistics. + * Note that if the original reply had an array with errors, and the module + * replied with just a portion of the original reply, and not the entire + * reply, the errors are currently not propagated and the errors stats + * will not get propagated. */ + list *errors = callReplyDeferredErrorList(reply); + if (errors) + deferredAfterErrorReply(c, errors); return REDISMODULE_OK; } @@ -5051,7 +5658,7 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch errno = ENOENT; goto cleanup; } - c->cmd = c->lastcmd = cmd; + c->cmd = c->lastcmd = c->realcmd = cmd; /* Basic arity checks. */ if ((cmd->arity > 0 && cmd->arity != argc) || (argc < -cmd->arity)) { @@ -5135,7 +5742,8 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch proto = sdscatlen(proto,o->buf,o->used); listDelNode(c->reply,listFirst(c->reply)); } - reply = callReplyCreate(proto, ctx); + reply = callReplyCreate(proto, c->deferred_reply_errors, ctx); + c->deferred_reply_errors = NULL; /* now the responsibility of the reply object. */ autoMemoryAdd(ctx,REDISMODULE_AM_REPLY,reply); cleanup: @@ -6572,6 +7180,7 @@ void moduleHandleBlockedClients(void) { * was blocked on keys (RM_BlockClientOnKeys()), because we already * called such callback in moduleTryServeClientBlockedOnKey() when * the key was signaled as ready. */ + long long prev_error_replies = server.stat_total_error_replies; uint64_t reply_us = 0; if (c && !bc->blocked_on_keys && bc->reply_callback) { RedisModuleCtx ctx; @@ -6586,13 +7195,6 @@ void moduleHandleBlockedClients(void) { reply_us = elapsedUs(replyTimer); moduleFreeContext(&ctx); } - /* Update stats now that we've finished the blocking operation. - * This needs to be out of the reply callback above given that a - * module might not define any callback and still do blocking ops. - */ - if (c && !bc->blocked_on_keys) { - updateStatsOnUnblock(c, bc->background_duration, reply_us); - } /* Free privdata if any. */ if (bc->privdata && bc->free_privdata) { @@ -6613,6 +7215,14 @@ void moduleHandleBlockedClients(void) { moduleReleaseTempClient(bc->reply_client); moduleReleaseTempClient(bc->thread_safe_ctx_client); + /* Update stats now that we've finished the blocking operation. + * This needs to be out of the reply callback above given that a + * module might not define any callback and still do blocking ops. + */ + if (c && !bc->blocked_on_keys) { + updateStatsOnUnblock(c, bc->background_duration, reply_us, server.stat_total_error_replies != prev_error_replies); + } + if (c != NULL) { /* Before unblocking the client, set the disconnect callback * to NULL, because if we reached this point, the client was @@ -6670,10 +7280,11 @@ void moduleBlockedClientTimedOut(client *c) { ctx.client = bc->client; ctx.blocked_client = bc; ctx.blocked_privdata = bc->privdata; + long long prev_error_replies = server.stat_total_error_replies; bc->timeout_callback(&ctx,(void**)c->argv,c->argc); moduleFreeContext(&ctx); if (!bc->blocked_on_keys) { - updateStatsOnUnblock(c, bc->background_duration, 0); + updateStatsOnUnblock(c, bc->background_duration, 0, server.stat_total_error_replies != prev_error_replies); } /* For timeout events, we do not want to call the disconnect callback, * because the blocked client will be automatically disconnected in @@ -7427,6 +8038,24 @@ int RM_GetTimerInfo(RedisModuleCtx *ctx, RedisModuleTimerID id, uint64_t *remain return REDISMODULE_OK; } +/* Query timers to see if any timer belongs to the module. + * Return 1 if any timer was found, otherwise 0 would be returned. */ +int moduleHoldsTimer(struct RedisModule *module) { + raxIterator iter; + int found = 0; + raxStart(&iter,Timers); + raxSeek(&iter,"^",NULL,0); + while (raxNext(&iter)) { + RedisModuleTimer *timer = iter.data; + if (timer->module == module) { + found = 1; + break; + } + } + raxStop(&iter); + return found; +} + /* -------------------------------------------------------------------------- * ## Modules EventLoop API * --------------------------------------------------------------------------*/ @@ -7808,28 +8437,34 @@ int RM_ACLCheckCommandPermissions(RedisModuleUser *user, RedisModuleString **arg return REDISMODULE_OK; } -/* Check if the key can be accessed by the user, according to the ACLs associated with it - * and the flags used. The supported flags are: - * - * REDISMODULE_KEY_PERMISSION_READ: Can the module read data from the key. - * REDISMODULE_KEY_PERMISSION_WRITE: Can the module write data to the key. +/* Check if the key can be accessed by the user according to the ACLs attached to the user + * and the flags representing the key access. The flags are the same that are used in the + * keyspec for logical operations. These flags are documented in RedisModule_SetCommandInfo as + * the REDISMODULE_CMD_KEY_ACCESS, REDISMODULE_CMD_KEY_UPDATE, REDISMODULE_CMD_KEY_INSERT, + * and REDISMODULE_CMD_KEY_DELETE flags. + * + * If no flags are supplied, the user is still required to have some access to the key for + * this command to return successfully. * - * On success a REDISMODULE_OK is returned, otherwise - * REDISMODULE_ERR is returned and errno is set to the following values: + * If the user is able to access the key then REDISMODULE_OK is returned, otherwise + * REDISMODULE_ERR is returned and errno is set to one of the following values: * * * EINVAL: The provided flags are invalid. * * EACCESS: The user does not have permission to access the key. */ int RM_ACLCheckKeyPermissions(RedisModuleUser *user, RedisModuleString *key, int flags) { - int acl_flags = 0; - if (flags & REDISMODULE_KEY_PERMISSION_READ) acl_flags |= ACL_READ_PERMISSION; - if (flags & REDISMODULE_KEY_PERMISSION_WRITE) acl_flags |= ACL_WRITE_PERMISSION; - if (!acl_flags || ((flags & REDISMODULE_KEY_PERMISSION_ALL) != flags)) { + const int allow_mask = (REDISMODULE_CMD_KEY_ACCESS + | REDISMODULE_CMD_KEY_INSERT + | REDISMODULE_CMD_KEY_DELETE + | REDISMODULE_CMD_KEY_UPDATE); + + if ((flags & allow_mask) != flags) { errno = EINVAL; return REDISMODULE_ERR; } - if (ACLUserCheckKeyPerm(user->user, key->ptr, sdslen(key->ptr), acl_flags) != ACL_OK) { + int keyspec_flags = moduleConvertKeySpecsFlags(flags, 0); + if (ACLUserCheckKeyPerm(user->user, key->ptr, sdslen(key->ptr), keyspec_flags) != ACL_OK) { errno = EACCES; return REDISMODULE_ERR; } @@ -7837,14 +8472,34 @@ int RM_ACLCheckKeyPermissions(RedisModuleUser *user, RedisModuleString *key, int return REDISMODULE_OK; } -/* Check if the pubsub channel can be accessed by the user, according to the ACLs associated with it. - * Glob-style pattern matching is employed, unless the literal flag is - * set. +/* Check if the pubsub channel can be accessed by the user based off of the given + * access flags. See RM_ChannelAtPosWithFlags for more information about the + * possible flags that can be passed in. * - * If the user can access the pubsub channel, REDISMODULE_OK is returned, otherwise - * REDISMODULE_ERR is returned. */ -int RM_ACLCheckChannelPermissions(RedisModuleUser *user, RedisModuleString *ch, int literal) { - if (ACLUserCheckChannelPerm(user->user, ch->ptr, literal) != ACL_OK) + * If the user is able to acecss the pubsub channel then REDISMODULE_OK is returned, otherwise + * REDISMODULE_ERR is returned and errno is set to one of the following values: + * + * * EINVAL: The provided flags are invalid. + * * EACCESS: The user does not have permission to access the pubsub channel. + */ +int RM_ACLCheckChannelPermissions(RedisModuleUser *user, RedisModuleString *ch, int flags) { + const int allow_mask = (REDISMODULE_CMD_CHANNEL_PUBLISH + | REDISMODULE_CMD_CHANNEL_SUBSCRIBE + | REDISMODULE_CMD_CHANNEL_UNSUBSCRIBE + | REDISMODULE_CMD_CHANNEL_PATTERN); + + if ((flags & allow_mask) != flags) { + errno = EINVAL; + return REDISMODULE_ERR; + } + + /* Unsubscribe permissions are currently always allowed. */ + if (flags & REDISMODULE_CMD_CHANNEL_UNSUBSCRIBE){ + return REDISMODULE_OK; + } + + int is_pattern = flags & REDISMODULE_CMD_CHANNEL_PATTERN; + if (ACLUserCheckChannelPerm(user->user, ch->ptr, is_pattern) != ACL_OK) return REDISMODULE_ERR; return REDISMODULE_OK; @@ -8254,9 +8909,10 @@ int RM_InfoAddSection(RedisModuleInfoCtx *ctx, const char *name) { * 1) no section was requested (emit all) * 2) the module name was requested (emit all) * 3) this specific section was requested. */ - if (ctx->requested_section) { - if (strcasecmp(ctx->requested_section, full_name) && - strcasecmp(ctx->requested_section, ctx->module->name)) { + if (ctx->requested_sections) { + if ((!full_name || !dictFind(ctx->requested_sections, full_name)) && + (!dictFind(ctx->requested_sections, ctx->module->name))) + { sdsfree(full_name); ctx->in_section = 0; return REDISMODULE_ERR; @@ -8405,7 +9061,7 @@ int RM_RegisterInfoFunc(RedisModuleCtx *ctx, RedisModuleInfoFunc cb) { return REDISMODULE_OK; } -sds modulesCollectInfo(sds info, const char *section, int for_crash_report, int sections) { +sds modulesCollectInfo(sds info, dict *sections_dict, int for_crash_report, int sections) { dictIterator *di = dictGetIterator(modules); dictEntry *de; @@ -8413,7 +9069,7 @@ sds modulesCollectInfo(sds info, const char *section, int for_crash_report, int struct RedisModule *module = dictGetVal(de); if (!module->info_cb) continue; - RedisModuleInfoCtx info_ctx = {module, section, info, sections, 0, 0}; + RedisModuleInfoCtx info_ctx = {module, sections_dict, info, sections, 0, 0}; module->info_cb(&info_ctx, for_crash_report); /* Implicitly end dicts (no way to handle errors, and we must add the newline). */ if (info_ctx.in_dict_field) @@ -8435,7 +9091,11 @@ RedisModuleServerInfoData *RM_GetServerInfo(RedisModuleCtx *ctx, const char *sec struct RedisModuleServerInfoData *d = zmalloc(sizeof(*d)); d->rax = raxNew(); if (ctx != NULL) autoMemoryAdd(ctx,REDISMODULE_AM_INFO,d); - sds info = genRedisInfoString(section); + int all = 0, everything = 0; + robj *argv[1]; + argv[0] = section ? createStringObject(section, strlen(section)) : NULL; + dict *section_dict = genInfoSectionDict(argv, section ? 1 : 0, NULL, &all, &everything); + sds info = genRedisInfoString(section_dict, all, everything); int totlines, i; sds *lines = sdssplitlen(info, sdslen(info), "\r\n", 2, &totlines); for(i=0; i<totlines; i++) { @@ -8451,6 +9111,8 @@ RedisModuleServerInfoData *RM_GetServerInfo(RedisModuleCtx *ctx, const char *sec } sdsfree(info); sdsfreesplitres(lines,totlines); + releaseInfoSectionDict(section_dict); + if(argv[0]) decrRefCount(argv[0]); return d; } @@ -9995,17 +10657,23 @@ int moduleFreeCommand(struct RedisModule *module, struct redisCommand *cmd) { return C_ERR; /* Free everything except cmd->fullname and cmd itself. */ + for (int j = 0; j < cmd->key_specs_num; j++) { + if (cmd->key_specs[j].notes) + zfree((char *)cmd->key_specs[j].notes); + if (cmd->key_specs[j].begin_search_type == KSPEC_BS_KEYWORD) + zfree((char *)cmd->key_specs[j].bs.keyword.keyword); + } if (cmd->key_specs != cmd->key_specs_static) zfree(cmd->key_specs); for (int j = 0; cmd->tips && cmd->tips[j]; j++) - sdsfree((sds)cmd->tips[j]); + zfree((char *)cmd->tips[j]); for (int j = 0; cmd->history && cmd->history[j].since; j++) { - sdsfree((sds)cmd->history[j].since); - sdsfree((sds)cmd->history[j].changes); + zfree((char *)cmd->history[j].since); + zfree((char *)cmd->history[j].changes); } - sdsfree((sds)cmd->summary); - sdsfree((sds)cmd->since); - sdsfree((sds)cmd->complexity); + zfree((char *)cmd->summary); + zfree((char *)cmd->since); + zfree((char *)cmd->complexity); if (cmd->latency_histogram) { hdr_close(cmd->latency_histogram); cmd->latency_histogram = NULL; @@ -10125,6 +10793,7 @@ int moduleLoad(const char *path, void **module_argv, int module_argc) { * * EBUSY: The module exports a new data type and can only be reloaded. * * EPERM: The module exports APIs which are used by other module. * * EAGAIN: The module has blocked clients. + * * EINPROGRESS: The module holds timer not fired. * * ECANCELED: Unload module error. */ int moduleUnload(sds name) { struct RedisModule *module = dictFetchValue(modules,name); @@ -10141,6 +10810,9 @@ int moduleUnload(sds name) { } else if (module->blocked_clients) { errno = EAGAIN; return C_ERR; + } else if (moduleHoldsTimer(module)) { + errno = EINPROGRESS; + return C_ERR; } /* Give module a chance to clean up. */ @@ -10255,6 +10927,8 @@ sds genModulesInfoStringRenderModuleOptions(struct RedisModule *module) { output = sdscat(output,"handle-io-errors|"); if (module->options & REDISMODULE_OPTIONS_HANDLE_REPL_ASYNC_LOAD) output = sdscat(output,"handle-repl-async-load|"); + if (module->options & REDISMODULE_OPTION_NO_IMPLICIT_SIGNAL_MODIFIED) + output = sdscat(output,"no-implicit-signal-modified|"); output = sdstrim(output,"|"); output = sdscat(output,"]"); return output; @@ -10345,6 +11019,10 @@ NULL errmsg = "the module has blocked clients. " "Please wait them unblocked and try again"; break; + case EINPROGRESS: + errmsg = "the module holds timer that is not fired. " + "Please stop the timer or wait until it fires."; + break; default: errmsg = "operation not possible."; break; @@ -10511,6 +11189,10 @@ int RM_ModuleTypeReplaceValue(RedisModuleKey *key, moduleType *mt, void *new_val * contains the indexes of all key name arguments. This function is * essentially a more efficient way to do `COMMAND GETKEYS`. * + * The out_flags argument is optional, and can be set to NULL. + * When provided it is filled with REDISMODULE_CMD_KEY_ flags in matching + * indexes with the key indexes of the returned array. + * * A NULL return value indicates the specified command has no keys, or * an error condition. Error conditions are indicated by setting errno * as follows: @@ -10520,9 +11202,10 @@ int RM_ModuleTypeReplaceValue(RedisModuleKey *key, moduleType *mt, void *new_val * * NOTE: The returned array is not a Redis Module object so it does not * get automatically freed even when auto-memory is used. The caller - * must explicitly call RM_Free() to free it. + * must explicitly call RM_Free() to free it, same as the out_flags pointer if + * used. */ -int *RM_GetCommandKeys(RedisModuleCtx *ctx, RedisModuleString **argv, int argc, int *num_keys) { +int *RM_GetCommandKeysWithFlags(RedisModuleCtx *ctx, RedisModuleString **argv, int argc, int *num_keys, int **out_flags) { UNUSED(ctx); struct redisCommand *cmd; int *res = NULL; @@ -10557,13 +11240,22 @@ int *RM_GetCommandKeys(RedisModuleCtx *ctx, RedisModuleString **argv, int argc, /* The return value here expects an array of key positions */ unsigned long int size = sizeof(int) * result.numkeys; res = zmalloc(size); + if (out_flags) + *out_flags = zmalloc(size); for (int i = 0; i < result.numkeys; i++) { res[i] = result.keys[i].pos; + if (out_flags) + (*out_flags)[i] = moduleConvertKeySpecsFlags(result.keys[i].flags, 0); } return res; } +/* Identinal to RM_GetCommandKeysWithFlags when flags are not needed. */ +int *RM_GetCommandKeys(RedisModuleCtx *ctx, RedisModuleString **argv, int argc, int *num_keys) { + return RM_GetCommandKeysWithFlags(ctx, argv, argc, num_keys, NULL); +} + /* Return the name of the command currently running */ const char *RM_GetCurrentCommandName(RedisModuleCtx *ctx) { if (!ctx || !ctx->client || !ctx->client->cmd) @@ -10803,6 +11495,7 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(CreateCommand); REGISTER_API(GetCommand); REGISTER_API(CreateSubcommand); + REGISTER_API(SetCommandInfo); REGISTER_API(SetModuleAttribs); REGISTER_API(IsModuleNameBusy); REGISTER_API(WrongArity); @@ -10913,6 +11606,9 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(StreamTrimByID); REGISTER_API(IsKeysPositionRequest); REGISTER_API(KeyAtPos); + REGISTER_API(KeyAtPosWithFlags); + REGISTER_API(IsChannelsPositionRequest); + REGISTER_API(ChannelAtPosWithFlags); REGISTER_API(GetClientId); REGISTER_API(GetClientUserNameById); REGISTER_API(GetContextFlags); @@ -11090,6 +11786,7 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(GetServerVersion); REGISTER_API(GetClientCertificate); REGISTER_API(GetCommandKeys); + REGISTER_API(GetCommandKeysWithFlags); REGISTER_API(GetCurrentCommandName); REGISTER_API(GetTypeMethodVersion); REGISTER_API(RegisterDefragFunc); @@ -11098,13 +11795,6 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(DefragShouldStop); REGISTER_API(DefragCursorSet); REGISTER_API(DefragCursorGet); -#ifdef INCLUDE_UNRELEASED_KEYSPEC_API - REGISTER_API(AddCommandKeySpec); - REGISTER_API(SetCommandKeySpecBeginSearchIndex); - REGISTER_API(SetCommandKeySpecBeginSearchKeyword); - REGISTER_API(SetCommandKeySpecFindKeysRange); - REGISTER_API(SetCommandKeySpecFindKeysKeynum); -#endif REGISTER_API(EventLoopAdd); REGISTER_API(EventLoopDel); REGISTER_API(EventLoopAddOneShot); diff --git a/src/modules/Makefile b/src/modules/Makefile index 3db19e79a..c4bc7eb1a 100644 --- a/src/modules/Makefile +++ b/src/modules/Makefile @@ -11,6 +11,13 @@ else SHOBJ_LDFLAGS ?= -bundle -undefined dynamic_lookup endif +# OS X 11.x doesn't have /usr/lib/libSystem.dylib and needs an explicit setting. +ifeq ($(uname_S),Darwin) +ifeq ("$(wildcard /usr/lib/libSystem.dylib)","") +LIBS = -L /Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/lib -lsystem +endif +endif + .SUFFIXES: .c .so .xo .o all: helloworld.so hellotype.so helloblock.so hellocluster.so hellotimer.so hellodict.so hellohook.so helloacl.so diff --git a/src/modules/gendoc.rb b/src/modules/gendoc.rb deleted file mode 100644 index f83b1ad9d..000000000 --- a/src/modules/gendoc.rb +++ /dev/null @@ -1,174 +0,0 @@ -# coding: utf-8 -# gendoc.rb -- Converts the top-comments inside module.c to modules API -# reference documentation in markdown format. - -# Convert the C comment to markdown -def markdown(s) - s = s.gsub(/\*\/$/,"") - s = s.gsub(/^ ?\* ?/,"") - s = s.gsub(/^\/\*\*? ?/,"") - s.chop! while s[-1] == "\n" || s[-1] == " " - lines = s.split("\n") - newlines = [] - # Fix some markdown, except in code blocks indented by 4 spaces. - lines.each{|l| - if not l.start_with?(' ') - # Rewrite RM_Xyz() to `RedisModule_Xyz()`. The () suffix is - # optional. Even RM_Xyz*() with * as wildcard is handled. - l = l.gsub(/(?<!`)RM_([A-z]+(?:\*?\(\))?)/, '`RedisModule_\1`') - # Add backquotes around RedisModule functions and type where missing. - l = l.gsub(/(?<!`)RedisModule[A-z]+(?:\*?\(\))?/){|x| "`#{x}`"} - # Add backquotes around c functions like malloc() where missing. - l = l.gsub(/(?<![`A-z])[a-z_]+\(\)/, '`\0`') - # Add backquotes around macro and var names containing underscores. - l = l.gsub(/(?<![`A-z\*])[A-Za-z]+_[A-Za-z0-9_]+/){|x| "`#{x}`"} - # Link URLs preceded by space or newline (not already linked) - l = l.gsub(/(^| )(https?:\/\/[A-Za-z0-9_\/\.\-]+[A-Za-z0-9\/])/, - '\1[\2](\2)') - # Replace double-dash with unicode ndash - l = l.gsub(/ -- /, ' – ') - end - # Link function names to their definition within the page - l = l.gsub(/`(RedisModule_[A-z0-9]+)[()]*`/) {|x| - $index[$1] ? "[#{x}](\##{$1})" : x - } - newlines << l - } - return newlines.join("\n") -end - -# Linebreak a prototype longer than 80 characters on the commas, but only -# between balanced parentheses so that we don't linebreak args which are -# function pointers, and then aligning each arg under each other. -def linebreak_proto(proto, indent) - if proto.bytesize <= 80 - return proto - end - parts = proto.split(/,\s*/); - if parts.length == 1 - return proto; - end - align_pos = proto.index("(") + 1; - align = " " * align_pos - result = parts.shift; - bracket_balance = 0; - parts.each{|part| - if bracket_balance == 0 - result += ",\n" + indent + align - else - result += ", " - end - result += part - bracket_balance += part.count("(") - part.count(")") - } - return result; -end - -# Given the source code array and the index at which an exported symbol was -# detected, extracts and outputs the documentation. -def docufy(src,i) - m = /RM_[A-z0-9]+/.match(src[i]) - name = m[0] - name = name.sub("RM_","RedisModule_") - proto = src[i].sub("{","").strip+";\n" - proto = proto.sub("RM_","RedisModule_") - proto = linebreak_proto(proto, " "); - # Add a link target with the function name. (We don't trust the exact id of - # the generated one, which depends on the Markdown implementation.) - puts "<span id=\"#{name}\"></span>\n\n" - puts "### `#{name}`\n\n" - puts " #{proto}\n" - comment = "" - while true - i = i-1 - comment = src[i]+comment - break if src[i] =~ /\/\*/ - end - comment = markdown(comment) - puts comment+"\n\n" -end - -# Print a comment from line until */ is found, as markdown. -def section_doc(src, i) - name = get_section_heading(src, i) - comment = "<span id=\"#{section_name_to_id(name)}\"></span>\n\n" - while true - # append line, except if it's a horizontal divider - comment = comment + src[i] if src[i] !~ /^[\/ ]?\*{1,2} ?-{50,}/ - break if src[i] =~ /\*\// - i = i+1 - end - comment = markdown(comment) - puts comment+"\n\n" -end - -# generates an id suitable for links within the page -def section_name_to_id(name) - return "section-" + - name.strip.downcase.gsub(/[^a-z0-9]+/, '-').gsub(/^-+|-+$/, '') -end - -# Returns the name of the first section heading in the comment block for which -# is_section_doc(src, i) is true -def get_section_heading(src, i) - if src[i] =~ /^\/\*\*? \#+ *(.*)/ - heading = $1 - elsif src[i+1] =~ /^ ?\* \#+ *(.*)/ - heading = $1 - end - return heading.gsub(' -- ', ' – ') -end - -# Returns true if the line is the start of a generic documentation section. Such -# section must start with the # symbol, i.e. a markdown heading, on the first or -# the second line. -def is_section_doc(src, i) - return src[i] =~ /^\/\*\*? \#/ || - (src[i] =~ /^\/\*/ && src[i+1] =~ /^ ?\* \#/) -end - -def is_func_line(src, i) - line = src[i] - return line =~ /RM_/ && - line[0] != ' ' && line[0] != '#' && line[0] != '/' && - src[i-1] =~ /\*\// -end - -puts "# Modules API reference\n\n" -puts "<!-- This file is generated from module.c using gendoc.rb -->\n\n" -src = File.open(File.dirname(__FILE__) ++ "/../module.c").to_a - -# Build function index -$index = {} -src.each_with_index do |line,i| - if is_func_line(src, i) - line =~ /RM_([A-z0-9]+)/ - name = "RedisModule_#{$1}" - $index[name] = true - end -end - -# Print TOC -puts "## Sections\n\n" -src.each_with_index do |_line,i| - if is_section_doc(src, i) - name = get_section_heading(src, i) - puts "* [#{name}](\##{section_name_to_id(name)})\n" - end -end -puts "* [Function index](#section-function-index)\n\n" - -# Docufy: Print function prototype and markdown docs -src.each_with_index do |_line,i| - if is_func_line(src, i) - docufy(src, i) - elsif is_section_doc(src, i) - section_doc(src, i) - end -end - -# Print function index -puts "<span id=\"section-function-index\"></span>\n\n" -puts "## Function index\n\n" -$index.keys.sort.each{|x| puts "* [`#{x}`](\##{x})\n"} -puts "\n" diff --git a/src/modules/hellocluster.c b/src/modules/hellocluster.c index 8f822e31e..a6508f837 100644 --- a/src/modules/hellocluster.c +++ b/src/modules/hellocluster.c @@ -76,7 +76,7 @@ int ListCommand_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int void PingReceiver(RedisModuleCtx *ctx, const char *sender_id, uint8_t type, const unsigned char *payload, uint32_t len) { RedisModule_Log(ctx,"notice","PING (type %d) RECEIVED from %.*s: '%.*s'", type,REDISMODULE_NODE_ID_LEN,sender_id,(int)len, payload); - RedisModule_SendClusterMessage(ctx,NULL,MSGTYPE_PONG,(unsigned char*)"Ohi!",4); + RedisModule_SendClusterMessage(ctx,NULL,MSGTYPE_PONG,"Ohi!",4); RedisModuleCallReply *reply = RedisModule_Call(ctx, "INCR", "c", "pings_received"); RedisModule_FreeCallReply(reply); } diff --git a/src/multi.c b/src/multi.c index a98195672..11f33f48f 100644 --- a/src/multi.c +++ b/src/multi.c @@ -189,7 +189,7 @@ void execCommand(client *c) { c->argc = c->mstate.commands[j].argc; c->argv = c->mstate.commands[j].argv; c->argv_len = c->mstate.commands[j].argv_len; - c->cmd = c->mstate.commands[j].cmd; + c->cmd = c->realcmd = c->mstate.commands[j].cmd; /* ACL permissions are also checked at the time of execution in case * they were changed after the commands were queued. */ @@ -240,7 +240,7 @@ void execCommand(client *c) { c->argv = orig_argv; c->argv_len = orig_argv_len; c->argc = orig_argc; - c->cmd = orig_cmd; + c->cmd = c->realcmd = orig_cmd; discardTransaction(c); server.in_exec = 0; @@ -257,10 +257,13 @@ void execCommand(client *c) { /* In the client->watched_keys list we need to use watchedKey structures * as in order to identify a key in Redis we need both the key name and the - * DB */ + * DB. This struct is also referenced from db->watched_keys dict, where the + * values are lists of watchedKey pointers. */ typedef struct watchedKey { robj *key; redisDb *db; + client *client; + unsigned expired:1; /* Flag that we're watching an already expired key. */ } watchedKey; /* Watch for the specified key */ @@ -284,13 +287,15 @@ void watchForKey(client *c, robj *key) { dictAdd(c->db->watched_keys,key,clients); incrRefCount(key); } - listAddNodeTail(clients,c); /* Add the new key to the list of keys watched by this client */ wk = zmalloc(sizeof(*wk)); wk->key = key; + wk->client = c; wk->db = c->db; + wk->expired = keyIsExpired(c->db, key); incrRefCount(key); listAddNodeTail(c->watched_keys,wk); + listAddNodeTail(clients,wk); } /* Unwatch all the keys watched by this client. To clean the EXEC dirty @@ -305,12 +310,12 @@ void unwatchAllKeys(client *c) { list *clients; watchedKey *wk; - /* Lookup the watched key -> clients list and remove the client + /* Lookup the watched key -> clients list and remove the client's wk * from the list */ wk = listNodeValue(ln); clients = dictFetchValue(wk->db->watched_keys, wk->key); serverAssertWithInfo(c,NULL,clients != NULL); - listDelNode(clients,listSearchKey(clients,c)); + listDelNode(clients,listSearchKey(clients,wk)); /* Kill the entry at all if this was the only client */ if (listLength(clients) == 0) dictDelete(wk->db->watched_keys, wk->key); @@ -321,8 +326,8 @@ void unwatchAllKeys(client *c) { } } -/* iterates over the watched_keys list and - * look for an expired key . */ +/* Iterates over the watched_keys list and looks for an expired key. Keys which + * were expired already when WATCH was called are ignored. */ int isWatchedKeyExpired(client *c) { listIter li; listNode *ln; @@ -331,6 +336,7 @@ int isWatchedKeyExpired(client *c) { listRewind(c->watched_keys,&li); while ((ln = listNext(&li))) { wk = listNodeValue(ln); + if (wk->expired) continue; /* was expired when WATCH was called */ if (keyIsExpired(wk->db, wk->key)) return 1; } @@ -352,13 +358,31 @@ void touchWatchedKey(redisDb *db, robj *key) { /* Check if we are already watching for this key */ listRewind(clients,&li); while((ln = listNext(&li))) { - client *c = listNodeValue(ln); + watchedKey *wk = listNodeValue(ln); + client *c = wk->client; + + if (wk->expired) { + /* The key was already expired when WATCH was called. */ + if (db == wk->db && + equalStringObjects(key, wk->key) && + dictFind(db->dict, key->ptr) == NULL) + { + /* Already expired key is deleted, so logically no change. Clear + * the flag. Deleted keys are not flagged as expired. */ + wk->expired = 0; + goto skip_client; + } + break; + } c->flags |= CLIENT_DIRTY_CAS; /* As the client is marked as dirty, there is no point in getting here * again in case that key (or others) are modified again (or keep the * memory overhead till EXEC). */ unwatchAllKeys(c); + + skip_client: + continue; } } @@ -379,14 +403,31 @@ void touchAllWatchedKeysInDb(redisDb *emptied, redisDb *replaced_with) { dictIterator *di = dictGetSafeIterator(emptied->watched_keys); while((de = dictNext(di)) != NULL) { robj *key = dictGetKey(de); - if (dictFind(emptied->dict, key->ptr) || + int exists_in_emptied = dictFind(emptied->dict, key->ptr) != NULL; + if (exists_in_emptied || (replaced_with && dictFind(replaced_with->dict, key->ptr))) { list *clients = dictGetVal(de); if (!clients) continue; listRewind(clients,&li); while((ln = listNext(&li))) { - client *c = listNodeValue(ln); + watchedKey *wk = listNodeValue(ln); + if (wk->expired) { + if (!replaced_with || !dictFind(replaced_with->dict, key->ptr)) { + /* Expired key now deleted. No logical change. Clear the + * flag. Deleted keys are not flagged as expired. */ + wk->expired = 0; + continue; + } else if (keyIsExpired(replaced_with, key)) { + /* Expired key remains expired. */ + continue; + } + } else if (!exists_in_emptied && keyIsExpired(replaced_with, key)) { + /* Non-existing key is replaced with an expired key. */ + wk->expired = 1; + continue; + } + client *c = wk->client; c->flags |= CLIENT_DIRTY_CAS; /* As the client is marked as dirty, there is no point in getting here * again for others keys (or keep the memory overhead till EXEC). */ diff --git a/src/networking.c b/src/networking.c index 05001c564..b05d02b1b 100644 --- a/src/networking.c +++ b/src/networking.c @@ -131,7 +131,7 @@ client *createClient(connection *conn) { connSetReadHandler(conn, readQueryFromClient); connSetPrivateData(conn, c); } - + c->buf = zmalloc(PROTO_REPLY_CHUNK_BYTES); selectDb(c,0); uint64_t client_id; atomicGetIncr(server.next_client_id, client_id, 1); @@ -140,7 +140,9 @@ client *createClient(connection *conn) { c->conn = conn; c->name = NULL; c->bufpos = 0; - c->buf_usable_size = zmalloc_usable_size(c)-offsetof(client,buf); + c->buf_usable_size = zmalloc_usable_size(c->buf); + c->buf_peak = c->buf_usable_size; + c->buf_peak_last_reset_time = server.unixtime; c->ref_repl_buf_node = NULL; c->ref_block_pos = 0; c->qb_pos = 0; @@ -154,7 +156,7 @@ client *createClient(connection *conn) { c->argv_len_sum = 0; c->original_argc = 0; c->original_argv = NULL; - c->cmd = c->lastcmd = NULL; + c->cmd = c->lastcmd = c->realcmd = NULL; c->multibulklen = 0; c->bulklen = -1; c->sentlen = 0; @@ -173,6 +175,7 @@ client *createClient(connection *conn) { c->slave_capa = SLAVE_CAPA_NONE; c->slave_req = SLAVE_REQ_NONE; c->reply = listCreate(); + c->deferred_reply_errors = NULL; c->reply_bytes = 0; c->obuf_soft_limit_reached_time = 0; listSetFreeMethod(c->reply,freeClientReplyValue); @@ -313,6 +316,9 @@ size_t _addReplyToBuffer(client *c, const char *s, size_t len) { size_t reply_len = len > available ? available : len; memcpy(c->buf+c->bufpos,s,reply_len); c->bufpos+=reply_len; + /* We update the buffer peak after appending the reply to the buffer */ + if(c->buf_peak < (size_t)c->bufpos) + c->buf_peak = (size_t)c->bufpos; return reply_len; } @@ -437,24 +443,46 @@ void addReplyErrorLength(client *c, const char *s, size_t len) { addReplyProto(c,"\r\n",2); } -/* Do some actions after an error reply was sent (Log if needed, updates stats, etc.) */ -void afterErrorReply(client *c, const char *s, size_t len) { - /* Increment the global error counter */ - server.stat_total_error_replies++; - /* Increment the error stats - * If the string already starts with "-..." then the error prefix - * is provided by the caller ( we limit the search to 32 chars). Otherwise we use "-ERR". */ - if (s[0] != '-') { - incrementErrorCount("ERR", 3); - } else { - char *spaceloc = memchr(s, ' ', len < 32 ? len : 32); - if (spaceloc) { - const size_t errEndPos = (size_t)(spaceloc - s); - incrementErrorCount(s+1, errEndPos-1); - } else { - /* Fallback to ERR if we can't retrieve the error prefix */ +/* Do some actions after an error reply was sent (Log if needed, updates stats, etc.) + * Possible flags: + * * ERR_REPLY_FLAG_NO_STATS_UPDATE - indicate not to update any error stats. */ +void afterErrorReply(client *c, const char *s, size_t len, int flags) { + /* Module clients fall into two categories: + * Calls to RM_Call, in which case the error isn't being returned to a client, so should not be counted. + * Module thread safe context calls to RM_ReplyWithError, which will be added to a real client by the main thread later. */ + if (c->flags & CLIENT_MODULE) { + if (!c->deferred_reply_errors) { + c->deferred_reply_errors = listCreate(); + listSetFreeMethod(c->deferred_reply_errors, (void (*)(void*))sdsfree); + } + listAddNodeTail(c->deferred_reply_errors, sdsnewlen(s, len)); + return; + } + + if (!(flags & ERR_REPLY_FLAG_NO_STATS_UPDATE)) { + /* Increment the global error counter */ + server.stat_total_error_replies++; + /* Increment the error stats + * If the string already starts with "-..." then the error prefix + * is provided by the caller ( we limit the search to 32 chars). Otherwise we use "-ERR". */ + if (s[0] != '-') { incrementErrorCount("ERR", 3); + } else { + char *spaceloc = memchr(s, ' ', len < 32 ? len : 32); + if (spaceloc) { + const size_t errEndPos = (size_t)(spaceloc - s); + incrementErrorCount(s+1, errEndPos-1); + } else { + /* Fallback to ERR if we can't retrieve the error prefix */ + incrementErrorCount("ERR", 3); + } } + } else { + /* stat_total_error_replies will not be updated, which means that + * the cmd stats will not be updated as well, we still want this command + * to be counted as failed so we update it here. We update c->realcmd in + * case c->cmd was changed (like in GEOADD). */ + c->realcmd->failed_calls++; } /* Sometimes it could be normal that a slave replies to a master with @@ -500,7 +528,7 @@ void afterErrorReply(client *c, const char *s, size_t len) { * Unlike addReplyErrorSds and others alike which rely on addReplyErrorLength. */ void addReplyErrorObject(client *c, robj *err) { addReply(c, err); - afterErrorReply(c, err->ptr, sdslen(err->ptr)-2); /* Ignore trailing \r\n */ + afterErrorReply(c, err->ptr, sdslen(err->ptr)-2, 0); /* Ignore trailing \r\n */ } /* Sends either a reply or an error reply by checking the first char. @@ -521,34 +549,57 @@ void addReplyOrErrorObject(client *c, robj *reply) { /* See addReplyErrorLength for expectations from the input string. */ void addReplyError(client *c, const char *err) { addReplyErrorLength(c,err,strlen(err)); - afterErrorReply(c,err,strlen(err)); + afterErrorReply(c,err,strlen(err),0); +} + +/* Add error reply to the given client. + * Supported flags: + * * ERR_REPLY_FLAG_NO_STATS_UPDATE - indicate not to perform any error stats updates */ +void addReplyErrorSdsEx(client *c, sds err, int flags) { + addReplyErrorLength(c,err,sdslen(err)); + afterErrorReply(c,err,sdslen(err),flags); + sdsfree(err); } /* See addReplyErrorLength for expectations from the input string. */ /* As a side effect the SDS string is freed. */ void addReplyErrorSds(client *c, sds err) { - addReplyErrorLength(c,err,sdslen(err)); - afterErrorReply(c,err,sdslen(err)); - sdsfree(err); + addReplyErrorSdsEx(c, err, 0); } -/* See addReplyErrorLength for expectations from the formatted string. - * The formatted string is safe to contain \r and \n anywhere. */ -void addReplyErrorFormat(client *c, const char *fmt, ...) { - va_list ap; - va_start(ap,fmt); - sds s = sdscatvprintf(sdsempty(),fmt,ap); - va_end(ap); +/* Internal function used by addReplyErrorFormat and addReplyErrorFormatEx. + * Refer to afterErrorReply for more information about the flags. */ +static void addReplyErrorFormatInternal(client *c, int flags, const char *fmt, va_list ap) { + va_list cpy; + va_copy(cpy,ap); + sds s = sdscatvprintf(sdsempty(),fmt,cpy); + va_end(cpy); /* Trim any newlines at the end (ones will be added by addReplyErrorLength) */ s = sdstrim(s, "\r\n"); /* Make sure there are no newlines in the middle of the string, otherwise * invalid protocol is emitted. */ s = sdsmapchars(s, "\r\n", " ", 2); addReplyErrorLength(c,s,sdslen(s)); - afterErrorReply(c,s,sdslen(s)); + afterErrorReply(c,s,sdslen(s),flags); sdsfree(s); } +void addReplyErrorFormatEx(client *c, int flags, const char *fmt, ...) { + va_list ap; + va_start(ap,fmt); + addReplyErrorFormatInternal(c, flags, fmt, ap); + va_end(ap); +} + +/* See addReplyErrorLength for expectations from the formatted string. + * The formatted string is safe to contain \r and \n anywhere. */ +void addReplyErrorFormat(client *c, const char *fmt, ...) { + va_list ap; + va_start(ap,fmt); + addReplyErrorFormatInternal(c, 0, fmt, ap); + va_end(ap); +} + void addReplyErrorArity(client *c) { addReplyErrorFormat(c, "wrong number of arguments for '%s' command", c->cmd->fullname); @@ -696,6 +747,24 @@ void setDeferredAggregateLen(client *c, void *node, long length, char prefix) { * we return NULL in addReplyDeferredLen() */ if (node == NULL) return; + /* Things like *2\r\n, %3\r\n or ~4\r\n are emitted very often by the protocol + * so we have a few shared objects to use if the integer is small + * like it is most of the times. */ + const size_t hdr_len = OBJ_SHARED_HDR_STRLEN(length); + const int opt_hdr = length < OBJ_SHARED_BULKHDR_LEN; + if (prefix == '*' && opt_hdr) { + setDeferredReply(c, node, shared.mbulkhdr[length]->ptr, hdr_len); + return; + } + if (prefix == '%' && opt_hdr) { + setDeferredReply(c, node, shared.maphdr[length]->ptr, hdr_len); + return; + } + if (prefix == '~' && opt_hdr) { + setDeferredReply(c, node, shared.sethdr[length]->ptr, hdr_len); + return; + } + char lenstr[128]; size_t lenstr_len = sprintf(lenstr, "%c%ld\r\n", prefix, length); setDeferredReply(c, node, lenstr, lenstr_len); @@ -788,11 +857,19 @@ void addReplyLongLongWithPrefix(client *c, long long ll, char prefix) { /* Things like $3\r\n or *2\r\n are emitted very often by the protocol * so we have a few shared objects to use if the integer is small * like it is most of the times. */ - if (prefix == '*' && ll < OBJ_SHARED_BULKHDR_LEN && ll >= 0) { - addReply(c,shared.mbulkhdr[ll]); + const int opt_hdr = ll < OBJ_SHARED_BULKHDR_LEN && ll >= 0; + const size_t hdr_len = OBJ_SHARED_HDR_STRLEN(ll); + if (prefix == '*' && opt_hdr) { + addReplyProto(c,shared.mbulkhdr[ll]->ptr,hdr_len); + return; + } else if (prefix == '$' && opt_hdr) { + addReplyProto(c,shared.bulkhdr[ll]->ptr,hdr_len); + return; + } else if (prefix == '%' && opt_hdr) { + addReplyProto(c,shared.maphdr[ll]->ptr,hdr_len); return; - } else if (prefix == '$' && ll < OBJ_SHARED_BULKHDR_LEN && ll >= 0) { - addReply(c,shared.bulkhdr[ll]); + } else if (prefix == '~' && opt_hdr) { + addReplyProto(c,shared.sethdr[ll]->ptr,hdr_len); return; } @@ -1024,10 +1101,28 @@ void AddReplyFromClient(client *dst, client *src) { src->reply_bytes = 0; src->bufpos = 0; + if (src->deferred_reply_errors) { + deferredAfterErrorReply(dst, src->deferred_reply_errors); + listRelease(src->deferred_reply_errors); + src->deferred_reply_errors = NULL; + } + /* Check output buffer limits */ closeClientOnOutputBufferLimitReached(dst, 1); } +/* Append the listed errors to the server error statistics. the input + * list is not modified and remains the responsibility of the caller. */ +void deferredAfterErrorReply(client *c, list *errors) { + listIter li; + listNode *ln; + listRewind(errors,&li); + while((ln = listNext(&li))) { + sds err = ln->value; + afterErrorReply(c, err, sdslen(err), 0); + } +} + /* Logically copy 'src' replica client buffers info to 'dst' replica. * Basically increase referenced buffer block node reference count. */ void copyReplicaOutputBuffer(client *dst, client *src) { @@ -1494,9 +1589,12 @@ void freeClient(client *c) { /* Free data structures. */ listRelease(c->reply); + zfree(c->buf); freeReplicaReferencedReplBuffer(c); freeClientArgv(c); freeClientOriginalArgv(c); + if (c->deferred_reply_errors) + listRelease(c->deferred_reply_errors); /* Unlink the client: this will close the socket, remove the I/O * handlers, and remove references of the client from different @@ -1658,10 +1756,82 @@ client *lookupClientByID(uint64_t id) { return (c == raxNotFound) ? NULL : c; } +/* This function should be called from _writeToClient when the reply list is not empty, + * it gathers the scattered buffers from reply list and sends them away with connWritev. + * If we write successfully, it returns C_OK, otherwise, C_ERR is returned, + * and 'nwritten' is an output parameter, it means how many bytes server write + * to client. */ +static int _writevToClient(client *c, ssize_t *nwritten) { + struct iovec iov[IOV_MAX]; + int iovcnt = 0; + size_t iov_bytes_len = 0; + /* If the static reply buffer is not empty, + * add it to the iov array for writev() as well. */ + if (c->bufpos > 0) { + iov[iovcnt].iov_base = c->buf + c->sentlen; + iov[iovcnt].iov_len = c->bufpos - c->sentlen; + iov_bytes_len += iov[iovcnt++].iov_len; + } + /* The first node of reply list might be incomplete from the last call, + * thus it needs to be calibrated to get the actual data address and length. */ + size_t offset = c->bufpos > 0 ? 0 : c->sentlen; + listIter iter; + listNode *next; + clientReplyBlock *o; + listRewind(c->reply, &iter); + while ((next = listNext(&iter)) && iovcnt < IOV_MAX && iov_bytes_len < NET_MAX_WRITES_PER_EVENT) { + o = listNodeValue(next); + if (o->used == 0) { /* empty node, just release it and skip. */ + c->reply_bytes -= o->size; + listDelNode(c->reply, next); + offset = 0; + continue; + } + + iov[iovcnt].iov_base = o->buf + offset; + iov[iovcnt].iov_len = o->used - offset; + iov_bytes_len += iov[iovcnt++].iov_len; + offset = 0; + } + if (iovcnt == 0) return C_OK; + *nwritten = connWritev(c->conn, iov, iovcnt); + if (*nwritten <= 0) return C_ERR; + + /* Locate the new node which has leftover data and + * release all nodes in front of it. */ + ssize_t remaining = *nwritten; + if (c->bufpos > 0) { /* deal with static reply buffer first. */ + int buf_len = c->bufpos - c->sentlen; + c->sentlen += remaining; + /* If the buffer was sent, set bufpos to zero to continue with + * the remainder of the reply. */ + if (remaining >= buf_len) { + c->bufpos = 0; + c->sentlen = 0; + } + remaining -= buf_len; + } + listRewind(c->reply, &iter); + while (remaining > 0) { + next = listNext(&iter); + o = listNodeValue(next); + if (remaining < (ssize_t)(o->used - c->sentlen)) { + c->sentlen += remaining; + break; + } + remaining -= (ssize_t)(o->used - c->sentlen); + c->reply_bytes -= o->size; + listDelNode(c->reply, next); + c->sentlen = 0; + } + + return C_OK; +} + /* This function does actual writing output buffers to different types of * clients, it is called by writeToClient. - * If we write successfully, it return C_OK, otherwise, C_ERR is returned, - * And 'nwritten' is a output parameter, it means how many bytes server write + * If we write successfully, it returns C_OK, otherwise, C_ERR is returned, + * and 'nwritten' is an output parameter, it means how many bytes server write * to client. */ int _writeToClient(client *c, ssize_t *nwritten) { *nwritten = 0; @@ -1690,8 +1860,18 @@ int _writeToClient(client *c, ssize_t *nwritten) { return C_OK; } - if (c->bufpos > 0) { - *nwritten = connWrite(c->conn,c->buf+c->sentlen,c->bufpos-c->sentlen); + /* When the reply list is not empty, it's better to use writev to save us some + * system calls and TCP packets. */ + if (listLength(c->reply) > 0) { + int ret = _writevToClient(c, nwritten); + if (ret != C_OK) return ret; + + /* If there are no longer objects in the list, we expect + * the count of reply bytes to be exactly zero. */ + if (listLength(c->reply) == 0) + serverAssert(c->reply_bytes == 0); + } else if (c->bufpos > 0) { + *nwritten = connWrite(c->conn, c->buf + c->sentlen, c->bufpos - c->sentlen); if (*nwritten <= 0) return C_ERR; c->sentlen += *nwritten; @@ -1701,31 +1881,8 @@ int _writeToClient(client *c, ssize_t *nwritten) { c->bufpos = 0; c->sentlen = 0; } - } else { - clientReplyBlock *o = listNodeValue(listFirst(c->reply)); - size_t objlen = o->used; - - if (objlen == 0) { - c->reply_bytes -= o->size; - listDelNode(c->reply,listFirst(c->reply)); - return C_OK; - } - - *nwritten = connWrite(c->conn, o->buf + c->sentlen, objlen - c->sentlen); - if (*nwritten <= 0) return C_ERR; - c->sentlen += *nwritten; + } - /* If we fully sent the object on head go to the next one */ - if (c->sentlen == objlen) { - c->reply_bytes -= o->size; - listDelNode(c->reply,listFirst(c->reply)); - c->sentlen = 0; - /* If there are no longer objects in the list, we expect - * the count of reply bytes to be exactly zero. */ - if (listLength(c->reply) == 0) - serverAssert(c->reply_bytes == 0); - } - } return C_OK; } @@ -1863,6 +2020,10 @@ void resetClient(client *c) { c->multibulklen = 0; c->bulklen = -1; + if (c->deferred_reply_errors) + listRelease(c->deferred_reply_errors); + c->deferred_reply_errors = NULL; + /* We clear the ASKING flag as well if we are not inside a MULTI, and * if what we just executed is not the ASKING command itself. */ if (!(c->flags & CLIENT_MULTI) && prevcmd != askingCommand) @@ -2556,7 +2717,7 @@ sds catClientInfoString(sds s, client *client) { } sds ret = sdscatfmt(s, - "id=%U addr=%s laddr=%s %s name=%s age=%I idle=%I flags=%s db=%i sub=%i psub=%i multi=%i qbuf=%U qbuf-free=%U argv-mem=%U multi-mem=%U obl=%U oll=%U omem=%U tot-mem=%U events=%s cmd=%s user=%s redir=%I resp=%i", + "id=%U addr=%s laddr=%s %s name=%s age=%I idle=%I flags=%s db=%i sub=%i psub=%i multi=%i qbuf=%U qbuf-free=%U argv-mem=%U multi-mem=%U rbs=%U rbp=%U obl=%U oll=%U omem=%U tot-mem=%U events=%s cmd=%s user=%s redir=%I resp=%i", (unsigned long long) client->id, getClientPeerId(client), getClientSockname(client), @@ -2573,6 +2734,8 @@ sds catClientInfoString(sds s, client *client) { (unsigned long long) sdsavail(client->querybuf), (unsigned long long) client->argv_len_sum, (unsigned long long) client->mstate.argv_len_sums, + (unsigned long long) client->buf_usable_size, + (unsigned long long) client->buf_peak, (unsigned long long) client->bufpos, (unsigned long long) listLength(client->reply) + used_blocks_of_repl_buf, (unsigned long long) obufmem, /* should not include client->buf since we want to see 0 for static clients. */ @@ -2919,6 +3082,7 @@ NULL else replyToBlockedClientTimedOut(target); unblockClient(target); + updateStatsOnUnblock(target, 0, 0, 1); addReply(c,shared.cone); } else { addReply(c,shared.czero); @@ -3414,6 +3578,7 @@ size_t getClientMemoryUsage(client *c, size_t *output_buffer_mem_usage) { *output_buffer_mem_usage = mem; mem += sdsZmallocSize(c->querybuf); mem += zmalloc_size(c); + mem += c->buf_usable_size; /* For efficiency (less work keeping track of the argv memory), it doesn't include the used memory * i.e. unused sds space and internal fragmentation, just the string length. but this is enough to * spot problematic clients. */ @@ -692,7 +692,7 @@ int rdbSaveObjectType(rio *rdb, robj *o) { else serverPanic("Unknown hash encoding"); case OBJ_STREAM: - return rdbSaveType(rdb,RDB_TYPE_STREAM_LISTPACKS); + return rdbSaveType(rdb,RDB_TYPE_STREAM_LISTPACKS_2); case OBJ_MODULE: return rdbSaveType(rdb,RDB_TYPE_MODULE_2); default: @@ -986,6 +986,19 @@ ssize_t rdbSaveObject(rio *rdb, robj *o, robj *key, int dbid) { nwritten += n; if ((n = rdbSaveLen(rdb,s->last_id.seq)) == -1) return -1; nwritten += n; + /* Save the first entry ID. */ + if ((n = rdbSaveLen(rdb,s->first_id.ms)) == -1) return -1; + nwritten += n; + if ((n = rdbSaveLen(rdb,s->first_id.seq)) == -1) return -1; + nwritten += n; + /* Save the maximal tombstone ID. */ + if ((n = rdbSaveLen(rdb,s->max_deleted_entry_id.ms)) == -1) return -1; + nwritten += n; + if ((n = rdbSaveLen(rdb,s->max_deleted_entry_id.seq)) == -1) return -1; + nwritten += n; + /* Save the offset. */ + if ((n = rdbSaveLen(rdb,s->entries_added)) == -1) return -1; + nwritten += n; /* The consumer groups and their clients are part of the stream * type, so serialize every consumer group. */ @@ -1020,6 +1033,13 @@ ssize_t rdbSaveObject(rio *rdb, robj *o, robj *key, int dbid) { return -1; } nwritten += n; + + /* Save the group's logical reads counter. */ + if ((n = rdbSaveLen(rdb,cg->entries_read)) == -1) { + raxStop(&ri); + return -1; + } + nwritten += n; /* Save the global PEL. */ if ((n = rdbSaveStreamPEL(rdb,cg->pel,1)) == -1) { @@ -1151,8 +1171,9 @@ ssize_t rdbSaveAuxFieldStrInt(rio *rdb, char *key, long long val) { /* Save a few default AUX fields with information about the RDB generated. */ int rdbSaveInfoAuxFields(rio *rdb, int rdbflags, rdbSaveInfo *rsi) { + UNUSED(rdbflags); int redis_bits = (sizeof(void*) == 8) ? 64 : 32; - int aof_preamble = (rdbflags & RDBFLAGS_AOF_PREAMBLE) != 0; + int aof_base = (rdbflags & RDBFLAGS_AOF_PREAMBLE) != 0; /* Add a few fields about the state when the RDB was created. */ if (rdbSaveAuxFieldStrStr(rdb,"redis-ver",REDIS_VERSION) == -1) return -1; @@ -1169,7 +1190,7 @@ int rdbSaveInfoAuxFields(rio *rdb, int rdbflags, rdbSaveInfo *rsi) { if (rdbSaveAuxFieldStrInt(rdb,"repl-offset",server.master_repl_offset) == -1) return -1; } - if (rdbSaveAuxFieldStrInt(rdb,"aof-preamble",aof_preamble) == -1) return -1; + if (rdbSaveAuxFieldStrInt(rdb, "aof-base", aof_base) == -1) return -1; return 1; } @@ -1470,6 +1491,7 @@ int rdbSaveBackground(int req, char *filename, rdbSaveInfo *rsi) { pid_t childpid; if (hasActiveChildProcess()) return C_ERR; + server.stat_rdb_saves++; server.dirty_before_bgsave = server.dirty; server.lastbgsave_try = time(NULL); @@ -2319,7 +2341,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) { rdbReportCorruptRDB("Unknown RDB encoding type %d",rdbtype); break; } - } else if (rdbtype == RDB_TYPE_STREAM_LISTPACKS) { + } else if (rdbtype == RDB_TYPE_STREAM_LISTPACKS || rdbtype == RDB_TYPE_STREAM_LISTPACKS_2) { o = createStreamObject(); stream *s = o->ptr; uint64_t listpacks = rdbLoadLen(rdb,NULL); @@ -2395,6 +2417,30 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) { /* Load the last entry ID. */ s->last_id.ms = rdbLoadLen(rdb,NULL); s->last_id.seq = rdbLoadLen(rdb,NULL); + + if (rdbtype == RDB_TYPE_STREAM_LISTPACKS_2) { + /* Load the first entry ID. */ + s->first_id.ms = rdbLoadLen(rdb,NULL); + s->first_id.seq = rdbLoadLen(rdb,NULL); + + /* Load the maximal deleted entry ID. */ + s->max_deleted_entry_id.ms = rdbLoadLen(rdb,NULL); + s->max_deleted_entry_id.seq = rdbLoadLen(rdb,NULL); + + /* Load the offset. */ + s->entries_added = rdbLoadLen(rdb,NULL); + } else { + /* During migration the offset can be initialized to the stream's + * length. At this point, we also don't care about tombstones + * because CG offsets will be later initialized as well. */ + s->max_deleted_entry_id.ms = 0; + s->max_deleted_entry_id.seq = 0; + s->entries_added = s->length; + + /* Since the rax is already loaded, we can find the first entry's + * ID. */ + streamGetEdgeID(s,1,1,&s->first_id); + } if (rioGetReadError(rdb)) { rdbReportReadError("Stream object metadata loading failed."); @@ -2430,8 +2476,22 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) { decrRefCount(o); return NULL; } + + /* Load group offset. */ + uint64_t cg_offset; + if (rdbtype == RDB_TYPE_STREAM_LISTPACKS_2) { + cg_offset = rdbLoadLen(rdb,NULL); + if (rioGetReadError(rdb)) { + rdbReportReadError("Stream cgroup offset loading failed."); + sdsfree(cgname); + decrRefCount(o); + return NULL; + } + } else { + cg_offset = streamEstimateDistanceFromFirstEverEntry(s,&cg_id); + } - streamCG *cgroup = streamCreateCG(s,cgname,sdslen(cgname),&cg_id); + streamCG *cgroup = streamCreateCG(s,cgname,sdslen(cgname),&cg_id,cg_offset); if (cgroup == NULL) { rdbReportCorruptRDB("Duplicated consumer group name %s", cgname); @@ -2962,6 +3022,9 @@ int rdbLoadRioWithLoadingCtx(rio *rdb, int rdbflags, rdbSaveInfo *rsi, rdbLoadin } else if (!strcasecmp(auxkey->ptr,"aof-preamble")) { long long haspreamble = strtoll(auxval->ptr,NULL,10); if (haspreamble) serverLog(LL_NOTICE,"RDB has an AOF tail"); + } else if (!strcasecmp(auxkey->ptr, "aof-base")) { + long long isbase = strtoll(auxval->ptr, NULL, 10); + if (isbase) serverLog(LL_NOTICE, "RDB is base AOF"); } else if (!strcasecmp(auxkey->ptr,"redis-bits")) { /* Just ignored. */ } else { @@ -3049,9 +3112,9 @@ int rdbLoadRioWithLoadingCtx(rio *rdb, int rdbflags, rdbSaveInfo *rsi, rdbLoadin * received from the master. In the latter case, the master is * responsible for key expiry. If we would expire keys here, the * snapshot taken by the master may not be reflected on the slave. - * Similarly if the RDB is the preamble of an AOF file, we want to - * load all the keys as they are, since the log of operations later - * assume to work in an exact keyspace state. */ + * Similarly, if the base AOF is RDB format, we want to load all + * the keys they are, since the log of operations in the incr AOF + * is assumed to work in the exact keyspace state. */ if (val == NULL) { /* Since we used to have bug that could lead to empty keys * (See #8453), we rather not fail when empty key is encountered @@ -94,10 +94,11 @@ #define RDB_TYPE_HASH_LISTPACK 16 #define RDB_TYPE_ZSET_LISTPACK 17 #define RDB_TYPE_LIST_QUICKLIST_2 18 +#define RDB_TYPE_STREAM_LISTPACKS_2 19 /* NOTE: WHEN ADDING NEW RDB TYPE, UPDATE rdbIsObjectType() BELOW */ /* Test if a type is an object type. */ -#define rdbIsObjectType(t) ((t >= 0 && t <= 7) || (t >= 9 && t <= 18)) +#define rdbIsObjectType(t) ((t >= 0 && t <= 7) || (t >= 9 && t <= 19)) /* Special RDB opcodes (saved/loaded with rdbSaveType/rdbLoadType). */ #define RDB_OPCODE_FUNCTION 246 /* engine data */ diff --git a/src/redis-benchmark.c b/src/redis-benchmark.c index 86421e33f..64893ac37 100644 --- a/src/redis-benchmark.c +++ b/src/redis-benchmark.c @@ -125,6 +125,7 @@ static struct config { int enable_tracking; pthread_mutex_t liveclients_mutex; pthread_mutex_t is_updating_slots_mutex; + int resp3; /* use RESP3 */ } config; typedef struct _client { @@ -632,6 +633,9 @@ static void writeHandler(aeEventLoop *el, int fd, void *privdata, int mask) { fprintf(stderr, "Error writing to the server: %s\n", strerror(errno)); freeClient(c); return; + } else if (nwritten > 0) { + c->written += nwritten; + return; } } else { aeDeleteFileEvent(el,c->context->fd,AE_WRITABLE); @@ -748,6 +752,15 @@ static client createClient(char *cmd, size_t len, client from, int thread_id) { (int)sdslen(config.input_dbnumstr),config.input_dbnumstr); c->prefix_pending++; } + + if (config.resp3) { + char *buf = NULL; + int len = redisFormatCommand(&buf, "HELLO 3"); + c->obuf = sdscatlen(c->obuf, buf, len); + free(buf); + c->prefix_pending++; + } + c->prefixlen = sdslen(c->obuf); /* Append the request itself. */ if (from) { @@ -1440,6 +1453,8 @@ int parseOptions(int argc, char **argv) { } else if (!strcmp(argv[i],"-u") && !lastarg) { parseRedisUri(argv[++i],"redis-benchmark",&config.conn_info,&config.tls); config.input_dbnumstr = sdsfromlonglong(config.conn_info.input_dbnum); + } else if (!strcmp(argv[i],"-3")) { + config.resp3 = 1; } else if (!strcmp(argv[i],"-d")) { if (lastarg) goto invalid; config.datasize = atoi(argv[++i]); @@ -1566,6 +1581,7 @@ usage: " -n <requests> Total number of requests (default 100000)\n" " -d <size> Data size of SET/GET value in bytes (default 3)\n" " --dbnum <db> SELECT the specified db number (default 0)\n" +" -3 Start session in RESP3 protocol mode.\n" " --threads <num> Enable multi-thread mode.\n" " --cluster Enable cluster mode.\n" " If the command is supplied on the command line in cluster\n" @@ -1739,6 +1755,7 @@ int main(int argc, char **argv) { config.is_updating_slots = 0; config.slots_last_update = 0; config.enable_tracking = 0; + config.resp3 = 0; i = parseOptions(argc,argv); argc -= i; diff --git a/src/redis-check-aof.c b/src/redis-check-aof.c index 01f42ec1b..a3da79dd4 100644 --- a/src/redis-check-aof.c +++ b/src/redis-check-aof.c @@ -30,6 +30,24 @@ #include "server.h" #include <sys/stat.h> +#include <sys/types.h> +#include <regex.h> +#include <libgen.h> + +#define AOF_CHECK_OK 0 +#define AOF_CHECK_EMPTY 1 +#define AOF_CHECK_TRUNCATED 2 +#define AOF_CHECK_TIMESTAMP_TRUNCATED 3 + +typedef enum { + AOF_RESP, + AOF_RDB_PREAMBLE, + AOF_MULTI_PART, +} input_file_type; + +aofManifest *aofManifestCreate(void); +void aofManifestFree(aofManifest *am); +aofManifest *aofLoadManifestFromFile(sds am_filepath); #define ERROR(...) { \ char __buf[1024]; \ @@ -51,47 +69,6 @@ int consumeNewline(char *buf) { return 1; } -int readAnnotations(FILE *fp) { - char buf[AOF_ANNOTATION_LINE_MAX_LEN]; - while (1) { - epos = ftello(fp); - if (fgets(buf, sizeof(buf), fp) == NULL) { - return 0; - } - if (buf[0] == '#') { - if (to_timestamp && strncmp(buf, "#TS:", 4) == 0) { - time_t ts = strtol(buf+4, NULL, 10); - if (ts <= to_timestamp) continue; - if (epos == 0) { - printf("AOF has nothing before timestamp %ld, " - "aborting...\n", to_timestamp); - fclose(fp); - exit(1); - } - /* Truncate remaining AOF if exceeding 'to_timestamp' */ - if (ftruncate(fileno(fp), epos) == -1) { - printf("Failed to truncate AOF to timestamp %ld\n", - to_timestamp); - exit(1); - } else { - printf("Successfully truncated AOF to timestamp %ld\n", - to_timestamp); - fclose(fp); - exit(0); - } - } - continue; - } else { - if (fseek(fp, -(ftello(fp)-epos), SEEK_CUR) == -1) { - ERROR("Fseek error: %s", strerror(errno)); - return 0; - } - return 1; - } - } - return 1; -} - int readLong(FILE *fp, char prefix, long *target) { char buf[128], *eptr; epos = ftello(fp); @@ -133,9 +110,13 @@ int readString(FILE *fp, char** target) { len += 2; *target = (char*)zmalloc(len); if (!readBytes(fp,*target,len)) { + zfree(*target); + *target = NULL; return 0; } if (!consumeNewline(*target+len-2)) { + zfree(*target); + *target = NULL; return 0; } (*target)[len-2] = '\0'; @@ -146,156 +127,454 @@ int readArgc(FILE *fp, long *target) { return readLong(fp,'*',target); } -off_t process(FILE *fp) { +/* Used to decode a RESP record in the AOF file to obtain the original + * redis command, and also check whether the command is MULTI/EXEC. If the + * command is MULTI, the parameter out_multi will be incremented by one, and + * if the command is EXEC, the parameter out_multi will be decremented + * by one. The parameter out_multi will be used by the upper caller to determine + * whether the AOF file contains unclosed transactions. + **/ +int processRESP(FILE *fp, char *filename, int *out_multi) { long argc; - off_t pos = 0; - int i, multi = 0; char *str; - while(1) { - if (!multi) pos = ftello(fp); - if (!readAnnotations(fp)) break; - if (!readArgc(fp, &argc)) break; - - for (i = 0; i < argc; i++) { - if (!readString(fp,&str)) break; - if (i == 0) { - if (strcasecmp(str, "multi") == 0) { - if (multi++) { - ERROR("Unexpected MULTI"); - break; - } - } else if (strcasecmp(str, "exec") == 0) { - if (--multi) { - ERROR("Unexpected EXEC"); - break; - } + if (!readArgc(fp, &argc)) return 0; + + for (int i = 0; i < argc; i++) { + if (!readString(fp, &str)) return 0; + if (i == 0) { + if (strcasecmp(str, "multi") == 0) { + if ((*out_multi)++) { + ERROR("Unexpected MULTI in AOF %s", filename); + zfree(str); + return 0; + } + } else if (strcasecmp(str, "exec") == 0) { + if (--(*out_multi)) { + ERROR("Unexpected EXEC in AOF %s", filename); + zfree(str); + return 0; } } - zfree(str); - } - - /* Stop if the loop did not finish */ - if (i < argc) { - if (str) zfree(str); - break; } + zfree(str); } - if (feof(fp) && multi && strlen(error) == 0) { - ERROR("Reached EOF before reading EXEC for MULTI"); - } - if (strlen(error) > 0) { - printf("%s\n", error); - } - return pos; + return 1; } -int redis_check_aof_main(int argc, char **argv) { - char *filename; - int fix = 0; +/* Used to parse an annotation in the AOF file, the annotation starts with '#' + * in AOF. Currently AOF only contains timestamp annotations, but this function + * can easily be extended to handle other annotations. + * + * The processing rule of time annotation is that once the timestamp is found to + * be greater than 'to_timestamp', the AOF after the annotation is truncated. + * Note that in Multi Part AOF, this truncation is only allowed when the last_file + * parameter is 1. + **/ +int processAnnotations(FILE *fp, char *filename, int last_file) { + char buf[AOF_ANNOTATION_LINE_MAX_LEN]; - if (argc < 2) { - goto invalid_args; - } else if (argc == 2) { - filename = argv[1]; - } else if (argc == 3) { - if (!strcmp(argv[1],"--fix")) { - filename = argv[2]; - fix = 1; - } else { - goto invalid_args; + epos = ftello(fp); + if (fgets(buf, sizeof(buf), fp) == NULL) { + printf("Failed to read annotations from AOF %s, aborting...\n", filename); + exit(1); + } + + if (to_timestamp && strncmp(buf, "#TS:", 4) == 0) { + char *endptr; + errno = 0; + time_t ts = strtol(buf+4, &endptr, 10); + if (errno != 0 || *endptr != '\r') { + printf("Invalid timestamp annotation\n"); + exit(1); } - } else if (argc == 4) { - if (!strcmp(argv[1], "--truncate-to-timestamp")) { - to_timestamp = strtol(argv[2],NULL,10); - filename = argv[3]; + if (ts <= to_timestamp) return 1; + if (epos == 0) { + printf("AOF %s has nothing before timestamp %ld, " + "aborting...\n", filename, to_timestamp); + exit(1); + } + if (!last_file) { + printf("Failed to truncate AOF %s to timestamp %ld to offset %ld because it is not the last file.\n", + filename, to_timestamp, (long int)epos); + printf("If you insist, please delete all files after this file according to the manifest " + "file and delete the corresponding records in manifest file manually. Then re-run redis-check-aof.\n"); + exit(1); + } + /* Truncate remaining AOF if exceeding 'to_timestamp' */ + if (ftruncate(fileno(fp), epos) == -1) { + printf("Failed to truncate AOF %s to timestamp %ld\n", + filename, to_timestamp); + exit(1); } else { - goto invalid_args; + return 0; } - } else { - goto invalid_args; } + return 1; +} + +/* Used to check the validity of a single AOF file. The AOF file can be: + * 1. Old-style AOF + * 2. Old-style RDB-preamble AOF + * 3. BASE or INCR in Multi Part AOF + * */ +int checkSingleAof(char *aof_filename, char *aof_filepath, int last_file, int fix, int preamble) { + off_t pos = 0, diff; + int multi = 0; + char buf[2]; - FILE *fp = fopen(filename,"r+"); + FILE *fp = fopen(aof_filepath, "r+"); if (fp == NULL) { - printf("Cannot open file: %s\n", filename); + printf("Cannot open file %s: %s, aborting...\n", aof_filepath, strerror(errno)); exit(1); } struct redis_stat sb; if (redis_fstat(fileno(fp),&sb) == -1) { - printf("Cannot stat file: %s\n", filename); + printf("Cannot stat file: %s, aborting...\n", aof_filename); exit(1); } off_t size = sb.st_size; if (size == 0) { - printf("Empty file: %s\n", filename); - exit(1); + return AOF_CHECK_EMPTY; } - /* This AOF file may have an RDB preamble. Check this to start, and if this - * is the case, start processing the RDB part. */ - if (size >= 8) { /* There must be at least room for the RDB header. */ - char sig[5]; - int has_preamble = fread(sig,sizeof(sig),1,fp) == 1 && - memcmp(sig,"REDIS",sizeof(sig)) == 0; - rewind(fp); - if (has_preamble) { - printf("The AOF appears to start with an RDB preamble.\n" - "Checking the RDB preamble to start:\n"); - if (redis_check_rdb_main(argc,argv,fp) == C_ERR) { - printf("RDB preamble of AOF file is not sane, aborting.\n"); - exit(1); - } else { - printf("RDB preamble is OK, proceeding with AOF tail...\n"); + if (preamble) { + char *argv[2] = {NULL, aof_filename}; + if (redis_check_rdb_main(2, argv, fp) == C_ERR) { + printf("RDB preamble of AOF file is not sane, aborting.\n"); + exit(1); + } else { + printf("RDB preamble is OK, proceeding with AOF tail...\n"); + } + } + + while(1) { + if (!multi) pos = ftello(fp); + if (fgets(buf, sizeof(buf), fp) == NULL) { + if (feof(fp)) { + break; + } + printf("Failed to read from AOF %s, aborting...\n", aof_filename); + exit(1); + } + + if (fseek(fp, -1, SEEK_CUR) == -1) { + printf("Failed to fseek in AOF %s: %s", aof_filename, strerror(errno)); + exit(1); + } + + if (buf[0] == '#') { + if (!processAnnotations(fp, aof_filepath, last_file)) { + fclose(fp); + return AOF_CHECK_TIMESTAMP_TRUNCATED; } + } else if (buf[0] == '*'){ + if (!processRESP(fp, aof_filepath, &multi)) break; + } else { + printf("AOF %s format error\n", aof_filename); + break; } } - off_t pos = process(fp); - off_t diff = size-pos; + if (feof(fp) && multi && strlen(error) == 0) { + ERROR("Reached EOF before reading EXEC for MULTI"); + } + + if (strlen(error) > 0) { + printf("%s\n", error); + } + + diff = size-pos; /* In truncate-to-timestamp mode, just exit if there is nothing to truncate. */ if (diff == 0 && to_timestamp) { - printf("Truncate nothing in AOF to timestamp %ld\n", to_timestamp); + printf("Truncate nothing in AOF %s to timestamp %ld\n", aof_filename, to_timestamp); fclose(fp); - exit(0); + return AOF_CHECK_OK; } - printf("AOF analyzed: size=%lld, ok_up_to=%lld, ok_up_to_line=%lld, diff=%lld\n", - (long long) size, (long long) pos, line, (long long) diff); + printf("AOF analyzed: filename=%s, size=%lld, ok_up_to=%lld, ok_up_to_line=%lld, diff=%lld\n", + aof_filename, (long long) size, (long long) pos, line, (long long) diff); if (diff > 0) { if (fix) { + if (!last_file) { + printf("Failed to truncate AOF %s because it is not the last file\n", aof_filename); + exit(1); + } + char buf[2]; - printf("This will shrink the AOF from %lld bytes, with %lld bytes, to %lld bytes\n",(long long)size,(long long)diff,(long long)pos); + printf("This will shrink the AOF %s from %lld bytes, with %lld bytes, to %lld bytes\n", + aof_filename, (long long)size, (long long)diff, (long long)pos); printf("Continue? [y/N]: "); - if (fgets(buf,sizeof(buf),stdin) == NULL || - strncasecmp(buf,"y",1) != 0) { - printf("Aborting...\n"); - exit(1); + if (fgets(buf, sizeof(buf), stdin) == NULL || strncasecmp(buf, "y", 1) != 0) { + printf("Aborting...\n"); + exit(1); } if (ftruncate(fileno(fp), pos) == -1) { - printf("Failed to truncate AOF\n"); + printf("Failed to truncate AOF %s\n", aof_filename); exit(1); } else { - printf("Successfully truncated AOF\n"); + fclose(fp); + return AOF_CHECK_TRUNCATED; } } else { - printf("AOF is not valid. " - "Use the --fix option to try fixing it.\n"); + printf("AOF %s is not valid. Use the --fix option to try fixing it.\n", aof_filename); exit(1); } - } else { - printf("AOF is valid\n"); + } + fclose(fp); + return AOF_CHECK_OK; +} + +/* Used to determine whether the file is a RDB file. These two possibilities: + * 1. The file is an old style RDB-preamble AOF + * 2. The file is a BASE AOF in Multi Part AOF + * */ +int fileIsRDB(char *filepath) { + FILE *fp = fopen(filepath, "r"); + if (fp == NULL) { + printf("Cannot open file %s: %s\n", filepath, strerror(errno)); + exit(1); + } + + struct redis_stat sb; + if (redis_fstat(fileno(fp), &sb) == -1) { + printf("Cannot stat file: %s\n", filepath); + exit(1); + } + + off_t size = sb.st_size; + if (size == 0) { + fclose(fp); + return 0; + } + + if (size >= 8) { /* There must be at least room for the RDB header. */ + char sig[5]; + int rdb_file = fread(sig, sizeof(sig), 1, fp) == 1 && + memcmp(sig, "REDIS", sizeof(sig)) == 0; + if (rdb_file) { + fclose(fp); + return 1; + } + } + + fclose(fp); + return 0; +} + +/* Used to determine whether the file is a manifest file. */ +#define MANIFEST_MAX_LINE 1024 +int fileIsManifest(char *filepath) { + int is_manifest = 0; + FILE *fp = fopen(filepath, "r"); + if (fp == NULL) { + printf("Cannot open file %s: %s\n", filepath, strerror(errno)); + exit(1); + } + + struct redis_stat sb; + if (redis_fstat(fileno(fp), &sb) == -1) { + printf("Cannot stat file: %s\n", filepath); + exit(1); + } + + off_t size = sb.st_size; + if (size == 0) { + fclose(fp); + return 0; + } + + char buf[MANIFEST_MAX_LINE+1]; + while (1) { + if (fgets(buf, MANIFEST_MAX_LINE+1, fp) == NULL) { + if (feof(fp)) { + break; + } else { + printf("Cannot read file: %s\n", filepath); + exit(1); + } + } + + /* Skip comments lines */ + if (buf[0] == '#') { + continue; + } else if (!memcmp(buf, "file", strlen("file"))) { + is_manifest = 1; + } } fclose(fp); + return is_manifest; +} + +/* Get the format of the file to be checked. It can be: + * AOF_RESP: Old-style AOF + * AOF_RDB_PREAMBLE: Old-style RDB-preamble AOF + * AOF_MULTI_PART: manifest in Multi Part AOF + * + * redis-check-aof tool will automatically perform different + * verification logic according to different file formats. + * */ +input_file_type getInputFileType(char *filepath) { + if (fileIsManifest(filepath)) { + return AOF_MULTI_PART; + } else if (fileIsRDB(filepath)) { + return AOF_RDB_PREAMBLE; + } else { + return AOF_RESP; + } +} + +/* Check if Multi Part AOF is valid. It will check the BASE file and INCR files + * at once according to the manifest instructions (this is somewhat similar to + * redis' AOF loading). + * + * When the verification is successful, we can guarantee: + * 1. The manifest file format is valid + * 2. Both BASE AOF and INCR AOFs format are valid + * 3. No BASE or INCR AOFs files are missing + * + * Note that in Multi Part AOF, we only allow truncation for the last AOF file. + * */ +void checkMultiPartAof(char *dirpath, char *manifest_filepath, int fix) { + int total_num = 0, aof_num = 0, last_file; + int ret; + + printf("Start checking Multi Part AOF\n"); + aofManifest *am = aofLoadManifestFromFile(manifest_filepath); + + if (am->base_aof_info) total_num++; + if (am->incr_aof_list) total_num += listLength(am->incr_aof_list); + + if (am->base_aof_info) { + sds aof_filename = am->base_aof_info->file_name; + sds aof_filepath = makePath(dirpath, aof_filename); + last_file = ++aof_num == total_num; + int aof_preable = fileIsRDB(aof_filepath); + + printf("Start to check BASE AOF (%s format).\n", aof_preable ? "RDB":"RESP"); + ret = checkSingleAof(aof_filename, aof_filepath, last_file, fix, aof_preable); + if (ret == AOF_CHECK_OK) { + printf("BASE AOF %s is valid\n", aof_filename); + } else if (ret == AOF_CHECK_EMPTY) { + printf("BASE AOF %s is empty\n", aof_filename); + } else if (ret == AOF_CHECK_TIMESTAMP_TRUNCATED) { + printf("Successfully truncated AOF %s to timestamp %ld\n", + aof_filename, to_timestamp); + } else if (ret == AOF_CHECK_TRUNCATED) { + printf("Successfully truncated AOF %s\n", aof_filename); + } + sdsfree(aof_filepath); + } + + if (listLength(am->incr_aof_list)) { + listNode *ln; + listIter li; + + printf("Start to check INCR files.\n"); + listRewind(am->incr_aof_list, &li); + while ((ln = listNext(&li)) != NULL) { + aofInfo *ai = (aofInfo*)ln->value; + sds aof_filename = (char*)ai->file_name; + sds aof_filepath = makePath(dirpath, aof_filename); + last_file = ++aof_num == total_num; + ret = checkSingleAof(aof_filename, aof_filepath, last_file, fix, 0); + if (ret == AOF_CHECK_OK) { + printf("INCR AOF %s is valid\n", aof_filename); + } else if (ret == AOF_CHECK_EMPTY) { + printf("INCR AOF %s is empty\n", aof_filename); + } else if (ret == AOF_CHECK_TIMESTAMP_TRUNCATED) { + printf("Successfully truncated AOF %s to timestamp %ld\n", + aof_filename, to_timestamp); + } else if (ret == AOF_CHECK_TRUNCATED) { + printf("Successfully truncated AOF %s\n", aof_filename); + } + sdsfree(aof_filepath); + } + } + + aofManifestFree(am); + printf("All AOF files and manifest are valid\n"); +} + +/* Check if old style AOF is valid. Internally, it will identify whether + * the AOF is in RDB-preamble format, and will eventually call `checkSingleAof` + * to do the check. */ +void checkOldStyleAof(char *filepath, int fix, int preamble) { + printf("Start checking Old-Style AOF\n"); + int ret = checkSingleAof(filepath, filepath, 1, fix, preamble); + if (ret == AOF_CHECK_OK) { + printf("AOF %s is valid\n", filepath); + } else if (ret == AOF_CHECK_EMPTY) { + printf("AOF %s is empty\n", filepath); + } else if (ret == AOF_CHECK_TIMESTAMP_TRUNCATED) { + printf("Successfully truncated AOF %s to timestamp %ld\n", + filepath, to_timestamp); + } else if (ret == AOF_CHECK_TRUNCATED) { + printf("Successfully truncated AOF %s\n", filepath); + } +} + +int redis_check_aof_main(int argc, char **argv) { + char *filepath; + char temp_filepath[PATH_MAX + 1]; + char *dirpath; + int fix = 0; + + if (argc < 2) { + goto invalid_args; + } else if (argc == 2) { + filepath = argv[1]; + } else if (argc == 3) { + if (!strcmp(argv[1], "--fix")) { + filepath = argv[2]; + fix = 1; + } else { + goto invalid_args; + } + } else if (argc == 4) { + if (!strcmp(argv[1], "--truncate-to-timestamp")) { + char *endptr; + errno = 0; + to_timestamp = strtol(argv[2], &endptr, 10); + if (errno != 0 || *endptr != '\0') { + printf("Invalid timestamp, aborting...\n"); + exit(1); + } + filepath = argv[3]; + } else { + goto invalid_args; + } + } else { + goto invalid_args; + } + + /* In the glibc implementation dirname may modify their argument. */ + memcpy(temp_filepath, filepath, strlen(filepath) + 1); + dirpath = dirname(temp_filepath); + + /* Select the corresponding verification method according to the input file type. */ + input_file_type type = getInputFileType(filepath); + switch (type) { + case AOF_MULTI_PART: + checkMultiPartAof(dirpath, filepath, fix); + break; + case AOF_RESP: + checkOldStyleAof(filepath, fix, 0); + break; + case AOF_RDB_PREAMBLE: + checkOldStyleAof(filepath, fix, 1); + break; + } + exit(0); invalid_args: - printf("Usage: %s [--fix|--truncate-to-timestamp $timestamp] <file.aof>\n", - argv[0]); + printf("Usage: %s [--fix|--truncate-to-timestamp $timestamp] <file.manifest|file.aof>\n", + argv[0]); exit(1); } diff --git a/src/redis-cli.c b/src/redis-cli.c index 31a2973c7..bbbe6d6ec 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -58,7 +58,7 @@ #include "adlist.h" #include "zmalloc.h" #include "linenoise.h" -#include "help.h" +#include "help.h" /* Used for backwards-compatibility with pre-7.0 servers that don't support COMMAND DOCS. */ #include "anet.h" #include "ae.h" #include "cli_common.h" @@ -167,13 +167,21 @@ int *spectrum_palette; int spectrum_palette_size; /* Dict Helpers */ - static uint64_t dictSdsHash(const void *key); static int dictSdsKeyCompare(dict *d, const void *key1, const void *key2); static void dictSdsDestructor(dict *d, void *val); static void dictListDestructor(dict *d, void *val); +/* Command documentation info used for help output */ +struct commandDocs { + char *name; + char *params; /* A string describing the syntax of the command arguments. */ + char *summary; + char *group; + char *since; +}; + /* Cluster Manager Command Info */ typedef struct clusterManagerCommand { char *name; @@ -398,11 +406,11 @@ typedef struct { sds full; /* Only used for help on commands */ - struct commandHelp *org; + struct commandDocs org; } helpEntry; -static helpEntry *helpEntries; -static int helpEntriesLen; +static helpEntry *helpEntries = NULL; +static int helpEntriesLen = 0; static sds cliVersion(void) { sds version; @@ -418,7 +426,8 @@ static sds cliVersion(void) { return version; } -static void cliInitHelp(void) { +/* For backwards compatibility with pre-7.0 servers. Initializes command help. */ +static void cliOldInitHelp(void) { int commandslen = sizeof(commandHelp)/sizeof(struct commandHelp); int groupslen = sizeof(commandGroups)/sizeof(char*); int i, len, pos = 0; @@ -433,7 +442,11 @@ static void cliInitHelp(void) { tmp.argv[0] = sdscatprintf(sdsempty(),"@%s",commandGroups[i]); tmp.full = tmp.argv[0]; tmp.type = CLI_HELP_GROUP; - tmp.org = NULL; + tmp.org.name = NULL; + tmp.org.params = NULL; + tmp.org.summary = NULL; + tmp.org.since = NULL; + tmp.org.group = NULL; helpEntries[pos++] = tmp; } @@ -441,17 +454,22 @@ static void cliInitHelp(void) { tmp.argv = sdssplitargs(commandHelp[i].name,&tmp.argc); tmp.full = sdsnew(commandHelp[i].name); tmp.type = CLI_HELP_COMMAND; - tmp.org = &commandHelp[i]; + tmp.org.name = commandHelp[i].name; + tmp.org.params = commandHelp[i].params; + tmp.org.summary = commandHelp[i].summary; + tmp.org.since = commandHelp[i].since; + tmp.org.group = commandGroups[commandHelp[i].group]; helpEntries[pos++] = tmp; } } -/* cliInitHelp() setups the helpEntries array with the command and group +/* For backwards compatibility with pre-7.0 servers. + * cliOldInitHelp() setups the helpEntries array with the command and group * names from the help.h file. However the Redis instance we are connecting * to may support more commands, so this function integrates the previous * entries with additional entries obtained using the COMMAND command * available in recent versions of Redis. */ -static void cliIntegrateHelp(void) { +static void cliOldIntegrateHelp(void) { if (cliConnect(CC_QUIET) == REDIS_ERR) return; redisReply *reply = redisCommand(context, "COMMAND"); @@ -486,33 +504,334 @@ static void cliIntegrateHelp(void) { new->type = CLI_HELP_COMMAND; sdstoupper(new->argv[0]); - struct commandHelp *ch = zmalloc(sizeof(*ch)); - ch->name = new->argv[0]; - ch->params = sdsempty(); + new->org.name = new->argv[0]; + new->org.params = sdsempty(); int args = llabs(entry->element[1]->integer); args--; /* Remove the command name itself. */ if (entry->element[3]->integer == 1) { - ch->params = sdscat(ch->params,"key "); + new->org.params = sdscat(new->org.params,"key "); args--; } - while(args-- > 0) ch->params = sdscat(ch->params,"arg "); + while(args-- > 0) new->org.params = sdscat(new->org.params,"arg "); if (entry->element[1]->integer < 0) - ch->params = sdscat(ch->params,"...options..."); - ch->summary = "Help not available"; - ch->group = 0; - ch->since = "not known"; - new->org = ch; + new->org.params = sdscat(new->org.params,"...options..."); + new->org.summary = "Help not available"; + new->org.since = "Not known"; + new->org.group = commandGroups[0]; } freeReplyObject(reply); } +/* Concatenate a string to an sds string, but if it's empty substitute double quote marks. */ +static sds sdscat_orempty(sds params, char *value) { + if (value[0] == '\0') { + return sdscat(params, "\"\""); + } + return sdscat(params, value); +} + +static sds cliAddArgument(sds params, redisReply *argMap); + +/* Concatenate a list of arguments to the parameter string, separated by a separator string. */ +static sds cliConcatArguments(sds params, redisReply *arguments, char *separator) { + for (size_t j = 0; j < arguments->elements; j++) { + params = cliAddArgument(params, arguments->element[j]); + if (j != arguments->elements - 1) { + params = sdscat(params, separator); + } + } + return params; +} + +/* Add an argument to the parameter string. */ +static sds cliAddArgument(sds params, redisReply *argMap) { + char *name = NULL; + char *type = NULL; + int optional = 0; + int multiple = 0; + int multipleToken = 0; + redisReply *arguments = NULL; + sds tokenPart = sdsempty(); + sds repeatPart = sdsempty(); + + /* First read the fields describing the argument. */ + if (argMap->type != REDIS_REPLY_MAP && argMap->type != REDIS_REPLY_ARRAY) { + return params; + } + for (size_t i = 0; i < argMap->elements; i += 2) { + assert(argMap->element[i]->type == REDIS_REPLY_STRING); + char *key = argMap->element[i]->str; + if (!strcmp(key, "name")) { + assert(argMap->element[i + 1]->type == REDIS_REPLY_STRING); + name = argMap->element[i + 1]->str; + } else if (!strcmp(key, "token")) { + assert(argMap->element[i + 1]->type == REDIS_REPLY_STRING); + char *token = argMap->element[i + 1]->str; + tokenPart = sdscat_orempty(tokenPart, token); + } else if (!strcmp(key, "type")) { + assert(argMap->element[i + 1]->type == REDIS_REPLY_STRING); + type = argMap->element[i + 1]->str; + } else if (!strcmp(key, "arguments")) { + arguments = argMap->element[i + 1]; + } else if (!strcmp(key, "flags")) { + redisReply *flags = argMap->element[i + 1]; + assert(flags->type == REDIS_REPLY_SET || flags->type == REDIS_REPLY_ARRAY); + for (size_t j = 0; j < flags->elements; j++) { + assert(flags->element[j]->type == REDIS_REPLY_STATUS); + char *flag = flags->element[j]->str; + if (!strcmp(flag, "optional")) { + optional = 1; + } else if (!strcmp(flag, "multiple")) { + multiple = 1; + } else if (!strcmp(flag, "multiple_token")) { + multipleToken = 1; + } + } + } + } + + /* Then build the "repeating part" of the argument string. */ + if (!strcmp(type, "key") || + !strcmp(type, "string") || + !strcmp(type, "integer") || + !strcmp(type, "double") || + !strcmp(type, "pattern") || + !strcmp(type, "unix-time") || + !strcmp(type, "token")) + { + repeatPart = sdscat_orempty(repeatPart, name); + } else if (!strcmp(type, "oneof")) { + repeatPart = cliConcatArguments(repeatPart, arguments, "|"); + } else if (!strcmp(type, "block")) { + repeatPart = cliConcatArguments(repeatPart, arguments, " "); + } else if (strcmp(type, "pure-token") != 0) { + fprintf(stderr, "Unknown type '%s' set for argument '%s'\n", type, name); + } + + /* Finally, build the parameter string. */ + if (tokenPart[0] != '\0' && strcmp(type, "pure-token") != 0) { + tokenPart = sdscat(tokenPart, " "); + } + if (optional) { + params = sdscat(params, "["); + } + params = sdscat(params, tokenPart); + params = sdscat(params, repeatPart); + if (multiple) { + params = sdscat(params, " ["); + if (multipleToken) { + params = sdscat(params, tokenPart); + } + params = sdscat(params, repeatPart); + params = sdscat(params, " ...]"); + } + if (optional) { + params = sdscat(params, "]"); + } + sdsfree(tokenPart); + sdsfree(repeatPart); + return params; +} + +/* Fill in the fields of a help entry for the command/subcommand name. */ +static void cliFillInCommandHelpEntry(helpEntry *help, char *cmdname, char *subcommandname) { + help->argc = subcommandname ? 2 : 1; + help->argv = zmalloc(sizeof(sds) * help->argc); + help->argv[0] = sdsnew(cmdname); + sdstoupper(help->argv[0]); + if (subcommandname) { + /* Subcommand name is two words separated by a pipe character. */ + help->argv[1] = sdsnew(strchr(subcommandname, '|') + 1); + sdstoupper(help->argv[1]); + } + sds fullname = sdsnew(help->argv[0]); + if (subcommandname) { + fullname = sdscat(fullname, " "); + fullname = sdscat(fullname, help->argv[1]); + } + help->full = fullname; + help->type = CLI_HELP_COMMAND; + + help->org.name = help->full; + help->org.params = sdsempty(); + help->org.since = NULL; +} + +/* Initialize a command help entry for the command/subcommand described in 'specs'. + * 'next' points to the next help entry to be filled in. + * 'groups' is a set of command group names to be filled in. + * Returns a pointer to the next available position in the help entries table. + * If the command has subcommands, this is called recursively for the subcommands. + */ +static helpEntry *cliInitCommandHelpEntry(char *cmdname, char *subcommandname, + helpEntry *next, redisReply *specs, + dict *groups) { + helpEntry *help = next++; + cliFillInCommandHelpEntry(help, cmdname, subcommandname); + + assert(specs->type == REDIS_REPLY_MAP || specs->type == REDIS_REPLY_ARRAY); + for (size_t j = 0; j < specs->elements; j += 2) { + assert(specs->element[j]->type == REDIS_REPLY_STRING); + char *key = specs->element[j]->str; + if (!strcmp(key, "summary")) { + redisReply *reply = specs->element[j + 1]; + assert(reply->type == REDIS_REPLY_STRING); + help->org.summary = sdsnew(reply->str); + } else if (!strcmp(key, "since")) { + redisReply *reply = specs->element[j + 1]; + assert(reply->type == REDIS_REPLY_STRING); + help->org.since = sdsnew(reply->str); + } else if (!strcmp(key, "group")) { + redisReply *reply = specs->element[j + 1]; + assert(reply->type == REDIS_REPLY_STRING); + help->org.group = sdsnew(reply->str); + sds group = sdsdup(help->org.group); + if (dictAdd(groups, group, NULL) != DICT_OK) { + sdsfree(group); + } + } else if (!strcmp(key, "arguments")) { + redisReply *args = specs->element[j + 1]; + assert(args->type == REDIS_REPLY_ARRAY); + help->org.params = cliConcatArguments(help->org.params, args, " "); + } else if (!strcmp(key, "subcommands")) { + redisReply *subcommands = specs->element[j + 1]; + assert(subcommands->type == REDIS_REPLY_MAP || subcommands->type == REDIS_REPLY_ARRAY); + for (size_t i = 0; i < subcommands->elements; i += 2) { + assert(subcommands->element[i]->type == REDIS_REPLY_STRING); + char *subcommandname = subcommands->element[i]->str; + redisReply *subcommand = subcommands->element[i + 1]; + assert(subcommand->type == REDIS_REPLY_MAP || subcommand->type == REDIS_REPLY_ARRAY); + next = cliInitCommandHelpEntry(cmdname, subcommandname, next, subcommand, groups); + } + } + } + return next; +} + +/* Returns the total number of commands and subcommands in the command docs table. */ +static size_t cliCountCommands(redisReply* commandTable) { + size_t numCommands = commandTable->elements / 2; + + /* The command docs table maps command names to a map of their specs. */ + for (size_t i = 0; i < commandTable->elements; i += 2) { + assert(commandTable->element[i]->type == REDIS_REPLY_STRING); /* Command name. */ + assert(commandTable->element[i + 1]->type == REDIS_REPLY_MAP || + commandTable->element[i + 1]->type == REDIS_REPLY_ARRAY); + redisReply *map = commandTable->element[i + 1]; + for (size_t j = 0; j < map->elements; j += 2) { + assert(map->element[j]->type == REDIS_REPLY_STRING); + char *key = map->element[j]->str; + if (!strcmp(key, "subcommands")) { + redisReply *subcommands = map->element[j + 1]; + assert(subcommands->type == REDIS_REPLY_MAP || subcommands->type == REDIS_REPLY_ARRAY); + numCommands += subcommands->elements / 2; + } + } + } + return numCommands; +} + +/* Comparator for sorting help table entries. */ +int helpEntryCompare(const void *entry1, const void *entry2) { + helpEntry *i1 = (helpEntry *)entry1; + helpEntry *i2 = (helpEntry *)entry2; + return strcmp(i1->full, i2->full); +} + +/* Initializes command help entries for command groups. + * Called after the command help entries have already been filled in. + * Extends the help table with new entries for the command groups. + */ +void cliInitGroupHelpEntries(dict *groups) { + dictIterator *iter = dictGetIterator(groups); + dictEntry *entry; + helpEntry tmp; + + int numGroups = dictSize(groups); + int pos = helpEntriesLen; + helpEntriesLen += numGroups; + helpEntries = zrealloc(helpEntries, sizeof(helpEntry)*helpEntriesLen); + + for (entry = dictNext(iter); entry != NULL; entry = dictNext(iter)) { + tmp.argc = 1; + tmp.argv = zmalloc(sizeof(sds)); + tmp.argv[0] = sdscatprintf(sdsempty(),"@%s",(char *)entry->key); + tmp.full = tmp.argv[0]; + tmp.type = CLI_HELP_GROUP; + tmp.org.name = NULL; + tmp.org.params = NULL; + tmp.org.summary = NULL; + tmp.org.since = NULL; + tmp.org.group = NULL; + helpEntries[pos++] = tmp; + } + dictReleaseIterator(iter); +} + +/* Initializes help entries for all commands in the COMMAND DOCS reply. */ +void cliInitCommandHelpEntries(redisReply *commandTable, dict *groups) { + helpEntry *next = helpEntries; + for (size_t i = 0; i < commandTable->elements; i += 2) { + assert(commandTable->element[i]->type == REDIS_REPLY_STRING); + char *cmdname = commandTable->element[i]->str; + + assert(commandTable->element[i + 1]->type == REDIS_REPLY_MAP || + commandTable->element[i + 1]->type == REDIS_REPLY_ARRAY); + redisReply *cmdspecs = commandTable->element[i + 1]; + next = cliInitCommandHelpEntry(cmdname, NULL, next, cmdspecs, groups); + } +} + +/* cliInitHelp() sets up the helpEntries array with the command and group + * names and command descriptions obtained using the COMMAND DOCS command. + */ +static void cliInitHelp(void) { + /* Dict type for a set of strings, used to collect names of command groups. */ + dictType groupsdt = { + dictSdsHash, /* hash function */ + NULL, /* key dup */ + NULL, /* val dup */ + dictSdsKeyCompare, /* key compare */ + dictSdsDestructor, /* key destructor */ + NULL, /* val destructor */ + NULL /* allow to expand */ + }; + redisReply *commandTable; + dict *groups; + + if (cliConnect(CC_QUIET) == REDIS_ERR) return; + commandTable = redisCommand(context, "COMMAND DOCS"); + if (commandTable == NULL || commandTable->type == REDIS_REPLY_ERROR) { + /* New COMMAND DOCS subcommand not supported - generate help from old help.h data instead. */ + freeReplyObject(commandTable); + cliOldInitHelp(); + cliOldIntegrateHelp(); + return; + }; + if (commandTable->type != REDIS_REPLY_MAP && commandTable->type != REDIS_REPLY_ARRAY) return; + + /* Scan the array reported by COMMAND DOCS and fill in the entries */ + helpEntriesLen = cliCountCommands(commandTable); + helpEntries = zmalloc(sizeof(helpEntry)*helpEntriesLen); + + groups = dictCreate(&groupsdt); + cliInitCommandHelpEntries(commandTable, groups); + cliInitGroupHelpEntries(groups); + + qsort(helpEntries, helpEntriesLen, sizeof(helpEntry), helpEntryCompare); + freeReplyObject(commandTable); + dictRelease(groups); +} + /* Output command help to stdout. */ -static void cliOutputCommandHelp(struct commandHelp *help, int group) { +static void cliOutputCommandHelp(struct commandDocs *help, int group) { printf("\r\n \x1b[1m%s\x1b[0m \x1b[90m%s\x1b[0m\r\n", help->name, help->params); printf(" \x1b[33msummary:\x1b[0m %s\r\n", help->summary); - printf(" \x1b[33msince:\x1b[0m %s\r\n", help->since); + if (help->since != NULL) { + printf(" \x1b[33msince:\x1b[0m %s\r\n", help->since); + } if (group) { - printf(" \x1b[33mgroup:\x1b[0m %s\r\n", commandGroups[help->group]); + printf(" \x1b[33mgroup:\x1b[0m %s\r\n", help->group); } } @@ -538,22 +857,16 @@ static void cliOutputGenericHelp(void) { /* Output all command help, filtering by group or command name. */ static void cliOutputHelp(int argc, char **argv) { - int i, j, len; - int group = -1; + int i, j; + char *group = NULL; helpEntry *entry; - struct commandHelp *help; + struct commandDocs *help; if (argc == 0) { cliOutputGenericHelp(); return; } else if (argc > 0 && argv[0][0] == '@') { - len = sizeof(commandGroups)/sizeof(char*); - for (i = 0; i < len; i++) { - if (strcasecmp(argv[0]+1,commandGroups[i]) == 0) { - group = i; - break; - } - } + group = argv[0]+1; } assert(argc > 0); @@ -561,8 +874,8 @@ static void cliOutputHelp(int argc, char **argv) { entry = &helpEntries[i]; if (entry->type != CLI_HELP_COMMAND) continue; - help = entry->org; - if (group == -1) { + help = &entry->org; + if (group == NULL) { /* Compare all arguments */ if (argc <= entry->argc) { for (j = 0; j < argc; j++) { @@ -572,10 +885,8 @@ static void cliOutputHelp(int argc, char **argv) { cliOutputCommandHelp(help,1); } } - } else { - if (group == help->group) { - cliOutputCommandHelp(help,0); - } + } else if (strcasecmp(group, help->group) == 0) { + cliOutputCommandHelp(help,0); } } printf("\r\n"); @@ -649,7 +960,7 @@ static char *hintsCallback(const char *buf, int *color, int *bold) { if (entry) { *color = 90; *bold = 0; - sds hint = sdsnew(entry->org->params); + sds hint = sdsnew(entry->org.params); /* Remove arguments from the returned hint to show only the * ones the user did not yet type. */ @@ -2229,10 +2540,8 @@ static void repl(void) { int argc; sds *argv; - /* Initialize the help and, if possible, use the COMMAND command in order - * to retrieve missing entries. */ + /* Initialize the help using the results of the COMMAND command. */ cliInitHelp(); - cliIntegrateHelp(); config.interactive = 1; linenoiseSetMultiLine(1); diff --git a/src/redismodule.h b/src/redismodule.h index 3255fcc30..79ce2c697 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -236,14 +236,16 @@ typedef uint64_t RedisModuleTimerID; /* Declare that the module can handle errors with RedisModule_SetModuleOptions. */ #define REDISMODULE_OPTIONS_HANDLE_IO_ERRORS (1<<0) -/* Declare that the module can handle diskless async replication with RedisModule_SetModuleOptions. */ -#define REDISMODULE_OPTIONS_HANDLE_REPL_ASYNC_LOAD (1<<1) - /* When set, Redis will not call RedisModule_SignalModifiedKey(), implicitly in * RedisModule_CloseKey, and the module needs to do that when manually when keys * are modified from the user's sperspective, to invalidate WATCH. */ #define REDISMODULE_OPTION_NO_IMPLICIT_SIGNAL_MODIFIED (1<<1) +/* Declare that the module can handle diskless async replication with RedisModule_SetModuleOptions. */ +#define REDISMODULE_OPTIONS_HANDLE_REPL_ASYNC_LOAD (1<<2) + +/* Definitions for RedisModule_SetCommandInfo. */ + typedef enum { REDISMODULE_ARG_TYPE_STRING, REDISMODULE_ARG_TYPE_INTEGER, @@ -260,12 +262,146 @@ typedef enum { #define REDISMODULE_CMD_ARG_OPTIONAL (1<<0) /* The argument is optional (like GET in SET command) */ #define REDISMODULE_CMD_ARG_MULTIPLE (1<<1) /* The argument may repeat itself (like key in DEL) */ #define REDISMODULE_CMD_ARG_MULTIPLE_TOKEN (1<<2) /* The argument may repeat itself, and so does its token (like `GET pattern` in SORT) */ +#define _REDISMODULE_CMD_ARG_NEXT (1<<3) -/* Redis ACL key permission flags, which specify which permissions a module - * needs on a key. */ -#define REDISMODULE_KEY_PERMISSION_READ (1<<0) -#define REDISMODULE_KEY_PERMISSION_WRITE (1<<1) -#define REDISMODULE_KEY_PERMISSION_ALL (REDISMODULE_KEY_PERMISSION_READ | REDISMODULE_KEY_PERMISSION_WRITE) +typedef enum { + REDISMODULE_KSPEC_BS_INVALID = 0, /* Must be zero. An implicitly value of + * zero is provided when the field is + * absent in a struct literal. */ + REDISMODULE_KSPEC_BS_UNKNOWN, + REDISMODULE_KSPEC_BS_INDEX, + REDISMODULE_KSPEC_BS_KEYWORD +} RedisModuleKeySpecBeginSearchType; + +typedef enum { + REDISMODULE_KSPEC_FK_OMITTED = 0, /* Used when the field is absent in a + * struct literal. Don't use this value + * explicitly. */ + REDISMODULE_KSPEC_FK_UNKNOWN, + REDISMODULE_KSPEC_FK_RANGE, + REDISMODULE_KSPEC_FK_KEYNUM +} RedisModuleKeySpecFindKeysType; + +/* Key-spec flags. For details, see the documentation of + * RedisModule_SetCommandInfo and the key-spec flags in server.h. */ +#define REDISMODULE_CMD_KEY_RO (1ULL<<0) +#define REDISMODULE_CMD_KEY_RW (1ULL<<1) +#define REDISMODULE_CMD_KEY_OW (1ULL<<2) +#define REDISMODULE_CMD_KEY_RM (1ULL<<3) +#define REDISMODULE_CMD_KEY_ACCESS (1ULL<<4) +#define REDISMODULE_CMD_KEY_UPDATE (1ULL<<5) +#define REDISMODULE_CMD_KEY_INSERT (1ULL<<6) +#define REDISMODULE_CMD_KEY_DELETE (1ULL<<7) +#define REDISMODULE_CMD_KEY_NOT_KEY (1ULL<<8) +#define REDISMODULE_CMD_KEY_INCOMPLETE (1ULL<<9) +#define REDISMODULE_CMD_KEY_VARIABLE_FLAGS (1ULL<<10) + +/* Channel flags, for details see the documentation of + * RedisModule_ChannelAtPosWithFlags. */ +#define REDISMODULE_CMD_CHANNEL_PATTERN (1ULL<<0) +#define REDISMODULE_CMD_CHANNEL_PUBLISH (1ULL<<1) +#define REDISMODULE_CMD_CHANNEL_SUBSCRIBE (1ULL<<2) +#define REDISMODULE_CMD_CHANNEL_UNSUBSCRIBE (1ULL<<3) + +typedef struct RedisModuleCommandArg { + const char *name; + RedisModuleCommandArgType type; + int key_spec_index; /* If type is KEY, this is a zero-based index of + * the key_spec in the command. For other types, + * you may specify -1. */ + const char *token; /* If type is PURE_TOKEN, this is the token. */ + const char *summary; + const char *since; + int flags; /* The REDISMODULE_CMD_ARG_* macros. */ + struct RedisModuleCommandArg *subargs; +} RedisModuleCommandArg; + +typedef struct { + const char *since; + const char *changes; +} RedisModuleCommandHistoryEntry; + +typedef struct { + const char *notes; + uint64_t flags; /* REDISMODULE_CMD_KEY_* macros. */ + RedisModuleKeySpecBeginSearchType begin_search_type; + union { + struct { + /* The index from which we start the search for keys */ + int pos; + } index; + struct { + /* The keyword that indicates the beginning of key args */ + const char *keyword; + /* An index in argv from which to start searching. + * Can be negative, which means start search from the end, in reverse + * (Example: -2 means to start in reverse from the panultimate arg) */ + int startfrom; + } keyword; + } bs; + RedisModuleKeySpecFindKeysType find_keys_type; + union { + struct { + /* Index of the last key relative to the result of the begin search + * step. Can be negative, in which case it's not relative. -1 + * indicating till the last argument, -2 one before the last and so + * on. */ + int lastkey; + /* How many args should we skip after finding a key, in order to + * find the next one. */ + int keystep; + /* If lastkey is -1, we use limit to stop the search by a factor. 0 + * and 1 mean no limit. 2 means 1/2 of the remaining args, 3 means + * 1/3, and so on. */ + int limit; + } range; + struct { + /* Index of the argument containing the number of keys to come + * relative to the result of the begin search step */ + int keynumidx; + /* Index of the fist key. (Usually it's just after keynumidx, in + * which case it should be set to keynumidx + 1.) */ + int firstkey; + /* How many args should we skip after finding a key, in order to + * find the next one, relative to the result of the begin search + * step. */ + int keystep; + } keynum; + } fk; +} RedisModuleCommandKeySpec; + +typedef struct { + int version; + size_t sizeof_historyentry; + size_t sizeof_keyspec; + size_t sizeof_arg; +} RedisModuleCommandInfoVersion; + +static const RedisModuleCommandInfoVersion RedisModule_CurrentCommandInfoVersion = { + .version = 1, + .sizeof_historyentry = sizeof(RedisModuleCommandHistoryEntry), + .sizeof_keyspec = sizeof(RedisModuleCommandKeySpec), + .sizeof_arg = sizeof(RedisModuleCommandArg) +}; + +#define REDISMODULE_COMMAND_INFO_VERSION (&RedisModule_CurrentCommandInfoVersion) + +typedef struct { + /* Always set version to REDISMODULE_COMMAND_INFO_VERSION */ + const RedisModuleCommandInfoVersion *version; + /* Version 1 fields (added in Redis 7.0.0) */ + const char *summary; /* Summary of the command */ + const char *complexity; /* Complexity description */ + const char *since; /* Debut module version of the command */ + RedisModuleCommandHistoryEntry *history; /* History */ + /* A string of space-separated tips meant for clients/proxies regarding this + * command */ + const char *tips; + /* Number of arguments, it is possible to use -N to say >= N */ + int arity; + RedisModuleCommandKeySpec *key_specs; + RedisModuleCommandArg *args; +} RedisModuleCommandInfo; /* Eventloop definitions. */ #define REDISMODULE_EVENTLOOP_READABLE 1 @@ -623,7 +759,6 @@ typedef struct RedisModuleScanCursor RedisModuleScanCursor; typedef struct RedisModuleDefragCtx RedisModuleDefragCtx; typedef struct RedisModuleUser RedisModuleUser; typedef struct RedisModuleKeyOptCtx RedisModuleKeyOptCtx; -typedef struct RedisModuleCommandArg RedisModuleCommandArg; typedef int (*RedisModuleCmdFunc)(RedisModuleCtx *ctx, RedisModuleString **argv, int argc); typedef void (*RedisModuleDisconnectFunc)(RedisModuleCtx *ctx, RedisModuleBlockedClient *bc); @@ -697,6 +832,7 @@ REDISMODULE_API int (*RedisModule_GetApi)(const char *, void *) REDISMODULE_ATTR REDISMODULE_API int (*RedisModule_CreateCommand)(RedisModuleCtx *ctx, const char *name, RedisModuleCmdFunc cmdfunc, const char *strflags, int firstkey, int lastkey, int keystep) REDISMODULE_ATTR; REDISMODULE_API RedisModuleCommand *(*RedisModule_GetCommand)(RedisModuleCtx *ctx, const char *name) REDISMODULE_ATTR; REDISMODULE_API int (*RedisModule_CreateSubcommand)(RedisModuleCommand *parent, const char *name, RedisModuleCmdFunc cmdfunc, const char *strflags, int firstkey, int lastkey, int keystep) REDISMODULE_ATTR; +REDISMODULE_API int (*RedisModule_SetCommandInfo)(RedisModuleCommand *command, const RedisModuleCommandInfo *info) REDISMODULE_ATTR; REDISMODULE_API void (*RedisModule_SetModuleAttribs)(RedisModuleCtx *ctx, const char *name, int ver, int apiver) REDISMODULE_ATTR; REDISMODULE_API int (*RedisModule_IsModuleNameBusy)(const char *name) REDISMODULE_ATTR; REDISMODULE_API int (*RedisModule_WrongArity)(RedisModuleCtx *ctx) REDISMODULE_ATTR; @@ -810,6 +946,9 @@ REDISMODULE_API long long (*RedisModule_StreamTrimByLength)(RedisModuleKey *key, REDISMODULE_API long long (*RedisModule_StreamTrimByID)(RedisModuleKey *key, int flags, RedisModuleStreamID *id) REDISMODULE_ATTR; REDISMODULE_API int (*RedisModule_IsKeysPositionRequest)(RedisModuleCtx *ctx) REDISMODULE_ATTR; REDISMODULE_API void (*RedisModule_KeyAtPos)(RedisModuleCtx *ctx, int pos) REDISMODULE_ATTR; +REDISMODULE_API void (*RedisModule_KeyAtPosWithFlags)(RedisModuleCtx *ctx, int pos, int flags) REDISMODULE_ATTR; +REDISMODULE_API int (*RedisModule_IsChannelsPositionRequest)(RedisModuleCtx *ctx) REDISMODULE_ATTR; +REDISMODULE_API void (*RedisModule_ChannelAtPosWithFlags)(RedisModuleCtx *ctx, int pos, int flags) REDISMODULE_ATTR; REDISMODULE_API unsigned long long (*RedisModule_GetClientId)(RedisModuleCtx *ctx) REDISMODULE_ATTR; REDISMODULE_API RedisModuleString * (*RedisModule_GetClientUserNameById)(RedisModuleCtx *ctx, uint64_t id) REDISMODULE_ATTR; REDISMODULE_API int (*RedisModule_GetClientInfoById)(void *ci, uint64_t id) REDISMODULE_ATTR; @@ -924,14 +1063,6 @@ REDISMODULE_API int (*RedisModule_GetKeyspaceNotificationFlagsAll)() REDISMODULE REDISMODULE_API int (*RedisModule_IsSubEventSupported)(RedisModuleEvent event, uint64_t subevent) REDISMODULE_ATTR; REDISMODULE_API int (*RedisModule_GetServerVersion)() REDISMODULE_ATTR; REDISMODULE_API int (*RedisModule_GetTypeMethodVersion)() REDISMODULE_ATTR; -#ifdef INCLUDE_UNRELEASED_KEYSPEC_API -REDISMODULE_API int (*RedisModule_AddCommandKeySpec)(RedisModuleCommand *command, const char *specflags, int *spec_id) REDISMODULE_ATTR; -REDISMODULE_API int (*RedisModule_SetCommandKeySpecBeginSearchIndex)(RedisModuleCommand *command, int spec_id, int index) REDISMODULE_ATTR; -REDISMODULE_API int (*RedisModule_SetCommandKeySpecBeginSearchKeyword)(RedisModuleCommand *command, int spec_id, const char *keyword, int startfrom) REDISMODULE_ATTR; -REDISMODULE_API int (*RedisModule_SetCommandKeySpecFindKeysRange)(RedisModuleCommand *command, int spec_id, int lastkey, int keystep, int limit) REDISMODULE_ATTR; -REDISMODULE_API int (*RedisModule_SetCommandKeySpecFindKeysKeynum)(RedisModuleCommand *command, int spec_id, int keynumidx, int firstkey, int keystep) REDISMODULE_ATTR; -#endif - REDISMODULE_API void (*RedisModule_Yield)(RedisModuleCtx *ctx, int flags, const char *busy_reply) REDISMODULE_ATTR; REDISMODULE_API RedisModuleBlockedClient * (*RedisModule_BlockClient)(RedisModuleCtx *ctx, RedisModuleCmdFunc reply_callback, RedisModuleCmdFunc timeout_callback, void (*free_privdata)(RedisModuleCtx*,void*), long long timeout_ms) REDISMODULE_ATTR; REDISMODULE_API int (*RedisModule_UnblockClient)(RedisModuleBlockedClient *bc, void *privdata) REDISMODULE_ATTR; @@ -995,6 +1126,7 @@ REDISMODULE_API int (*RedisModule_AuthenticateClientWithUser)(RedisModuleCtx *ct REDISMODULE_API int (*RedisModule_DeauthenticateAndCloseClient)(RedisModuleCtx *ctx, uint64_t client_id) REDISMODULE_ATTR; REDISMODULE_API RedisModuleString * (*RedisModule_GetClientCertificate)(RedisModuleCtx *ctx, uint64_t id) REDISMODULE_ATTR; REDISMODULE_API int *(*RedisModule_GetCommandKeys)(RedisModuleCtx *ctx, RedisModuleString **argv, int argc, int *num_keys) REDISMODULE_ATTR; +REDISMODULE_API int *(*RedisModule_GetCommandKeysWithFlags)(RedisModuleCtx *ctx, RedisModuleString **argv, int argc, int *num_keys, int **out_flags) REDISMODULE_ATTR; REDISMODULE_API const char *(*RedisModule_GetCurrentCommandName)(RedisModuleCtx *ctx) REDISMODULE_ATTR; REDISMODULE_API int (*RedisModule_RegisterDefragFunc)(RedisModuleCtx *ctx, RedisModuleDefragFunc func) REDISMODULE_ATTR; REDISMODULE_API void *(*RedisModule_DefragAlloc)(RedisModuleDefragCtx *ctx, void *ptr) REDISMODULE_ATTR; @@ -1023,6 +1155,7 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(CreateCommand); REDISMODULE_GET_API(GetCommand); REDISMODULE_GET_API(CreateSubcommand); + REDISMODULE_GET_API(SetCommandInfo); REDISMODULE_GET_API(SetModuleAttribs); REDISMODULE_GET_API(IsModuleNameBusy); REDISMODULE_GET_API(WrongArity); @@ -1136,6 +1269,9 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(StreamTrimByID); REDISMODULE_GET_API(IsKeysPositionRequest); REDISMODULE_GET_API(KeyAtPos); + REDISMODULE_GET_API(KeyAtPosWithFlags); + REDISMODULE_GET_API(IsChannelsPositionRequest); + REDISMODULE_GET_API(ChannelAtPosWithFlags); REDISMODULE_GET_API(GetClientId); REDISMODULE_GET_API(GetClientUserNameById); REDISMODULE_GET_API(GetContextFlags); @@ -1250,13 +1386,6 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(IsSubEventSupported); REDISMODULE_GET_API(GetServerVersion); REDISMODULE_GET_API(GetTypeMethodVersion); -#ifdef INCLUDE_UNRELEASED_KEYSPEC_API - REDISMODULE_GET_API(AddCommandKeySpec); - REDISMODULE_GET_API(SetCommandKeySpecBeginSearchIndex); - REDISMODULE_GET_API(SetCommandKeySpecBeginSearchKeyword); - REDISMODULE_GET_API(SetCommandKeySpecFindKeysRange); - REDISMODULE_GET_API(SetCommandKeySpecFindKeysKeynum); -#endif REDISMODULE_GET_API(Yield); REDISMODULE_GET_API(GetThreadSafeContext); REDISMODULE_GET_API(GetDetachedThreadSafeContext); @@ -1320,6 +1449,7 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(AuthenticateClientWithUser); REDISMODULE_GET_API(GetClientCertificate); REDISMODULE_GET_API(GetCommandKeys); + REDISMODULE_GET_API(GetCommandKeysWithFlags); REDISMODULE_GET_API(GetCurrentCommandName); REDISMODULE_GET_API(RegisterDefragFunc); REDISMODULE_GET_API(DefragAlloc); @@ -1347,7 +1477,6 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int /* Things only defined for the modules core, not exported to modules * including this file. */ #define RedisModuleString robj -#define RedisModuleCommandArg redisCommandArg #endif /* REDISMODULE_CORE */ #endif /* REDISMODULE_H */ diff --git a/src/replication.c b/src/replication.c index e387a8fd4..9c2d110b0 100644 --- a/src/replication.c +++ b/src/replication.c @@ -705,18 +705,12 @@ int replicationSetupSlaveForFullResync(client *slave, long long offset) { * * On success return C_OK, otherwise C_ERR is returned and we proceed * with the usual full resync. */ -int masterTryPartialResynchronization(client *c) { - long long psync_offset, psync_len; +int masterTryPartialResynchronization(client *c, long long psync_offset) { + long long psync_len; char *master_replid = c->argv[1]->ptr; char buf[128]; int buflen; - /* Parse the replication offset asked by the slave. Go to full sync - * on parse error: this should never happen but we try to handle - * it in a robust way compared to aborting. */ - if (getLongLongFromObjectOrReply(c,c->argv[2],&psync_offset,NULL) != - C_OK) goto need_full_resync; - /* Is the replication ID of this master the same advertised by the wannabe * slave via PSYNC? If the replication ID changed this master has a * different replication history, and there is no way to continue. @@ -977,7 +971,14 @@ void syncCommand(client *c) { * So the slave knows the new replid and offset to try a PSYNC later * if the connection with the master is lost. */ if (!strcasecmp(c->argv[0]->ptr,"psync")) { - if (masterTryPartialResynchronization(c) == C_OK) { + long long psync_offset; + if (getLongLongFromObjectOrReply(c, c->argv[2], &psync_offset, NULL) != C_OK) { + serverLog(LL_WARNING, "Replica %s asks for synchronization but with a wrong offset", + replicationGetSlaveName(c)); + return; + } + + if (masterTryPartialResynchronization(c, psync_offset) == C_OK) { server.stat_sync_partial_ok++; return; /* No full resync needed, return. */ } else { @@ -1545,6 +1546,9 @@ void updateSlavesWaitingBgsave(int bgsaveerr, int type) { listNode *ln; listIter li; + /* Note: there's a chance we got here from within the REPLCONF ACK command + * so we must avoid using freeClient, otherwise we'll crash on our way up. */ + listRewind(server.slaves,&li); while((ln = listNext(&li))) { client *slave = ln->value; @@ -1553,7 +1557,7 @@ void updateSlavesWaitingBgsave(int bgsaveerr, int type) { struct redis_stat buf; if (bgsaveerr != C_OK) { - freeClient(slave); + freeClientAsync(slave); serverLog(LL_WARNING,"SYNC failed. BGSAVE child returned an error"); continue; } @@ -1597,7 +1601,7 @@ void updateSlavesWaitingBgsave(int bgsaveerr, int type) { } else { if ((slave->repldbfd = open(server.rdb_filename,O_RDONLY)) == -1 || redis_fstat(slave->repldbfd,&buf) == -1) { - freeClient(slave); + freeClientAsync(slave); serverLog(LL_WARNING,"SYNC failed. Can't open/stat DB after BGSAVE: %s", strerror(errno)); continue; } @@ -1609,7 +1613,7 @@ void updateSlavesWaitingBgsave(int bgsaveerr, int type) { connSetWriteHandler(slave->conn,NULL); if (connSetWriteHandler(slave->conn,sendBulkToSlave) == C_ERR) { - freeClient(slave); + freeClientAsync(slave); continue; } } @@ -2716,7 +2720,7 @@ void syncWithMaster(connection *conn) { return; } - /* If reached this point, we should be in REPL_STATE_RECEIVE_PSYNC. */ + /* If reached this point, we should be in REPL_STATE_RECEIVE_PSYNC_REPLY. */ if (server.repl_state != REPL_STATE_RECEIVE_PSYNC_REPLY) { serverLog(LL_WARNING,"syncWithMaster(): state machine error, " "state should be RECEIVE_PSYNC but is %d", diff --git a/src/script.c b/src/script.c index 14a64b961..d78d9fd6b 100644 --- a/src/script.c +++ b/src/script.c @@ -146,6 +146,7 @@ int scriptPrepareForRun(scriptRunCtx *run_ctx, client *engine_client, client *ca return C_ERR; } + /* Deny writes if we're unale to persist. */ int deny_write_type = writeCommandsDeniedByDiskError(); if (deny_write_type != DISK_ERROR_TYPE_NONE && server.masterhost == NULL) { if (deny_write_type == DISK_ERROR_TYPE_RDB) @@ -164,6 +165,17 @@ int scriptPrepareForRun(scriptRunCtx *run_ctx, client *engine_client, client *ca addReplyError(caller, "Can not execute a script with write flag using *_ro command."); return C_ERR; } + + /* Don't accept write commands if there are not enough good slaves and + * user configured the min-slaves-to-write option. */ + if (server.masterhost == NULL && + server.repl_min_slaves_max_lag && + server.repl_min_slaves_to_write && + server.repl_good_slaves_count < server.repl_min_slaves_to_write) + { + addReplyErrorObject(caller, shared.noreplicaserr); + return C_ERR; + } } } else { /* Special handling for backwards compatibility (no shebang eval[sha]) mode */ @@ -359,6 +371,19 @@ static int scriptVerifyWriteCommandAllow(scriptRunCtx *run_ctx, char **err) { return C_ERR; } + /* Don't accept write commands if there are not enough good slaves and + * user configured the min-slaves-to-write option. Note this only reachable + * for Eval scripts that didn't declare flags, see the other check in + * scriptPrepareForRun */ + if (server.masterhost == NULL && + server.repl_min_slaves_max_lag && + server.repl_min_slaves_to_write && + server.repl_good_slaves_count < server.repl_min_slaves_to_write) + { + *err = sdsdup(shared.noreplicaserr->ptr); + return C_ERR; + } + return C_OK; } @@ -480,32 +505,31 @@ void scriptCall(scriptRunCtx *run_ctx, robj* *argv, int argc, sds *err) { argc = c->argc; struct redisCommand *cmd = lookupCommand(argv, argc); + c->cmd = c->lastcmd = c->realcmd = cmd; if (scriptVerifyCommandArity(cmd, argc, err) != C_OK) { - return; + goto error; } - c->cmd = c->lastcmd = cmd; - /* There are commands that are not allowed inside scripts. */ if (!server.script_disable_deny_script && (cmd->flags & CMD_NOSCRIPT)) { *err = sdsnew("This Redis command is not allowed from script"); - return; + goto error; } if (scriptVerifyAllowStale(c, err) != C_OK) { - return; + goto error; } if (scriptVerifyACL(c, err) != C_OK) { - return; + goto error; } if (scriptVerifyWriteCommandAllow(run_ctx, err) != C_OK) { - return; + goto error; } if (scriptVerifyOOM(run_ctx, err) != C_OK) { - return; + goto error; } if (cmd->flags & CMD_WRITE) { @@ -514,7 +538,7 @@ void scriptCall(scriptRunCtx *run_ctx, robj* *argv, int argc, sds *err) { } if (scriptVerifyClusterState(c, run_ctx->original_client, err) != C_OK) { - return; + goto error; } int call_flags = CMD_CALL_SLOWLOG | CMD_CALL_STATS; @@ -526,6 +550,11 @@ void scriptCall(scriptRunCtx *run_ctx, robj* *argv, int argc, sds *err) { } call(c, call_flags); serverAssert((c->flags & CLIENT_BLOCKED) == 0); + return; + +error: + afterErrorReply(c, *err, sdslen(*err), 0); + incrCommandStatsOnError(cmd, ERROR_COMMAND_REJECTED); } /* Returns the time when the script invocation started */ diff --git a/src/script_lua.c b/src/script_lua.c index d7332cf86..9a08a7e47 100644 --- a/src/script_lua.c +++ b/src/script_lua.c @@ -238,9 +238,12 @@ static void redisProtocolToLuaType_Error(void *ctx, const char *str, size_t len, * to push elements to the stack. On failure, exit with panic. */ serverPanic("lua stack limit reach when parsing redis.call reply"); } - lua_newtable(lua); - lua_pushstring(lua,"err"); - lua_pushlstring(lua,str,len); + sds err_msg = sdscatlen(sdsnew("-"), str, len); + luaPushErrorBuff(lua,err_msg); + /* push a field indicate to ignore updating the stats on this error + * because it was already updated when executing the command. */ + lua_pushstring(lua,"ignore_error_stats_update"); + lua_pushboolean(lua, true); lua_settable(lua,-3); } @@ -428,40 +431,66 @@ static void redisProtocolToLuaType_Double(void *ctx, double d, const char *proto /* This function is used in order to push an error on the Lua stack in the * format used by redis.pcall to return errors, which is a lua table - * with a single "err" field set to the error string. Note that this - * table is never a valid reply by proper commands, since the returned - * tables are otherwise always indexed by integers, never by strings. */ -void luaPushError(lua_State *lua, char *error) { - lua_Debug dbg; + * with an "err" field set to the error string including the error code. + * Note that this table is never a valid reply by proper commands, + * since the returned tables are otherwise always indexed by integers, never by strings. + * + * The function takes ownership on the given err_buffer. */ +void luaPushErrorBuff(lua_State *lua, sds err_buffer) { + sds msg; + sds error_code; /* If debugging is active and in step mode, log errors resulting from * Redis commands. */ if (ldbIsEnabled()) { - ldbLog(sdscatprintf(sdsempty(),"<error> %s",error)); + ldbLog(sdscatprintf(sdsempty(),"<error> %s",err_buffer)); + } + + /* There are two possible formats for the received `error` string: + * 1) "-CODE msg": in this case we remove the leading '-' since we don't store it as part of the lua error format. + * 2) "msg": in this case we prepend a generic 'ERR' code since all error statuses need some error code. + * We support format (1) so this function can reuse the error messages used in other places in redis. + * We support format (2) so it'll be easy to pass descriptive errors to this function without worrying about format. + */ + if (err_buffer[0] == '-') { + /* derive error code from the message */ + char *err_msg = strstr(err_buffer, " "); + if (!err_msg) { + msg = sdsnew(err_buffer+1); + error_code = sdsnew("ERR"); + } else { + *err_msg = '\0'; + msg = sdsnew(err_msg+1); + error_code = sdsnew(err_buffer + 1); + } + sdsfree(err_buffer); + } else { + msg = err_buffer; + error_code = sdsnew("ERR"); } + /* Trim newline at end of string. If we reuse the ready-made Redis error objects (case 1 above) then we might + * have a newline that needs to be trimmed. In any case the lua Redis error table shouldn't end with a newline. */ + msg = sdstrim(msg, "\r\n"); + sds final_msg = sdscatfmt(error_code, " %s", msg); lua_newtable(lua); lua_pushstring(lua,"err"); - - /* Attempt to figure out where this function was called, if possible */ - if(lua_getstack(lua, 1, &dbg) && lua_getinfo(lua, "nSl", &dbg)) { - sds msg = sdscatprintf(sdsempty(), "%s: %d: %s", - dbg.source, dbg.currentline, error); - lua_pushstring(lua, msg); - sdsfree(msg); - } else { - lua_pushstring(lua, error); - } + lua_pushstring(lua, final_msg); lua_settable(lua,-3); + + sdsfree(msg); + sdsfree(final_msg); +} + +void luaPushError(lua_State *lua, const char *error) { + luaPushErrorBuff(lua, sdsnew(error)); } /* In case the error set into the Lua stack by luaPushError() was generated * by the non-error-trapping version of redis.pcall(), which is redis.call(), * this function will raise the Lua error so that the execution of the * script will be halted. */ -int luaRaiseError(lua_State *lua) { - lua_pushstring(lua,"err"); - lua_gettable(lua,-2); +int luaError(lua_State *lua) { return lua_error(lua); } @@ -511,8 +540,15 @@ static void luaReplyToRedisReply(client *c, client* script_client, lua_State *lu lua_gettable(lua,-2); t = lua_type(lua,-1); if (t == LUA_TSTRING) { - addReplyErrorFormat(c,"-%s",lua_tostring(lua,-1)); - lua_pop(lua,2); + lua_pop(lua, 1); /* pop the error message, we will use luaExtractErrorInformation to get error information */ + errorInfo err_info = {0}; + luaExtractErrorInformation(lua, &err_info); + addReplyErrorFormatEx(c, + err_info.ignore_err_stats_update? ERR_REPLY_FLAG_NO_STATS_UPDATE: 0, + "-%s", + err_info.msg); + luaErrorInformationDiscard(&err_info); + lua_pop(lua,1); /* pop the result table */ return; } lua_pop(lua,1); /* Discard field name pushed before. */ @@ -655,55 +691,19 @@ static void luaReplyToRedisReply(client *c, client* script_client, lua_State *lu * Lua redis.* functions implementations. * ------------------------------------------------------------------------- */ -#define LUA_CMD_OBJCACHE_SIZE 32 -#define LUA_CMD_OBJCACHE_MAX_LEN 64 -static int luaRedisGenericCommand(lua_State *lua, int raise_error) { - int j, argc = lua_gettop(lua); - scriptRunCtx* rctx = luaGetFromRegistry(lua, REGISTRY_RUN_CTX_NAME); - if (!rctx) { - luaPushError(lua, "redis.call/pcall can only be called inside a script invocation"); - return luaRaiseError(lua); - } - sds err = NULL; - client* c = rctx->c; - sds reply; - - /* Cached across calls. */ - static robj **argv = NULL; - static int argv_size = 0; - static robj *cached_objects[LUA_CMD_OBJCACHE_SIZE]; - static size_t cached_objects_len[LUA_CMD_OBJCACHE_SIZE]; - static int inuse = 0; /* Recursive calls detection. */ - - /* By using Lua debug hooks it is possible to trigger a recursive call - * to luaRedisGenericCommand(), which normally should never happen. - * To make this function reentrant is futile and makes it slower, but - * we should at least detect such a misuse, and abort. */ - if (inuse) { - char *recursion_warning = - "luaRedisGenericCommand() recursive call detected. " - "Are you doing funny stuff with Lua debug hooks?"; - serverLog(LL_WARNING,"%s",recursion_warning); - luaPushError(lua,recursion_warning); - return 1; - } - inuse++; - +static robj **luaArgsToRedisArgv(lua_State *lua, int *argc) { + int j; /* Require at least one argument */ - if (argc == 0) { - luaPushError(lua, - "Please specify at least one argument for redis.call()"); - inuse--; - return raise_error ? luaRaiseError(lua) : 1; + *argc = lua_gettop(lua); + if (*argc == 0) { + luaPushError(lua, "Please specify at least one argument for this redis lib call"); + return NULL; } /* Build the arguments vector */ - if (argv_size < argc) { - argv = zrealloc(argv,sizeof(robj*)*argc); - argv_size = argc; - } + robj **argv = zcalloc(sizeof(robj*) * *argc); - for (j = 0; j < argc; j++) { + for (j = 0; j < *argc; j++) { char *obj_s; size_t obj_len; char dbuf[64]; @@ -720,38 +720,62 @@ static int luaRedisGenericCommand(lua_State *lua, int raise_error) { if (obj_s == NULL) break; /* Not a string. */ } - /* Try to use a cached object. */ - if (j < LUA_CMD_OBJCACHE_SIZE && cached_objects[j] && - cached_objects_len[j] >= obj_len) - { - sds s = cached_objects[j]->ptr; - argv[j] = cached_objects[j]; - cached_objects[j] = NULL; - memcpy(s,obj_s,obj_len+1); - sdssetlen(s, obj_len); - } else { - argv[j] = createStringObject(obj_s, obj_len); - } + argv[j] = createStringObject(obj_s, obj_len); } + /* Pop all arguments from the stack, we do not need them anymore + * and this way we guaranty we will have room on the stack for the result. */ + lua_pop(lua, *argc); + /* Check if one of the arguments passed by the Lua script * is not a string or an integer (lua_isstring() return true for * integers as well). */ - if (j != argc) { + if (j != *argc) { j--; while (j >= 0) { decrRefCount(argv[j]); j--; } - luaPushError(lua, - "Lua redis() command arguments must be strings or integers"); - inuse--; - return raise_error ? luaRaiseError(lua) : 1; + zfree(argv); + luaPushError(lua, "Lua redis lib command arguments must be strings or integers"); + return NULL; } - /* Pop all arguments from the stack, we do not need them anymore - * and this way we guaranty we will have room on the stack for the result. */ - lua_pop(lua, argc); + return argv; +} + +static int luaRedisGenericCommand(lua_State *lua, int raise_error) { + int j; + scriptRunCtx* rctx = luaGetFromRegistry(lua, REGISTRY_RUN_CTX_NAME); + if (!rctx) { + luaPushError(lua, "redis.call/pcall can only be called inside a script invocation"); + return luaError(lua); + } + sds err = NULL; + client* c = rctx->c; + sds reply; + + int argc; + robj **argv = luaArgsToRedisArgv(lua, &argc); + if (argv == NULL) { + return raise_error ? luaError(lua) : 1; + } + + static int inuse = 0; /* Recursive calls detection. */ + + /* By using Lua debug hooks it is possible to trigger a recursive call + * to luaRedisGenericCommand(), which normally should never happen. + * To make this function reentrant is futile and makes it slower, but + * we should at least detect such a misuse, and abort. */ + if (inuse) { + char *recursion_warning = + "luaRedisGenericCommand() recursive call detected. " + "Are you doing funny stuff with Lua debug hooks?"; + serverLog(LL_WARNING,"%s",recursion_warning); + luaPushError(lua,recursion_warning); + return 1; + } + inuse++; /* Log the command if debugging is active. */ if (ldbIsEnabled()) { @@ -769,11 +793,15 @@ static int luaRedisGenericCommand(lua_State *lua, int raise_error) { ldbLog(cmdlog); } - scriptCall(rctx, argv, argc, &err); if (err) { luaPushError(lua, err); sdsfree(err); + /* push a field indicate to ignore updating the stats on this error + * because it was already updated when executing the command. */ + lua_pushstring(lua,"ignore_error_stats_update"); + lua_pushboolean(lua, true); + lua_settable(lua,-3); goto cleanup; } @@ -810,48 +838,48 @@ static int luaRedisGenericCommand(lua_State *lua, int raise_error) { cleanup: /* Clean up. Command code may have changed argv/argc so we use the * argv/argc of the client instead of the local variables. */ - for (j = 0; j < c->argc; j++) { - robj *o = c->argv[j]; - - /* Try to cache the object in the cached_objects array. - * The object must be small, SDS-encoded, and with refcount = 1 - * (we must be the only owner) for us to cache it. */ - if (j < LUA_CMD_OBJCACHE_SIZE && - o->refcount == 1 && - (o->encoding == OBJ_ENCODING_RAW || - o->encoding == OBJ_ENCODING_EMBSTR) && - sdslen(o->ptr) <= LUA_CMD_OBJCACHE_MAX_LEN) - { - sds s = o->ptr; - if (cached_objects[j]) decrRefCount(cached_objects[j]); - cached_objects[j] = o; - cached_objects_len[j] = sdsalloc(s); - } else { - decrRefCount(o); - } - } - - if (c->argv != argv) { - zfree(c->argv); - argv = NULL; - argv_size = 0; - } - + freeClientArgv(c); c->user = NULL; - c->argv = NULL; - c->argc = 0; + inuse--; if (raise_error) { /* If we are here we should have an error in the stack, in the * form of a table with an "err" field. Extract the string to * return the plain error. */ - inuse--; - return luaRaiseError(lua); + return luaError(lua); } - inuse--; return 1; } +/* Our implementation to lua pcall. + * We need this implementation for backward + * comparability with older Redis versions. + * + * On Redis 7, the error object is a table, + * compare to older version where the error + * object is a string. To keep backward + * comparability we catch the table object + * and just return the error message. */ +static int luaRedisPcall(lua_State *lua) { + int argc = lua_gettop(lua); + lua_pushboolean(lua, 1); /* result place holder */ + lua_insert(lua, 1); + if (lua_pcall(lua, argc - 1, LUA_MULTRET, 0)) { + /* Error */ + lua_remove(lua, 1); /* remove the result place holder, now we have room for at least one element */ + if (lua_istable(lua, -1)) { + lua_getfield(lua, -1, "err"); + if (lua_isstring(lua, -1)) { + lua_replace(lua, -2); /* replace the error message with the table */ + } + } + lua_pushboolean(lua, 0); /* push result */ + lua_insert(lua, 1); + } + return lua_gettop(lua); + +} + /* redis.call() */ static int luaRedisCallCommand(lua_State *lua) { return luaRedisGenericCommand(lua,1); @@ -871,8 +899,8 @@ static int luaRedisSha1hexCommand(lua_State *lua) { char *s; if (argc != 1) { - lua_pushstring(lua, "wrong number of arguments"); - return lua_error(lua); + luaPushError(lua, "wrong number of arguments"); + return luaError(lua); } s = (char*)lua_tolstring(lua,1,&len); @@ -903,7 +931,21 @@ static int luaRedisReturnSingleFieldTable(lua_State *lua, char *field) { /* redis.error_reply() */ static int luaRedisErrorReplyCommand(lua_State *lua) { - return luaRedisReturnSingleFieldTable(lua,"err"); + if (lua_gettop(lua) != 1 || lua_type(lua,-1) != LUA_TSTRING) { + luaPushError(lua, "wrong number or type of arguments"); + return 1; + } + + /* add '-' if not exists */ + const char *err = lua_tostring(lua, -1); + sds err_buff = NULL; + if (err[0] != '-') { + err_buff = sdscatfmt(sdsempty(), "-%s", err); + } else { + err_buff = sdsnew(err); + } + luaPushErrorBuff(lua, err_buff); + return 1; } /* redis.status_reply() */ @@ -920,25 +962,65 @@ static int luaRedisSetReplCommand(lua_State *lua) { scriptRunCtx* rctx = luaGetFromRegistry(lua, REGISTRY_RUN_CTX_NAME); if (!rctx) { - lua_pushstring(lua, "redis.set_repl can only be called inside a script invocation"); - return lua_error(lua); + luaPushError(lua, "redis.set_repl can only be called inside a script invocation"); + return luaError(lua); } if (argc != 1) { - lua_pushstring(lua, "redis.set_repl() requires two arguments."); - return lua_error(lua); + luaPushError(lua, "redis.set_repl() requires two arguments."); + return luaError(lua); } flags = lua_tonumber(lua,-1); if ((flags & ~(PROPAGATE_AOF|PROPAGATE_REPL)) != 0) { - lua_pushstring(lua, "Invalid replication flags. Use REPL_AOF, REPL_REPLICA, REPL_ALL or REPL_NONE."); - return lua_error(lua); + luaPushError(lua, "Invalid replication flags. Use REPL_AOF, REPL_REPLICA, REPL_ALL or REPL_NONE."); + return luaError(lua); } scriptSetRepl(rctx, flags); return 0; } +/* redis.acl_check_cmd() + * + * Checks ACL permissions for given command for the current user. */ +static int luaRedisAclCheckCmdPermissionsCommand(lua_State *lua) { + scriptRunCtx* rctx = luaGetFromRegistry(lua, REGISTRY_RUN_CTX_NAME); + if (!rctx) { + luaPushError(lua, "redis.acl_check_cmd can only be called inside a script invocation"); + return luaError(lua); + } + int raise_error = 0; + + int argc; + robj **argv = luaArgsToRedisArgv(lua, &argc); + + /* Require at least one argument */ + if (argv == NULL) return luaError(lua); + + /* Find command */ + struct redisCommand *cmd; + if ((cmd = lookupCommand(argv, argc)) == NULL) { + luaPushError(lua, "Invalid command passed to redis.acl_check_cmd()"); + raise_error = 1; + } else { + int keyidxptr; + if (ACLCheckAllUserCommandPerm(rctx->original_client->user, cmd, argv, argc, &keyidxptr) != ACL_OK) { + lua_pushboolean(lua, 0); + } else { + lua_pushboolean(lua, 1); + } + } + + while (argc--) decrRefCount(argv[argc]); + zfree(argv); + if (raise_error) + return luaError(lua); + else + return 1; +} + + /* redis.log() */ static int luaLogCommand(lua_State *lua) { int j, argc = lua_gettop(lua); @@ -946,16 +1028,16 @@ static int luaLogCommand(lua_State *lua) { sds log; if (argc < 2) { - lua_pushstring(lua, "redis.log() requires two arguments or more."); - return lua_error(lua); + luaPushError(lua, "redis.log() requires two arguments or more."); + return luaError(lua); } else if (!lua_isnumber(lua,-argc)) { - lua_pushstring(lua, "First argument must be a number (log level)."); - return lua_error(lua); + luaPushError(lua, "First argument must be a number (log level)."); + return luaError(lua); } level = lua_tonumber(lua,-argc); if (level < LL_DEBUG || level > LL_WARNING) { - lua_pushstring(lua, "Invalid debug level."); - return lua_error(lua); + luaPushError(lua, "Invalid debug level."); + return luaError(lua); } if (level < server.verbosity) return 0; @@ -980,20 +1062,20 @@ static int luaLogCommand(lua_State *lua) { static int luaSetResp(lua_State *lua) { scriptRunCtx* rctx = luaGetFromRegistry(lua, REGISTRY_RUN_CTX_NAME); if (!rctx) { - lua_pushstring(lua, "redis.setresp can only be called inside a script invocation"); - return lua_error(lua); + luaPushError(lua, "redis.setresp can only be called inside a script invocation"); + return luaError(lua); } int argc = lua_gettop(lua); if (argc != 1) { - lua_pushstring(lua, "redis.setresp() requires one argument."); - return lua_error(lua); + luaPushError(lua, "redis.setresp() requires one argument."); + return luaError(lua); } int resp = lua_tonumber(lua,-argc); if (resp != 2 && resp != 3) { - lua_pushstring(lua, "RESP version must be 2 or 3."); - return lua_error(lua); + luaPushError(lua, "RESP version must be 2 or 3."); + return luaError(lua); } scriptSetResp(rctx, resp); return 0; @@ -1193,6 +1275,9 @@ void luaRegisterRedisAPI(lua_State* lua) { luaLoadLibraries(lua); luaRemoveUnsupportedFunctions(lua); + lua_pushcfunction(lua,luaRedisPcall); + lua_setglobal(lua, "pcall"); + /* Register the redis commands table and fields */ lua_newtable(lua); @@ -1251,8 +1336,13 @@ void luaRegisterRedisAPI(lua_State* lua) { lua_pushstring(lua,"REPL_ALL"); lua_pushnumber(lua,PROPAGATE_AOF|PROPAGATE_REPL); + lua_settable(lua,-3); + /* redis.acl_check_cmd */ + lua_pushstring(lua,"acl_check_cmd"); + lua_pushcfunction(lua,luaRedisAclCheckCmdPermissionsCommand); lua_settable(lua,-3); + /* Finally set the table as 'redis' global var. */ lua_setglobal(lua,REDIS_API_NAME); @@ -1348,11 +1438,50 @@ static void luaMaskCountHook(lua_State *lua, lua_Debug *ar) { */ lua_sethook(lua, luaMaskCountHook, LUA_MASKLINE, 0); - lua_pushstring(lua,"Script killed by user with SCRIPT KILL..."); - lua_error(lua); + luaPushError(lua,"Script killed by user with SCRIPT KILL..."); + luaError(lua); } } +void luaErrorInformationDiscard(errorInfo *err_info) { + if (err_info->msg) sdsfree(err_info->msg); + if (err_info->source) sdsfree(err_info->source); + if (err_info->line) sdsfree(err_info->line); +} + +void luaExtractErrorInformation(lua_State *lua, errorInfo *err_info) { + if (lua_isstring(lua, -1)) { + err_info->msg = sdscatfmt(sdsempty(), "ERR %s", lua_tostring(lua, -1)); + err_info->line = NULL; + err_info->source = NULL; + err_info->ignore_err_stats_update = 0; + } + + lua_getfield(lua, -1, "err"); + if (lua_isstring(lua, -1)) { + err_info->msg = sdsnew(lua_tostring(lua, -1)); + } + lua_pop(lua, 1); + + lua_getfield(lua, -1, "source"); + if (lua_isstring(lua, -1)) { + err_info->source = sdsnew(lua_tostring(lua, -1)); + } + lua_pop(lua, 1); + + lua_getfield(lua, -1, "line"); + if (lua_isstring(lua, -1)) { + err_info->line = sdsnew(lua_tostring(lua, -1)); + } + lua_pop(lua, 1); + + lua_getfield(lua, -1, "ignore_error_stats_update"); + if (lua_isboolean(lua, -1)) { + err_info->ignore_err_stats_update = lua_toboolean(lua, -1); + } + lua_pop(lua, 1); +} + void luaCallFunction(scriptRunCtx* run_ctx, lua_State *lua, robj** keys, size_t nkeys, robj** args, size_t nargs, int debug_enabled) { client* c = run_ctx->original_client; int delhook = 0; @@ -1410,9 +1539,28 @@ void luaCallFunction(scriptRunCtx* run_ctx, lua_State *lua, robj** keys, size_t } if (err) { - addReplyErrorFormat(c,"Error running script (call to %s): %s\n", - run_ctx->funcname, lua_tostring(lua,-1)); - lua_pop(lua,1); /* Consume the Lua reply and remove error handler. */ + /* Error object is a table of the following format: + * {err='<error msg>', source='<source file>', line=<line>} + * We can construct the error message from this information */ + if (!lua_istable(lua, -1)) { + /* Should not happened, and we should considered assert it */ + addReplyErrorFormat(c,"Error running script (call to %s)\n", run_ctx->funcname); + } else { + errorInfo err_info = {0}; + sds final_msg = sdsempty(); + luaExtractErrorInformation(lua, &err_info); + final_msg = sdscatfmt(final_msg, "-%s", + err_info.msg); + if (err_info.line && err_info.source) { + final_msg = sdscatfmt(final_msg, " script: %s, on %s:%s.", + run_ctx->funcname, + err_info.source, + err_info.line); + } + addReplyErrorSdsEx(c, final_msg, err_info.ignore_err_stats_update? ERR_REPLY_FLAG_NO_STATS_UPDATE : 0); + luaErrorInformationDiscard(&err_info); + } + lua_pop(lua,1); /* Consume the Lua error */ } else { /* On success convert the Lua return value into Redis protocol, and * send it to * the client. */ diff --git a/src/script_lua.h b/src/script_lua.h index ac13178ca..5a4533784 100644 --- a/src/script_lua.h +++ b/src/script_lua.h @@ -58,6 +58,13 @@ #define REGISTRY_SET_GLOBALS_PROTECTION_NAME "__GLOBAL_PROTECTION__" #define REDIS_API_NAME "redis" +typedef struct errorInfo { + sds msg; + sds source; + sds line; + int ignore_err_stats_update; +}errorInfo; + void luaRegisterRedisAPI(lua_State* lua); sds luaGetStringSds(lua_State *lua, int index); void luaEnableGlobalsProtection(lua_State *lua, int is_eval); @@ -65,11 +72,14 @@ void luaRegisterGlobalProtectionFunction(lua_State *lua); void luaSetGlobalProtection(lua_State *lua); void luaRegisterLogFunction(lua_State* lua); void luaRegisterVersion(lua_State* lua); -void luaPushError(lua_State *lua, char *error); -int luaRaiseError(lua_State *lua); +void luaPushErrorBuff(lua_State *lua, sds err_buff); +void luaPushError(lua_State *lua, const char *error); +int luaError(lua_State *lua); void luaSaveOnRegistry(lua_State* lua, const char* name, void* ptr); void* luaGetFromRegistry(lua_State* lua, const char* name); void luaCallFunction(scriptRunCtx* r_ctx, lua_State *lua, robj** keys, size_t nkeys, robj** args, size_t nargs, int debug_enabled); +void luaExtractErrorInformation(lua_State *lua, errorInfo *err_info); +void luaErrorInformationDiscard(errorInfo *err_info); unsigned long luaMemory(lua_State *lua); diff --git a/src/sentinel.c b/src/sentinel.c index f65e29876..eb37e5ede 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -1190,10 +1190,6 @@ int sentinelUpdateSentinelAddressInAllMasters(sentinelRedisInstance *ri) { if (match->link->pc != NULL) instanceLinkCloseConnection(match->link,match->link->pc); - /* Remove any sentinel with port number set to 0 */ - if (match->addr->port == 0) - dictDelete(master->sentinels,match->name); - if (match == ri) continue; /* Address already updated for it. */ /* Update the address of the matching Sentinel by copying the address @@ -2281,6 +2277,16 @@ werr: return C_ERR; } +/* Call sentinelFlushConfig() produce a success/error reply to the + * calling client. + */ +static void sentinelFlushConfigAndReply(client *c) { + if (sentinelFlushConfig() == C_ERR) + addReplyError(c, "Failed to save config file. Check server logs."); + else + addReply(c, shared.ok); +} + /* ====================== hiredis connection handling ======================= */ /* Send the AUTH command with the specified master password if needed. @@ -2868,9 +2874,22 @@ void sentinelProcessHelloMessage(char *hello, int hello_len) { getSentinelRedisInstanceByAddrAndRunID( master->sentinels, token[0],port,NULL); if (other) { + /* If there is already other sentinel with same address (but + * different runid) then remove the old one across all masters */ sentinelEvent(LL_NOTICE,"+sentinel-invalid-addr",other,"%@"); - other->addr->port = 0; /* It means: invalid address. */ - sentinelUpdateSentinelAddressInAllMasters(other); + dictIterator *di; + dictEntry *de; + + /* Keep a copy of runid. 'other' about to be deleted in loop. */ + sds runid_obsolete = sdsnew(other->runid); + + di = dictGetIterator(sentinel.masters); + while((de = dictNext(di)) != NULL) { + sentinelRedisInstance *master = dictGetVal(de); + removeMatchingSentinelFromMaster(master, runid_obsolete); + } + dictReleaseIterator(di); + sdsfree(runid_obsolete); } } @@ -3174,8 +3193,7 @@ void sentinelConfigSetCommand(client *c) { return; } - sentinelFlushConfig(); - addReply(c, shared.ok); + sentinelFlushConfigAndReply(c); /* Drop Sentinel connections to initiate a reconnect if needed. */ if (drop_conns) @@ -3445,7 +3463,6 @@ void addReplySentinelRedisInstance(client *c, sentinelRedisInstance *ri) { } void sentinelSetDebugConfigParameters(client *c){ - int j; int badarg = 0; /* Bad argument position for error reporting. */ char *option; @@ -3464,7 +3481,7 @@ void sentinelSetDebugConfigParameters(client *c){ goto badfmt; } sentinel_info_period = ll; - + } else if (!strcasecmp(option,"ping-period") && moreargs > 0) { /* ping-period <milliseconds> */ robj *o = c->argv[++j]; @@ -3473,7 +3490,7 @@ void sentinelSetDebugConfigParameters(client *c){ goto badfmt; } sentinel_ping_period = ll; - + } else if (!strcasecmp(option,"ask-period") && moreargs > 0) { /* ask-period <milliseconds> */ robj *o = c->argv[++j]; @@ -3482,7 +3499,7 @@ void sentinelSetDebugConfigParameters(client *c){ goto badfmt; } sentinel_ask_period = ll; - + } else if (!strcasecmp(option,"publish-period") && moreargs > 0) { /* publish-period <milliseconds> */ robj *o = c->argv[++j]; @@ -3491,8 +3508,8 @@ void sentinelSetDebugConfigParameters(client *c){ goto badfmt; } sentinel_publish_period = ll; - - }else if (!strcasecmp(option,"default-down-after") && moreargs > 0) { + + } else if (!strcasecmp(option,"default-down-after") && moreargs > 0) { /* default-down-after <milliseconds> */ robj *o = c->argv[++j]; if (getLongLongFromObject(o,&ll) == C_ERR || ll <= 0) { @@ -3500,7 +3517,7 @@ void sentinelSetDebugConfigParameters(client *c){ goto badfmt; } sentinel_default_down_after = ll; - + } else if (!strcasecmp(option,"tilt-trigger") && moreargs > 0) { /* tilt-trigger <milliseconds> */ robj *o = c->argv[++j]; @@ -3509,7 +3526,7 @@ void sentinelSetDebugConfigParameters(client *c){ goto badfmt; } sentinel_tilt_trigger = ll; - + } else if (!strcasecmp(option,"tilt-period") && moreargs > 0) { /* tilt-period <milliseconds> */ robj *o = c->argv[++j]; @@ -3518,7 +3535,7 @@ void sentinelSetDebugConfigParameters(client *c){ goto badfmt; } sentinel_tilt_period = ll; - + } else if (!strcasecmp(option,"slave-reconf-timeout") && moreargs > 0) { /* slave-reconf-timeout <milliseconds> */ robj *o = c->argv[++j]; @@ -3527,7 +3544,7 @@ void sentinelSetDebugConfigParameters(client *c){ goto badfmt; } sentinel_slave_reconf_timeout = ll; - + } else if (!strcasecmp(option,"min-link-reconnect-period") && moreargs > 0) { /* min-link-reconnect-period <milliseconds> */ robj *o = c->argv[++j]; @@ -3536,7 +3553,7 @@ void sentinelSetDebugConfigParameters(client *c){ goto badfmt; } sentinel_min_link_reconnect_period = ll; - + } else if (!strcasecmp(option,"default-failover-timeout") && moreargs > 0) { /* default-failover-timeout <milliseconds> */ robj *o = c->argv[++j]; @@ -3545,7 +3562,7 @@ void sentinelSetDebugConfigParameters(client *c){ goto badfmt; } sentinel_default_failover_timeout = ll; - + } else if (!strcasecmp(option,"election-timeout") && moreargs > 0) { /* election-timeout <milliseconds> */ robj *o = c->argv[++j]; @@ -3554,7 +3571,7 @@ void sentinelSetDebugConfigParameters(client *c){ goto badfmt; } sentinel_election_timeout = ll; - + } else if (!strcasecmp(option,"script-max-runtime") && moreargs > 0) { /* script-max-runtime <milliseconds> */ robj *o = c->argv[++j]; @@ -3563,7 +3580,7 @@ void sentinelSetDebugConfigParameters(client *c){ goto badfmt; } sentinel_script_max_runtime = ll; - + } else if (!strcasecmp(option,"script-retry-delay") && moreargs > 0) { /* script-retry-delay <milliseconds> */ robj *o = c->argv[++j]; @@ -3572,27 +3589,25 @@ void sentinelSetDebugConfigParameters(client *c){ goto badfmt; } sentinel_script_retry_delay = ll; - + } else { addReplyErrorFormat(c,"Unknown option or number of arguments for " - "SENTINEL SET '%s'", option); + "SENTINEL DEBUG '%s'", option); + return; } - } addReply(c,shared.ok); return; badfmt: /* Bad format errors */ - addReplyErrorFormat(c,"Invalid argument '%s' for SENTINEL SET '%s'", + addReplyErrorFormat(c,"Invalid argument '%s' for SENTINEL DEBUG '%s'", (char*)c->argv[badarg]->ptr,option); return; } - void addReplySentinelDebugInfo(client *c) { - void *mbl; int fields = 0; @@ -3621,7 +3636,7 @@ void addReplySentinelDebugInfo(client *c) { addReplyBulkCString(c,"DEFAULT-FAILOVER-TIMEOUT"); addReplyBulkLongLong(c,sentinel_default_failover_timeout); fields++; - + addReplyBulkCString(c,"TILT-TRIGGER"); addReplyBulkLongLong(c,sentinel_tilt_trigger); fields++; @@ -3726,8 +3741,9 @@ void sentinelCommand(client *c) { " Set a global Sentinel configuration parameter.", "CONFIG GET <param>", " Get global Sentinel configuration parameter.", -"DEBUG", +"DEBUG [<param> <value> ...]", " Show a list of configurable time parameters and their values (milliseconds).", +" Or update current configurable parameters values (one or more).", "GET-MASTER-ADDR-BY-NAME <master-name>", " Return the ip and port number of the master with that name.", "FAILOVER <master-name>", @@ -3929,14 +3945,12 @@ NULL if (ri == NULL) { addReplyError(c,sentinelCheckCreateInstanceErrors(SRI_MASTER)); } else { - sentinelFlushConfig(); + sentinelFlushConfigAndReply(c); sentinelEvent(LL_WARNING,"+monitor",ri,"%@ quorum %d",ri->quorum); - addReply(c,shared.ok); } } else if (!strcasecmp(c->argv[1]->ptr,"flushconfig")) { if (c->argc != 2) goto numargserr; - sentinelFlushConfig(); - addReply(c,shared.ok); + sentinelFlushConfigAndReply(c); return; } else if (!strcasecmp(c->argv[1]->ptr,"remove")) { /* SENTINEL REMOVE <name> */ @@ -3947,8 +3961,7 @@ NULL == NULL) return; sentinelEvent(LL_WARNING,"-monitor",ri,"%@"); dictDelete(sentinel.masters,c->argv[2]->ptr); - sentinelFlushConfig(); - addReply(c,shared.ok); + sentinelFlushConfigAndReply(c); } else if (!strcasecmp(c->argv[1]->ptr,"ckquorum")) { /* SENTINEL CKQUORUM <name> */ sentinelRedisInstance *ri; @@ -4092,46 +4105,51 @@ numargserr: addReplyErrorArity(c); } -#define info_section_from_redis(section_name) do { \ - if (defsections || allsections || !strcasecmp(section,section_name)) { \ - sds redissection; \ - if (sections++) info = sdscat(info,"\r\n"); \ - redissection = genRedisInfoString(section_name); \ - info = sdscatlen(info,redissection,sdslen(redissection)); \ - sdsfree(redissection); \ - } \ -} while(0) +void addInfoSectionsToDict(dict *section_dict, char **sections); /* SENTINEL INFO [section] */ void sentinelInfoCommand(client *c) { - if (c->argc > 2) { - addReplyErrorObject(c,shared.syntaxerr); - return; - } + char *sentinel_sections[] = {"server", "clients", "cpu", "stats", "sentinel", NULL}; + int sec_all = 0, sec_everything = 0; + static dict *cached_all_info_sections = NULL; - int defsections = 0, allsections = 0; - char *section = c->argc == 2 ? c->argv[1]->ptr : NULL; - if (section) { - allsections = !strcasecmp(section,"all"); - defsections = !strcasecmp(section,"default"); - } else { - defsections = 1; - } + /* Get requested section list. */ + dict *sections_dict = genInfoSectionDict(c->argv+1, c->argc-1, sentinel_sections, &sec_all, &sec_everything); - int sections = 0; - sds info = sdsempty(); + /* Purge unsupported sections from the requested ones. */ + dictEntry *de; + dictIterator *di = dictGetSafeIterator(sections_dict); + while((de = dictNext(di)) != NULL) { + int i; + sds sec = dictGetKey(de); + for (i=0; sentinel_sections[i]; i++) + if (!strcasecmp(sentinel_sections[i], sec)) + break; + /* section not found? remove it */ + if (!sentinel_sections[i]) + dictDelete(sections_dict, sec); + } + dictReleaseIterator(di); - info_section_from_redis("server"); - info_section_from_redis("clients"); - info_section_from_redis("cpu"); - info_section_from_redis("stats"); + /* Insert explicit all sections (don't pass these vars to genRedisInfoString) */ + if (sec_all || sec_everything) { + releaseInfoSectionDict(sections_dict); + /* We cache this dict as an optimization. */ + if (!cached_all_info_sections) { + cached_all_info_sections = dictCreate(&stringSetDictType); + addInfoSectionsToDict(cached_all_info_sections, sentinel_sections); + } + sections_dict = cached_all_info_sections; + } - if (defsections || allsections || !strcasecmp(section,"sentinel")) { + sds info = genRedisInfoString(sections_dict, 0, 0); + if (sec_all || (dictFind(sections_dict, "sentinel") != NULL)) { dictIterator *di; dictEntry *de; int master_id = 0; - if (sections++) info = sdscat(info,"\r\n"); + if (sdslen(info) != 0) + info = sdscat(info,"\r\n"); info = sdscatprintf(info, "# Sentinel\r\n" "sentinel_masters:%lu\r\n" @@ -4164,7 +4182,8 @@ void sentinelInfoCommand(client *c) { } dictReleaseIterator(di); } - + if (sections_dict != cached_all_info_sections) + releaseInfoSectionDict(sections_dict); addReplyBulkSds(c, info); } @@ -4348,22 +4367,16 @@ void sentinelSetCommand(client *c) { break; } } - if (changes && sentinelFlushConfig() == C_ERR) { - addReplyErrorFormat(c,"Failed to save Sentinel new configuration on disk"); - return; - } - addReply(c,shared.ok); + if (changes) sentinelFlushConfigAndReply(c); return; badfmt: /* Bad format errors */ addReplyErrorFormat(c,"Invalid argument '%s' for SENTINEL SET '%s'", (char*)c->argv[badarg]->ptr,option); seterr: - if (changes && sentinelFlushConfig() == C_ERR) { - addReplyErrorFormat(c,"Failed to save Sentinel new configuration on disk"); - return; - } - return; + /* TODO: Handle the case of both bad input and save error, possibly handling + * SENTINEL SET atomically. */ + if (changes) sentinelFlushConfig(); } /* Our fake PUBLISH command: it is actually useful only to receive hello messages diff --git a/src/server.c b/src/server.c index 565cd9467..00c279837 100644 --- a/src/server.c +++ b/src/server.c @@ -289,6 +289,33 @@ uint64_t dictSdsCaseHash(const void *key) { return dictGenCaseHashFunction((unsigned char*)key, sdslen((char*)key)); } +/* Dict hash function for null terminated string */ +uint64_t distCStrHash(const void *key) { + return dictGenHashFunction((unsigned char*)key, strlen((char*)key)); +} + +/* Dict hash function for null terminated string */ +uint64_t distCStrCaseHash(const void *key) { + return dictGenCaseHashFunction((unsigned char*)key, strlen((char*)key)); +} + +/* Dict compare function for null terminated string */ +int distCStrKeyCompare(dict *d, const void *key1, const void *key2) { + int l1,l2; + UNUSED(d); + + l1 = strlen((char*)key1); + l2 = strlen((char*)key2); + if (l1 != l2) return 0; + return memcmp(key1, key2, l1) == 0; +} + +/* Dict case insensitive compare function for null terminated string */ +int distCStrKeyCaseCompare(dict *d, const void *key1, const void *key2) { + UNUSED(d); + return strcasecmp(key1, key2) == 0; +} + int dictEncObjKeyCompare(dict *d, const void *key1, const void *key2) { robj *o1 = (robj*) key1, *o2 = (robj*) key2; @@ -487,14 +514,13 @@ dictType migrateCacheDictType = { NULL /* allow to expand */ }; -/* Replication cached script dict (server.repl_scriptcache_dict). - * Keys are sds SHA1 strings, while values are not used at all in the current - * implementation. */ -dictType replScriptCacheDictType = { - dictSdsCaseHash, /* hash function */ +/* Dict for for case-insensitive search using null terminated C strings. + * The keys stored in dict are sds though. */ +dictType stringSetDictType = { + distCStrCaseHash, /* hash function */ NULL, /* key dup */ NULL, /* val dup */ - dictSdsKeyCaseCompare, /* key compare */ + distCStrKeyCaseCompare, /* key compare */ dictSdsDestructor, /* key destructor */ NULL, /* val destructor */ NULL /* allow to expand */ @@ -680,6 +706,51 @@ int clientsCronResizeQueryBuffer(client *c) { return 0; } +/* The client output buffer can be adjusted to better fit the memory requirements. + * + * the logic is: + * in case the last observed peak size of the buffer equals the buffer size - we double the size + * in case the last observed peak size of the buffer is less than half the buffer size - we shrink by half. + * The buffer peak will be reset back to the buffer position every server.reply_buffer_peak_reset_time milliseconds + * The function always returns 0 as it never terminates the client. */ +int clientsCronResizeOutputBuffer(client *c, mstime_t now_ms) { + + size_t new_buffer_size = 0; + char *oldbuf = NULL; + const size_t buffer_target_shrink_size = c->buf_usable_size/2; + const size_t buffer_target_expand_size = c->buf_usable_size*2; + + if (buffer_target_shrink_size >= PROTO_REPLY_MIN_BYTES && + c->buf_peak < buffer_target_shrink_size ) + { + new_buffer_size = max(PROTO_REPLY_MIN_BYTES,c->buf_peak+1); + server.stat_reply_buffer_shrinks++; + } else if (buffer_target_expand_size < PROTO_REPLY_CHUNK_BYTES*2 && + c->buf_peak == c->buf_usable_size) + { + new_buffer_size = min(PROTO_REPLY_CHUNK_BYTES,buffer_target_expand_size); + server.stat_reply_buffer_expands++; + } + + /* reset the peak value each server.reply_buffer_peak_reset_time seconds. in case the client will be idle + * it will start to shrink. + */ + if (server.reply_buffer_peak_reset_time >=0 && + now_ms - c->buf_peak_last_reset_time >= server.reply_buffer_peak_reset_time) + { + c->buf_peak = c->bufpos; + c->buf_peak_last_reset_time = now_ms; + } + + if (new_buffer_size) { + oldbuf = c->buf; + c->buf = zmalloc_usable(new_buffer_size, &c->buf_usable_size); + memcpy(c->buf,oldbuf,c->bufpos); + zfree(oldbuf); + } + return 0; +} + /* This function is used in order to track clients using the biggest amount * of memory in the latest few seconds. This way we can provide such information * in the INFO output (clients section), without having to do an O(N) scan for @@ -873,6 +944,8 @@ void clientsCron(void) { * terminated. */ if (clientsCronHandleTimeout(c,now)) continue; if (clientsCronResizeQueryBuffer(c)) continue; + if (clientsCronResizeOutputBuffer(c,now)) continue; + if (clientsCronTrackExpansiveClients(c, curr_peak_mem_usage_slot)) continue; /* Iterating all the clients in getMemoryOverheadData() is too slow and @@ -1234,7 +1307,7 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { } } /* Just for the sake of defensive programming, to avoid forgetting to - * call this function when need. */ + * call this function when needed. */ updateDictResizePolicy(); @@ -1694,6 +1767,7 @@ void createSharedObjects(void) { shared.retrycount = createStringObject("RETRYCOUNT",10); shared.force = createStringObject("FORCE",5); shared.justid = createStringObject("JUSTID",6); + shared.entriesread = createStringObject("ENTRIESREAD",11); shared.lastid = createStringObject("LASTID",6); shared.default_username = createStringObject("default",7); shared.ping = createStringObject("ping",4); @@ -1717,6 +1791,10 @@ void createSharedObjects(void) { sdscatprintf(sdsempty(),"*%d\r\n",j)); shared.bulkhdr[j] = createObject(OBJ_STRING, sdscatprintf(sdsempty(),"$%d\r\n",j)); + shared.maphdr[j] = createObject(OBJ_STRING, + sdscatprintf(sdsempty(),"%%%d\r\n",j)); + shared.sethdr[j] = createObject(OBJ_STRING, + sdscatprintf(sdsempty(),"~%d\r\n",j)); } /* The following two shared objects, minstring and maxstring, are not * actually used for their value but as a special object meaning @@ -1891,7 +1969,7 @@ int restartServer(int flags, mstime_t delay) { rewriteConfig(server.configfile, 0) == -1) { serverLog(LL_WARNING,"Can't restart: configuration rewrite process " - "failed"); + "failed: %s", strerror(errno)); return C_ERR; } @@ -2256,12 +2334,16 @@ void resetServerStats(void) { memset(server.inst_metric[j].samples,0, sizeof(server.inst_metric[j].samples)); } + server.stat_aof_rewrites = 0; + server.stat_rdb_saves = 0; atomicSet(server.stat_net_input_bytes, 0); atomicSet(server.stat_net_output_bytes, 0); server.stat_unexpected_error_replies = 0; server.stat_total_error_replies = 0; server.stat_dump_payload_sanitizations = 0; server.aof_delayed_fsync = 0; + server.stat_reply_buffer_shrinks = 0; + server.stat_reply_buffer_expands = 0; lazyfreeResetStats(); } @@ -2322,6 +2404,7 @@ void initServer(void) { server.blocking_op_nesting = 0; server.thp_enabled = 0; server.cluster_drop_packet_filter = -1; + server.reply_buffer_peak_reset_time = REPLY_BUFFER_DEFAULT_PEAK_RESET_TIME; resetReplicationBuffer(); if ((server.tls_port || server.tls_replication || server.tls_cluster) @@ -2631,12 +2714,14 @@ void setImplicitACLCategories(struct redisCommand *c) { c->acl_categories |= ACL_CATEGORY_SLOW; } -/* Recursively populate the args structure and return the number of args. */ +/* Recursively populate the args structure (setting num_args to the number of + * subargs) and return the number of args. */ int populateArgsStructure(struct redisCommandArg *args) { if (!args) return 0; int count = 0; while (args->name) { + serverAssert(count < INT_MAX); args->num_args = populateArgsStructure(args->subargs); count++; args++; @@ -3050,6 +3135,34 @@ void propagatePendingCommands() { redisOpArrayFree(&server.also_propagate); } +/* Increment the command failure counters (either rejected_calls or failed_calls). + * The decision which counter to increment is done using the flags argument, options are: + * * ERROR_COMMAND_REJECTED - update rejected_calls + * * ERROR_COMMAND_FAILED - update failed_calls + * + * The function also reset the prev_err_count to make sure we will not count the same error + * twice, its possible to pass a NULL cmd value to indicate that the error was counted elsewhere. + * + * The function returns true if stats was updated and false if not. */ +int incrCommandStatsOnError(struct redisCommand *cmd, int flags) { + /* hold the prev error count captured on the last command execution */ + static long long prev_err_count = 0; + int res = 0; + if (cmd) { + if ((server.stat_total_error_replies - prev_err_count) > 0) { + if (flags & ERROR_COMMAND_REJECTED) { + cmd->rejected_calls++; + res = 1; + } else if (flags & ERROR_COMMAND_FAILED) { + cmd->failed_calls++; + res = 1; + } + } + } + prev_err_count = server.stat_total_error_replies; + return res; +} + /* Call() is the core of Redis execution of a command. * * The following flags can be passed: @@ -3091,8 +3204,7 @@ void call(client *c, int flags) { long long dirty; monotime call_timer; uint64_t client_old_flags = c->flags; - struct redisCommand *real_cmd = c->cmd; - static long long prev_err_count; + struct redisCommand *real_cmd = c->realcmd; /* Initialization: clear the flags that must be set by the command on * demand, and initialize the array for additional commands propagation. */ @@ -3113,7 +3225,7 @@ void call(client *c, int flags) { /* Call the command. */ dirty = server.dirty; - prev_err_count = server.stat_total_error_replies; + incrCommandStatsOnError(NULL, 0); /* Update cache time, in case we have nested calls we want to * update only on the first call*/ @@ -3131,11 +3243,13 @@ void call(client *c, int flags) { server.in_nested_call--; - /* Update failed command calls if required. - * We leverage a static variable (prev_err_count) to retain - * the counter across nested function calls and avoid logging - * the same error twice. */ - if ((server.stat_total_error_replies - prev_err_count) > 0) { + /* Update failed command calls if required. */ + + if (!incrCommandStatsOnError(real_cmd, ERROR_COMMAND_FAILED) && c->deferred_reply_errors) { + /* When call is used from a module client, error stats, and total_error_replies + * isn't updated since these errors, if handled by the module, are internal, + * and not reflected to users. however, the commandstats does show these calls + * (made by RM_Call), so it should log if they failed or succeeded. */ real_cmd->failed_calls++; } @@ -3257,7 +3371,6 @@ void call(client *c, int flags) { server.fixed_time_expire--; server.stat_numcommands++; - prev_err_count = server.stat_total_error_replies; /* Record peak memory after each command and before the eviction that runs * before the next command. */ @@ -3393,7 +3506,7 @@ int processCommand(client *c) { /* Now lookup the command and check ASAP about trivial error conditions * such as wrong arity, bad command name and so forth. */ - c->cmd = c->lastcmd = lookupCommand(c->argv,c->argc); + c->cmd = c->lastcmd = c->realcmd = lookupCommand(c->argv,c->argc); if (!c->cmd) { if (isContainerCommandBySds(c->argv[0]->ptr)) { /* If we can't find the command but argv[0] by itself is a command @@ -4146,8 +4259,9 @@ void addReplyFlagsForKeyArgs(client *c, uint64_t flags) { {CMD_KEY_UPDATE, "update"}, {CMD_KEY_INSERT, "insert"}, {CMD_KEY_DELETE, "delete"}, - {CMD_KEY_CHANNEL, "channel"}, + {CMD_KEY_NOT_KEY, "not_key"}, {CMD_KEY_INCOMPLETE, "incomplete"}, + {CMD_KEY_VARIABLE_FLAGS, "variable_flags"}, {0,NULL} }; addReplyCommandFlags(c, flags, docFlagNames); @@ -4435,7 +4549,9 @@ void addReplyCommandInfo(client *c, struct redisCommand *cmd) { /* Output the representation of a Redis command. Used by the COMMAND DOCS. */ void addReplyCommandDocs(client *c, struct redisCommand *cmd) { /* Count our reply len so we don't have to use deferred reply. */ - long maplen = 3; + long maplen = 1; + if (cmd->summary) maplen++; + if (cmd->since) maplen++; if (cmd->complexity) maplen++; if (cmd->doc_flags) maplen++; if (cmd->deprecated_since) maplen++; @@ -4445,12 +4561,16 @@ void addReplyCommandDocs(client *c, struct redisCommand *cmd) { if (cmd->subcommands_dict) maplen++; addReplyMapLen(c, maplen); - addReplyBulkCString(c, "summary"); - addReplyBulkCString(c, cmd->summary); - - addReplyBulkCString(c, "since"); - addReplyBulkCString(c, cmd->since); + if (cmd->summary) { + addReplyBulkCString(c, "summary"); + addReplyBulkCString(c, cmd->summary); + } + if (cmd->since) { + addReplyBulkCString(c, "since"); + addReplyBulkCString(c, cmd->since); + } + /* Always have the group, for module commands the group is always "module". */ addReplyBulkCString(c, "group"); addReplyBulkCString(c, COMMAND_GROUP_STR[cmd->group]); @@ -4484,10 +4604,8 @@ void addReplyCommandDocs(client *c, struct redisCommand *cmd) { } } -/* Helper for COMMAND(S) command - * - * COMMAND(S) GETKEYS cmd arg1 arg2 ... */ -void getKeysSubcommand(client *c) { +/* Helper for COMMAND GETKEYS and GETKEYSANDFLAGS */ +void getKeysSubcommandImpl(client *c, int with_flags) { struct redisCommand *cmd = lookupCommand(c->argv+2,c->argc-2); getKeysResult result = GETKEYS_RESULT_INIT; int j; @@ -4505,7 +4623,7 @@ void getKeysSubcommand(client *c) { return; } - if (!getKeysFromCommand(cmd,c->argv+2,c->argc-2,&result)) { + if (!getKeysFromCommandWithSpecs(cmd,c->argv+2,c->argc-2,GET_KEYSPEC_DEFAULT,&result)) { if (cmd->flags & CMD_NO_MANDATORY_KEYS) { addReplyArrayLen(c,0); } else { @@ -4513,11 +4631,29 @@ void getKeysSubcommand(client *c) { } } else { addReplyArrayLen(c,result.numkeys); - for (j = 0; j < result.numkeys; j++) addReplyBulk(c,c->argv[result.keys[j].pos+2]); + for (j = 0; j < result.numkeys; j++) { + if (!with_flags) { + addReplyBulk(c,c->argv[result.keys[j].pos+2]); + } else { + addReplyArrayLen(c,2); + addReplyBulk(c,c->argv[result.keys[j].pos+2]); + addReplyFlagsForKeyArgs(c,result.keys[j].flags); + } + } } getKeysFreeResult(&result); } +/* COMMAND GETKEYSANDFLAGS cmd arg1 arg2 ... */ +void commandGetKeysAndFlagsCommand(client *c) { + getKeysSubcommandImpl(c, 1); +} + +/* COMMAND GETKEYS cmd arg1 arg2 ... */ +void getKeysSubcommand(client *c) { + getKeysSubcommandImpl(c, 0); +} + /* COMMAND (no args) */ void commandCommand(client *c) { dictIterator *di; @@ -4734,6 +4870,8 @@ void commandHelpCommand(client *c) { " commands are returned.", "GETKEYS <full-command>", " Return the keys from a full Redis command.", +"GETKEYSANDFLAGS <full-command>", +" Return the keys and the access flags from a full Redis command.", NULL }; @@ -4867,25 +5005,78 @@ sds genRedisInfoStringLatencyStats(sds info, dict *commands) { return info; } +/* Takes a null terminated sections list, and adds them to the dict. */ +void addInfoSectionsToDict(dict *section_dict, char **sections) { + while (*sections) { + sds section = sdsnew(*sections); + if (dictAdd(section_dict, section, NULL)==DICT_ERR) + sdsfree(section); + sections++; + } +} + +/* Cached copy of the default sections, as an optimization. */ +static dict *cached_default_info_sections = NULL; + +void releaseInfoSectionDict(dict *sec) { + if (sec != cached_default_info_sections) + dictRelease(sec); +} + +/* Create a dictionary with unique section names to be used by genRedisInfoString. + * 'argv' and 'argc' are list of arguments for INFO. + * 'defaults' is an optional null terminated list of default sections. + * 'out_all' and 'out_everything' are optional. + * The resulting dictionary should be released with releaseInfoSectionDict. */ +dict *genInfoSectionDict(robj **argv, int argc, char **defaults, int *out_all, int *out_everything) { + char *default_sections[] = { + "server", "clients", "memory", "persistence", "stats", "replication", + "cpu", "module_list", "errorstats", "cluster", "keyspace", NULL}; + if (!defaults) + defaults = default_sections; + + if (argc == 0) { + /* In this case we know the dict is not gonna be modified, so we cache + * it as an optimization for a common case. */ + if (cached_default_info_sections) + return cached_default_info_sections; + cached_default_info_sections = dictCreate(&stringSetDictType); + dictExpand(cached_default_info_sections, 16); + addInfoSectionsToDict(cached_default_info_sections, defaults); + return cached_default_info_sections; + } + + dict *section_dict = dictCreate(&stringSetDictType); + dictExpand(section_dict, min(argc,16)); + for (int i = 0; i < argc; i++) { + if (!strcasecmp(argv[i]->ptr,"default")) { + addInfoSectionsToDict(section_dict, defaults); + } else if (!strcasecmp(argv[i]->ptr,"all")) { + if (out_all) *out_all = 1; + } else if (!strcasecmp(argv[i]->ptr,"everything")) { + if (out_everything) *out_everything = 1; + if (out_all) *out_all = 1; + } else { + sds section = sdsnew(argv[i]->ptr); + if (dictAdd(section_dict, section, NULL) != DICT_OK) + sdsfree(section); + } + } + return section_dict; +} + /* Create the string returned by the INFO command. This is decoupled * by the INFO command itself as we need to report the same information * on memory corruption problems. */ -sds genRedisInfoString(const char *section) { +sds genRedisInfoString(dict *section_dict, int all_sections, int everything) { sds info = sdsempty(); time_t uptime = server.unixtime-server.stat_starttime; int j; - int allsections = 0, defsections = 0, everything = 0, modules = 0; int sections = 0; - - if (section == NULL) section = "default"; - allsections = strcasecmp(section,"all") == 0; - defsections = strcasecmp(section,"default") == 0; - everything = strcasecmp(section,"everything") == 0; - modules = strcasecmp(section,"modules") == 0; - if (everything) allsections = 1; + if (everything) all_sections = 1; /* Server */ - if (allsections || defsections || !strcasecmp(section,"server")) { + if (all_sections || (dictFind(section_dict,"server") != NULL)) { static int call_uname = 1; static struct utsname name; char *mode; @@ -4975,7 +5166,7 @@ sds genRedisInfoString(const char *section) { } /* Clients */ - if (allsections || defsections || !strcasecmp(section,"clients")) { + if (all_sections || (dictFind(section_dict,"clients") != NULL)) { size_t maxin, maxout; getExpansiveClientsInfo(&maxin,&maxout); if (sections++) info = sdscat(info,"\r\n"); @@ -4999,7 +5190,7 @@ sds genRedisInfoString(const char *section) { } /* Memory */ - if (allsections || defsections || !strcasecmp(section,"memory")) { + if (all_sections || (dictFind(section_dict,"memory") != NULL)) { char hmem[64]; char peak_hmem[64]; char total_system_hmem[64]; @@ -5144,7 +5335,7 @@ sds genRedisInfoString(const char *section) { } /* Persistence */ - if (allsections || defsections || !strcasecmp(section,"persistence")) { + if (all_sections || (dictFind(section_dict,"persistence") != NULL)) { if (sections++) info = sdscat(info,"\r\n"); double fork_perc = 0; if (server.stat_module_progress) { @@ -5171,6 +5362,7 @@ sds genRedisInfoString(const char *section) { "rdb_last_bgsave_status:%s\r\n" "rdb_last_bgsave_time_sec:%jd\r\n" "rdb_current_bgsave_time_sec:%jd\r\n" + "rdb_saves:%lld\r\n" "rdb_last_cow_size:%zu\r\n" "rdb_last_load_keys_expired:%lld\r\n" "rdb_last_load_keys_loaded:%lld\r\n" @@ -5180,6 +5372,7 @@ sds genRedisInfoString(const char *section) { "aof_last_rewrite_time_sec:%jd\r\n" "aof_current_rewrite_time_sec:%jd\r\n" "aof_last_bgrewrite_status:%s\r\n" + "aof_rewrites:%lld\r\n" "aof_last_write_status:%s\r\n" "aof_last_cow_size:%zu\r\n" "module_fork_in_progress:%d\r\n" @@ -5199,6 +5392,7 @@ sds genRedisInfoString(const char *section) { (intmax_t)server.rdb_save_time_last, (intmax_t)((server.child_type != CHILD_TYPE_RDB) ? -1 : time(NULL)-server.rdb_save_time_start), + server.stat_rdb_saves, server.stat_rdb_cow_bytes, server.rdb_last_load_keys_expired, server.rdb_last_load_keys_loaded, @@ -5209,6 +5403,7 @@ sds genRedisInfoString(const char *section) { (intmax_t)((server.child_type != CHILD_TYPE_AOF) ? -1 : time(NULL)-server.aof_rewrite_time_start), (server.aof_lastbgrewrite_status == C_OK) ? "ok" : "err", + server.stat_aof_rewrites, (server.aof_last_write_status == C_OK && aof_bio_fsync_status == C_OK) ? "ok" : "err", server.stat_aof_cow_bytes, @@ -5273,7 +5468,7 @@ sds genRedisInfoString(const char *section) { } /* Stats */ - if (allsections || defsections || !strcasecmp(section,"stats")) { + if (all_sections || (dictFind(section_dict,"stats") != NULL)) { long long stat_total_reads_processed, stat_total_writes_processed; long long stat_net_input_bytes, stat_net_output_bytes; long long current_eviction_exceeded_time = server.stat_last_eviction_exceeded_time ? @@ -5330,7 +5525,9 @@ sds genRedisInfoString(const char *section) { "total_reads_processed:%lld\r\n" "total_writes_processed:%lld\r\n" "io_threaded_reads_processed:%lld\r\n" - "io_threaded_writes_processed:%lld\r\n", + "io_threaded_writes_processed:%lld\r\n" + "reply_buffer_shrinks:%lld\r\n" + "reply_buffer_expands:%lld\r\n", server.stat_numconnections, server.stat_numcommands, getInstantaneousMetric(STATS_METRIC_COMMAND), @@ -5373,11 +5570,13 @@ sds genRedisInfoString(const char *section) { stat_total_reads_processed, stat_total_writes_processed, server.stat_io_reads_processed, - server.stat_io_writes_processed); + server.stat_io_writes_processed, + server.stat_reply_buffer_shrinks, + server.stat_reply_buffer_expands); } /* Replication */ - if (allsections || defsections || !strcasecmp(section,"replication")) { + if (all_sections || (dictFind(section_dict,"replication") != NULL)) { if (sections++) info = sdscat(info,"\r\n"); info = sdscatprintf(info, "# Replication\r\n" @@ -5513,7 +5712,7 @@ sds genRedisInfoString(const char *section) { } /* CPU */ - if (allsections || defsections || !strcasecmp(section,"cpu")) { + if (all_sections || (dictFind(section_dict,"cpu") != NULL)) { if (sections++) info = sdscat(info,"\r\n"); struct rusage self_ru, c_ru; @@ -5541,20 +5740,21 @@ sds genRedisInfoString(const char *section) { } /* Modules */ - if (allsections || defsections || !strcasecmp(section,"modules")) { + if (all_sections || (dictFind(section_dict,"module_list") != NULL) || (dictFind(section_dict,"modules") != NULL)) { if (sections++) info = sdscat(info,"\r\n"); info = sdscatprintf(info,"# Modules\r\n"); info = genModulesInfoString(info); } /* Command statistics */ - if (allsections || !strcasecmp(section,"commandstats")) { + if (all_sections || (dictFind(section_dict,"commandstats") != NULL)) { if (sections++) info = sdscat(info,"\r\n"); info = sdscatprintf(info, "# Commandstats\r\n"); info = genRedisInfoStringCommandStats(info, server.commands); } + /* Error statistics */ - if (allsections || defsections || !strcasecmp(section,"errorstats")) { + if (all_sections || (dictFind(section_dict,"errorstats") != NULL)) { if (sections++) info = sdscat(info,"\r\n"); info = sdscat(info, "# Errorstats\r\n"); raxIterator ri; @@ -5573,7 +5773,7 @@ sds genRedisInfoString(const char *section) { } /* Latency by percentile distribution per command */ - if (allsections || !strcasecmp(section,"latencystats")) { + if (all_sections || (dictFind(section_dict,"latencystats") != NULL)) { if (sections++) info = sdscat(info,"\r\n"); info = sdscatprintf(info, "# Latencystats\r\n"); if (server.latency_tracking_enabled) { @@ -5582,7 +5782,7 @@ sds genRedisInfoString(const char *section) { } /* Cluster */ - if (allsections || defsections || !strcasecmp(section,"cluster")) { + if (all_sections || (dictFind(section_dict,"cluster") != NULL)) { if (sections++) info = sdscat(info,"\r\n"); info = sdscatprintf(info, "# Cluster\r\n" @@ -5591,7 +5791,7 @@ sds genRedisInfoString(const char *section) { } /* Key space */ - if (allsections || defsections || !strcasecmp(section,"keyspace")) { + if (all_sections || (dictFind(section_dict,"keyspace") != NULL)) { if (sections++) info = sdscat(info,"\r\n"); info = sdscatprintf(info, "# Keyspace\r\n"); for (j = 0; j < server.dbnum; j++) { @@ -5610,10 +5810,10 @@ sds genRedisInfoString(const char *section) { /* Get info from modules. * if user asked for "everything" or "modules", or a specific section * that's not found yet. */ - if (everything || modules || - (!allsections && !defsections && sections==0)) { + if (everything || dictFind(section_dict, "modules") != NULL || sections < (int)dictSize(section_dict)) { + info = modulesCollectInfo(info, - everything || modules ? NULL: section, + everything || dictFind(section_dict, "modules") != NULL ? NULL: section_dict, 0, /* not a crash report */ sections); } @@ -5625,16 +5825,14 @@ void infoCommand(client *c) { sentinelInfoCommand(c); return; } - - char *section = c->argc == 2 ? c->argv[1]->ptr : "default"; - - if (c->argc > 2) { - addReplyErrorObject(c,shared.syntaxerr); - return; - } - sds info = genRedisInfoString(section); + int all_sections = 0; + int everything = 0; + dict *sections_dict = genInfoSectionDict(c->argv+1, c->argc-1, NULL, &all_sections, &everything); + sds info = genRedisInfoString(sections_dict, all_sections, everything); addReplyVerbatim(c,info,sdslen(info),"txt"); sdsfree(info); + releaseInfoSectionDict(sections_dict); + return; } void monitorCommand(client *c) { diff --git a/src/server.h b/src/server.h index 485c051fd..4da7a010f 100644 --- a/src/server.h +++ b/src/server.h @@ -57,6 +57,10 @@ #include <systemd/sd-daemon.h> #endif +#ifndef static_assert +#define static_assert(expr, lit) extern char __static_assert_failure[(expr) ? 1:-1] +#endif + typedef long long mstime_t; /* millisecond time type. */ typedef long long ustime_t; /* microsecond time type. */ @@ -105,6 +109,7 @@ typedef long long ustime_t; /* microsecond time type. */ #define PROTO_SHARED_SELECT_CMDS 10 #define OBJ_SHARED_INTEGERS 10000 #define OBJ_SHARED_BULKHDR_LEN 32 +#define OBJ_SHARED_HDR_STRLEN(_len_) (((_len_) < 10) ? 4 : 5) /* see shared.mbulkhdr etc. */ #define LOG_MAX_LEN 1024 /* Default maximum length of syslog messages.*/ #define AOF_REWRITE_ITEMS_PER_CMD 64 #define AOF_ANNOTATION_LINE_MAX_LEN 1024 @@ -159,11 +164,14 @@ typedef long long ustime_t; /* microsecond time type. */ #define PROTO_INLINE_MAX_SIZE (1024*64) /* Max size of inline reads */ #define PROTO_MBULK_BIG_ARG (1024*32) #define PROTO_RESIZE_THRESHOLD (1024*32) /* Threshold for determining whether to resize query buffer */ +#define PROTO_REPLY_MIN_BYTES (1024) /* the lower limit on reply buffer size */ #define LONG_STR_SIZE 21 /* Bytes needed for long -> str + '\0' */ #define REDIS_AUTOSYNC_BYTES (1024*1024*4) /* Sync file every 4MB. */ #define LIMIT_PENDING_QUERYBUF (4*1024*1024) /* 4mb */ +#define REPLY_BUFFER_DEFAULT_PEAK_RESET_TIME 5000 /* 5 seconds */ + /* When configuring the server eventloop, we setup it so that the total number * of file descriptors we can handle are server.maxclients + RESERVED_FDS + * a few more to stay safe. Since RESERVED_FDS defaults to 32, we add 96 @@ -210,6 +218,7 @@ extern int configOOMScoreAdjValuesDefaults[CONFIG_OOM_COUNT]; #define CMD_NO_MULTI (1ULL<<24) #define CMD_MOVABLE_KEYS (1ULL<<25) /* populated by populateCommandMovableKeys */ #define CMD_ALLOW_BUSY ((1ULL<<26)) +#define CMD_MODULE_GETCHANNELS (1ULL<<27) /* Use the modules getchannels interface. */ /* Command flags that describe ACLs categories. */ #define ACL_CATEGORY_KEYSPACE (1ULL<<0) @@ -262,12 +271,23 @@ extern int configOOMScoreAdjValuesDefaults[CONFIG_OOM_COUNT]; #define CMD_KEY_DELETE (1ULL<<7) /* Explicitly deletes some content * from the value of the key. */ /* Other flags: */ -#define CMD_KEY_CHANNEL (1ULL<<8) /* PUBSUB shard channel */ +#define CMD_KEY_NOT_KEY (1ULL<<8) /* A 'fake' key that should be routed + * like a key in cluster mode but is + * excluded from other key checks. */ #define CMD_KEY_INCOMPLETE (1ULL<<9) /* Means that the keyspec might not point * out to all keys it should cover */ #define CMD_KEY_VARIABLE_FLAGS (1ULL<<10) /* Means that some keys might have * different flags depending on arguments */ +/* Key flags for when access type is unknown */ +#define CMD_KEY_FULL_ACCESS (CMD_KEY_RW | CMD_KEY_ACCESS | CMD_KEY_UPDATE) + +/* Channel flags share the same flag space as the key flags */ +#define CMD_CHANNEL_PATTERN (1ULL<<11) /* The argument is a channel pattern */ +#define CMD_CHANNEL_SUBSCRIBE (1ULL<<12) /* The command subscribes to channels */ +#define CMD_CHANNEL_UNSUBSCRIBE (1ULL<<13) /* The command unsubscribes to channels */ +#define CMD_CHANNEL_PUBLISH (1ULL<<14) /* The command publishes to channels. */ + /* AOF states */ #define AOF_OFF 0 /* AOF is off */ #define AOF_ON 1 /* AOF is on */ @@ -1076,6 +1096,9 @@ typedef struct client { robj **original_argv; /* Arguments of original command if arguments were rewritten. */ size_t argv_len_sum; /* Sum of lengths of objects in argv list. */ struct redisCommand *cmd, *lastcmd; /* Last command executed. */ + struct redisCommand *realcmd; /* The original command that was executed by the client, + Used to update error stats in case the c->cmd was modified + during the command invocation (like on GEOADD for example). */ user *user; /* User associated with this connection. If the user is set to NULL the connection can do anything (admin). */ @@ -1084,6 +1107,7 @@ typedef struct client { long bulklen; /* Length of bulk argument in multi bulk request. */ list *reply; /* List of reply objects to send to the client. */ unsigned long long reply_bytes; /* Tot bytes of objects in reply list. */ + list *deferred_reply_errors; /* Used for module thread safe contexts. */ size_t sentlen; /* Amount of bytes already sent in the current buffer or object being sent. */ time_t ctime; /* Client creation time. */ @@ -1159,14 +1183,11 @@ typedef struct client { * i.e. the next offset to send. */ /* Response buffer */ + size_t buf_peak; /* Peak used size of buffer in last 5 sec interval. */ + mstime_t buf_peak_last_reset_time; /* keeps the last time the buffer peak value was reset */ int bufpos; size_t buf_usable_size; /* Usable size of buffer. */ - /* Note that 'buf' must be the last field of client struct, because memory - * allocator may give us more memory than our apply for reducing fragments, - * but we want to make full use of given memory, i.e. we may access the - * memory after 'buf'. To avoid make others fields corrupt, 'buf' must be - * the last one. */ - char buf[PROTO_REPLY_CHUNK_BYTES]; + char *buf; } client; struct saveparam { @@ -1205,14 +1226,16 @@ struct sharedObjectsStruct { *rpop, *lpop, *lpush, *rpoplpush, *lmove, *blmove, *zpopmin, *zpopmax, *emptyscan, *multi, *exec, *left, *right, *hset, *srem, *xgroup, *xclaim, *script, *replconf, *eval, *persist, *set, *pexpireat, *pexpire, - *time, *pxat, *absttl, *retrycount, *force, *justid, + *time, *pxat, *absttl, *retrycount, *force, *justid, *entriesread, *lastid, *ping, *setid, *keepttl, *load, *createconsumer, *getack, *special_asterick, *special_equals, *default_username, *redacted, *ssubscribebulk,*sunsubscribebulk, *select[PROTO_SHARED_SELECT_CMDS], *integers[OBJ_SHARED_INTEGERS], *mbulkhdr[OBJ_SHARED_BULKHDR_LEN], /* "*<value>\r\n" */ - *bulkhdr[OBJ_SHARED_BULKHDR_LEN]; /* "$<value>\r\n" */ + *bulkhdr[OBJ_SHARED_BULKHDR_LEN], /* "$<value>\r\n" */ + *maphdr[OBJ_SHARED_BULKHDR_LEN], /* "%<value>\r\n" */ + *sethdr[OBJ_SHARED_BULKHDR_LEN]; /* "~<value>\r\n" */ sds minstring, maxstring; }; @@ -1536,6 +1559,8 @@ struct redisServer { long long stat_total_active_defrag_time; /* Total time memory fragmentation over the limit, unit us */ monotime stat_last_active_defrag_time; /* Timestamp of current active defrag start */ size_t stat_peak_memory; /* Max used memory record */ + long long stat_aof_rewrites; /* number of aof file rewrites performed */ + long long stat_rdb_saves; /* number of rdb saves performed */ long long stat_fork_time; /* Time needed to perform latest fork() */ double stat_fork_rate; /* Fork rate in GB/sec. */ long long stat_total_forks; /* Total count of fork. */ @@ -1576,6 +1601,9 @@ struct redisServer { long long samples[STATS_METRIC_SAMPLES]; int idx; } inst_metric[STATS_METRIC_COUNT]; + long long stat_reply_buffer_shrinks; /* Total number of output buffer shrinks */ + long long stat_reply_buffer_expands; /* Total number of output buffer expands */ + /* Configuration */ int verbosity; /* Loglevel in redis.conf */ int maxidletime; /* Client timeout in seconds */ @@ -1635,7 +1663,7 @@ struct redisServer { int aof_last_write_status; /* C_OK or C_ERR */ int aof_last_write_errno; /* Valid if aof write/fsync status is ERR */ int aof_load_truncated; /* Don't stop on unexpected AOF EOF. */ - int aof_use_rdb_preamble; /* Use RDB preamble on AOF rewrites. */ + int aof_use_rdb_preamble; /* Specify base AOF to use RDB encoding on AOF rewrites. */ redisAtomic int aof_bio_fsync_status; /* Status of AOF fsync in bio job. */ redisAtomic int aof_bio_fsync_errno; /* Errno of AOF fsync in bio job. */ aofManifest *aof_manifest; /* Used to track AOFs. */ @@ -1752,10 +1780,6 @@ struct redisServer { char master_replid[CONFIG_RUN_ID_SIZE+1]; /* Master PSYNC runid. */ long long master_initial_offset; /* Master PSYNC offset. */ int repl_slave_lazy_flush; /* Lazy FLUSHALL before loading DB? */ - /* Replication script cache. */ - dict *repl_scriptcache_dict; /* SHA1 all slaves are aware of. */ - list *repl_scriptcache_fifo; /* First in, first out LRU eviction. */ - unsigned int repl_scriptcache_size; /* Max number of elements. */ /* Synchronous replication. */ list *clients_waiting_acks; /* Clients waiting in WAIT command. */ int get_ack_from_slaves; /* If true we send REPLCONF GETACK. */ @@ -1889,6 +1913,7 @@ struct redisServer { int failover_state; /* Failover state */ int cluster_allow_pubsubshard_when_down; /* Is pubsubshard allowed when the cluster is down, doesn't affect pubsub global. */ + long reply_buffer_peak_reset_time; /* The amount of time (in milliseconds) to wait between reply buffer peak resets */ }; #define MAX_KEYS_BUFFER 256 @@ -1900,7 +1925,8 @@ typedef struct { } keyReference; /* A result structure for the various getkeys function calls. It lists the - * keys as indices to the provided argv. + * keys as indices to the provided argv. This functionality is also re-used + * for returning channel information. */ typedef struct { keyReference keysbuf[MAX_KEYS_BUFFER]; /* Pre-allocated buffer, to save heap allocations */ @@ -2294,11 +2320,12 @@ extern struct sharedObjectsStruct shared; extern dictType objectKeyPointerValueDictType; extern dictType objectKeyHeapPointerValueDictType; extern dictType setDictType; +extern dictType BenchmarkDictType; extern dictType zsetDictType; extern dictType dbDictType; extern double R_Zero, R_PosInf, R_NegInf, R_Nan; extern dictType hashDictType; -extern dictType replScriptCacheDictType; +extern dictType stringSetDictType; extern dictType dbExpiresDictType; extern dictType modulesDictType; extern dictType sdsReplyDictType; @@ -2308,8 +2335,9 @@ extern dict *modules; * Functions prototypes *----------------------------------------------------------------------------*/ -/* Key arguments specs */ +/* Command metadata */ void populateCommandLegacyRangeSpec(struct redisCommand *c); +int populateArgsStructure(struct redisCommandArg *args); /* Modules */ void moduleInitModulesSystem(void); @@ -2318,6 +2346,7 @@ void modulesCron(void); int moduleLoad(const char *path, void **argv, int argc); void moduleLoadFromQueue(void); int moduleGetCommandKeysViaAPI(struct redisCommand *cmd, robj **argv, int argc, getKeysResult *result); +int moduleGetCommandChannelsViaAPI(struct redisCommand *cmd, robj **argv, int argc, getKeysResult *result); moduleType *moduleTypeLookupModuleByID(uint64_t id); void moduleTypeNameByID(char *name, uint64_t moduleid); const char *moduleTypeModuleName(moduleType *mt); @@ -2337,7 +2366,7 @@ int TerminateModuleForkChild(int child_pid, int wait); ssize_t rdbSaveModulesAux(rio *rdb, int when); int moduleAllDatatypesHandleErrors(); int moduleAllModulesHandleReplAsyncLoad(); -sds modulesCollectInfo(sds info, const char *section, int for_crash_report, int sections); +sds modulesCollectInfo(sds info, dict *sections_dict, int for_crash_report, int sections); void moduleFireServerEvent(uint64_t eid, int subid, void *data); void processModuleLoadingProgressEvent(int is_aof); int moduleTryServeClientBlockedOnKey(client *c, robj *key); @@ -2368,6 +2397,9 @@ int validateProcTitleTemplate(const char *template); int redisCommunicateSystemd(const char *sd_notify_msg); void redisSetCpuAffinity(const char *cpulist); +/* afterErrorReply flags */ +#define ERR_REPLY_FLAG_NO_STATS_UPDATE (1ULL<<0) /* Indicating that we should not update + error stats after sending error reply */ /* networking.c -- Networking and Client related operations */ client *createClient(connection *conn); void freeClient(client *c); @@ -2377,6 +2409,7 @@ int beforeNextClient(client *c); void clearClientConnectionState(client *c); void resetClient(client *c); void freeClientOriginalArgv(client *c); +void freeClientArgv(client *c); void sendReplyToClient(connection *conn); void *addReplyDeferredLen(client *c); void setDeferredArrayLen(client *c, void *node, long length); @@ -2406,6 +2439,8 @@ void addReplyBulkSds(client *c, sds s); void setDeferredReplyBulkSds(client *c, void *node, sds s); void addReplyErrorObject(client *c, robj *err); void addReplyOrErrorObject(client *c, robj *reply); +void afterErrorReply(client *c, const char *s, size_t len, int flags); +void addReplyErrorSdsEx(client *c, sds err, int flags); void addReplyErrorSds(client *c, sds err); void addReplyError(client *c, const char *err); void addReplyErrorArity(client *c); @@ -2426,6 +2461,7 @@ void addReplySubcommandSyntaxError(client *c); void addReplyLoadedModules(client *c); void copyReplicaOutputBuffer(client *dst, client *src); void addListRangeReply(client *c, robj *o, long start, long end, int reverse); +void deferredAfterErrorReply(client *c, list *errors); size_t sdsZmallocSize(sds s); size_t getStringObjectSdsUsedMemory(robj *o); void freeClientReplyValue(void *o); @@ -2477,11 +2513,14 @@ int authRequired(client *c); void clientInstallWriteHandler(client *c); #ifdef __GNUC__ +void addReplyErrorFormatEx(client *c, int flags, const char *fmt, ...) + __attribute__((format(printf, 3, 4))); void addReplyErrorFormat(client *c, const char *fmt, ...) __attribute__((format(printf, 2, 3))); void addReplyStatusFormat(client *c, const char *fmt, ...) __attribute__((format(printf, 2, 3))); #else +void addReplyErrorFormatEx(client *c, int flags, const char *fmt, ...); void addReplyErrorFormat(client *c, const char *fmt, ...); void addReplyStatusFormat(client *c, const char *fmt, ...); #endif @@ -2760,6 +2799,10 @@ typedef struct { int minex, maxex; /* are min or max exclusive? */ } zlexrangespec; +/* flags for incrCommandFailedCalls */ +#define ERROR_COMMAND_REJECTED (1<<0) /* Indicate to update the command rejected stats */ +#define ERROR_COMMAND_FAILED (1<<1) /* Indicate to update the command failed stats */ + zskiplist *zslCreate(void); void zslFree(zskiplist *zsl); zskiplistNode *zslInsert(zskiplist *zsl, double score, sds ele); @@ -2814,6 +2857,8 @@ struct redisCommand *lookupCommandBySds(sds s); struct redisCommand *lookupCommandByCStringLogic(dict *commands, const char *s); struct redisCommand *lookupCommandByCString(const char *s); struct redisCommand *lookupCommandOrOriginal(robj **argv, int argc); +void startCommandExecution(); +int incrCommandStatsOnError(struct redisCommand *cmd, int flags); void call(client *c, int flags); void alsoPropagate(int dbid, robj **argv, int argc, int target); void propagatePendingCommands(); @@ -2932,7 +2977,7 @@ void resetServerSaveParams(void); struct rewriteConfigState; /* Forward declaration to export API. */ void rewriteConfigRewriteLine(struct rewriteConfigState *state, const char *option, sds line, int force); void rewriteConfigMarkAsProcessed(struct rewriteConfigState *state, const char *option); -int rewriteConfig(char *path, int force_all); +int rewriteConfig(char *path, int force_write); void initConfigValues(); sds getConfigDebugInfo(); int allowProtectedAction(int config, client *c); @@ -3001,13 +3046,15 @@ void freeReplicationBacklogRefMemAsync(list *blocks, rax *index); /* API to get key arguments from commands */ #define GET_KEYSPEC_DEFAULT 0 -#define GET_KEYSPEC_INCLUDE_CHANNELS (1<<0) /* Consider channels as keys */ +#define GET_KEYSPEC_INCLUDE_NOT_KEYS (1<<0) /* Consider 'fake' keys as keys */ #define GET_KEYSPEC_RETURN_PARTIAL (1<<1) /* Return all keys that can be found */ int getKeysFromCommandWithSpecs(struct redisCommand *cmd, robj **argv, int argc, int search_flags, getKeysResult *result); keyReference *getKeysPrepareResult(getKeysResult *result, int numkeys); int getKeysFromCommand(struct redisCommand *cmd, robj **argv, int argc, getKeysResult *result); int doesCommandHaveKeys(struct redisCommand *cmd); +int getChannelsFromCommand(struct redisCommand *cmd, robj **argv, int argc, getKeysResult *result); +int doesCommandHaveChannelsWithFlags(struct redisCommand *cmd, int flags); void getKeysFreeResult(getKeysResult *result); int sintercardGetKeys(struct redisCommand *cmd,robj **argv, int argc, getKeysResult *result); int zunionInterDiffGetKeys(struct redisCommand *cmd,robj **argv, int argc, getKeysResult *result); @@ -3064,6 +3111,11 @@ unsigned long evalMemory(); dict* evalScriptsDict(); unsigned long evalScriptsMemory(); +typedef struct luaScript { + uint64_t flags; + robj *body; +} luaScript; + /* Blocked clients */ void processUnblockedClients(void); void blockClient(client *c, int btype); @@ -3075,7 +3127,7 @@ void disconnectAllBlockedClients(void); void handleClientsBlockedOnKeys(void); void signalKeyAsReady(redisDb *db, robj *key, int type); void blockForKeys(client *c, int btype, robj **keys, int numkeys, long count, mstime_t timeout, robj *target, struct blockPos *blockpos, streamID *ids); -void updateStatsOnUnblock(client *c, long blocked_us, long reply_us); +void updateStatsOnUnblock(client *c, long blocked_us, long reply_us, int had_errors); /* timeout.c -- Blocked clients timeout and connections timeout. */ void addClientToTimeoutTable(client *c); @@ -3125,6 +3177,7 @@ void commandCountCommand(client *c); void commandListCommand(client *c); void commandInfoCommand(client *c); void commandGetKeysCommand(client *c); +void commandGetKeysAndFlagsCommand(client *c); void commandHelpCommand(client *c); void commandDocsCommand(client *c); void setCommand(client *c); @@ -3390,7 +3443,9 @@ void _serverPanic(const char *file, int line, const char *msg, ...); void serverLogObjectDebugInfo(const robj *o); void sigsegvHandler(int sig, siginfo_t *info, void *secret); const char *getSafeInfoString(const char *s, size_t len, char **tmp); -sds genRedisInfoString(const char *section); +dict *genInfoSectionDict(robj **argv, int argc, char **defaults, int *out_all, int *out_everything); +void releaseInfoSectionDict(dict *sec); +sds genRedisInfoString(dict *section_dict, int all_sections, int everything); sds genModulesInfoString(sds info); void applyWatchdogPeriod(); void watchdogScheduleSignal(int period); diff --git a/src/stream.h b/src/stream.h index 724f2c2ad..2d4997919 100644 --- a/src/stream.h +++ b/src/stream.h @@ -15,8 +15,11 @@ typedef struct streamID { typedef struct stream { rax *rax; /* The radix tree holding the stream. */ - uint64_t length; /* Number of elements inside this stream. */ + uint64_t length; /* Current number of elements inside this stream. */ streamID last_id; /* Zero if there are yet no items. */ + streamID first_id; /* The first non-tombstone entry, zero if empty. */ + streamID max_deleted_entry_id; /* The maximal ID that was deleted. */ + uint64_t entries_added; /* All time count of elements added. */ rax *cgroups; /* Consumer groups dictionary: name -> streamCG */ } stream; @@ -34,6 +37,7 @@ typedef struct streamIterator { unsigned char *master_fields_ptr; /* Master field to emit next. */ int entry_flags; /* Flags of entry we are emitting. */ int rev; /* True if iterating end to start (reverse). */ + int skip_tombstones; /* True if not emitting tombstone entries. */ uint64_t start_key[2]; /* Start key as 128 bit big endian. */ uint64_t end_key[2]; /* End key as 128 bit big endian. */ raxIterator ri; /* Rax iterator. */ @@ -52,6 +56,11 @@ typedef struct streamCG { streamID last_id; /* Last delivered (not acknowledged) ID for this group. Consumers that will just ask for more messages will served with IDs > than this. */ + long long entries_read; /* In a perfect world (CG starts at 0-0, no dels, no + XGROUP SETID, ...), this is the total number of + group reads. In the real world, the reasoning behind + this value is detailed at the top comment of + streamEstimateDistanceFromFirstEverEntry(). */ rax *pel; /* Pending entries list. This is a radix tree that has every message delivered to consumers (without the NOACK option) that was yet not acknowledged @@ -105,6 +114,8 @@ struct client; #define SCC_NO_NOTIFY (1<<0) /* Do not notify key space if consumer created */ #define SCC_NO_DIRTIFY (1<<1) /* Do not dirty++ if consumer created */ +#define SCG_INVALID_ENTRIES_READ -1 + stream *streamNew(void); void freeStream(stream *s); unsigned long streamLength(const robj *subject); @@ -117,7 +128,7 @@ void streamIteratorStop(streamIterator *si); streamCG *streamLookupCG(stream *s, sds groupname); streamConsumer *streamLookupConsumer(streamCG *cg, sds name, int flags); streamConsumer *streamCreateConsumer(streamCG *cg, sds name, robj *key, int dbid, int flags); -streamCG *streamCreateCG(stream *s, char *name, size_t namelen, streamID *id); +streamCG *streamCreateCG(stream *s, char *name, size_t namelen, streamID *id, long long entries_read); streamNACK *streamCreateNACK(streamConsumer *consumer); void streamDecodeID(void *buf, streamID *id); int streamCompareID(streamID *a, streamID *b); @@ -131,6 +142,8 @@ int streamParseID(const robj *o, streamID *id); robj *createObjectFromStreamID(streamID *id); int streamAppendItem(stream *s, robj **argv, int64_t numfields, streamID *added_id, streamID *use_id, int seq_given); int streamDeleteItem(stream *s, streamID *id); +void streamGetEdgeID(stream *s, int first, int skip_tombstones, streamID *edge_id); +long long streamEstimateDistanceFromFirstEverEntry(stream *s, streamID *id); int64_t streamTrimByLength(stream *s, long long maxlen, int approx); int64_t streamTrimByID(stream *s, streamID minid, int approx); diff --git a/src/t_stream.c b/src/t_stream.c index e47194926..cd7d9723e 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -68,8 +68,13 @@ stream *streamNew(void) { stream *s = zmalloc(sizeof(*s)); s->rax = raxNew(); s->length = 0; + s->first_id.ms = 0; + s->first_id.seq = 0; s->last_id.ms = 0; s->last_id.seq = 0; + s->max_deleted_entry_id.seq = 0; + s->max_deleted_entry_id.ms = 0; + s->entries_added = 0; s->cgroups = NULL; /* Created on demand to save memory when not used. */ return s; } @@ -184,7 +189,10 @@ robj *streamDup(robj *o) { new_lp, NULL); } new_s->length = s->length; + new_s->first_id = s->first_id; new_s->last_id = s->last_id; + new_s->max_deleted_entry_id = s->max_deleted_entry_id; + new_s->entries_added = s->entries_added; raxStop(&ri); if (s->cgroups == NULL) return sobj; @@ -196,7 +204,8 @@ robj *streamDup(robj *o) { while (raxNext(&ri_cgroups)) { streamCG *cg = ri_cgroups.data; streamCG *new_cg = streamCreateCG(new_s, (char *)ri_cgroups.key, - ri_cgroups.key_len, &cg->last_id); + ri_cgroups.key_len, &cg->last_id, + cg->entries_read); serverAssert(new_cg != NULL); @@ -378,37 +387,21 @@ int streamCompareID(streamID *a, streamID *b) { return 0; } -void streamGetEdgeID(stream *s, int first, streamID *edge_id) +/* Retrieves the ID of the stream edge entry. An edge is either the first or + * the last ID in the stream, and may be a tombstone. To filter out tombstones, + * set the'skip_tombstones' argument to 1. */ +void streamGetEdgeID(stream *s, int first, int skip_tombstones, streamID *edge_id) { - raxIterator ri; - raxStart(&ri, s->rax); - int empty; - if (first) { - raxSeek(&ri, "^", NULL, 0); - empty = !raxNext(&ri); - } else { - raxSeek(&ri, "$", NULL, 0); - empty = !raxPrev(&ri); - } - - if (empty) { - /* Stream is empty, mark edge ID as lowest/highest possible. */ - edge_id->ms = first ? UINT64_MAX : 0; - edge_id->seq = first ? UINT64_MAX : 0; - raxStop(&ri); - return; + streamIterator si; + int64_t numfields; + streamIteratorStart(&si,s,NULL,NULL,!first); + si.skip_tombstones = skip_tombstones; + int found = streamIteratorGetID(&si,edge_id,&numfields); + if (!found) { + streamID min_id = {0, 0}, max_id = {UINT64_MAX, UINT64_MAX}; + *edge_id = first ? max_id : min_id; } - unsigned char *lp = ri.data; - - /* Read the master ID from the radix tree key. */ - streamID master_id; - streamDecodeID(ri.key, &master_id); - - /* Construct edge ID. */ - lpGetEdgeStreamID(lp, first, &master_id, edge_id); - - raxStop(&ri); } /* Adds a new item into the stream 's' having the specified number of @@ -664,7 +657,9 @@ int streamAppendItem(stream *s, robj **argv, int64_t numfields, streamID *added_ if (ri.data != lp) raxInsert(s->rax,(unsigned char*)&rax_key,sizeof(rax_key),lp,NULL); s->length++; + s->entries_added++; s->last_id = id; + if (s->length == 1) s->first_id = id; if (added_id) *added_id = id; return C_OK; } @@ -842,7 +837,7 @@ int64_t streamTrim(stream *s, streamAddTrimArgs *args) { } deleted += deleted_from_lp; - /* Now we the entries/deleted counters. */ + /* Now we update the entries/deleted counters. */ p = lpFirst(lp); lp = lpReplaceInteger(lp,&p,entries-deleted_from_lp); p = lpNext(lp,p); /* Skip deleted field. */ @@ -864,8 +859,16 @@ int64_t streamTrim(stream *s, streamAddTrimArgs *args) { break; /* If we are here, there was enough to delete in the current node, so no need to go to the next node. */ } - raxStop(&ri); + + /* Update the stream's first ID after the trimming. */ + if (s->length == 0) { + s->first_id.ms = 0; + s->first_id.seq = 0; + } else if (deleted) { + streamGetEdgeID(s,1,1,&s->first_id); + } + return deleted; } @@ -1089,9 +1092,10 @@ void streamIteratorStart(streamIterator *si, stream *s, streamID *start, streamI } } si->stream = s; - si->lp = NULL; /* There is no current listpack right now. */ + si->lp = NULL; /* There is no current listpack right now. */ si->lp_ele = NULL; /* Current listpack cursor. */ - si->rev = rev; /* Direction, if non-zero reversed, from end to start. */ + si->rev = rev; /* Direction, if non-zero reversed, from end to start. */ + si->skip_tombstones = 1; /* By default tombstones aren't emitted. */ } /* Return 1 and store the current item ID at 'id' if there are still @@ -1189,10 +1193,10 @@ int streamIteratorGetID(streamIterator *si, streamID *id, int64_t *numfields) { serverAssert(*numfields>=0); /* If current >= start, and the entry is not marked as - * deleted, emit it. */ + * deleted or tombstones are included, emit it. */ if (!si->rev) { if (memcmp(buf,si->start_key,sizeof(streamID)) >= 0 && - !(flags & STREAM_ITEM_FLAG_DELETED)) + (!si->skip_tombstones || !(flags & STREAM_ITEM_FLAG_DELETED))) { if (memcmp(buf,si->end_key,sizeof(streamID)) > 0) return 0; /* We are already out of range. */ @@ -1203,7 +1207,7 @@ int streamIteratorGetID(streamIterator *si, streamID *id, int64_t *numfields) { } } else { if (memcmp(buf,si->end_key,sizeof(streamID)) <= 0 && - !(flags & STREAM_ITEM_FLAG_DELETED)) + (!si->skip_tombstones || !(flags & STREAM_ITEM_FLAG_DELETED))) { if (memcmp(buf,si->start_key,sizeof(streamID)) < 0) return 0; /* We are already out of range. */ @@ -1270,7 +1274,7 @@ void streamIteratorRemoveEntry(streamIterator *si, streamID *current) { int64_t aux; /* We do not really delete the entry here. Instead we mark it as - * deleted flagging it, and also incrementing the count of the + * deleted by flagging it, and also incrementing the count of the * deleted entries in the listpack header. * * We start flagging: */ @@ -1314,7 +1318,7 @@ void streamIteratorRemoveEntry(streamIterator *si, streamID *current) { streamIteratorStop(si); streamIteratorStart(si,si->stream,&start,&end,si->rev); - /* TODO: perform a garbage collection here if the ration between + /* TODO: perform a garbage collection here if the ratio between * deleted and valid goes over a certain limit. */ } @@ -1325,6 +1329,20 @@ void streamIteratorStop(streamIterator *si) { raxStop(&si->ri); } +/* Return 1 if `id` exists in `s` (and not marked as deleted) */ +int streamEntryExists(stream *s, streamID *id) { + streamIterator si; + streamIteratorStart(&si,s,id,id,0); + streamID myid; + int64_t numfields; + int found = streamIteratorGetID(&si,&myid,&numfields); + streamIteratorStop(&si); + if (!found) + return 0; + serverAssert(streamCompareID(id,&myid) == 0); + return 1; +} + /* Delete the specified item ID from the stream, returning 1 if the item * was deleted 0 otherwise (if it does not exist). */ int streamDeleteItem(stream *s, streamID *id) { @@ -1372,6 +1390,148 @@ robj *createObjectFromStreamID(streamID *id) { id->ms,id->seq)); } +/* Returns non-zero if the ID is 0-0. */ +int streamIDEqZero(streamID *id) { + return !(id->ms || id->seq); +} + +/* A helper that returns non-zero if the range from 'start' to `end` + * contains a tombstone. + * + * NOTE: this assumes that the caller had verified that 'start' is less than + * 's->last_id'. */ +int streamRangeHasTombstones(stream *s, streamID *start, streamID *end) { + streamID start_id, end_id; + + if (!s->length || streamIDEqZero(&s->max_deleted_entry_id)) { + /* The stream is empty or has no tombstones. */ + return 0; + } + + if (streamCompareID(&s->first_id,&s->max_deleted_entry_id) > 0) { + /* The latest tombstone is before the first entry. */ + return 0; + } + + if (start) { + start_id = *start; + } else { + start_id.ms = 0; + start_id.seq = 0; + } + + if (end) { + end_id = *end; + } else { + end_id.ms = UINT64_MAX; + end_id.seq = UINT64_MAX; + } + + if (streamCompareID(&start_id,&s->max_deleted_entry_id) <= 0 && + streamCompareID(&s->max_deleted_entry_id,&end_id) <= 0) + { + /* start_id <= max_deleted_entry_id <= end_id: The range does include a tombstone. */ + return 1; + } + + /* The range doesn't includes a tombstone. */ + return 0; +} + +/* Replies with a consumer group's current lag, that is the number of messages + * in the stream that are yet to be delivered. In case that the lag isn't + * available due to fragmentation, the reply to the client is a null. */ +void streamReplyWithCGLag(client *c, stream *s, streamCG *cg) { + int valid = 0; + long long lag = 0; + + if (!s->entries_added) { + /* The lag of a newly-initialized stream is 0. */ + lag = 0; + valid = 1; + } else if (cg->entries_read != SCG_INVALID_ENTRIES_READ && !streamRangeHasTombstones(s,&cg->last_id,NULL)) { + /* No fragmentation ahead means that the group's logical reads counter + * is valid for performing the lag calculation. */ + lag = (long long)s->entries_added - cg->entries_read; + valid = 1; + } else { + /* Attempt to retrieve the group's last ID logical read counter. */ + long long entries_read = streamEstimateDistanceFromFirstEverEntry(s,&cg->last_id); + if (entries_read != SCG_INVALID_ENTRIES_READ) { + /* A valid counter was obtained. */ + lag = (long long)s->entries_added - entries_read; + valid = 1; + } + } + + if (valid) { + addReplyLongLong(c,lag); + } else { + addReplyNull(c); + } +} + +/* This function returns a value that is the ID's logical read counter, or its + * distance (the number of entries) from the first entry ever to have been added + * to the stream. + * + * A counter is returned only in one of the following cases: + * 1. The ID is the same as the stream's last ID. In this case, the returned + * is the same as the stream's entries_added counter. + * 2. The ID equals that of the currently first entry in the stream, and the + * stream has no tombstones. The returned value, in this case, is the result + * of subtracting the stream's length from its added_entries, incremented by + * one. + * 3. The ID less than the stream's first current entry's ID, and there are no + * tombstones. Here the estimated counter is the result of subtracting the + * stream's length from its added_entries. + * 4. The stream's added_entries is zero, meaning that no entries were ever + * added. + * + * The special return value of ULLONG_MAX signals that the counter's value isn't + * obtainable. It is returned in these cases: + * 1. The provided ID, if it even exists, is somewhere between the stream's + * current first and last entries' IDs, or in the future. + * 2. The stream contains one or more tombstones. */ +long long streamEstimateDistanceFromFirstEverEntry(stream *s, streamID *id) { + /* The counter of any ID in an empty, never-before-used stream is 0. */ + if (!s->entries_added) { + return 0; + } + + /* In the empty stream, if the ID is smaller or equal to the last ID, + * it can set to the current added_entries value. */ + if (!s->length && streamCompareID(id,&s->last_id) < 1) { + return s->entries_added; + } + + int cmp_last = streamCompareID(id,&s->last_id); + if (cmp_last == 0) { + /* Return the exact counter of the last entry in the stream. */ + return s->entries_added; + } else if (cmp_last > 0) { + /* The counter of a future ID is unknown. */ + return SCG_INVALID_ENTRIES_READ; + } + + int cmp_id_first = streamCompareID(id,&s->first_id); + int cmp_xdel_first = streamCompareID(&s->max_deleted_entry_id,&s->first_id); + if (streamIDEqZero(&s->max_deleted_entry_id) || cmp_xdel_first < 0) { + /* There's definitely no fragmentation ahead. */ + if (cmp_id_first < 0) { + /* Return the estimated counter. */ + return s->entries_added - s->length; + } else if (cmp_id_first == 0) { + /* Return the exact counter of the first entry in the stream. */ + return s->entries_added - s->length + 1; + } + } + + /* The ID is either before an XDEL that fragments the stream or an arbitrary + * ID. Either case, so we can't make a prediction. */ + return SCG_INVALID_ENTRIES_READ; +} + /* As a result of an explicit XCLAIM or XREADGROUP command, new entries * are created in the pending list of the stream and consumers. We need * to propagate this changes in the form of XCLAIM commands. */ @@ -1411,19 +1571,22 @@ void streamPropagateXCLAIM(client *c, robj *key, streamCG *group, robj *groupnam * that was consumed by XREADGROUP with the NOACK option: in that case we can't * propagate the last ID just using the XCLAIM LASTID option, so we emit * - * XGROUP SETID <key> <groupname> <id> + * XGROUP SETID <key> <groupname> <id> ENTRIESREAD <entries_read> */ void streamPropagateGroupID(client *c, robj *key, streamCG *group, robj *groupname) { - robj *argv[5]; + robj *argv[7]; argv[0] = shared.xgroup; argv[1] = shared.setid; argv[2] = key; argv[3] = groupname; argv[4] = createObjectFromStreamID(&group->last_id); + argv[5] = shared.entriesread; + argv[6] = createStringObjectFromLongLong(group->entries_read); - alsoPropagate(c->db->id,argv,5,PROPAGATE_AOF|PROPAGATE_REPL); + alsoPropagate(c->db->id,argv,7,PROPAGATE_AOF|PROPAGATE_REPL); decrRefCount(argv[4]); + decrRefCount(argv[6]); } /* We need this when we want to propagate creation of consumer that was created @@ -1462,6 +1625,10 @@ void streamPropagateConsumerCreation(client *c, robj *key, robj *groupname, sds * function will not return it to the client. * 3. An entry in the pending list will be created for every entry delivered * for the first time to this consumer. + * 4. The group's read counter is incremented if it is already valid and there + * are no future tombstones, or is invalidated (set to 0) otherwise. If the + * counter is invalid to begin with, we try to obtain it for the last + * delivered ID. * * The behavior may be modified passing non-zero flags: * @@ -1518,6 +1685,15 @@ size_t streamReplyWithRange(client *c, stream *s, streamID *start, streamID *end while(streamIteratorGetID(&si,&id,&numfields)) { /* Update the group last_id if needed. */ if (group && streamCompareID(&id,&group->last_id) > 0) { + if (group->entries_read != SCG_INVALID_ENTRIES_READ && !streamRangeHasTombstones(s,&id,NULL)) { + /* A valid counter and no future tombstones mean we can + * increment the read counter to keep tracking the group's + * progress. */ + group->entries_read++; + } else if (s->entries_added) { + /* The group's counter may be invalid, so we try to obtain it. */ + group->entries_read = streamEstimateDistanceFromFirstEverEntry(s,&id); + } group->last_id = id; /* Group last ID should be propagated only if NOACK was * specified, otherwise the last id will be included @@ -1791,7 +1967,7 @@ void streamRewriteTrimArgument(client *c, stream *s, int trim_strategy, int idx) arg = createStringObjectFromLongLong(s->length); } else { streamID first_id; - streamGetEdgeID(s, 1, &first_id); + streamGetEdgeID(s,1,0,&first_id); arg = createObjectFromStreamID(&first_id); } @@ -2284,10 +2460,10 @@ void streamFreeConsumer(streamConsumer *sc) { } /* Create a new consumer group in the context of the stream 's', having the - * specified name and last server ID. If a consumer group with the same name - * already existed NULL is returned, otherwise the pointer to the consumer - * group is returned. */ -streamCG *streamCreateCG(stream *s, char *name, size_t namelen, streamID *id) { + * specified name, last server ID and reads counter. If a consumer group with + * the same name already exists NULL is returned, otherwise the pointer to the + * consumer group is returned. */ +streamCG *streamCreateCG(stream *s, char *name, size_t namelen, streamID *id, long long entries_read) { if (s->cgroups == NULL) s->cgroups = raxNew(); if (raxFind(s->cgroups,(unsigned char*)name,namelen) != raxNotFound) return NULL; @@ -2296,6 +2472,7 @@ streamCG *streamCreateCG(stream *s, char *name, size_t namelen, streamID *id) { cg->pel = raxNew(); cg->consumers = raxNew(); cg->last_id = *id; + cg->entries_read = entries_read; raxInsert(s->cgroups,(unsigned char*)name,namelen,cg,NULL); return cg; } @@ -2375,8 +2552,8 @@ void streamDelConsumer(streamCG *cg, streamConsumer *consumer) { * Consumer groups commands * ----------------------------------------------------------------------- */ -/* XGROUP CREATE <key> <groupname> <id or $> [MKSTREAM] - * XGROUP SETID <key> <groupname> <id or $> +/* XGROUP CREATE <key> <groupname> <id or $> [MKSTREAM] [ENTRIESADDED count] + * XGROUP SETID <key> <groupname> <id or $> [ENTRIESADDED count] * XGROUP DESTROY <key> <groupname> * XGROUP CREATECONSUMER <key> <groupname> <consumer> * XGROUP DELCONSUMER <key> <groupname> <consumername> */ @@ -2386,21 +2563,33 @@ void xgroupCommand(client *c) { streamCG *cg = NULL; char *opt = c->argv[1]->ptr; /* Subcommand name. */ int mkstream = 0; + long long entries_read = SCG_INVALID_ENTRIES_READ; robj *o; - /* CREATE has an MKSTREAM option that creates the stream if it - * does not exist. */ - if (c->argc == 6 && !strcasecmp(opt,"CREATE")) { - if (strcasecmp(c->argv[5]->ptr,"MKSTREAM")) { - addReplySubcommandSyntaxError(c); - return; - } - mkstream = 1; - grpname = c->argv[3]->ptr; - } - /* Everything but the "HELP" option requires a key and group name. */ if (c->argc >= 4) { + /* Parse optional arguments for CREATE and SETID */ + int i = 5; + int create_subcmd = !strcasecmp(opt,"CREATE"); + int setid_subcmd = !strcasecmp(opt,"SETID"); + while (i < c->argc) { + if (create_subcmd && !strcasecmp(c->argv[i]->ptr,"MKSTREAM")) { + mkstream = 1; + i++; + } else if ((create_subcmd || setid_subcmd) && !strcasecmp(c->argv[i]->ptr,"ENTRIESREAD") && i + 1 < c->argc) { + if (getLongLongFromObjectOrReply(c,c->argv[i+1],&entries_read,NULL) != C_OK) + return; + if (entries_read < 0 && entries_read != SCG_INVALID_ENTRIES_READ) { + addReplyError(c,"value for ENTRIESREAD must be positive or -1"); + return; + } + i += 2; + } else { + addReplySubcommandSyntaxError(c); + return; + } + } + o = lookupKeyWrite(c->db,c->argv[2]); if (o) { if (checkType(c,o,OBJ_STREAM)) return; @@ -2440,18 +2629,20 @@ void xgroupCommand(client *c) { " Create a new consumer group. Options are:", " * MKSTREAM", " Create the empty stream if it does not exist.", +" * ENTRIESREAD entries_read", +" Set the group's entries_read counter (internal use).", "CREATECONSUMER <key> <groupname> <consumer>", " Create a new consumer in the specified group.", "DELCONSUMER <key> <groupname> <consumer>", " Remove the specified consumer.", "DESTROY <key> <groupname>", " Remove the specified group.", -"SETID <key> <groupname> <id|$>", -" Set the current group ID.", +"SETID <key> <groupname> <id|$> [ENTRIESREAD entries_read]", +" Set the current group ID and entries_read counter.", NULL }; addReplyHelp(c, help); - } else if (!strcasecmp(opt,"CREATE") && (c->argc == 5 || c->argc == 6)) { + } else if (!strcasecmp(opt,"CREATE") && (c->argc >= 5 && c->argc <= 8)) { streamID id; if (!strcmp(c->argv[4]->ptr,"$")) { if (s) { @@ -2473,7 +2664,7 @@ NULL signalModifiedKey(c,c->db,c->argv[2]); } - streamCG *cg = streamCreateCG(s,grpname,sdslen(grpname),&id); + streamCG *cg = streamCreateCG(s,grpname,sdslen(grpname),&id,entries_read); if (cg) { addReply(c,shared.ok); server.dirty++; @@ -2482,7 +2673,7 @@ NULL } else { addReplyError(c,"-BUSYGROUP Consumer Group name already exists"); } - } else if (!strcasecmp(opt,"SETID") && c->argc == 5) { + } else if (!strcasecmp(opt,"SETID") && (c->argc == 5 || c->argc == 7)) { streamID id; if (!strcmp(c->argv[4]->ptr,"$")) { id = s->last_id; @@ -2490,6 +2681,7 @@ NULL return; } cg->last_id = id; + cg->entries_read = entries_read; addReply(c,shared.ok); server.dirty++; notifyKeyspaceEvent(NOTIFY_STREAM,"xgroup-setid",c->argv[2],c->db->id); @@ -2528,16 +2720,46 @@ NULL } } -/* XSETID <stream> <id> +/* XSETID <stream> <id> [ENTRIESADDED entries_added] [MAXDELETEDID max_deleted_entry_id] * - * Set the internal "last ID" of a stream. */ + * Set the internal "last ID", "added entries" and "maximal deleted entry ID" + * of a stream. */ void xsetidCommand(client *c) { + streamID id, max_xdel_id = {0, 0}; + long long entries_added = -1; + + if (streamParseStrictIDOrReply(c,c->argv[2],&id,0,NULL) != C_OK) + return; + + int i = 3; + while (i < c->argc) { + int moreargs = (c->argc-1) - i; /* Number of additional arguments. */ + char *opt = c->argv[i]->ptr; + if (!strcasecmp(opt,"ENTRIESADDED") && moreargs) { + if (getLongLongFromObjectOrReply(c,c->argv[i+1],&entries_added,NULL) != C_OK) { + return; + } else if (entries_added < 0) { + addReplyError(c,"entries_added must be positive"); + return; + } + i += 2; + } else if (!strcasecmp(opt,"MAXDELETEDID") && moreargs) { + if (streamParseStrictIDOrReply(c,c->argv[i+1],&max_xdel_id,0,NULL) != C_OK) { + return; + } else if (streamCompareID(&id,&max_xdel_id) < 0) { + addReplyError(c,"The ID specified in XSETID is smaller than the provided max_deleted_entry_id"); + return; + } + i += 2; + } else { + addReplyErrorObject(c,shared.syntaxerr); + return; + } + } + robj *o = lookupKeyWriteOrReply(c,c->argv[1],shared.nokeyerr); if (o == NULL || checkType(c,o,OBJ_STREAM)) return; - stream *s = o->ptr; - streamID id; - if (streamParseStrictIDOrReply(c,c->argv[2],&id,0,NULL) != C_OK) return; /* If the stream has at least one item, we want to check that the user * is setting a last ID that is equal or greater than the current top @@ -2547,12 +2769,22 @@ void xsetidCommand(client *c) { streamLastValidID(s,&maxid); if (streamCompareID(&id,&maxid) < 0) { - addReplyError(c,"The ID specified in XSETID is smaller than the " - "target stream top item"); + addReplyError(c,"The ID specified in XSETID is smaller than the target stream top item"); + return; + } + + /* If an entries_added was provided, it can't be lower than the length. */ + if (entries_added != -1 && s->length > (uint64_t)entries_added) { + addReplyError(c,"The entries_added specified in XSETID is smaller than the target stream length"); return; } } + s->last_id = id; + if (entries_added != -1) + s->entries_added = entries_added; + if (!streamIDEqZero(&max_xdel_id)) + s->max_deleted_entry_id = max_xdel_id; addReply(c,shared.ok); server.dirty++; notifyKeyspaceEvent(NOTIFY_STREAM,"xsetid",c->argv[1],c->db->id); @@ -2980,23 +3212,28 @@ void xclaimCommand(client *c) { /* Lookup the ID in the group PEL. */ streamNACK *nack = raxFind(group->pel,buf,sizeof(buf)); + /* Item must exist for us to transfer it to another consumer. */ + if (!streamEntryExists(o->ptr,&id)) { + /* Clear this entry from the PEL, it no longer exists */ + if (nack != raxNotFound) { + /* Propagate this change (we are going to delete the NACK). */ + streamPropagateXCLAIM(c,c->argv[1],group,c->argv[2],c->argv[j],nack); + propagate_last_id = 0; /* Will be propagated by XCLAIM itself. */ + server.dirty++; + /* Release the NACK */ + raxRemove(group->pel,buf,sizeof(buf),NULL); + raxRemove(nack->consumer->pel,buf,sizeof(buf),NULL); + streamFreeNACK(nack); + } + continue; + } + /* If FORCE is passed, let's check if at least the entry * exists in the Stream. In such case, we'll create a new * entry in the PEL from scratch, so that XCLAIM can also * be used to create entries in the PEL. Useful for AOF * and replication of consumer groups. */ if (force && nack == raxNotFound) { - streamIterator myiterator; - streamIteratorStart(&myiterator,o->ptr,&id,&id,0); - int64_t numfields; - int found = 0; - streamID item_id; - if (streamIteratorGetID(&myiterator,&item_id,&numfields)) found = 1; - streamIteratorStop(&myiterator); - - /* Item must exist for us to create a NACK for it. */ - if (!found) continue; - /* Create the NACK. */ nack = streamCreateNACK(NULL); raxInsert(group->pel,buf,sizeof(buf),nack,NULL); @@ -3013,6 +3250,7 @@ void xclaimCommand(client *c) { mstime_t this_idle = now - nack->delivery_time; if (this_idle < minidle) continue; } + if (consumer == NULL && (consumer = streamLookupConsumer(group,name,SLC_DEFAULT)) == NULL) { @@ -3042,9 +3280,7 @@ void xclaimCommand(client *c) { if (justid) { addReplyStreamID(c,&id); } else { - size_t emitted = streamReplyWithRange(c,o->ptr,&id,&id,1,0, - NULL,NULL,STREAM_RWR_RAWENTRIES,NULL); - if (!emitted) addReplyNull(c); + serverAssert(streamReplyWithRange(c,o->ptr,&id,&id,1,0,NULL,NULL,STREAM_RWR_RAWENTRIES,NULL) == 1); } arraylen++; @@ -3138,9 +3374,9 @@ void xautoclaimCommand(client *c) { streamConsumer *consumer = NULL; long long attempts = count*10; - addReplyArrayLen(c, 2); - void *endidptr = addReplyDeferredLen(c); - void *arraylenptr = addReplyDeferredLen(c); + addReplyArrayLen(c, 3); /* We add another reply later */ + void *endidptr = addReplyDeferredLen(c); /* reply[0] */ + void *arraylenptr = addReplyDeferredLen(c); /* reply[1] */ unsigned char startkey[sizeof(streamID)]; streamEncodeID(startkey,&startid); @@ -3150,18 +3386,37 @@ void xautoclaimCommand(client *c) { size_t arraylen = 0; mstime_t now = mstime(); sds name = c->argv[3]->ptr; + streamID *deleted_ids = zmalloc(count * sizeof(streamID)); + int deleted_id_num = 0; while (attempts-- && count && raxNext(&ri)) { streamNACK *nack = ri.data; + streamID id; + streamDecodeID(ri.key, &id); + + /* Item must exist for us to transfer it to another consumer. */ + if (!streamEntryExists(o->ptr,&id)) { + /* Propagate this change (we are going to delete the NACK). */ + robj *idstr = createObjectFromStreamID(&id); + streamPropagateXCLAIM(c,c->argv[1],group,c->argv[2],idstr,nack); + decrRefCount(idstr); + server.dirty++; + /* Clear this entry from the PEL, it no longer exists */ + raxRemove(group->pel,ri.key,ri.key_len,NULL); + raxRemove(nack->consumer->pel,ri.key,ri.key_len,NULL); + streamFreeNACK(nack); + /* Remember the ID for later */ + deleted_ids[deleted_id_num++] = id; + raxSeek(&ri,">=",ri.key,ri.key_len); + continue; + } + if (minidle) { mstime_t this_idle = now - nack->delivery_time; if (this_idle < minidle) continue; } - streamID id; - streamDecodeID(ri.key, &id); - if (consumer == NULL && (consumer = streamLookupConsumer(group,name,SLC_DEFAULT)) == NULL) { @@ -3191,11 +3446,7 @@ void xautoclaimCommand(client *c) { if (justid) { addReplyStreamID(c,&id); } else { - size_t emitted = - streamReplyWithRange(c,o->ptr,&id,&id,1,0,NULL,NULL, - STREAM_RWR_RAWENTRIES,NULL); - if (!emitted) - addReplyNull(c); + serverAssert(streamReplyWithRange(c,o->ptr,&id,&id,1,0,NULL,NULL,STREAM_RWR_RAWENTRIES,NULL) == 1); } arraylen++; count--; @@ -3221,6 +3472,12 @@ void xautoclaimCommand(client *c) { setDeferredArrayLen(c,arraylenptr,arraylen); setDeferredReplyStreamID(c,endidptr,&endid); + addReplyArrayLen(c, deleted_id_num); /* reply[2] */ + for (int i = 0; i < deleted_id_num; i++) { + addReplyStreamID(c, &deleted_ids[i]); + } + zfree(deleted_ids); + preventCommandPropagation(c); } @@ -3250,8 +3507,31 @@ void xdelCommand(client *c) { /* Actually apply the command. */ int deleted = 0; + int first_entry = 0; for (int j = 2; j < c->argc; j++) { - deleted += streamDeleteItem(s,&ids[j-2]); + streamID *id = &ids[j-2]; + if (streamDeleteItem(s,id)) { + /* We want to know if the first entry in the stream was deleted + * so we can later set the new one. */ + if (streamCompareID(id,&s->first_id) == 0) { + first_entry = 1; + } + /* Update the stream's maximal tombstone if needed. */ + if (streamCompareID(id,&s->max_deleted_entry_id) > 0) { + s->max_deleted_entry_id = *id; + } + deleted++; + }; + } + + /* Update the stream's first ID. */ + if (deleted) { + if (s->length == 0) { + s->first_id.ms = 0; + s->first_id.seq = 0; + } else if (first_entry) { + streamGetEdgeID(s,1,1,&s->first_id); + } } /* Propagate the write if needed. */ @@ -3359,7 +3639,7 @@ void xinfoReplyWithStreamInfo(client *c, stream *s) { } } - addReplyMapLen(c,full ? 6 : 7); + addReplyMapLen(c,full ? 9 : 10); addReplyBulkCString(c,"length"); addReplyLongLong(c,s->length); addReplyBulkCString(c,"radix-tree-keys"); @@ -3368,6 +3648,12 @@ void xinfoReplyWithStreamInfo(client *c, stream *s) { addReplyLongLong(c,s->rax->numnodes); addReplyBulkCString(c,"last-generated-id"); addReplyStreamID(c,&s->last_id); + addReplyBulkCString(c,"max-deleted-entry-id"); + addReplyStreamID(c,&s->max_deleted_entry_id); + addReplyBulkCString(c,"entries-added"); + addReplyLongLong(c,s->entries_added); + addReplyBulkCString(c,"recorded-first-entry-id"); + addReplyStreamID(c,&s->first_id); if (!full) { /* XINFO STREAM <key> */ @@ -3406,7 +3692,7 @@ void xinfoReplyWithStreamInfo(client *c, stream *s) { raxSeek(&ri_cgroups,"^",NULL,0); while(raxNext(&ri_cgroups)) { streamCG *cg = ri_cgroups.data; - addReplyMapLen(c,5); + addReplyMapLen(c,7); /* Name */ addReplyBulkCString(c,"name"); @@ -3416,6 +3702,18 @@ void xinfoReplyWithStreamInfo(client *c, stream *s) { addReplyBulkCString(c,"last-delivered-id"); addReplyStreamID(c,&cg->last_id); + /* Read counter of the last delivered ID */ + addReplyBulkCString(c,"entries-read"); + if (cg->entries_read != SCG_INVALID_ENTRIES_READ) { + addReplyLongLong(c,cg->entries_read); + } else { + addReplyNull(c); + } + + /* Group lag */ + addReplyBulkCString(c,"lag"); + streamReplyWithCGLag(c,s,cg); + /* Group PEL count */ addReplyBulkCString(c,"pel-count"); addReplyLongLong(c,raxSize(cg->pel)); @@ -3593,7 +3891,7 @@ NULL raxSeek(&ri,"^",NULL,0); while(raxNext(&ri)) { streamCG *cg = ri.data; - addReplyMapLen(c,4); + addReplyMapLen(c,6); addReplyBulkCString(c,"name"); addReplyBulkCBuffer(c,ri.key,ri.key_len); addReplyBulkCString(c,"consumers"); @@ -3602,6 +3900,14 @@ NULL addReplyLongLong(c,raxSize(cg->pel)); addReplyBulkCString(c,"last-delivered-id"); addReplyStreamID(c,&cg->last_id); + addReplyBulkCString(c,"entries-read"); + if (cg->entries_read != SCG_INVALID_ENTRIES_READ) { + addReplyLongLong(c,cg->entries_read); + } else { + addReplyNull(c); + } + addReplyBulkCString(c,"lag"); + streamReplyWithCGLag(c,s,cg); } raxStop(&ri); } else if (!strcasecmp(opt,"STREAM")) { diff --git a/src/t_zset.c b/src/t_zset.c index e09d4528b..bc947c965 100644 --- a/src/t_zset.c +++ b/src/t_zset.c @@ -2848,7 +2848,7 @@ typedef enum { typedef struct zrange_result_handler zrange_result_handler; -typedef void (*zrangeResultBeginFunction)(zrange_result_handler *c); +typedef void (*zrangeResultBeginFunction)(zrange_result_handler *c, long length); typedef void (*zrangeResultFinalizeFunction)( zrange_result_handler *c, size_t result_count); typedef void (*zrangeResultEmitCBufferFunction)( @@ -2876,8 +2876,22 @@ struct zrange_result_handler { zrangeResultEmitLongLongFunction emitResultFromLongLong; }; -/* Result handler methods for responding the ZRANGE to clients. */ -static void zrangeResultBeginClient(zrange_result_handler *handler) { +/* Result handler methods for responding the ZRANGE to clients. + * length can be used to provide the result length in advance (avoids deferred reply overhead). + * length can be set to -1 if the result length is not know in advance. + */ +static void zrangeResultBeginClient(zrange_result_handler *handler, long length) { + if (length > 0) { + /* In case of WITHSCORES, respond with a single array in RESP2, and + * nested arrays in RESP3. We can't use a map response type since the + * client library needs to know to respect the order. */ + if (handler->withscores && (handler->client->resp == 2)) { + length *= 2; + } + addReplyArrayLen(handler->client, length); + handler->userdata = NULL; + return; + } handler->userdata = addReplyDeferredLen(handler->client); } @@ -2912,6 +2926,9 @@ static void zrangeResultEmitLongLongToClient(zrange_result_handler *handler, static void zrangeResultFinalizeClient(zrange_result_handler *handler, size_t result_count) { + /* If the reply size was know at start there's nothing left to do */ + if (!handler->userdata) + return; /* In case of WITHSCORES, respond with a single array in RESP2, and * nested arrays in RESP3. We can't use a map response type since the * client library needs to know to respect the order. */ @@ -2923,8 +2940,9 @@ static void zrangeResultFinalizeClient(zrange_result_handler *handler, } /* Result handler methods for storing the ZRANGESTORE to a zset. */ -static void zrangeResultBeginStore(zrange_result_handler *handler) +static void zrangeResultBeginStore(zrange_result_handler *handler, long length) { + UNUSED(length); handler->dstobj = createZsetListpackObject(); } @@ -3019,11 +3037,11 @@ void genericZrangebyrankCommand(zrange_result_handler *handler, if (end < 0) end = llen+end; if (start < 0) start = 0; - handler->beginResultEmission(handler); /* Invariant: start >= 0, so this test will be true when end < 0. * The range is empty when start > end or start >= length. */ if (start > end || start >= llen) { + handler->beginResultEmission(handler, 0); handler->finalizeResultEmission(handler, 0); return; } @@ -3031,6 +3049,7 @@ void genericZrangebyrankCommand(zrange_result_handler *handler, rangelen = (end-start)+1; result_cardinality = rangelen; + handler->beginResultEmission(handler, rangelen); if (zobj->encoding == OBJ_ENCODING_LISTPACK) { unsigned char *zl = zobj->ptr; unsigned char *eptr, *sptr; @@ -3124,7 +3143,7 @@ void genericZrangebyscoreCommand(zrange_result_handler *handler, int reverse) { unsigned long rangelen = 0; - handler->beginResultEmission(handler); + handler->beginResultEmission(handler, -1); /* For invalid offset, return directly. */ if (offset > 0 && offset >= (long)zsetLength(zobj)) { @@ -3409,7 +3428,7 @@ void genericZrangebylexCommand(zrange_result_handler *handler, { unsigned long rangelen = 0; - handler->beginResultEmission(handler); + handler->beginResultEmission(handler, -1); if (zobj->encoding == OBJ_ENCODING_LISTPACK) { unsigned char *zl = zobj->ptr; @@ -3647,7 +3666,7 @@ void zrangeGenericCommand(zrange_result_handler *handler, int argc_start, int st zobj = lookupKeyRead(c->db, key); if (zobj == NULL) { if (store) { - handler->beginResultEmission(handler); + handler->beginResultEmission(handler, -1); handler->finalizeResultEmission(handler, 0); } else { addReply(c, shared.emptyarray); @@ -39,6 +39,10 @@ #include <openssl/err.h> #include <openssl/rand.h> #include <openssl/pem.h> +#if OPENSSL_VERSION_NUMBER >= 0x30000000L +#include <openssl/decoder.h> +#endif +#include <sys/uio.h> #define REDIS_TLS_PROTO_TLSv1 (1<<0) #define REDIS_TLS_PROTO_TLSv1_1 (1<<1) @@ -146,14 +150,13 @@ void tlsInit(void) { */ #if OPENSSL_VERSION_NUMBER < 0x10100000L OPENSSL_config(NULL); + SSL_load_error_strings(); + SSL_library_init(); #elif OPENSSL_VERSION_NUMBER < 0x10101000L OPENSSL_init_crypto(OPENSSL_INIT_LOAD_CONFIG, NULL); #else OPENSSL_init_crypto(OPENSSL_INIT_LOAD_CONFIG|OPENSSL_INIT_ATFORK, NULL); #endif - ERR_load_crypto_strings(); - SSL_load_error_strings(); - SSL_library_init(); #ifdef USE_CRYPTO_LOCKS initCryptoLocks(); @@ -323,20 +326,46 @@ int tlsConfigure(redisTLSContextConfig *ctx_config) { if (ctx_config->prefer_server_ciphers) SSL_CTX_set_options(ctx, SSL_OP_CIPHER_SERVER_PREFERENCE); -#if defined(SSL_CTX_set_ecdh_auto) +#if ((OPENSSL_VERSION_NUMBER < 0x30000000L) && defined(SSL_CTX_set_ecdh_auto)) SSL_CTX_set_ecdh_auto(ctx, 1); #endif SSL_CTX_set_options(ctx, SSL_OP_SINGLE_DH_USE); if (ctx_config->dh_params_file) { FILE *dhfile = fopen(ctx_config->dh_params_file, "r"); - DH *dh = NULL; if (!dhfile) { serverLog(LL_WARNING, "Failed to load %s: %s", ctx_config->dh_params_file, strerror(errno)); goto error; } - dh = PEM_read_DHparams(dhfile, NULL, NULL, NULL); +#if (OPENSSL_VERSION_NUMBER >= 0x30000000L) + EVP_PKEY *pkey = NULL; + OSSL_DECODER_CTX *dctx = OSSL_DECODER_CTX_new_for_pkey( + &pkey, "PEM", NULL, "DH", OSSL_KEYMGMT_SELECT_DOMAIN_PARAMETERS, NULL, NULL); + if (!dctx) { + serverLog(LL_WARNING, "No decoder for DH params."); + fclose(dhfile); + goto error; + } + if (!OSSL_DECODER_from_fp(dctx, dhfile)) { + serverLog(LL_WARNING, "%s: failed to read DH params.", ctx_config->dh_params_file); + OSSL_DECODER_CTX_free(dctx); + fclose(dhfile); + goto error; + } + + OSSL_DECODER_CTX_free(dctx); + fclose(dhfile); + + if (SSL_CTX_set0_tmp_dh_pkey(ctx, pkey) <= 0) { + ERR_error_string_n(ERR_get_error(), errbuf, sizeof(errbuf)); + serverLog(LL_WARNING, "Failed to load DH params file: %s: %s", ctx_config->dh_params_file, errbuf); + EVP_PKEY_free(pkey); + goto error; + } + /* Not freeing pkey, it is owned by OpenSSL now */ +#else + DH *dh = PEM_read_DHparams(dhfile, NULL, NULL, NULL); fclose(dhfile); if (!dh) { serverLog(LL_WARNING, "%s: failed to read DH params.", ctx_config->dh_params_file); @@ -351,6 +380,11 @@ int tlsConfigure(redisTLSContextConfig *ctx_config) { } DH_free(dh); +#endif + } else { +#if (OPENSSL_VERSION_NUMBER >= 0x30000000L) + SSL_CTX_set_dh_auto(ctx, 1); +#endif } /* If a client-side certificate is configured, create an explicit client context */ @@ -786,6 +820,43 @@ static int connTLSWrite(connection *conn_, const void *data, size_t data_len) { return ret; } +static int connTLSWritev(connection *conn_, const struct iovec *iov, int iovcnt) { + if (iovcnt == 1) return connTLSWrite(conn_, iov[0].iov_base, iov[0].iov_len); + + /* Accumulate the amount of bytes of each buffer and check if it exceeds NET_MAX_WRITES_PER_EVENT. */ + size_t iov_bytes_len = 0; + for (int i = 0; i < iovcnt; i++) { + iov_bytes_len += iov[i].iov_len; + if (iov_bytes_len > NET_MAX_WRITES_PER_EVENT) break; + } + + /* The amount of all buffers is greater than NET_MAX_WRITES_PER_EVENT, + * which is not worth doing so much memory copying to reduce system calls, + * therefore, invoke connTLSWrite() multiple times to avoid memory copies. */ + if (iov_bytes_len > NET_MAX_WRITES_PER_EVENT) { + size_t tot_sent = 0; + for (int i = 0; i < iovcnt; i++) { + size_t sent = connTLSWrite(conn_, iov[i].iov_base, iov[i].iov_len); + if (sent <= 0) return tot_sent > 0 ? tot_sent : sent; + tot_sent += sent; + if (sent != iov[i].iov_len) break; + } + return tot_sent; + } + + /* The amount of all buffers is less than NET_MAX_WRITES_PER_EVENT, + * which is worth doing more memory copies in exchange for fewer system calls, + * so concatenate these scattered buffers into a contiguous piece of memory + * and send it away by one call to connTLSWrite(). */ + char buf[iov_bytes_len]; + size_t offset = 0; + for (int i = 0; i < iovcnt; i++) { + memcpy(buf + offset, iov[i].iov_base, iov[i].iov_len); + offset += iov[i].iov_len; + } + return connTLSWrite(conn_, buf, iov_bytes_len); +} + static int connTLSRead(connection *conn_, void *buf, size_t buf_len) { tls_connection *conn = (tls_connection *) conn_; int ret; @@ -949,6 +1020,7 @@ ConnectionType CT_TLS = { .blocking_connect = connTLSBlockingConnect, .read = connTLSRead, .write = connTLSWrite, + .writev = connTLSWritev, .close = connTLSClose, .set_write_handler = connTLSSetWriteHandler, .set_read_handler = connTLSSetReadHandler, diff --git a/src/version.h b/src/version.h index cddaae2b1..0f750bad3 100644 --- a/src/version.h +++ b/src/version.h @@ -1,2 +1,2 @@ -#define REDIS_VERSION "6.9.240" -#define REDIS_VERSION_NUM 0x000609f0 +#define REDIS_VERSION "6.9.241" +#define REDIS_VERSION_NUM 0x000609f1 |