diff options
144 files changed, 8139 insertions, 1923 deletions
diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml index 88e31f52a..da266fdce 100644 --- a/.github/workflows/daily.yml +++ b/.github/workflows/daily.yml @@ -11,10 +11,10 @@ on: inputs: skipjobs: description: 'jobs to skip (delete the ones you wanna keep, do not leave empty)' - default: 'valgrind,sanitizer,tls,freebsd,macos,alpine,32bit' + default: 'valgrind,sanitizer,tls,freebsd,macos,alpine,32bit,ubuntu,centos,malloc' skiptests: description: 'tests to skip (delete the ones you wanna keep, do not leave empty)' - default: 'redis,modules,sentinel,cluster' + default: 'redis,modules,sentinel,cluster,unittest' test_args: description: 'extra test arguments' default: '' @@ -35,7 +35,7 @@ jobs: runs-on: ubuntu-latest if: | github.event_name == 'workflow_dispatch' || - (github.event_name == 'schedule' && github.repository == 'redis/redis') + (github.event_name == 'schedule' && github.repository == 'redis/redis') && !contains(github.event.inputs.skipjobs, 'ubuntu') timeout-minutes: 14400 steps: - name: prep @@ -65,13 +65,14 @@ jobs: if: true && !contains(github.event.inputs.skiptests, 'cluster') run: ./runtest-cluster ${{github.event.inputs.cluster_test_args}} - name: unittest + if: true && !contains(github.event.inputs.skiptests, 'unittest') run: ./src/redis-server test all --accurate test-ubuntu-libc-malloc: runs-on: ubuntu-latest if: | github.event_name == 'workflow_dispatch' || - (github.event_name == 'schedule' && github.repository == 'redis/redis') + (github.event_name == 'schedule' && github.repository == 'redis/redis') && !contains(github.event.inputs.skipjobs, 'malloc') timeout-minutes: 14400 steps: - name: prep @@ -104,7 +105,7 @@ jobs: runs-on: ubuntu-latest if: | github.event_name == 'workflow_dispatch' || - (github.event_name == 'schedule' && github.repository == 'redis/redis') + (github.event_name == 'schedule' && github.repository == 'redis/redis') && !contains(github.event.inputs.skipjobs, 'malloc') timeout-minutes: 14400 steps: - name: prep @@ -170,6 +171,7 @@ jobs: if: true && !contains(github.event.inputs.skiptests, 'cluster') run: ./runtest-cluster ${{github.event.inputs.cluster_test_args}} - name: unittest + if: true && !contains(github.event.inputs.skiptests, 'unittest') run: ./src/redis-server test all --accurate test-ubuntu-tls: @@ -273,6 +275,7 @@ jobs: if: true && !contains(github.event.inputs.skiptests, 'modules') run: ./runtest-moduleapi --valgrind --no-latency --verbose --clients 1 --timeout 2400 --dump-logs ${{github.event.inputs.test_args}} - name: unittest + if: true && !contains(github.event.inputs.skiptests, 'unittest') run: | valgrind --track-origins=yes --suppressions=./src/valgrind.sup --show-reachable=no --show-possibly-lost=no --leak-check=full --log-file=err.txt ./src/redis-server test all if grep -q 0x err.txt; then cat err.txt; exit 1; fi @@ -346,6 +349,7 @@ jobs: if: true && !contains(github.event.inputs.skiptests, 'cluster') run: ./runtest-cluster ${{github.event.inputs.cluster_test_args}} - name: unittest + if: true && !contains(github.event.inputs.skiptests, 'unittest') run: ./src/redis-server test all test-sanitizer-undefined: @@ -388,13 +392,14 @@ jobs: if: true && !contains(github.event.inputs.skiptests, 'cluster') run: ./runtest-cluster ${{github.event.inputs.cluster_test_args}} - name: unittest + if: true && !contains(github.event.inputs.skiptests, 'unittest') run: ./src/redis-server test all --accurate test-centos7-jemalloc: runs-on: ubuntu-latest if: | github.event_name == 'workflow_dispatch' || - (github.event_name == 'schedule' && github.repository == 'redis/redis') + (github.event_name == 'schedule' && github.repository == 'redis/redis') && !contains(github.event.inputs.skipjobs, 'centos') container: centos:7 timeout-minutes: 14400 steps: @@ -521,10 +526,11 @@ jobs: repository: ${{ env.GITHUB_REPOSITORY }} ref: ${{ env.GITHUB_HEAD_REF }} - name: test - uses: vmactions/freebsd-vm@v0.1.5 + uses: vmactions/freebsd-vm@v0.1.6 with: usesh: true sync: rsync + copyback: false prepare: pkg install -y bash gmake lang/tcl86 lang/tclx run: > gmake || exit 1 ; diff --git a/.gitignore b/.gitignore index 363acfca1..e03c834d0 100644 --- a/.gitignore +++ b/.gitignore @@ -16,7 +16,7 @@ doc-tools release misc/* src/release.h -appendonly.aof.* +appendonly.aof* appendonlydir SHORT_TERM_TODO release.h diff --git a/00-RELEASENOTES b/00-RELEASENOTES index 1541278c4..2caa0492d 100644 --- a/00-RELEASENOTES +++ b/00-RELEASENOTES @@ -1,4 +1,95 @@ ================================================================================ +Redis 7.0 RC1 Released Mon Feb 28 12:00:00 IST 2022 +================================================================================ + +Upgrade urgency LOW: This is another Release Candidate of Redis 7.0. + +New Features +============ + +* Add stream consumer group lag tracking and reporting (#9127) +* Add API for functions and eval Lua scripts to check ACL explicitly (#10220) + +New user commands or command arguments +-------------------------------------- + +* COMMAND GETKEYSANDFLAGS sub-command (#10237) +* INFO command can take multiple section arguments (#6891) +* XGROUP CREATE and SETID: new ENTRIESREAD optional argument (#9127) +* XSETID new ENTRIESADDED and MAXDELETEDID optional arguments (#9127) + +Command replies that have been extended +--------------------------------------- + +* XINFO reports consumer group lag and a few other fields (#9127) +* XAUTOCLAIM returns a new element with a list of deletes IDs (#10227) + +Potentially Breaking Changes +============================ + +* X[AUTO]CLAIM skips deleted entries instead of replying with Nil, and deletes + them from the pending entry list (#10227) +* Fix messed up error codes returned from EVAL scripts (#10218, #10329) +* COMMAND INFO, Renames key-spec "CHANNEL" flag to be "NOT_KEY" (#10299) + +Performance and resource utilization improvements +================================================= + +* Reduce system calls and small packets for client replies (#9934) +* Reduce memory usage of stale clients (#9822) +* Fix regression in Z[REV]RANGE commands (by-rank) introduced in Redis 6.2 (#10337) + +Changes in CLI tools +=================== + +* Adapt redis-check-aof tool for Multi Part AOF (#10061) +* Enable redis-benchmark to use RESP3 protocol mode (#10335) + +Platform / toolchain support related improvements +================================================= + +* Fix OpenSSL 3.0.x related issues (#10291) + +INFO fields and introspection changes +===================================== + +* COMMAND INFO key-specs has new variable_flags flag (#10237, #10148) +* INFO stats: add aof_rewrites and rdb_snapshots counters (#10178) +* INFO stats: add reply_buffer_shrinks and reply_buffer_expends (#9822) +* INFO modules: add no-implicit-signal-modified module option (#10284) + +Module API changes +================== + +* Add RM_SetCommandInfo API to set command metadata for the new COMMAND + introspection features and ACL key permissions (#10108) +* Add RM_KeyAtPosWithFlags and RM_GetCommandKeysWithFlags APIs (#10237) +* Add getchannels-api command flag and RM_IsChannelsPositionRequest, + RM_ChannelAtPosWithFlags APIs (#10299) +* Change RM_ACLCheckChannelPermissions and RM_ACLCheckKeyPermissions APIs + (released in RC1) to take different flags (#10299) +* Fix RM_SetModuleOptions flag collision. Bug in 7.0 RC1 header file, modules + that used OPTIONS_HANDLE_REPL_ASYNC_LOAD will mess up key invalidations (#10284) + +Bug Fixes +========= + +* Modules: Fix thread safety violation when a module thread adds an error reply, + broken in 6.2 (#10278) +* Lua: Fix Eval scripts active defrag, broken 7.0 in RC1 (#10271) +* Fix geo search bounding box check causing missing results (#10018) +* Lua: Add checks for min-slave-* configs when evaluating Lua scripts and + Functions (#10160) +* Modules: Prevent crashes and memory leaks when MODULE UNLOAD is used on module + with a pending timer (#10187) +* Fix error stats and failed command stats for blocked clients (#10309) +* Lua/Modules: Fix missing and duplicate error stats for scripts and modules (#10329, #10278) +* Check target node is a primary during cluster setslot (#10277) +* Fix key deletion not to invalidate WATCH when used on a logically expired key (#10256) +* Sentinel: return an error if configuration save fails (#10151) +* Sentinel: fix a free-after-use issue re-registering Sentinels (#10333) + +================================================================================ Redis 7.0 RC1 Released Mon Jan 31 12:00:00 IST 2022 ================================================================================ diff --git a/deps/hiredis/.github/workflows/build.yml b/deps/hiredis/.github/workflows/build.yml new file mode 100644 index 000000000..362bc77b7 --- /dev/null +++ b/deps/hiredis/.github/workflows/build.yml @@ -0,0 +1,205 @@ +name: Build and test +on: [push, pull_request] + +jobs: + ubuntu: + name: Ubuntu + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + repository: ${{ env.GITHUB_REPOSITORY }} + ref: ${{ env.GITHUB_HEAD_REF }} + + - name: Install dependencies + run: | + sudo add-apt-repository -y ppa:chris-lea/redis-server + sudo apt-get update + sudo apt-get install -y redis-server valgrind libevent-dev + + - name: Build using cmake + env: + EXTRA_CMAKE_OPTS: -DENABLE_EXAMPLES:BOOL=ON -DENABLE_SSL:BOOL=ON -DENABLE_SSL_TESTS:BOOL=ON -DENABLE_ASYNC_TESTS:BOOL=ON + CFLAGS: -Werror + CXXFLAGS: -Werror + run: mkdir build-ubuntu && cd build-ubuntu && cmake .. + + - name: Build using makefile + run: USE_SSL=1 TEST_ASYNC=1 make + + - name: Run tests + env: + SKIPS_AS_FAILS: 1 + TEST_SSL: 1 + run: $GITHUB_WORKSPACE/test.sh + + # - name: Run tests under valgrind + # env: + # SKIPS_AS_FAILS: 1 + # TEST_PREFIX: valgrind --error-exitcode=99 --track-origins=yes --leak-check=full + # run: $GITHUB_WORKSPACE/test.sh + + centos7: + name: CentOS 7 + runs-on: ubuntu-latest + container: centos:7 + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + repository: ${{ env.GITHUB_REPOSITORY }} + ref: ${{ env.GITHUB_HEAD_REF }} + + - name: Install dependencies + run: | + yum -y install http://rpms.remirepo.net/enterprise/remi-release-7.rpm + yum -y --enablerepo=remi install redis + yum -y install gcc gcc-c++ make openssl openssl-devel cmake3 valgrind libevent-devel + + - name: Build using cmake + env: + EXTRA_CMAKE_OPTS: -DENABLE_EXAMPLES:BOOL=ON -DENABLE_SSL:BOOL=ON -DENABLE_SSL_TESTS:BOOL=ON -DENABLE_ASYNC_TESTS:BOOL=ON + CFLAGS: -Werror + CXXFLAGS: -Werror + run: mkdir build-centos7 && cd build-centos7 && cmake3 .. + + - name: Build using Makefile + run: USE_SSL=1 TEST_ASYNC=1 make + + - name: Run tests + env: + SKIPS_AS_FAILS: 1 + TEST_SSL: 1 + run: $GITHUB_WORKSPACE/test.sh + + - name: Run tests under valgrind + env: + SKIPS_AS_FAILS: 1 + TEST_SSL: 1 + TEST_PREFIX: valgrind --error-exitcode=99 --track-origins=yes --leak-check=full + run: $GITHUB_WORKSPACE/test.sh + + centos8: + name: RockyLinux 8 + runs-on: ubuntu-latest + container: rockylinux:8 + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + repository: ${{ env.GITHUB_REPOSITORY }} + ref: ${{ env.GITHUB_HEAD_REF }} + + - name: Install dependencies + run: | + dnf -y install https://rpms.remirepo.net/enterprise/remi-release-8.rpm + dnf -y module install redis:remi-6.0 + dnf -y group install "Development Tools" + dnf -y install openssl-devel cmake valgrind libevent-devel + + - name: Build using cmake + env: + EXTRA_CMAKE_OPTS: -DENABLE_EXAMPLES:BOOL=ON -DENABLE_SSL:BOOL=ON -DENABLE_SSL_TESTS:BOOL=ON -DENABLE_ASYNC_TESTS:BOOL=ON + CFLAGS: -Werror + CXXFLAGS: -Werror + run: mkdir build-centos8 && cd build-centos8 && cmake .. + + - name: Build using Makefile + run: USE_SSL=1 TEST_ASYNC=1 make + + - name: Run tests + env: + SKIPS_AS_FAILS: 1 + TEST_SSL: 1 + run: $GITHUB_WORKSPACE/test.sh + + - name: Run tests under valgrind + env: + SKIPS_AS_FAILS: 1 + TEST_SSL: 1 + TEST_PREFIX: valgrind --error-exitcode=99 --track-origins=yes --leak-check=full + run: $GITHUB_WORKSPACE/test.sh + + freebsd: + runs-on: macos-10.15 + name: FreeBSD + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + repository: ${{ env.GITHUB_REPOSITORY }} + ref: ${{ env.GITHUB_HEAD_REF }} + + - name: Build in FreeBSD + uses: vmactions/freebsd-vm@v0.1.5 + with: + prepare: pkg install -y gmake cmake + run: | + mkdir build && cd build && cmake .. && make && cd .. + gmake + + macos: + name: macOS + runs-on: macos-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + repository: ${{ env.GITHUB_REPOSITORY }} + ref: ${{ env.GITHUB_HEAD_REF }} + + - name: Install dependencies + run: | + brew install openssl redis + + - name: Build hiredis + run: USE_SSL=1 make + + - name: Run tests + env: + TEST_SSL: 1 + run: $GITHUB_WORKSPACE/test.sh + + windows: + name: Windows + runs-on: windows-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + repository: ${{ env.GITHUB_REPOSITORY }} + ref: ${{ env.GITHUB_HEAD_REF }} + + - name: Install dependencies + run: | + choco install -y ninja memurai-developer + + - uses: ilammy/msvc-dev-cmd@v1 + - name: Build hiredis + run: | + mkdir build && cd build + cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release -DENABLE_EXAMPLES=ON + ninja -v + + - name: Run tests + run: | + ./build/hiredis-test.exe + + - name: Setup cygwin + uses: egor-tensin/setup-cygwin@v3 + with: + platform: x64 + packages: make git gcc-core + + - name: Build in cygwin + env: + HIREDIS_PATH: ${{ github.workspace }} + run: | + build_hiredis() { + cd $(cygpath -u $HIREDIS_PATH) + git clean -xfd + make + } + build_hiredis + shell: C:\tools\cygwin\bin\bash.exe --login --norc -eo pipefail -o igncr '{0}' diff --git a/deps/hiredis/.travis.yml b/deps/hiredis/.travis.yml index f9a9460ff..1e9b5569f 100644 --- a/deps/hiredis/.travis.yml +++ b/deps/hiredis/.travis.yml @@ -17,11 +17,11 @@ branches: - /^release\/.*$/ install: - - if [ "$BITS" == "64" ]; then + - if [ "$TRAVIS_COMPILER" != "mingw" ]; then wget https://github.com/redis/redis/archive/6.0.6.tar.gz; tar -xzvf 6.0.6.tar.gz; pushd redis-6.0.6 && BUILD_TLS=yes make && export PATH=$PWD/src:$PATH && popd; - fi + fi; before_script: - if [ "$TRAVIS_OS_NAME" == "osx" ]; then @@ -33,8 +33,6 @@ before_script: addons: apt: - sources: - - sourceline: 'ppa:chris-lea/redis-server' packages: - libc6-dbg - libc6-dev @@ -46,17 +44,13 @@ addons: - libssl-dev - libssl-dev:i386 - valgrind - - redis env: - BITS="32" - BITS="64" script: - - EXTRA_CMAKE_OPTS="-DENABLE_EXAMPLES:BOOL=ON -DENABLE_SSL:BOOL=ON"; - if [ "$BITS" == "64" ]; then - EXTRA_CMAKE_OPTS="$EXTRA_CMAKE_OPTS -DENABLE_SSL_TESTS:BOOL=ON"; - fi; + - EXTRA_CMAKE_OPTS="-DENABLE_EXAMPLES:BOOL=ON -DENABLE_SSL:BOOL=ON -DENABLE_SSL_TESTS:BOOL=ON"; if [ "$TRAVIS_OS_NAME" == "osx" ]; then if [ "$BITS" == "32" ]; then CFLAGS="-m32 -Werror"; diff --git a/deps/hiredis/CHANGELOG.md b/deps/hiredis/CHANGELOG.md index 271f1fcf3..2a2bc314a 100644 --- a/deps/hiredis/CHANGELOG.md +++ b/deps/hiredis/CHANGELOG.md @@ -1,3 +1,22 @@ +## [1.0.2](https://github.com/redis/hiredis/tree/v1.0.2) - (2021-10-07) + +Announcing Hiredis v1.0.2, which fixes CVE-2021-32765 but returns the SONAME to the correct value of `1.0.0`. + +- [Revert SONAME bump](https://github.com/redis/hiredis/commit/d4e6f109a064690cde64765c654e679fea1d3548) + ([Michael Grunder](https://github.com/michael-grunder)) + +## [1.0.1](https://github.com/redis/hiredis/tree/v1.0.1) - (2021-10-04) + +<span style="color:red">This release erroneously bumped the SONAME, please use [1.0.2](https://github.com/redis/hiredis/tree/v1.0.2)</span> + +Announcing Hiredis v1.0.1, a security release fixing CVE-2021-32765 + +- Fix for [CVE-2021-32765](https://github.com/redis/hiredis/security/advisories/GHSA-hfm9-39pp-55p2) + [commit](https://github.com/redis/hiredis/commit/76a7b10005c70babee357a7d0f2becf28ec7ed1e) + ([Yossi Gottlieb](https://github.com/yossigo)) + +_Thanks to [Yossi Gottlieb](https://github.com/yossigo) for the security fix and to [Microsoft Security Vulnerability Research](https://www.microsoft.com/en-us/msrc/msvr) for finding the bug._ :sparkling_heart: + ## [1.0.0](https://github.com/redis/hiredis/tree/v1.0.0) - (2020-08-03) Announcing Hiredis v1.0.0, which adds support for RESP3, SSL connections, allocator injection, and better Windows support! :tada: diff --git a/deps/hiredis/CMakeLists.txt b/deps/hiredis/CMakeLists.txt index f86c9b70b..fe6720b28 100644 --- a/deps/hiredis/CMakeLists.txt +++ b/deps/hiredis/CMakeLists.txt @@ -1,10 +1,9 @@ CMAKE_MINIMUM_REQUIRED(VERSION 3.4.0) -INCLUDE(GNUInstallDirs) -PROJECT(hiredis) OPTION(ENABLE_SSL "Build hiredis_ssl for SSL support" OFF) OPTION(DISABLE_TESTS "If tests should be compiled or not" OFF) -OPTION(ENABLE_SSL_TESTS, "Should we test SSL connections" OFF) +OPTION(ENABLE_SSL_TESTS "Should we test SSL connections" OFF) +OPTION(ENABLE_ASYNC_TESTS "Should we run all asynchronous API tests" OFF) MACRO(getVersionBit name) SET(VERSION_REGEX "^#define ${name} (.+)$") @@ -20,7 +19,13 @@ getVersionBit(HIREDIS_SONAME) SET(VERSION "${HIREDIS_MAJOR}.${HIREDIS_MINOR}.${HIREDIS_PATCH}") MESSAGE("Detected version: ${VERSION}") -PROJECT(hiredis VERSION "${VERSION}") +PROJECT(hiredis LANGUAGES "C" VERSION "${VERSION}") +INCLUDE(GNUInstallDirs) + +# Hiredis requires C99 +SET(CMAKE_C_STANDARD 99) +SET(CMAKE_POSITION_INDEPENDENT_CODE ON) +SET(CMAKE_DEBUG_POSTFIX d) SET(ENABLE_EXAMPLES OFF CACHE BOOL "Enable building hiredis examples") @@ -41,30 +46,84 @@ IF(WIN32) ENDIF() ADD_LIBRARY(hiredis SHARED ${hiredis_sources}) +ADD_LIBRARY(hiredis_static STATIC ${hiredis_sources}) +ADD_LIBRARY(hiredis::hiredis ALIAS hiredis) +ADD_LIBRARY(hiredis::hiredis_static ALIAS hiredis_static) SET_TARGET_PROPERTIES(hiredis PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS TRUE VERSION "${HIREDIS_SONAME}") +SET_TARGET_PROPERTIES(hiredis_static + PROPERTIES COMPILE_PDB_NAME hiredis_static) +SET_TARGET_PROPERTIES(hiredis_static + PROPERTIES COMPILE_PDB_NAME_DEBUG hiredis_static${CMAKE_DEBUG_POSTFIX}) IF(WIN32 OR MINGW) - TARGET_LINK_LIBRARIES(hiredis PRIVATE ws2_32) + TARGET_LINK_LIBRARIES(hiredis PUBLIC ws2_32 crypt32) + TARGET_LINK_LIBRARIES(hiredis_static PUBLIC ws2_32 crypt32) +ELSEIF(CMAKE_SYSTEM_NAME MATCHES "FreeBSD") + TARGET_LINK_LIBRARIES(hiredis PUBLIC m) + TARGET_LINK_LIBRARIES(hiredis_static PUBLIC m) +ELSEIF(CMAKE_SYSTEM_NAME MATCHES "SunOS") + TARGET_LINK_LIBRARIES(hiredis PUBLIC socket) + TARGET_LINK_LIBRARIES(hiredis_static PUBLIC socket) ENDIF() -TARGET_INCLUDE_DIRECTORIES(hiredis PUBLIC $<INSTALL_INTERFACE:.> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>) +TARGET_INCLUDE_DIRECTORIES(hiredis PUBLIC $<INSTALL_INTERFACE:include> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>) +TARGET_INCLUDE_DIRECTORIES(hiredis_static PUBLIC $<INSTALL_INTERFACE:include> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>) CONFIGURE_FILE(hiredis.pc.in hiredis.pc @ONLY) -INSTALL(TARGETS hiredis +set(CPACK_PACKAGE_VENDOR "Redis") +set(CPACK_PACKAGE_DESCRIPTION "\ +Hiredis is a minimalistic C client library for the Redis database. + +It is minimalistic because it just adds minimal support for the protocol, \ +but at the same time it uses a high level printf-alike API in order to make \ +it much higher level than otherwise suggested by its minimal code base and the \ +lack of explicit bindings for every Redis command. + +Apart from supporting sending commands and receiving replies, it comes with a \ +reply parser that is decoupled from the I/O layer. It is a stream parser designed \ +for easy reusability, which can for instance be used in higher level language bindings \ +for efficient reply parsing. + +Hiredis only supports the binary-safe Redis protocol, so you can use it with any Redis \ +version >= 1.2.0. + +The library comes with multiple APIs. There is the synchronous API, the asynchronous API \ +and the reply parsing API.") +set(CPACK_PACKAGE_HOMEPAGE_URL "https://github.com/redis/hiredis") +set(CPACK_PACKAGE_CONTACT "michael dot grunder at gmail dot com") +set(CPACK_DEBIAN_PACKAGE_SHLIBDEPS ON) +set(CPACK_RPM_PACKAGE_AUTOREQPROV ON) + +include(CPack) + +INSTALL(TARGETS hiredis hiredis_static EXPORT hiredis-targets RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}) +if (MSVC) + INSTALL(FILES $<TARGET_PDB_FILE:hiredis> + DESTINATION ${CMAKE_INSTALL_BINDIR} + CONFIGURATIONS Debug RelWithDebInfo) + INSTALL(FILES $<TARGET_FILE_DIR:hiredis_static>/$<TARGET_FILE_BASE_NAME:hiredis_static>.pdb + DESTINATION ${CMAKE_INSTALL_LIBDIR} + CONFIGURATIONS Debug RelWithDebInfo) +endif() + +# For NuGet packages +INSTALL(FILES hiredis.targets + DESTINATION build/native) + INSTALL(FILES hiredis.h read.h sds.h async.h alloc.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/hiredis) - + INSTALL(DIRECTORY adapters DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/hiredis) - + INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/hiredis.pc DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig) @@ -95,10 +154,12 @@ IF(ENABLE_SSL) ENDIF() ENDIF() FIND_PACKAGE(OpenSSL REQUIRED) - SET(hiredis_ssl_sources + SET(hiredis_ssl_sources ssl.c) ADD_LIBRARY(hiredis_ssl SHARED ${hiredis_ssl_sources}) + ADD_LIBRARY(hiredis_ssl_static STATIC + ${hiredis_ssl_sources}) IF (APPLE) SET_PROPERTY(TARGET hiredis_ssl PROPERTY LINK_FLAGS "-Wl,-undefined -Wl,dynamic_lookup") @@ -108,23 +169,39 @@ IF(ENABLE_SSL) PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS TRUE VERSION "${HIREDIS_SONAME}") + SET_TARGET_PROPERTIES(hiredis_ssl_static + PROPERTIES COMPILE_PDB_NAME hiredis_ssl_static) + SET_TARGET_PROPERTIES(hiredis_ssl_static + PROPERTIES COMPILE_PDB_NAME_DEBUG hiredis_ssl_static${CMAKE_DEBUG_POSTFIX}) TARGET_INCLUDE_DIRECTORIES(hiredis_ssl PRIVATE "${OPENSSL_INCLUDE_DIR}") + TARGET_INCLUDE_DIRECTORIES(hiredis_ssl_static PRIVATE "${OPENSSL_INCLUDE_DIR}") + TARGET_LINK_LIBRARIES(hiredis_ssl PRIVATE ${OPENSSL_LIBRARIES}) IF (WIN32 OR MINGW) TARGET_LINK_LIBRARIES(hiredis_ssl PRIVATE hiredis) + TARGET_LINK_LIBRARIES(hiredis_ssl_static PUBLIC hiredis_static) ENDIF() CONFIGURE_FILE(hiredis_ssl.pc.in hiredis_ssl.pc @ONLY) - INSTALL(TARGETS hiredis_ssl + INSTALL(TARGETS hiredis_ssl hiredis_ssl_static EXPORT hiredis_ssl-targets RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}) + if (MSVC) + INSTALL(FILES $<TARGET_PDB_FILE:hiredis_ssl> + DESTINATION ${CMAKE_INSTALL_BINDIR} + CONFIGURATIONS Debug RelWithDebInfo) + INSTALL(FILES $<TARGET_FILE_DIR:hiredis_ssl_static>/$<TARGET_FILE_BASE_NAME:hiredis_ssl_static>.pdb + DESTINATION ${CMAKE_INSTALL_LIBDIR} + CONFIGURATIONS Debug RelWithDebInfo) + endif() + INSTALL(FILES hiredis_ssl.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/hiredis) - + INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/hiredis_ssl.pc DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig) @@ -149,11 +226,14 @@ ENDIF() IF(NOT DISABLE_TESTS) ENABLE_TESTING() ADD_EXECUTABLE(hiredis-test test.c) + TARGET_LINK_LIBRARIES(hiredis-test hiredis) IF(ENABLE_SSL_TESTS) ADD_DEFINITIONS(-DHIREDIS_TEST_SSL=1) - TARGET_LINK_LIBRARIES(hiredis-test hiredis hiredis_ssl) - ELSE() - TARGET_LINK_LIBRARIES(hiredis-test hiredis) + TARGET_LINK_LIBRARIES(hiredis-test hiredis_ssl) + ENDIF() + IF(ENABLE_ASYNC_TESTS) + ADD_DEFINITIONS(-DHIREDIS_TEST_ASYNC=1) + TARGET_LINK_LIBRARIES(hiredis-test event) ENDIF() ADD_TEST(NAME hiredis-test COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/test.sh) diff --git a/deps/hiredis/Makefile b/deps/hiredis/Makefile index 7e41c97a5..a2ad84c6b 100644 --- a/deps/hiredis/Makefile +++ b/deps/hiredis/Makefile @@ -4,16 +4,10 @@ # This file is released under the BSD license, see the COPYING file OBJ=alloc.o net.o hiredis.o sds.o async.o read.o sockcompat.o -SSL_OBJ=ssl.o EXAMPLES=hiredis-example hiredis-example-libevent hiredis-example-libev hiredis-example-glib hiredis-example-push -ifeq ($(USE_SSL),1) -EXAMPLES+=hiredis-example-ssl hiredis-example-libevent-ssl -endif TESTS=hiredis-test LIBNAME=libhiredis PKGCONFNAME=hiredis.pc -SSL_LIBNAME=libhiredis_ssl -SSL_PKGCONFNAME=hiredis_ssl.pc HIREDIS_MAJOR=$(shell grep HIREDIS_MAJOR hiredis.h | awk '{print $$3}') HIREDIS_MINOR=$(shell grep HIREDIS_MINOR hiredis.h | awk '{print $$3}') @@ -60,33 +54,66 @@ DYLIB_MAKE_CMD=$(CC) -shared -Wl,-soname,$(DYLIB_MINOR_NAME) STLIBNAME=$(LIBNAME).$(STLIBSUFFIX) STLIB_MAKE_CMD=$(AR) rcs +#################### SSL variables start #################### +SSL_OBJ=ssl.o +SSL_LIBNAME=libhiredis_ssl +SSL_PKGCONFNAME=hiredis_ssl.pc +SSL_INSTALLNAME=install-ssl SSL_DYLIB_MINOR_NAME=$(SSL_LIBNAME).$(DYLIBSUFFIX).$(HIREDIS_SONAME) SSL_DYLIB_MAJOR_NAME=$(SSL_LIBNAME).$(DYLIBSUFFIX).$(HIREDIS_MAJOR) SSL_DYLIBNAME=$(SSL_LIBNAME).$(DYLIBSUFFIX) SSL_STLIBNAME=$(SSL_LIBNAME).$(STLIBSUFFIX) SSL_DYLIB_MAKE_CMD=$(CC) -shared -Wl,-soname,$(SSL_DYLIB_MINOR_NAME) +USE_SSL?=0 +ifeq ($(USE_SSL),1) + # This is required for test.c only + CFLAGS+=-DHIREDIS_TEST_SSL + EXAMPLES+=hiredis-example-ssl hiredis-example-libevent-ssl + SSL_STLIB=$(SSL_STLIBNAME) + SSL_DYLIB=$(SSL_DYLIBNAME) + SSL_PKGCONF=$(SSL_PKGCONFNAME) + SSL_INSTALL=$(SSL_INSTALLNAME) +else + SSL_STLIB= + SSL_DYLIB= + SSL_PKGCONF= + SSL_INSTALL= +endif +##################### SSL variables end ##################### + + # Platform-specific overrides uname_S := $(shell sh -c 'uname -s 2>/dev/null || echo not') -USE_SSL?=0 - # This is required for test.c only -ifeq ($(USE_SSL),1) - CFLAGS+=-DHIREDIS_TEST_SSL +ifeq ($(TEST_ASYNC),1) + export CFLAGS+=-DHIREDIS_TEST_ASYNC endif -ifeq ($(uname_S),Linux) - ifdef OPENSSL_PREFIX +ifeq ($(USE_SSL),1) + ifeq ($(uname_S),Linux) + ifdef OPENSSL_PREFIX + CFLAGS+=-I$(OPENSSL_PREFIX)/include + SSL_LDFLAGS+=-L$(OPENSSL_PREFIX)/lib -lssl -lcrypto + else + SSL_LDFLAGS=-lssl -lcrypto + endif + else + OPENSSL_PREFIX?=/usr/local/opt/openssl CFLAGS+=-I$(OPENSSL_PREFIX)/include SSL_LDFLAGS+=-L$(OPENSSL_PREFIX)/lib -lssl -lcrypto - else - SSL_LDFLAGS=-lssl -lcrypto + endif +endif + +ifeq ($(uname_S),FreeBSD) + LDFLAGS+=-lm + IS_GCC=$(shell sh -c '$(CC) --version 2>/dev/null |egrep -i -c "gcc"') + ifeq ($(IS_GCC),1) + REAL_CFLAGS+=-pedantic endif else - OPENSSL_PREFIX?=/usr/local/opt/openssl - CFLAGS+=-I$(OPENSSL_PREFIX)/include - SSL_LDFLAGS+=-L$(OPENSSL_PREFIX)/lib -lssl -lcrypto + REAL_CFLAGS+=-pedantic endif ifeq ($(uname_S),SunOS) @@ -108,10 +135,13 @@ ifeq ($(uname_S),Darwin) DYLIB_PLUGIN=-Wl,-undefined -Wl,dynamic_lookup endif -all: $(DYLIBNAME) $(STLIBNAME) hiredis-test $(PKGCONFNAME) -ifeq ($(USE_SSL),1) -all: $(SSL_DYLIBNAME) $(SSL_STLIBNAME) $(SSL_PKGCONFNAME) -endif +all: dynamic static hiredis-test pkgconfig + +dynamic: $(DYLIBNAME) $(SSL_DYLIB) + +static: $(STLIBNAME) $(SSL_STLIB) + +pkgconfig: $(PKGCONFNAME) $(SSL_PKGCONF) # Deps (use make dep to generate this) alloc.o: alloc.c fmacros.h alloc.h @@ -122,7 +152,6 @@ net.o: net.c fmacros.h net.h hiredis.h read.h sds.h alloc.h sockcompat.h win32.h read.o: read.c fmacros.h alloc.h read.h sds.h win32.h sds.o: sds.c sds.h sdsalloc.h alloc.h sockcompat.o: sockcompat.c sockcompat.h -ssl.o: ssl.c hiredis.h read.h sds.h alloc.h async.h win32.h async_private.h test.o: test.c fmacros.h hiredis.h read.h sds.h alloc.h net.h sockcompat.h win32.h $(DYLIBNAME): $(OBJ) @@ -131,18 +160,15 @@ $(DYLIBNAME): $(OBJ) $(STLIBNAME): $(OBJ) $(STLIB_MAKE_CMD) $(STLIBNAME) $(OBJ) +#################### SSL building rules start #################### $(SSL_DYLIBNAME): $(SSL_OBJ) $(SSL_DYLIB_MAKE_CMD) $(DYLIB_PLUGIN) -o $(SSL_DYLIBNAME) $(SSL_OBJ) $(REAL_LDFLAGS) $(LDFLAGS) $(SSL_LDFLAGS) $(SSL_STLIBNAME): $(SSL_OBJ) $(STLIB_MAKE_CMD) $(SSL_STLIBNAME) $(SSL_OBJ) -dynamic: $(DYLIBNAME) -static: $(STLIBNAME) -ifeq ($(USE_SSL),1) -dynamic: $(SSL_DYLIBNAME) -static: $(SSL_STLIBNAME) -endif +$(SSL_OBJ): ssl.c hiredis.h read.h sds.h alloc.h async.h win32.h async_private.h +#################### SSL building rules end #################### # Binaries: hiredis-example-libevent: examples/example-libevent.c adapters/libevent.h $(STLIBNAME) @@ -166,7 +192,6 @@ hiredis-example-macosx: examples/example-macosx.c adapters/macosx.h $(STLIBNAME) hiredis-example-ssl: examples/example-ssl.c $(STLIBNAME) $(SSL_STLIBNAME) $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< $(STLIBNAME) $(SSL_STLIBNAME) $(REAL_LDFLAGS) $(SSL_LDFLAGS) - ifndef AE_DIR hiredis-example-ae: @echo "Please specify AE_DIR (e.g. <redis repository>/src)" @@ -177,10 +202,11 @@ hiredis-example-ae: examples/example-ae.c adapters/ae.h $(STLIBNAME) endif ifndef LIBUV_DIR -hiredis-example-libuv: - @echo "Please specify LIBUV_DIR (e.g. ../libuv/)" - @false +# dynamic link libuv.so +hiredis-example-libuv: examples/example-libuv.c adapters/libuv.h $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. -I$(LIBUV_DIR)/include $< -luv -lpthread -lrt $(STLIBNAME) $(REAL_LDFLAGS) else +# use user provided static lib hiredis-example-libuv: examples/example-libuv.c adapters/libuv.h $(STLIBNAME) $(CC) -o examples/$@ $(REAL_CFLAGS) -I. -I$(LIBUV_DIR)/include $< $(LIBUV_DIR)/.libs/libuv.a -lpthread -lrt $(STLIBNAME) $(REAL_LDFLAGS) endif @@ -206,10 +232,13 @@ hiredis-example-push: examples/example-push.c $(STLIBNAME) examples: $(EXAMPLES) -TEST_LIBS = $(STLIBNAME) +TEST_LIBS = $(STLIBNAME) $(SSL_STLIB) +TEST_LDFLAGS = $(SSL_LDFLAGS) ifeq ($(USE_SSL),1) - TEST_LIBS += $(SSL_STLIBNAME) - TEST_LDFLAGS = $(SSL_LDFLAGS) -lssl -lcrypto -lpthread + TEST_LDFLAGS += -pthread +endif +ifeq ($(TEST_ASYNC),1) + TEST_LDFLAGS += -levent endif hiredis-test: test.o $(TEST_LIBS) @@ -225,7 +254,7 @@ check: hiredis-test TEST_SSL=$(USE_SSL) ./test.sh .c.o: - $(CC) -std=c99 -pedantic -c $(REAL_CFLAGS) $< + $(CC) -std=c99 -c $(REAL_CFLAGS) $< clean: rm -rf $(DYLIBNAME) $(STLIBNAME) $(SSL_DYLIBNAME) $(SSL_STLIBNAME) $(TESTS) $(PKGCONFNAME) examples/hiredis-example* *.o *.gcda *.gcno *.gcov @@ -262,7 +291,7 @@ $(SSL_PKGCONFNAME): hiredis_ssl.h @echo Libs: -L\$${libdir} -lhiredis_ssl >> $@ @echo Libs.private: -lssl -lcrypto >> $@ -install: $(DYLIBNAME) $(STLIBNAME) $(PKGCONFNAME) +install: $(DYLIBNAME) $(STLIBNAME) $(PKGCONFNAME) $(SSL_INSTALL) mkdir -p $(INSTALL_INCLUDE_PATH) $(INSTALL_INCLUDE_PATH)/adapters $(INSTALL_LIBRARY_PATH) $(INSTALL) hiredis.h async.h read.h sds.h alloc.h $(INSTALL_INCLUDE_PATH) $(INSTALL) adapters/*.h $(INSTALL_INCLUDE_PATH)/adapters @@ -272,9 +301,6 @@ install: $(DYLIBNAME) $(STLIBNAME) $(PKGCONFNAME) mkdir -p $(INSTALL_PKGCONF_PATH) $(INSTALL) $(PKGCONFNAME) $(INSTALL_PKGCONF_PATH) -ifeq ($(USE_SSL),1) -install: install-ssl - install-ssl: $(SSL_DYLIBNAME) $(SSL_STLIBNAME) $(SSL_PKGCONFNAME) mkdir -p $(INSTALL_INCLUDE_PATH) $(INSTALL_LIBRARY_PATH) $(INSTALL) hiredis_ssl.h $(INSTALL_INCLUDE_PATH) @@ -283,7 +309,6 @@ install-ssl: $(SSL_DYLIBNAME) $(SSL_STLIBNAME) $(SSL_PKGCONFNAME) $(INSTALL) $(SSL_STLIBNAME) $(INSTALL_LIBRARY_PATH) mkdir -p $(INSTALL_PKGCONF_PATH) $(INSTALL) $(SSL_PKGCONFNAME) $(INSTALL_PKGCONF_PATH) -endif 32bit: @echo "" @@ -299,12 +324,12 @@ gprof: $(MAKE) CFLAGS="-pg" LDFLAGS="-pg" gcov: - $(MAKE) CFLAGS="-fprofile-arcs -ftest-coverage" LDFLAGS="-fprofile-arcs" + $(MAKE) CFLAGS+="-fprofile-arcs -ftest-coverage" LDFLAGS="-fprofile-arcs" coverage: gcov make check mkdir -p tmp/lcov - lcov -d . -c -o tmp/lcov/hiredis.info + lcov -d . -c --exclude '/usr*' -o tmp/lcov/hiredis.info genhtml --legend -o tmp/lcov/report tmp/lcov/hiredis.info noopt: diff --git a/deps/hiredis/README.md b/deps/hiredis/README.md index 3a22553ea..ed66220c7 100644 --- a/deps/hiredis/README.md +++ b/deps/hiredis/README.md @@ -1,10 +1,11 @@ -[](https://travis-ci.org/redis/hiredis) + +[](https://github.com/redis/hiredis/actions/workflows/build.yml) **This Readme reflects the latest changed in the master branch. See [v1.0.0](https://github.com/redis/hiredis/tree/v1.0.0) for the Readme and documentation for the latest release ([API/ABI history](https://abi-laboratory.pro/?view=timeline&l=hiredis)).** # HIREDIS -Hiredis is a minimalistic C client library for the [Redis](http://redis.io/) database. +Hiredis is a minimalistic C client library for the [Redis](https://redis.io/) database. It is minimalistic because it just adds minimal support for the protocol, but at the same time it uses a high level printf-alike API in order to make it @@ -22,6 +23,12 @@ Redis version >= 1.2.0. The library comes with multiple APIs. There is the *synchronous API*, the *asynchronous API* and the *reply parsing API*. +## Upgrading to `1.0.2` + +<span style="color:red">NOTE: v1.0.1 erroneously bumped SONAME, which is why it is skipped here.</span> + +Version 1.0.2 is simply 1.0.0 with a fix for [CVE-2021-32765](https://github.com/redis/hiredis/security/advisories/GHSA-hfm9-39pp-55p2). They are otherwise identical. + ## Upgrading to `1.0.0` Version 1.0.0 marks the first stable release of Hiredis. @@ -169,7 +176,7 @@ Hiredis also supports every new `RESP3` data type which are as follows. For mor * **`REDIS_REPLY_MAP`**: * An array with the added invariant that there will always be an even number of elements. - The MAP is functionally equivelant to `REDIS_REPLY_ARRAY` except for the previously mentioned invariant. + The MAP is functionally equivalent to `REDIS_REPLY_ARRAY` except for the previously mentioned invariant. * **`REDIS_REPLY_SET`**: * An array response where each entry is unique. @@ -189,7 +196,7 @@ Hiredis also supports every new `RESP3` data type which are as follows. For mor * **`REDIS_REPLY_VERB`**: * A verbatim string, intended to be presented to the user without modification. - The string payload is stored in the `str` memeber, and type data is stored in the `vtype` member (e.g. `txt` for raw text or `md` for markdown). + The string payload is stored in the `str` member, and type data is stored in the `vtype` member (e.g. `txt` for raw text or `md` for markdown). Replies should be freed using the `freeReplyObject()` function. Note that this function will take care of freeing sub-reply objects @@ -261,9 +268,9 @@ a single call to `read(2)`): redisReply *reply; redisAppendCommand(context,"SET foo bar"); redisAppendCommand(context,"GET foo"); -redisGetReply(context,(void *)&reply); // reply for SET +redisGetReply(context,(void**)&reply); // reply for SET freeReplyObject(reply); -redisGetReply(context,(void *)&reply); // reply for GET +redisGetReply(context,(void**)&reply); // reply for GET freeReplyObject(reply); ``` This API can also be used to implement a blocking subscriber: @@ -517,7 +524,7 @@ initialize OpenSSL and create a context. You can do that in two ways: /* An Hiredis SSL context. It holds SSL configuration and can be reused across * many contexts. */ -redisSSLContext *ssl; +redisSSLContext *ssl_context; /* An error variable to indicate what went wrong, if the context fails to * initialize. @@ -532,17 +539,23 @@ redisSSLContextError ssl_error; redisInitOpenSSL(); /* Create SSL context */ -ssl = redisCreateSSLContext( +ssl_context = redisCreateSSLContext( "cacertbundle.crt", /* File name of trusted CA/ca bundle file, optional */ "/path/to/certs", /* Path of trusted certificates, optional */ "client_cert.pem", /* File name of client certificate file, optional */ "client_key.pem", /* File name of client private key, optional */ "redis.mydomain.com", /* Server name to request (SNI), optional */ - &ssl_error - ) != REDIS_OK) { - printf("SSL error: %s\n", redisSSLContextGetError(ssl_error); - /* Abort... */ - } + &ssl_error); + +if(ssl_context == NULL || ssl_error != 0) { + /* Handle error and abort... */ + /* e.g. + printf("SSL error: %s\n", + (ssl_error != 0) ? + redisSSLContextGetError(ssl_error) : "Unknown error"); + // Abort + */ +} /* Create Redis context and establish connection */ c = redisConnect("localhost", 6443); @@ -551,7 +564,7 @@ if (c == NULL || c->err) { } /* Negotiate SSL/TLS */ -if (redisInitiateSSLWithContext(c, ssl) != REDIS_OK) { +if (redisInitiateSSLWithContext(c, ssl_context) != REDIS_OK) { /* Handle error, in c->err / c->errstr */ } ``` diff --git a/deps/hiredis/adapters/libev.h b/deps/hiredis/adapters/libev.h index e1e7bbd99..c59d3da77 100644 --- a/deps/hiredis/adapters/libev.h +++ b/deps/hiredis/adapters/libev.h @@ -46,7 +46,7 @@ typedef struct redisLibevEvents { static void redisLibevReadEvent(EV_P_ ev_io *watcher, int revents) { #if EV_MULTIPLICITY - ((void)loop); + ((void)EV_A); #endif ((void)revents); @@ -56,7 +56,7 @@ static void redisLibevReadEvent(EV_P_ ev_io *watcher, int revents) { static void redisLibevWriteEvent(EV_P_ ev_io *watcher, int revents) { #if EV_MULTIPLICITY - ((void)loop); + ((void)EV_A); #endif ((void)revents); @@ -66,8 +66,9 @@ static void redisLibevWriteEvent(EV_P_ ev_io *watcher, int revents) { static void redisLibevAddRead(void *privdata) { redisLibevEvents *e = (redisLibevEvents*)privdata; +#if EV_MULTIPLICITY struct ev_loop *loop = e->loop; - ((void)loop); +#endif if (!e->reading) { e->reading = 1; ev_io_start(EV_A_ &e->rev); @@ -76,8 +77,9 @@ static void redisLibevAddRead(void *privdata) { static void redisLibevDelRead(void *privdata) { redisLibevEvents *e = (redisLibevEvents*)privdata; +#if EV_MULTIPLICITY struct ev_loop *loop = e->loop; - ((void)loop); +#endif if (e->reading) { e->reading = 0; ev_io_stop(EV_A_ &e->rev); @@ -86,8 +88,9 @@ static void redisLibevDelRead(void *privdata) { static void redisLibevAddWrite(void *privdata) { redisLibevEvents *e = (redisLibevEvents*)privdata; +#if EV_MULTIPLICITY struct ev_loop *loop = e->loop; - ((void)loop); +#endif if (!e->writing) { e->writing = 1; ev_io_start(EV_A_ &e->wev); @@ -96,8 +99,9 @@ static void redisLibevAddWrite(void *privdata) { static void redisLibevDelWrite(void *privdata) { redisLibevEvents *e = (redisLibevEvents*)privdata; +#if EV_MULTIPLICITY struct ev_loop *loop = e->loop; - ((void)loop); +#endif if (e->writing) { e->writing = 0; ev_io_stop(EV_A_ &e->wev); @@ -106,8 +110,9 @@ static void redisLibevDelWrite(void *privdata) { static void redisLibevStopTimer(void *privdata) { redisLibevEvents *e = (redisLibevEvents*)privdata; +#if EV_MULTIPLICITY struct ev_loop *loop = e->loop; - ((void)loop); +#endif ev_timer_stop(EV_A_ &e->timer); } @@ -120,6 +125,9 @@ static void redisLibevCleanup(void *privdata) { } static void redisLibevTimeout(EV_P_ ev_timer *timer, int revents) { +#if EV_MULTIPLICITY + ((void)EV_A); +#endif ((void)revents); redisLibevEvents *e = (redisLibevEvents*)timer->data; redisAsyncHandleTimeout(e->context); @@ -127,8 +135,9 @@ static void redisLibevTimeout(EV_P_ ev_timer *timer, int revents) { static void redisLibevSetTimeout(void *privdata, struct timeval tv) { redisLibevEvents *e = (redisLibevEvents*)privdata; +#if EV_MULTIPLICITY struct ev_loop *loop = e->loop; - ((void)loop); +#endif if (!ev_is_active(&e->timer)) { ev_init(&e->timer, redisLibevTimeout); @@ -154,7 +163,7 @@ static int redisLibevAttach(EV_P_ redisAsyncContext *ac) { e->context = ac; #if EV_MULTIPLICITY - e->loop = loop; + e->loop = EV_A; #else e->loop = NULL; #endif diff --git a/deps/hiredis/adapters/libevent.h b/deps/hiredis/adapters/libevent.h index 9150979bc..73bb8ed75 100644 --- a/deps/hiredis/adapters/libevent.h +++ b/deps/hiredis/adapters/libevent.h @@ -50,7 +50,7 @@ static void redisLibeventDestroy(redisLibeventEvents *e) { hi_free(e); } -static void redisLibeventHandler(int fd, short event, void *arg) { +static void redisLibeventHandler(evutil_socket_t fd, short event, void *arg) { ((void)fd); redisLibeventEvents *e = (redisLibeventEvents*)arg; e->state |= REDIS_LIBEVENT_ENTERED; diff --git a/deps/hiredis/adapters/libuv.h b/deps/hiredis/adapters/libuv.h index c120b1b39..df0a84578 100644 --- a/deps/hiredis/adapters/libuv.h +++ b/deps/hiredis/adapters/libuv.h @@ -7,111 +7,157 @@ #include <string.h> typedef struct redisLibuvEvents { - redisAsyncContext* context; - uv_poll_t handle; - int events; + redisAsyncContext* context; + uv_poll_t handle; + uv_timer_t timer; + int events; } redisLibuvEvents; static void redisLibuvPoll(uv_poll_t* handle, int status, int events) { - redisLibuvEvents* p = (redisLibuvEvents*)handle->data; - int ev = (status ? p->events : events); - - if (p->context != NULL && (ev & UV_READABLE)) { - redisAsyncHandleRead(p->context); - } - if (p->context != NULL && (ev & UV_WRITABLE)) { - redisAsyncHandleWrite(p->context); - } + redisLibuvEvents* p = (redisLibuvEvents*)handle->data; + int ev = (status ? p->events : events); + + if (p->context != NULL && (ev & UV_READABLE)) { + redisAsyncHandleRead(p->context); + } + if (p->context != NULL && (ev & UV_WRITABLE)) { + redisAsyncHandleWrite(p->context); + } } static void redisLibuvAddRead(void *privdata) { - redisLibuvEvents* p = (redisLibuvEvents*)privdata; + redisLibuvEvents* p = (redisLibuvEvents*)privdata; - p->events |= UV_READABLE; + p->events |= UV_READABLE; - uv_poll_start(&p->handle, p->events, redisLibuvPoll); + uv_poll_start(&p->handle, p->events, redisLibuvPoll); } static void redisLibuvDelRead(void *privdata) { - redisLibuvEvents* p = (redisLibuvEvents*)privdata; + redisLibuvEvents* p = (redisLibuvEvents*)privdata; - p->events &= ~UV_READABLE; + p->events &= ~UV_READABLE; - if (p->events) { - uv_poll_start(&p->handle, p->events, redisLibuvPoll); - } else { - uv_poll_stop(&p->handle); - } + if (p->events) { + uv_poll_start(&p->handle, p->events, redisLibuvPoll); + } else { + uv_poll_stop(&p->handle); + } } static void redisLibuvAddWrite(void *privdata) { - redisLibuvEvents* p = (redisLibuvEvents*)privdata; + redisLibuvEvents* p = (redisLibuvEvents*)privdata; - p->events |= UV_WRITABLE; + p->events |= UV_WRITABLE; - uv_poll_start(&p->handle, p->events, redisLibuvPoll); + uv_poll_start(&p->handle, p->events, redisLibuvPoll); } static void redisLibuvDelWrite(void *privdata) { - redisLibuvEvents* p = (redisLibuvEvents*)privdata; + redisLibuvEvents* p = (redisLibuvEvents*)privdata; - p->events &= ~UV_WRITABLE; + p->events &= ~UV_WRITABLE; - if (p->events) { - uv_poll_start(&p->handle, p->events, redisLibuvPoll); - } else { - uv_poll_stop(&p->handle); - } + if (p->events) { + uv_poll_start(&p->handle, p->events, redisLibuvPoll); + } else { + uv_poll_stop(&p->handle); + } } +static void on_timer_close(uv_handle_t *handle) { + redisLibuvEvents* p = (redisLibuvEvents*)handle->data; + p->timer.data = NULL; + if (!p->handle.data) { + // both timer and handle are closed + hi_free(p); + } + // else, wait for `on_handle_close` +} -static void on_close(uv_handle_t* handle) { - redisLibuvEvents* p = (redisLibuvEvents*)handle->data; +static void on_handle_close(uv_handle_t *handle) { + redisLibuvEvents* p = (redisLibuvEvents*)handle->data; + p->handle.data = NULL; + if (!p->timer.data) { + // timer never started, or timer already destroyed + hi_free(p); + } + // else, wait for `on_timer_close` +} - hi_free(p); +// libuv removed `status` parameter since v0.11.23 +// see: https://github.com/libuv/libuv/blob/v0.11.23/include/uv.h +#if (UV_VERSION_MAJOR == 0 && UV_VERSION_MINOR < 11) || \ + (UV_VERSION_MAJOR == 0 && UV_VERSION_MINOR == 11 && UV_VERSION_PATCH < 23) +static void redisLibuvTimeout(uv_timer_t *timer, int status) { + (void)status; // unused +#else +static void redisLibuvTimeout(uv_timer_t *timer) { +#endif + redisLibuvEvents *e = (redisLibuvEvents*)timer->data; + redisAsyncHandleTimeout(e->context); } +static void redisLibuvSetTimeout(void *privdata, struct timeval tv) { + redisLibuvEvents* p = (redisLibuvEvents*)privdata; + + uint64_t millsec = tv.tv_sec * 1000 + tv.tv_usec / 1000.0; + if (!p->timer.data) { + // timer is uninitialized + if (uv_timer_init(p->handle.loop, &p->timer) != 0) { + return; + } + p->timer.data = p; + } + // updates the timeout if the timer has already started + // or start the timer + uv_timer_start(&p->timer, redisLibuvTimeout, millsec, 0); +} static void redisLibuvCleanup(void *privdata) { - redisLibuvEvents* p = (redisLibuvEvents*)privdata; + redisLibuvEvents* p = (redisLibuvEvents*)privdata; - p->context = NULL; // indicate that context might no longer exist - uv_close((uv_handle_t*)&p->handle, on_close); + p->context = NULL; // indicate that context might no longer exist + if (p->timer.data) { + uv_close((uv_handle_t*)&p->timer, on_timer_close); + } + uv_close((uv_handle_t*)&p->handle, on_handle_close); } static int redisLibuvAttach(redisAsyncContext* ac, uv_loop_t* loop) { - redisContext *c = &(ac->c); + redisContext *c = &(ac->c); - if (ac->ev.data != NULL) { - return REDIS_ERR; - } + if (ac->ev.data != NULL) { + return REDIS_ERR; + } - ac->ev.addRead = redisLibuvAddRead; - ac->ev.delRead = redisLibuvDelRead; - ac->ev.addWrite = redisLibuvAddWrite; - ac->ev.delWrite = redisLibuvDelWrite; - ac->ev.cleanup = redisLibuvCleanup; + ac->ev.addRead = redisLibuvAddRead; + ac->ev.delRead = redisLibuvDelRead; + ac->ev.addWrite = redisLibuvAddWrite; + ac->ev.delWrite = redisLibuvDelWrite; + ac->ev.cleanup = redisLibuvCleanup; + ac->ev.scheduleTimer = redisLibuvSetTimeout; - redisLibuvEvents* p = (redisLibuvEvents*)hi_malloc(sizeof(*p)); - if (p == NULL) - return REDIS_ERR; + redisLibuvEvents* p = (redisLibuvEvents*)hi_malloc(sizeof(*p)); + if (p == NULL) + return REDIS_ERR; - memset(p, 0, sizeof(*p)); + memset(p, 0, sizeof(*p)); - if (uv_poll_init_socket(loop, &p->handle, c->fd) != 0) { - return REDIS_ERR; - } + if (uv_poll_init_socket(loop, &p->handle, c->fd) != 0) { + return REDIS_ERR; + } - ac->ev.data = p; - p->handle.data = p; - p->context = ac; + ac->ev.data = p; + p->handle.data = p; + p->context = ac; - return REDIS_OK; + return REDIS_OK; } #endif diff --git a/deps/hiredis/async.c b/deps/hiredis/async.c index 64ab601c9..d73d09fb1 100644 --- a/deps/hiredis/async.c +++ b/deps/hiredis/async.c @@ -47,6 +47,11 @@ #include "async_private.h" +#ifdef NDEBUG +#undef assert +#define assert(e) (void)(e) +#endif + /* Forward declarations of hiredis.c functions */ int __redisAppendCommand(redisContext *c, const char *cmd, size_t len); void __redisSetError(redisContext *c, int type, const char *str); @@ -139,8 +144,8 @@ static redisAsyncContext *redisAsyncInitialize(redisContext *c) { ac->replies.head = NULL; ac->replies.tail = NULL; - ac->sub.invalid.head = NULL; - ac->sub.invalid.tail = NULL; + ac->sub.replies.head = NULL; + ac->sub.replies.tail = NULL; ac->sub.channels = channels; ac->sub.patterns = patterns; @@ -301,36 +306,28 @@ static void __redisRunPushCallback(redisAsyncContext *ac, redisReply *reply) { static void __redisAsyncFree(redisAsyncContext *ac) { redisContext *c = &(ac->c); redisCallback cb; - dictIterator *it; + dictIterator it; dictEntry *de; /* Execute pending callbacks with NULL reply. */ while (__redisShiftCallback(&ac->replies,&cb) == REDIS_OK) __redisRunCallback(ac,&cb,NULL); - - /* Execute callbacks for invalid commands */ - while (__redisShiftCallback(&ac->sub.invalid,&cb) == REDIS_OK) + while (__redisShiftCallback(&ac->sub.replies,&cb) == REDIS_OK) __redisRunCallback(ac,&cb,NULL); /* Run subscription callbacks with NULL reply */ if (ac->sub.channels) { - it = dictGetIterator(ac->sub.channels); - if (it != NULL) { - while ((de = dictNext(it)) != NULL) - __redisRunCallback(ac,dictGetEntryVal(de),NULL); - dictReleaseIterator(it); - } + dictInitIterator(&it,ac->sub.channels); + while ((de = dictNext(&it)) != NULL) + __redisRunCallback(ac,dictGetEntryVal(de),NULL); dictRelease(ac->sub.channels); } if (ac->sub.patterns) { - it = dictGetIterator(ac->sub.patterns); - if (it != NULL) { - while ((de = dictNext(it)) != NULL) - __redisRunCallback(ac,dictGetEntryVal(de),NULL); - dictReleaseIterator(it); - } + dictInitIterator(&it,ac->sub.patterns); + while ((de = dictNext(&it)) != NULL) + __redisRunCallback(ac,dictGetEntryVal(de),NULL); dictRelease(ac->sub.patterns); } @@ -420,10 +417,11 @@ static int __redisGetSubscribeCallback(redisAsyncContext *ac, redisReply *reply, char *stype; hisds sname; - /* Custom reply functions are not supported for pub/sub. This will fail - * very hard when they are used... */ - if (reply->type == REDIS_REPLY_ARRAY || reply->type == REDIS_REPLY_PUSH) { - assert(reply->elements >= 2); + /* Match reply with the expected format of a pushed message. + * The type and number of elements (3 to 4) are specified at: + * https://redis.io/topics/pubsub#format-of-pushed-messages */ + if ((reply->type == REDIS_REPLY_ARRAY && !(c->flags & REDIS_SUPPORTS_PUSH) && reply->elements >= 3) || + reply->type == REDIS_REPLY_PUSH) { assert(reply->element[0]->type == REDIS_REPLY_STRING); stype = reply->element[0]->str; pvariant = (tolower(stype[0]) == 'p') ? 1 : 0; @@ -462,14 +460,21 @@ static int __redisGetSubscribeCallback(redisAsyncContext *ac, redisReply *reply, /* Unset subscribed flag only when no pipelined pending subscribe. */ if (reply->element[2]->integer == 0 && dictSize(ac->sub.channels) == 0 - && dictSize(ac->sub.patterns) == 0) + && dictSize(ac->sub.patterns) == 0) { c->flags &= ~REDIS_SUBSCRIBED; + + /* Move ongoing regular command callbacks. */ + redisCallback cb; + while (__redisShiftCallback(&ac->sub.replies,&cb) == REDIS_OK) { + __redisPushCallback(&ac->replies,&cb); + } + } } } hi_sdsfree(sname); } else { - /* Shift callback for invalid commands. */ - __redisShiftCallback(&ac->sub.invalid,dstcb); + /* Shift callback for pending command in subscribed context. */ + __redisShiftCallback(&ac->sub.replies,dstcb); } return REDIS_OK; oom: @@ -497,13 +502,12 @@ static int redisIsSubscribeReply(redisReply *reply) { len = reply->element[0]->len - off; return !strncasecmp(str, "subscribe", len) || - !strncasecmp(str, "message", len); - + !strncasecmp(str, "message", len) || + !strncasecmp(str, "unsubscribe", len); } void redisProcessCallbacks(redisAsyncContext *ac) { redisContext *c = &(ac->c); - redisCallback cb = {NULL, NULL, 0, NULL}; void *reply = NULL; int status; @@ -516,17 +520,14 @@ void redisProcessCallbacks(redisAsyncContext *ac) { __redisAsyncDisconnect(ac); return; } - - /* If monitor mode, repush callback */ - if(c->flags & REDIS_MONITORING) { - __redisPushCallback(&ac->replies,&cb); - } - /* When the connection is not being disconnected, simply stop * trying to get replies and wait for the next loop tick. */ break; } + /* Keep track of push message support for subscribe handling */ + if (redisIsPushReply(reply)) c->flags |= REDIS_SUPPORTS_PUSH; + /* Send any non-subscribe related PUSH messages to our PUSH handler * while allowing subscribe related PUSH messages to pass through. * This allows existing code to be backward compatible and work in @@ -539,6 +540,7 @@ void redisProcessCallbacks(redisAsyncContext *ac) { /* Even if the context is subscribed, pending regular * callbacks will get a reply before pub/sub messages arrive. */ + redisCallback cb = {NULL, NULL, 0, NULL}; if (__redisShiftCallback(&ac->replies,&cb) != REDIS_OK) { /* * A spontaneous reply in a not-subscribed context can be the error @@ -562,15 +564,17 @@ void redisProcessCallbacks(redisAsyncContext *ac) { __redisAsyncDisconnect(ac); return; } - /* No more regular callbacks and no errors, the context *must* be subscribed or monitoring. */ - assert((c->flags & REDIS_SUBSCRIBED || c->flags & REDIS_MONITORING)); - if(c->flags & REDIS_SUBSCRIBED) + /* No more regular callbacks and no errors, the context *must* be subscribed. */ + assert(c->flags & REDIS_SUBSCRIBED); + if (c->flags & REDIS_SUBSCRIBED) __redisGetSubscribeCallback(ac,reply,&cb); } if (cb.fn != NULL) { __redisRunCallback(ac,&cb,reply); - c->reader->fn->freeObject(reply); + if (!(c->flags & REDIS_NO_AUTO_FREE_REPLIES)){ + c->reader->fn->freeObject(reply); + } /* Proceed with free'ing when redisAsyncFree() was called. */ if (c->flags & REDIS_FREEING) { @@ -584,6 +588,11 @@ void redisProcessCallbacks(redisAsyncContext *ac) { * doesn't know what the server will spit out over the wire. */ c->reader->fn->freeObject(reply); } + + /* If in monitor mode, repush the callback */ + if (c->flags & REDIS_MONITORING) { + __redisPushCallback(&ac->replies,&cb); + } } /* Disconnect when there was an error reading the reply */ @@ -605,7 +614,8 @@ static int __redisAsyncHandleConnect(redisAsyncContext *ac) { if (redisCheckConnectDone(c, &completed) == REDIS_ERR) { /* Error! */ - redisCheckSocketError(c); + if (redisCheckSocketError(c) == REDIS_ERR) + __redisAsyncCopyError(ac); __redisAsyncHandleConnectFailure(ac); return REDIS_ERR; } else if (completed == 1) { @@ -691,13 +701,22 @@ void redisAsyncHandleTimeout(redisAsyncContext *ac) { redisContext *c = &(ac->c); redisCallback cb; - if ((c->flags & REDIS_CONNECTED) && ac->replies.head == NULL) { - /* Nothing to do - just an idle timeout */ - return; + if ((c->flags & REDIS_CONNECTED)) { + if (ac->replies.head == NULL && ac->sub.replies.head == NULL) { + /* Nothing to do - just an idle timeout */ + return; + } + + if (!ac->c.command_timeout || + (!ac->c.command_timeout->tv_sec && !ac->c.command_timeout->tv_usec)) { + /* A belated connect timeout arriving, ignore */ + return; + } } if (!c->err) { __redisSetError(c, REDIS_ERR_TIMEOUT, "Timeout"); + __redisAsyncCopyError(ac); } if (!(c->flags & REDIS_CONNECTED) && ac->onConnect) { @@ -796,17 +815,19 @@ static int __redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void /* (P)UNSUBSCRIBE does not have its own response: every channel or * pattern that is unsubscribed will receive a message. This means we * should not append a callback function for this command. */ - } else if(strncasecmp(cstr,"monitor\r\n",9) == 0) { - /* Set monitor flag and push callback */ - c->flags |= REDIS_MONITORING; - __redisPushCallback(&ac->replies,&cb); + } else if (strncasecmp(cstr,"monitor\r\n",9) == 0) { + /* Set monitor flag and push callback */ + c->flags |= REDIS_MONITORING; + if (__redisPushCallback(&ac->replies,&cb) != REDIS_OK) + goto oom; } else { - if (c->flags & REDIS_SUBSCRIBED) - /* This will likely result in an error reply, but it needs to be - * received and passed to the callback. */ - __redisPushCallback(&ac->sub.invalid,&cb); - else - __redisPushCallback(&ac->replies,&cb); + if (c->flags & REDIS_SUBSCRIBED) { + if (__redisPushCallback(&ac->sub.replies,&cb) != REDIS_OK) + goto oom; + } else { + if (__redisPushCallback(&ac->replies,&cb) != REDIS_OK) + goto oom; + } } __redisAppendCommand(c,cmd,len); @@ -817,6 +838,7 @@ static int __redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void return REDIS_OK; oom: __redisSetError(&(ac->c), REDIS_ERR_OOM, "Out of memory"); + __redisAsyncCopyError(ac); return REDIS_ERR; } @@ -846,7 +868,7 @@ int redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata int redisAsyncCommandArgv(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, int argc, const char **argv, const size_t *argvlen) { hisds cmd; - int len; + long long len; int status; len = redisFormatSdsCommandArgv(&cmd,argc,argv,argvlen); if (len < 0) diff --git a/deps/hiredis/async.h b/deps/hiredis/async.h index b1d2cb263..4c65203c1 100644 --- a/deps/hiredis/async.h +++ b/deps/hiredis/async.h @@ -102,7 +102,7 @@ typedef struct redisAsyncContext { /* Subscription callbacks */ struct { - redisCallbackList invalid; + redisCallbackList replies; struct dict *channels; struct dict *patterns; } sub; diff --git a/deps/hiredis/dict.c b/deps/hiredis/dict.c index 34a33ead9..ad571818e 100644 --- a/deps/hiredis/dict.c +++ b/deps/hiredis/dict.c @@ -267,16 +267,11 @@ static dictEntry *dictFind(dict *ht, const void *key) { return NULL; } -static dictIterator *dictGetIterator(dict *ht) { - dictIterator *iter = hi_malloc(sizeof(*iter)); - if (iter == NULL) - return NULL; - +static void dictInitIterator(dictIterator *iter, dict *ht) { iter->ht = ht; iter->index = -1; iter->entry = NULL; iter->nextEntry = NULL; - return iter; } static dictEntry *dictNext(dictIterator *iter) { @@ -299,10 +294,6 @@ static dictEntry *dictNext(dictIterator *iter) { return NULL; } -static void dictReleaseIterator(dictIterator *iter) { - hi_free(iter); -} - /* ------------------------- private functions ------------------------------ */ /* Expand the hash table if needed */ diff --git a/deps/hiredis/dict.h b/deps/hiredis/dict.h index 95fcd280e..6ad0acd8d 100644 --- a/deps/hiredis/dict.h +++ b/deps/hiredis/dict.h @@ -119,8 +119,7 @@ static int dictReplace(dict *ht, void *key, void *val); static int dictDelete(dict *ht, const void *key); static void dictRelease(dict *ht); static dictEntry * dictFind(dict *ht, const void *key); -static dictIterator *dictGetIterator(dict *ht); +static void dictInitIterator(dictIterator *iter, dict *ht); static dictEntry *dictNext(dictIterator *iter); -static void dictReleaseIterator(dictIterator *iter); #endif /* __DICT_H */ diff --git a/deps/hiredis/examples/CMakeLists.txt b/deps/hiredis/examples/CMakeLists.txt index 1d5bc56e0..49cd8d440 100644 --- a/deps/hiredis/examples/CMakeLists.txt +++ b/deps/hiredis/examples/CMakeLists.txt @@ -21,7 +21,7 @@ ENDIF() FIND_PATH(LIBEVENT event.h) if (LIBEVENT) - ADD_EXECUTABLE(example-libevent example-libevent) + ADD_EXECUTABLE(example-libevent example-libevent.c) TARGET_LINK_LIBRARIES(example-libevent hiredis event) ENDIF() diff --git a/deps/hiredis/examples/example-libuv.c b/deps/hiredis/examples/example-libuv.c index cbde452b9..53fd04a8e 100644 --- a/deps/hiredis/examples/example-libuv.c +++ b/deps/hiredis/examples/example-libuv.c @@ -7,18 +7,33 @@ #include <async.h> #include <adapters/libuv.h> +void debugCallback(redisAsyncContext *c, void *r, void *privdata) { + (void)privdata; //unused + redisReply *reply = r; + if (reply == NULL) { + /* The DEBUG SLEEP command will almost always fail, because we have set a 1 second timeout */ + printf("`DEBUG SLEEP` error: %s\n", c->errstr ? c->errstr : "unknown error"); + return; + } + /* Disconnect after receiving the reply of DEBUG SLEEP (which will not)*/ + redisAsyncDisconnect(c); +} + void getCallback(redisAsyncContext *c, void *r, void *privdata) { redisReply *reply = r; - if (reply == NULL) return; - printf("argv[%s]: %s\n", (char*)privdata, reply->str); + if (reply == NULL) { + printf("`GET key` error: %s\n", c->errstr ? c->errstr : "unknown error"); + return; + } + printf("`GET key` result: argv[%s]: %s\n", (char*)privdata, reply->str); - /* Disconnect after receiving the reply to GET */ - redisAsyncDisconnect(c); + /* start another request that demonstrate timeout */ + redisAsyncCommand(c, debugCallback, NULL, "DEBUG SLEEP %f", 1.5); } void connectCallback(const redisAsyncContext *c, int status) { if (status != REDIS_OK) { - printf("Error: %s\n", c->errstr); + printf("connect error: %s\n", c->errstr); return; } printf("Connected...\n"); @@ -26,7 +41,7 @@ void connectCallback(const redisAsyncContext *c, int status) { void disconnectCallback(const redisAsyncContext *c, int status) { if (status != REDIS_OK) { - printf("Error: %s\n", c->errstr); + printf("disconnect because of error: %s\n", c->errstr); return; } printf("Disconnected...\n"); @@ -49,8 +64,18 @@ int main (int argc, char **argv) { redisLibuvAttach(c,loop); redisAsyncSetConnectCallback(c,connectCallback); redisAsyncSetDisconnectCallback(c,disconnectCallback); + redisAsyncSetTimeout(c, (struct timeval){ .tv_sec = 1, .tv_usec = 0}); + + /* + In this demo, we first `set key`, then `get key` to demonstrate the basic usage of libuv adapter. + Then in `getCallback`, we start a `debug sleep` command to create 1.5 second long request. + Because we have set a 1 second timeout to the connection, the command will always fail with a + timeout error, which is shown in the `debugCallback`. + */ + redisAsyncCommand(c, NULL, NULL, "SET key %b", argv[argc-1], strlen(argv[argc-1])); redisAsyncCommand(c, getCallback, (char*)"end-1", "GET key"); + uv_run(loop, UV_RUN_DEFAULT); return 0; } diff --git a/deps/hiredis/examples/example-push.c b/deps/hiredis/examples/example-push.c index 2d4ab4dc0..6bc12055e 100644 --- a/deps/hiredis/examples/example-push.c +++ b/deps/hiredis/examples/example-push.c @@ -31,7 +31,6 @@ #include <stdlib.h> #include <string.h> #include <hiredis.h> -#include <win32.h> #define KEY_COUNT 5 diff --git a/deps/hiredis/examples/example-ssl.c b/deps/hiredis/examples/example-ssl.c index c754177cf..b8ca44281 100644 --- a/deps/hiredis/examples/example-ssl.c +++ b/deps/hiredis/examples/example-ssl.c @@ -4,7 +4,10 @@ #include <hiredis.h> #include <hiredis_ssl.h> -#include <win32.h> + +#ifdef _MSC_VER +#include <winsock2.h> /* For struct timeval */ +#endif int main(int argc, char **argv) { unsigned int j; diff --git a/deps/hiredis/examples/example.c b/deps/hiredis/examples/example.c index 15dacbd18..f1b8b4a85 100644 --- a/deps/hiredis/examples/example.c +++ b/deps/hiredis/examples/example.c @@ -2,7 +2,10 @@ #include <stdlib.h> #include <string.h> #include <hiredis.h> -#include <win32.h> + +#ifdef _MSC_VER +#include <winsock2.h> /* For struct timeval */ +#endif int main(int argc, char **argv) { unsigned int j, isunix = 0; diff --git a/deps/hiredis/fmacros.h b/deps/hiredis/fmacros.h index 3227faafd..754a53c21 100644 --- a/deps/hiredis/fmacros.h +++ b/deps/hiredis/fmacros.h @@ -1,8 +1,10 @@ #ifndef __HIREDIS_FMACRO_H #define __HIREDIS_FMACRO_H +#ifndef _AIX #define _XOPEN_SOURCE 600 #define _POSIX_C_SOURCE 200112L +#endif #if defined(__APPLE__) && defined(__MACH__) /* Enable TCP_KEEPALIVE */ diff --git a/deps/hiredis/fuzzing/format_command_fuzzer.c b/deps/hiredis/fuzzing/format_command_fuzzer.c new file mode 100644 index 000000000..91adeac58 --- /dev/null +++ b/deps/hiredis/fuzzing/format_command_fuzzer.c @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2020, Salvatore Sanfilippo <antirez at gmail dot com> + * Copyright (c) 2020, Pieter Noordhuis <pcnoordhuis at gmail dot com> + * Copyright (c) 2020, Matt Stancliff <matt at genges dot com>, + * Jan-Erik Rediger <janerik at fnordig dot com> + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdlib.h> +#include <string.h> +#include "hiredis.h" + +int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { + char *new_str, *cmd; + + if (size < 3) + return 0; + + new_str = malloc(size+1); + if (new_str == NULL) + return 0; + + memcpy(new_str, data, size); + new_str[size] = '\0'; + + redisFormatCommand(&cmd, new_str); + + if (cmd != NULL) + hi_free(cmd); + free(new_str); + return 0; +} diff --git a/deps/hiredis/hiredis.c b/deps/hiredis/hiredis.c index 51f22a665..f7b61c206 100644 --- a/deps/hiredis/hiredis.c +++ b/deps/hiredis/hiredis.c @@ -96,6 +96,8 @@ void freeReplyObject(void *reply) { switch(r->type) { case REDIS_REPLY_INTEGER: + case REDIS_REPLY_NIL: + case REDIS_REPLY_BOOL: break; /* Nothing to free */ case REDIS_REPLY_ARRAY: case REDIS_REPLY_MAP: @@ -112,6 +114,7 @@ void freeReplyObject(void *reply) { case REDIS_REPLY_STRING: case REDIS_REPLY_DOUBLE: case REDIS_REPLY_VERB: + case REDIS_REPLY_BIGNUM: hi_free(r->str); break; } @@ -129,7 +132,8 @@ static void *createStringObject(const redisReadTask *task, char *str, size_t len assert(task->type == REDIS_REPLY_ERROR || task->type == REDIS_REPLY_STATUS || task->type == REDIS_REPLY_STRING || - task->type == REDIS_REPLY_VERB); + task->type == REDIS_REPLY_VERB || + task->type == REDIS_REPLY_BIGNUM); /* Copy string value */ if (task->type == REDIS_REPLY_VERB) { @@ -235,12 +239,14 @@ static void *createDoubleObject(const redisReadTask *task, double value, char *s * decimal string conversion artifacts. */ memcpy(r->str, str, len); r->str[len] = '\0'; + r->len = len; if (task->parent) { parent = task->parent->obj; assert(parent->type == REDIS_REPLY_ARRAY || parent->type == REDIS_REPLY_MAP || - parent->type == REDIS_REPLY_SET); + parent->type == REDIS_REPLY_SET || + parent->type == REDIS_REPLY_PUSH); parent->element[task->idx] = r; } return r; @@ -277,7 +283,8 @@ static void *createBoolObject(const redisReadTask *task, int bval) { parent = task->parent->obj; assert(parent->type == REDIS_REPLY_ARRAY || parent->type == REDIS_REPLY_MAP || - parent->type == REDIS_REPLY_SET); + parent->type == REDIS_REPLY_SET || + parent->type == REDIS_REPLY_PUSH); parent->element[task->idx] = r; } return r; @@ -565,13 +572,12 @@ int redisFormatCommand(char **target, const char *format, ...) { * lengths. If the latter is set to NULL, strlen will be used to compute the * argument lengths. */ -int redisFormatSdsCommandArgv(hisds *target, int argc, const char **argv, - const size_t *argvlen) +long long redisFormatSdsCommandArgv(hisds *target, int argc, const char **argv, + const size_t *argvlen) { hisds cmd, aux; - unsigned long long totlen; + unsigned long long totlen, len; int j; - size_t len; /* Abort on a NULL target */ if (target == NULL) @@ -602,7 +608,7 @@ int redisFormatSdsCommandArgv(hisds *target, int argc, const char **argv, cmd = hi_sdscatfmt(cmd, "*%i\r\n", argc); for (j=0; j < argc; j++) { len = argvlen ? argvlen[j] : strlen(argv[j]); - cmd = hi_sdscatfmt(cmd, "$%u\r\n", len); + cmd = hi_sdscatfmt(cmd, "$%U\r\n", len); cmd = hi_sdscatlen(cmd, argv[j], len); cmd = hi_sdscatlen(cmd, "\r\n", sizeof("\r\n")-1); } @@ -622,11 +628,11 @@ void redisFreeSdsCommand(hisds cmd) { * lengths. If the latter is set to NULL, strlen will be used to compute the * argument lengths. */ -int redisFormatCommandArgv(char **target, int argc, const char **argv, const size_t *argvlen) { +long long redisFormatCommandArgv(char **target, int argc, const char **argv, const size_t *argvlen) { char *cmd = NULL; /* final command */ - int pos; /* position in final command */ - size_t len; - int totlen, j; + size_t pos; /* position in final command */ + size_t len, totlen; + int j; /* Abort on a NULL target */ if (target == NULL) @@ -797,6 +803,9 @@ redisContext *redisConnectWithOptions(const redisOptions *options) { if (options->options & REDIS_OPT_NOAUTOFREE) { c->flags |= REDIS_NO_AUTO_FREE; } + if (options->options & REDIS_OPT_NOAUTOFREEREPLIES) { + c->flags |= REDIS_NO_AUTO_FREE_REPLIES; + } /* Set any user supplied RESP3 PUSH handler or use freeReplyObject * as a default unless specifically flagged that we don't want one. */ @@ -825,7 +834,7 @@ redisContext *redisConnectWithOptions(const redisOptions *options) { c->fd = options->endpoint.fd; c->flags |= REDIS_CONNECTED; } else { - // Unknown type - FIXME - FREE + redisFree(c); return NULL; } @@ -939,13 +948,11 @@ int redisBufferRead(redisContext *c) { return REDIS_ERR; nread = c->funcs->read(c, buf, sizeof(buf)); - if (nread > 0) { - if (redisReaderFeed(c->reader, buf, nread) != REDIS_OK) { - __redisSetError(c, c->reader->err, c->reader->errstr); - return REDIS_ERR; - } else { - } - } else if (nread < 0) { + if (nread < 0) { + return REDIS_ERR; + } + if (nread > 0 && redisReaderFeed(c->reader, buf, nread) != REDIS_OK) { + __redisSetError(c, c->reader->err, c->reader->errstr); return REDIS_ERR; } return REDIS_OK; @@ -989,17 +996,6 @@ oom: return REDIS_ERR; } -/* Internal helper function to try and get a reply from the reader, - * or set an error in the context otherwise. */ -int redisGetReplyFromReader(redisContext *c, void **reply) { - if (redisReaderGetReply(c->reader,reply) == REDIS_ERR) { - __redisSetError(c,c->reader->err,c->reader->errstr); - return REDIS_ERR; - } - - return REDIS_OK; -} - /* Internal helper that returns 1 if the reply was a RESP3 PUSH * message and we handled it with a user-provided callback. */ static int redisHandledPushReply(redisContext *c, void *reply) { @@ -1011,12 +1007,34 @@ static int redisHandledPushReply(redisContext *c, void *reply) { return 0; } +/* Get a reply from our reader or set an error in the context. */ +int redisGetReplyFromReader(redisContext *c, void **reply) { + if (redisReaderGetReply(c->reader, reply) == REDIS_ERR) { + __redisSetError(c,c->reader->err,c->reader->errstr); + return REDIS_ERR; + } + + return REDIS_OK; +} + +/* Internal helper to get the next reply from our reader while handling + * any PUSH messages we encounter along the way. This is separate from + * redisGetReplyFromReader so as to not change its behavior. */ +static int redisNextInBandReplyFromReader(redisContext *c, void **reply) { + do { + if (redisGetReplyFromReader(c, reply) == REDIS_ERR) + return REDIS_ERR; + } while (redisHandledPushReply(c, *reply)); + + return REDIS_OK; +} + int redisGetReply(redisContext *c, void **reply) { int wdone = 0; void *aux = NULL; /* Try to read pending replies */ - if (redisGetReplyFromReader(c,&aux) == REDIS_ERR) + if (redisNextInBandReplyFromReader(c,&aux) == REDIS_ERR) return REDIS_ERR; /* For the blocking context, flush output buffer and read reply */ @@ -1032,12 +1050,8 @@ int redisGetReply(redisContext *c, void **reply) { if (redisBufferRead(c) == REDIS_ERR) return REDIS_ERR; - /* We loop here in case the user has specified a RESP3 - * PUSH handler (e.g. for client tracking). */ - do { - if (redisGetReplyFromReader(c,&aux) == REDIS_ERR) - return REDIS_ERR; - } while (redisHandledPushReply(c, aux)); + if (redisNextInBandReplyFromReader(c,&aux) == REDIS_ERR) + return REDIS_ERR; } while (aux == NULL); } @@ -1114,7 +1128,7 @@ int redisAppendCommand(redisContext *c, const char *format, ...) { int redisAppendCommandArgv(redisContext *c, int argc, const char **argv, const size_t *argvlen) { hisds cmd; - int len; + long long len; len = redisFormatSdsCommandArgv(&cmd,argc,argv,argvlen); if (len == -1) { diff --git a/deps/hiredis/hiredis.h b/deps/hiredis/hiredis.h index b597394d4..9c65901bd 100644 --- a/deps/hiredis/hiredis.h +++ b/deps/hiredis/hiredis.h @@ -47,8 +47,8 @@ typedef long long ssize_t; #define HIREDIS_MAJOR 1 #define HIREDIS_MINOR 0 -#define HIREDIS_PATCH 0 -#define HIREDIS_SONAME 1.0.0 +#define HIREDIS_PATCH 3 +#define HIREDIS_SONAME 1.0.3-dev /* Connection type can be blocking or non-blocking and is set in the * least significant bit of the flags field in redisContext. */ @@ -80,12 +80,18 @@ typedef long long ssize_t; /* Flag that is set when we should set SO_REUSEADDR before calling bind() */ #define REDIS_REUSEADDR 0x80 +/* Flag that is set when the async connection supports push replies. */ +#define REDIS_SUPPORTS_PUSH 0x100 + /** * Flag that indicates the user does not want the context to * be automatically freed upon error */ #define REDIS_NO_AUTO_FREE 0x200 +/* Flag that indicates the user does not want replies to be automatically freed */ +#define REDIS_NO_AUTO_FREE_REPLIES 0x400 + #define REDIS_KEEPALIVE_INTERVAL 15 /* seconds */ /* number of times we retry to connect in the case of EADDRNOTAVAIL and @@ -112,7 +118,8 @@ typedef struct redisReply { double dval; /* The double when type is REDIS_REPLY_DOUBLE */ size_t len; /* Length of string */ char *str; /* Used for REDIS_REPLY_ERROR, REDIS_REPLY_STRING - REDIS_REPLY_VERB, and REDIS_REPLY_DOUBLE (in additional to dval). */ + REDIS_REPLY_VERB, REDIS_REPLY_DOUBLE (in additional to dval), + and REDIS_REPLY_BIGNUM. */ char vtype[4]; /* Used for REDIS_REPLY_VERB, contains the null terminated 3 character content type, such as "txt". */ size_t elements; /* number of elements, for REDIS_REPLY_ARRAY */ @@ -127,8 +134,8 @@ void freeReplyObject(void *reply); /* Functions to format a command according to the protocol. */ int redisvFormatCommand(char **target, const char *format, va_list ap); int redisFormatCommand(char **target, const char *format, ...); -int redisFormatCommandArgv(char **target, int argc, const char **argv, const size_t *argvlen); -int redisFormatSdsCommandArgv(hisds *target, int argc, const char ** argv, const size_t *argvlen); +long long redisFormatCommandArgv(char **target, int argc, const char **argv, const size_t *argvlen); +long long redisFormatSdsCommandArgv(hisds *target, int argc, const char ** argv, const size_t *argvlen); void redisFreeCommand(char *cmd); void redisFreeSdsCommand(hisds cmd); @@ -152,6 +159,11 @@ struct redisSsl; /* Don't automatically intercept and free RESP3 PUSH replies. */ #define REDIS_OPT_NO_PUSH_AUTOFREE 0x08 +/** + * Don't automatically free replies + */ +#define REDIS_OPT_NOAUTOFREEREPLIES 0x10 + /* In Unix systems a file descriptor is a regular signed int, with -1 * representing an invalid descriptor. In Windows it is a SOCKET * (32- or 64-bit unsigned integer depending on the architecture), where @@ -255,7 +267,7 @@ typedef struct redisContext { } unix_sock; /* For non-blocking connect */ - struct sockadr *saddr; + struct sockaddr *saddr; size_t addrlen; /* Optional data and corresponding destructor users can use to provide diff --git a/deps/hiredis/hiredis.targets b/deps/hiredis/hiredis.targets new file mode 100644 index 000000000..effd8a561 --- /dev/null +++ b/deps/hiredis/hiredis.targets @@ -0,0 +1,11 @@ +<?xml version="1.0" encoding="utf-8"?> +<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> + <ItemDefinitionGroup> + <ClCompile> + <AdditionalIncludeDirectories>$(MSBuildThisFileDirectory)\..\..\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> + </ClCompile> + <Link> + <AdditionalLibraryDirectories>$(MSBuildThisFileDirectory)\..\..\lib;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories> + </Link> + </ItemDefinitionGroup> +</Project>
\ No newline at end of file diff --git a/deps/hiredis/hiredis_ssl.h b/deps/hiredis/hiredis_ssl.h index 604efe0c1..e3d3e1cf5 100644 --- a/deps/hiredis/hiredis_ssl.h +++ b/deps/hiredis/hiredis_ssl.h @@ -56,7 +56,9 @@ typedef enum { REDIS_SSL_CTX_CERT_KEY_REQUIRED, /* Client cert and key must both be specified or skipped */ REDIS_SSL_CTX_CA_CERT_LOAD_FAILED, /* Failed to load CA Certificate or CA Path */ REDIS_SSL_CTX_CLIENT_CERT_LOAD_FAILED, /* Failed to load client certificate */ - REDIS_SSL_CTX_PRIVATE_KEY_LOAD_FAILED /* Failed to load private key */ + REDIS_SSL_CTX_PRIVATE_KEY_LOAD_FAILED, /* Failed to load private key */ + REDIS_SSL_CTX_OS_CERTSTORE_OPEN_FAILED, /* Failed to open system certifcate store */ + REDIS_SSL_CTX_OS_CERT_ADD_FAILED /* Failed to add CA certificates obtained from system to the SSL context */ } redisSSLContextError; /** diff --git a/deps/hiredis/read.c b/deps/hiredis/read.c index 682b9a6b9..6c19c5a5e 100644 --- a/deps/hiredis/read.c +++ b/deps/hiredis/read.c @@ -123,29 +123,28 @@ static char *readBytes(redisReader *r, unsigned int bytes) { /* Find pointer to \r\n. */ static char *seekNewline(char *s, size_t len) { - int pos = 0; - int _len = len-1; - - /* Position should be < len-1 because the character at "pos" should be - * followed by a \n. Note that strchr cannot be used because it doesn't - * allow to search a limited length and the buffer that is being searched - * might not have a trailing NULL character. */ - while (pos < _len) { - while(pos < _len && s[pos] != '\r') pos++; - if (pos==_len) { - /* Not found. */ - return NULL; - } else { - if (s[pos+1] == '\n') { - /* Found. */ - return s+pos; - } else { - /* Continue searching. */ - pos++; - } + char *ret; + + /* We cannot match with fewer than 2 bytes */ + if (len < 2) + return NULL; + + /* Search up to len - 1 characters */ + len--; + + /* Look for the \r */ + while ((ret = memchr(s, '\r', len)) != NULL) { + if (ret[1] == '\n') { + /* Found. */ + break; } + /* Continue searching. */ + ret++; + len -= ret - s; + s = ret; } - return NULL; + + return ret; } /* Convert a string into a long long. Returns REDIS_OK if the string could be @@ -274,60 +273,104 @@ static int processLineItem(redisReader *r) { if ((p = readLine(r,&len)) != NULL) { if (cur->type == REDIS_REPLY_INTEGER) { + long long v; + + if (string2ll(p, len, &v) == REDIS_ERR) { + __redisReaderSetError(r,REDIS_ERR_PROTOCOL, + "Bad integer value"); + return REDIS_ERR; + } + if (r->fn && r->fn->createInteger) { - long long v; - if (string2ll(p, len, &v) == REDIS_ERR) { - __redisReaderSetError(r,REDIS_ERR_PROTOCOL, - "Bad integer value"); - return REDIS_ERR; - } obj = r->fn->createInteger(cur,v); } else { obj = (void*)REDIS_REPLY_INTEGER; } } else if (cur->type == REDIS_REPLY_DOUBLE) { - if (r->fn && r->fn->createDouble) { - char buf[326], *eptr; - double d; + char buf[326], *eptr; + double d; - if ((size_t)len >= sizeof(buf)) { + if ((size_t)len >= sizeof(buf)) { + __redisReaderSetError(r,REDIS_ERR_PROTOCOL, + "Double value is too large"); + return REDIS_ERR; + } + + memcpy(buf,p,len); + buf[len] = '\0'; + + if (len == 3 && strcasecmp(buf,"inf") == 0) { + d = INFINITY; /* Positive infinite. */ + } else if (len == 4 && strcasecmp(buf,"-inf") == 0) { + d = -INFINITY; /* Negative infinite. */ + } else { + d = strtod((char*)buf,&eptr); + /* RESP3 only allows "inf", "-inf", and finite values, while + * strtod() allows other variations on infinity, NaN, + * etc. We explicity handle our two allowed infinite cases + * above, so strtod() should only result in finite values. */ + if (buf[0] == '\0' || eptr != &buf[len] || !isfinite(d)) { __redisReaderSetError(r,REDIS_ERR_PROTOCOL, - "Double value is too large"); + "Bad double value"); return REDIS_ERR; } + } - memcpy(buf,p,len); - buf[len] = '\0'; - - if (strcasecmp(buf,",inf") == 0) { - d = INFINITY; /* Positive infinite. */ - } else if (strcasecmp(buf,",-inf") == 0) { - d = -INFINITY; /* Negative infinite. */ - } else { - d = strtod((char*)buf,&eptr); - if (buf[0] == '\0' || eptr[0] != '\0' || isnan(d)) { - __redisReaderSetError(r,REDIS_ERR_PROTOCOL, - "Bad double value"); - return REDIS_ERR; - } - } + if (r->fn && r->fn->createDouble) { obj = r->fn->createDouble(cur,d,buf,len); } else { obj = (void*)REDIS_REPLY_DOUBLE; } } else if (cur->type == REDIS_REPLY_NIL) { + if (len != 0) { + __redisReaderSetError(r,REDIS_ERR_PROTOCOL, + "Bad nil value"); + return REDIS_ERR; + } + if (r->fn && r->fn->createNil) obj = r->fn->createNil(cur); else obj = (void*)REDIS_REPLY_NIL; } else if (cur->type == REDIS_REPLY_BOOL) { - int bval = p[0] == 't' || p[0] == 'T'; + int bval; + + if (len != 1 || !strchr("tTfF", p[0])) { + __redisReaderSetError(r,REDIS_ERR_PROTOCOL, + "Bad bool value"); + return REDIS_ERR; + } + + bval = p[0] == 't' || p[0] == 'T'; if (r->fn && r->fn->createBool) obj = r->fn->createBool(cur,bval); else obj = (void*)REDIS_REPLY_BOOL; + } else if (cur->type == REDIS_REPLY_BIGNUM) { + /* Ensure all characters are decimal digits (with possible leading + * minus sign). */ + for (int i = 0; i < len; i++) { + /* XXX Consider: Allow leading '+'? Error on leading '0's? */ + if (i == 0 && p[0] == '-') continue; + if (p[i] < '0' || p[i] > '9') { + __redisReaderSetError(r,REDIS_ERR_PROTOCOL, + "Bad bignum value"); + return REDIS_ERR; + } + } + if (r->fn && r->fn->createString) + obj = r->fn->createString(cur,p,len); + else + obj = (void*)REDIS_REPLY_BIGNUM; } else { /* Type will be error or status. */ + for (int i = 0; i < len; i++) { + if (p[i] == '\r' || p[i] == '\n') { + __redisReaderSetError(r,REDIS_ERR_PROTOCOL, + "Bad simple string value"); + return REDIS_ERR; + } + } if (r->fn && r->fn->createString) obj = r->fn->createString(cur,p,len); else @@ -453,7 +496,6 @@ static int processAggregateItem(redisReader *r) { long long elements; int root = 0, len; - /* Set error for nested multi bulks with depth > 7 */ if (r->ridx == r->tasks - 1) { if (redisReaderGrow(r) == REDIS_ERR) return REDIS_ERR; @@ -569,6 +611,9 @@ static int processItem(redisReader *r) { case '>': cur->type = REDIS_REPLY_PUSH; break; + case '(': + cur->type = REDIS_REPLY_BIGNUM; + break; default: __redisReaderSetErrorProtocolByte(r,*p); return REDIS_ERR; @@ -587,6 +632,7 @@ static int processItem(redisReader *r) { case REDIS_REPLY_DOUBLE: case REDIS_REPLY_NIL: case REDIS_REPLY_BOOL: + case REDIS_REPLY_BIGNUM: return processLineItem(r); case REDIS_REPLY_STRING: case REDIS_REPLY_VERB: diff --git a/deps/hiredis/sds.c b/deps/hiredis/sds.c index 675e7649f..114fa49a6 100644 --- a/deps/hiredis/sds.c +++ b/deps/hiredis/sds.c @@ -72,7 +72,7 @@ static inline char hi_sdsReqType(size_t string_size) { * and 'initlen'. * If NULL is used for 'init' the string is initialized with zero bytes. * - * The string is always null-termined (all the hisds strings are, always) so + * The string is always null-terminated (all the hisds strings are, always) so * even if you create an hisds string with: * * mystring = hi_sdsnewlen("abc",3); @@ -415,7 +415,7 @@ hisds hi_sdscpylen(hisds s, const char *t, size_t len) { return s; } -/* Like hi_sdscpylen() but 't' must be a null-termined string so that the length +/* Like hi_sdscpylen() but 't' must be a null-terminated string so that the length * of the string is obtained with strlen(). */ hisds hi_sdscpy(hisds s, const char *t) { return hi_sdscpylen(s, t, strlen(t)); diff --git a/deps/hiredis/ssl.c b/deps/hiredis/ssl.c index fe9a2fdce..a709ea7cc 100644 --- a/deps/hiredis/ssl.c +++ b/deps/hiredis/ssl.c @@ -38,6 +38,7 @@ #include <string.h> #ifdef _WIN32 #include <windows.h> +#include <wincrypt.h> #else #include <pthread.h> #endif @@ -182,6 +183,10 @@ const char *redisSSLContextGetError(redisSSLContextError error) return "Failed to load client certificate"; case REDIS_SSL_CTX_PRIVATE_KEY_LOAD_FAILED: return "Failed to load private key"; + case REDIS_SSL_CTX_OS_CERTSTORE_OPEN_FAILED: + return "Failed to open system certifcate store"; + case REDIS_SSL_CTX_OS_CERT_ADD_FAILED: + return "Failed to add CA certificates obtained from system to the SSL context"; default: return "Unknown error code"; } @@ -214,6 +219,11 @@ redisSSLContext *redisCreateSSLContext(const char *cacert_filename, const char * const char *cert_filename, const char *private_key_filename, const char *server_name, redisSSLContextError *error) { +#ifdef _WIN32 + HCERTSTORE win_store = NULL; + PCCERT_CONTEXT win_ctx = NULL; +#endif + redisSSLContext *ctx = hi_calloc(1, sizeof(redisSSLContext)); if (ctx == NULL) goto error; @@ -234,6 +244,31 @@ redisSSLContext *redisCreateSSLContext(const char *cacert_filename, const char * } if (capath || cacert_filename) { +#ifdef _WIN32 + if (0 == strcmp(cacert_filename, "wincert")) { + win_store = CertOpenSystemStore(NULL, "Root"); + if (!win_store) { + if (error) *error = REDIS_SSL_CTX_OS_CERTSTORE_OPEN_FAILED; + goto error; + } + X509_STORE* store = SSL_CTX_get_cert_store(ctx->ssl_ctx); + while (win_ctx = CertEnumCertificatesInStore(win_store, win_ctx)) { + X509* x509 = NULL; + x509 = d2i_X509(NULL, (const unsigned char**)&win_ctx->pbCertEncoded, win_ctx->cbCertEncoded); + if (x509) { + if ((1 != X509_STORE_add_cert(store, x509)) || + (1 != SSL_CTX_add_client_CA(ctx->ssl_ctx, x509))) + { + if (error) *error = REDIS_SSL_CTX_OS_CERT_ADD_FAILED; + goto error; + } + X509_free(x509); + } + } + CertFreeCertificateContext(win_ctx); + CertCloseStore(win_store, 0); + } else +#endif if (!SSL_CTX_load_verify_locations(ctx->ssl_ctx, cacert_filename, capath)) { if (error) *error = REDIS_SSL_CTX_CA_CERT_LOAD_FAILED; goto error; @@ -257,6 +292,10 @@ redisSSLContext *redisCreateSSLContext(const char *cacert_filename, const char * return ctx; error: +#ifdef _WIN32 + CertFreeCertificateContext(win_ctx); + CertCloseStore(win_store, 0); +#endif redisFreeSSLContext(ctx); return NULL; } @@ -353,7 +392,11 @@ int redisInitiateSSLWithContext(redisContext *c, redisSSLContext *redis_ssl_ctx) } } - return redisSSLConnect(c, ssl); + if (redisSSLConnect(c, ssl) != REDIS_OK) { + goto error; + } + + return REDIS_OK; error: if (ssl) diff --git a/deps/hiredis/test.c b/deps/hiredis/test.c index bdff74e88..306aa5559 100644 --- a/deps/hiredis/test.c +++ b/deps/hiredis/test.c @@ -11,12 +11,17 @@ #include <signal.h> #include <errno.h> #include <limits.h> +#include <math.h> #include "hiredis.h" #include "async.h" #ifdef HIREDIS_TEST_SSL #include "hiredis_ssl.h" #endif +#ifdef HIREDIS_TEST_ASYNC +#include "adapters/libevent.h" +#include <event2/event.h> +#endif #include "net.h" #include "win32.h" @@ -58,6 +63,8 @@ struct pushCounters { int str; }; +static int insecure_calloc_calls; + #ifdef HIREDIS_TEST_SSL redisSSLContext *_ssl_ctx = NULL; #endif @@ -597,6 +604,147 @@ static void test_reply_reader(void) { ((redisReply*)reply)->element[1]->integer == 42); freeReplyObject(reply); redisReaderFree(reader); + + test("Can parse RESP3 doubles: "); + reader = redisReaderCreate(); + redisReaderFeed(reader, ",3.14159265358979323846\r\n",25); + ret = redisReaderGetReply(reader,&reply); + test_cond(ret == REDIS_OK && + ((redisReply*)reply)->type == REDIS_REPLY_DOUBLE && + fabs(((redisReply*)reply)->dval - 3.14159265358979323846) < 0.00000001 && + ((redisReply*)reply)->len == 22 && + strcmp(((redisReply*)reply)->str, "3.14159265358979323846") == 0); + freeReplyObject(reply); + redisReaderFree(reader); + + test("Set error on invalid RESP3 double: "); + reader = redisReaderCreate(); + redisReaderFeed(reader, ",3.14159\000265358979323846\r\n",26); + ret = redisReaderGetReply(reader,&reply); + test_cond(ret == REDIS_ERR && + strcasecmp(reader->errstr,"Bad double value") == 0); + freeReplyObject(reply); + redisReaderFree(reader); + + test("Correctly parses RESP3 double INFINITY: "); + reader = redisReaderCreate(); + redisReaderFeed(reader, ",inf\r\n",6); + ret = redisReaderGetReply(reader,&reply); + test_cond(ret == REDIS_OK && + ((redisReply*)reply)->type == REDIS_REPLY_DOUBLE && + isinf(((redisReply*)reply)->dval) && + ((redisReply*)reply)->dval > 0); + freeReplyObject(reply); + redisReaderFree(reader); + + test("Set error when RESP3 double is NaN: "); + reader = redisReaderCreate(); + redisReaderFeed(reader, ",nan\r\n",6); + ret = redisReaderGetReply(reader,&reply); + test_cond(ret == REDIS_ERR && + strcasecmp(reader->errstr,"Bad double value") == 0); + freeReplyObject(reply); + redisReaderFree(reader); + + test("Can parse RESP3 nil: "); + reader = redisReaderCreate(); + redisReaderFeed(reader, "_\r\n",3); + ret = redisReaderGetReply(reader,&reply); + test_cond(ret == REDIS_OK && + ((redisReply*)reply)->type == REDIS_REPLY_NIL); + freeReplyObject(reply); + redisReaderFree(reader); + + test("Set error on invalid RESP3 nil: "); + reader = redisReaderCreate(); + redisReaderFeed(reader, "_nil\r\n",6); + ret = redisReaderGetReply(reader,&reply); + test_cond(ret == REDIS_ERR && + strcasecmp(reader->errstr,"Bad nil value") == 0); + freeReplyObject(reply); + redisReaderFree(reader); + + test("Can parse RESP3 bool (true): "); + reader = redisReaderCreate(); + redisReaderFeed(reader, "#t\r\n",4); + ret = redisReaderGetReply(reader,&reply); + test_cond(ret == REDIS_OK && + ((redisReply*)reply)->type == REDIS_REPLY_BOOL && + ((redisReply*)reply)->integer); + freeReplyObject(reply); + redisReaderFree(reader); + + test("Can parse RESP3 bool (false): "); + reader = redisReaderCreate(); + redisReaderFeed(reader, "#f\r\n",4); + ret = redisReaderGetReply(reader,&reply); + test_cond(ret == REDIS_OK && + ((redisReply*)reply)->type == REDIS_REPLY_BOOL && + !((redisReply*)reply)->integer); + freeReplyObject(reply); + redisReaderFree(reader); + + test("Set error on invalid RESP3 bool: "); + reader = redisReaderCreate(); + redisReaderFeed(reader, "#foobar\r\n",9); + ret = redisReaderGetReply(reader,&reply); + test_cond(ret == REDIS_ERR && + strcasecmp(reader->errstr,"Bad bool value") == 0); + freeReplyObject(reply); + redisReaderFree(reader); + + test("Can parse RESP3 map: "); + reader = redisReaderCreate(); + redisReaderFeed(reader, "%2\r\n+first\r\n:123\r\n$6\r\nsecond\r\n#t\r\n",34); + ret = redisReaderGetReply(reader,&reply); + test_cond(ret == REDIS_OK && + ((redisReply*)reply)->type == REDIS_REPLY_MAP && + ((redisReply*)reply)->elements == 4 && + ((redisReply*)reply)->element[0]->type == REDIS_REPLY_STATUS && + ((redisReply*)reply)->element[0]->len == 5 && + !strcmp(((redisReply*)reply)->element[0]->str,"first") && + ((redisReply*)reply)->element[1]->type == REDIS_REPLY_INTEGER && + ((redisReply*)reply)->element[1]->integer == 123 && + ((redisReply*)reply)->element[2]->type == REDIS_REPLY_STRING && + ((redisReply*)reply)->element[2]->len == 6 && + !strcmp(((redisReply*)reply)->element[2]->str,"second") && + ((redisReply*)reply)->element[3]->type == REDIS_REPLY_BOOL && + ((redisReply*)reply)->element[3]->integer); + freeReplyObject(reply); + redisReaderFree(reader); + + test("Can parse RESP3 set: "); + reader = redisReaderCreate(); + redisReaderFeed(reader, "~5\r\n+orange\r\n$5\r\napple\r\n#f\r\n:100\r\n:999\r\n",40); + ret = redisReaderGetReply(reader,&reply); + test_cond(ret == REDIS_OK && + ((redisReply*)reply)->type == REDIS_REPLY_SET && + ((redisReply*)reply)->elements == 5 && + ((redisReply*)reply)->element[0]->type == REDIS_REPLY_STATUS && + ((redisReply*)reply)->element[0]->len == 6 && + !strcmp(((redisReply*)reply)->element[0]->str,"orange") && + ((redisReply*)reply)->element[1]->type == REDIS_REPLY_STRING && + ((redisReply*)reply)->element[1]->len == 5 && + !strcmp(((redisReply*)reply)->element[1]->str,"apple") && + ((redisReply*)reply)->element[2]->type == REDIS_REPLY_BOOL && + !((redisReply*)reply)->element[2]->integer && + ((redisReply*)reply)->element[3]->type == REDIS_REPLY_INTEGER && + ((redisReply*)reply)->element[3]->integer == 100 && + ((redisReply*)reply)->element[4]->type == REDIS_REPLY_INTEGER && + ((redisReply*)reply)->element[4]->integer == 999); + freeReplyObject(reply); + redisReaderFree(reader); + + test("Can parse RESP3 bignum: "); + reader = redisReaderCreate(); + redisReaderFeed(reader,"(3492890328409238509324850943850943825024385\r\n",46); + ret = redisReaderGetReply(reader,&reply); + test_cond(ret == REDIS_OK && + ((redisReply*)reply)->type == REDIS_REPLY_BIGNUM && + ((redisReply*)reply)->len == 43 && + !strcmp(((redisReply*)reply)->str,"3492890328409238509324850943850943825024385")); + freeReplyObject(reply); + redisReaderFree(reader); } static void test_free_null(void) { @@ -623,6 +771,13 @@ static void *hi_calloc_fail(size_t nmemb, size_t size) { return NULL; } +static void *hi_calloc_insecure(size_t nmemb, size_t size) { + (void)nmemb; + (void)size; + insecure_calloc_calls++; + return (void*)0xdeadc0de; +} + static void *hi_realloc_fail(void *ptr, size_t size) { (void)ptr; (void)size; @@ -630,6 +785,8 @@ static void *hi_realloc_fail(void *ptr, size_t size) { } static void test_allocator_injection(void) { + void *ptr; + hiredisAllocFuncs ha = { .mallocFn = hi_malloc_fail, .callocFn = hi_calloc_fail, @@ -649,6 +806,13 @@ static void test_allocator_injection(void) { redisReader *reader = redisReaderCreate(); test_cond(reader == NULL); + /* Make sure hiredis itself protects against a non-overflow checking calloc */ + test("hiredis calloc wrapper protects against overflow: "); + ha.callocFn = hi_calloc_insecure; + hiredisSetAllocators(&ha); + ptr = hi_calloc((SIZE_MAX / sizeof(void*)) + 3, sizeof(void*)); + test_cond(ptr == NULL && insecure_calloc_calls == 0); + // Return allocators to default hiredisResetAllocators(); } @@ -1283,6 +1447,440 @@ static void test_throughput(struct config config) { // redisFree(c); // } +#ifdef HIREDIS_TEST_ASYNC +struct event_base *base; + +typedef struct TestState { + redisOptions *options; + int checkpoint; + int resp3; + int disconnect; +} TestState; + +/* Helper to disconnect and stop event loop */ +void async_disconnect(redisAsyncContext *ac) { + redisAsyncDisconnect(ac); + event_base_loopbreak(base); +} + +/* Testcase timeout, will trigger a failure */ +void timeout_cb(int fd, short event, void *arg) { + (void) fd; (void) event; (void) arg; + printf("Timeout in async testing!\n"); + exit(1); +} + +/* Unexpected call, will trigger a failure */ +void unexpected_cb(redisAsyncContext *ac, void *r, void *privdata) { + (void) ac; (void) r; + printf("Unexpected call: %s\n",(char*)privdata); + exit(1); +} + +/* Helper function to publish a message via own client. */ +void publish_msg(redisOptions *options, const char* channel, const char* msg) { + redisContext *c = redisConnectWithOptions(options); + assert(c != NULL); + redisReply *reply = redisCommand(c,"PUBLISH %s %s",channel,msg); + assert(reply->type == REDIS_REPLY_INTEGER && reply->integer == 1); + freeReplyObject(reply); + disconnect(c, 0); +} + +/* Expect a reply of type INTEGER */ +void integer_cb(redisAsyncContext *ac, void *r, void *privdata) { + redisReply *reply = r; + TestState *state = privdata; + assert(reply != NULL && reply->type == REDIS_REPLY_INTEGER); + state->checkpoint++; + if (state->disconnect) async_disconnect(ac); +} + +/* Subscribe callback for test_pubsub_handling and test_pubsub_handling_resp3: + * - a published message triggers an unsubscribe + * - a command is sent before the unsubscribe response is received. */ +void subscribe_cb(redisAsyncContext *ac, void *r, void *privdata) { + redisReply *reply = r; + TestState *state = privdata; + + assert(reply != NULL && + reply->type == (state->resp3 ? REDIS_REPLY_PUSH : REDIS_REPLY_ARRAY) && + reply->elements == 3); + + if (strcmp(reply->element[0]->str,"subscribe") == 0) { + assert(strcmp(reply->element[1]->str,"mychannel") == 0 && + reply->element[2]->str == NULL); + publish_msg(state->options,"mychannel","Hello!"); + } else if (strcmp(reply->element[0]->str,"message") == 0) { + assert(strcmp(reply->element[1]->str,"mychannel") == 0 && + strcmp(reply->element[2]->str,"Hello!") == 0); + state->checkpoint++; + + /* Unsubscribe after receiving the published message. Send unsubscribe + * which should call the callback registered during subscribe */ + redisAsyncCommand(ac,unexpected_cb, + (void*)"unsubscribe should call subscribe_cb()", + "unsubscribe"); + /* Send a regular command after unsubscribing, then disconnect */ + state->disconnect = 1; + redisAsyncCommand(ac,integer_cb,state,"LPUSH mylist foo"); + + } else if (strcmp(reply->element[0]->str,"unsubscribe") == 0) { + assert(strcmp(reply->element[1]->str,"mychannel") == 0 && + reply->element[2]->str == NULL); + } else { + printf("Unexpected pubsub command: %s\n", reply->element[0]->str); + exit(1); + } +} + +/* Expect a reply of type ARRAY */ +void array_cb(redisAsyncContext *ac, void *r, void *privdata) { + redisReply *reply = r; + TestState *state = privdata; + assert(reply != NULL && reply->type == REDIS_REPLY_ARRAY); + state->checkpoint++; + if (state->disconnect) async_disconnect(ac); +} + +/* Expect a NULL reply */ +void null_cb(redisAsyncContext *ac, void *r, void *privdata) { + (void) ac; + assert(r == NULL); + TestState *state = privdata; + state->checkpoint++; +} + +static void test_pubsub_handling(struct config config) { + test("Subscribe, handle published message and unsubscribe: "); + /* Setup event dispatcher with a testcase timeout */ + base = event_base_new(); + struct event *timeout = evtimer_new(base, timeout_cb, NULL); + assert(timeout != NULL); + + evtimer_assign(timeout,base,timeout_cb,NULL); + struct timeval timeout_tv = {.tv_sec = 10}; + evtimer_add(timeout, &timeout_tv); + + /* Connect */ + redisOptions options = get_redis_tcp_options(config); + redisAsyncContext *ac = redisAsyncConnectWithOptions(&options); + assert(ac != NULL && ac->err == 0); + redisLibeventAttach(ac,base); + + /* Start subscribe */ + TestState state = {.options = &options}; + redisAsyncCommand(ac,subscribe_cb,&state,"subscribe mychannel"); + + /* Make sure non-subscribe commands are handled */ + redisAsyncCommand(ac,array_cb,&state,"PING"); + + /* Start event dispatching loop */ + test_cond(event_base_dispatch(base) == 0); + event_free(timeout); + event_base_free(base); + + /* Verify test checkpoints */ + assert(state.checkpoint == 3); +} + +/* Unexpected push message, will trigger a failure */ +void unexpected_push_cb(redisAsyncContext *ac, void *r) { + (void) ac; (void) r; + printf("Unexpected call to the PUSH callback!\n"); + exit(1); +} + +static void test_pubsub_handling_resp3(struct config config) { + test("Subscribe, handle published message and unsubscribe using RESP3: "); + /* Setup event dispatcher with a testcase timeout */ + base = event_base_new(); + struct event *timeout = evtimer_new(base, timeout_cb, NULL); + assert(timeout != NULL); + + evtimer_assign(timeout,base,timeout_cb,NULL); + struct timeval timeout_tv = {.tv_sec = 10}; + evtimer_add(timeout, &timeout_tv); + + /* Connect */ + redisOptions options = get_redis_tcp_options(config); + redisAsyncContext *ac = redisAsyncConnectWithOptions(&options); + assert(ac != NULL && ac->err == 0); + redisLibeventAttach(ac,base); + + /* Not expecting any push messages in this test */ + redisAsyncSetPushCallback(ac, unexpected_push_cb); + + /* Switch protocol */ + redisAsyncCommand(ac,NULL,NULL,"HELLO 3"); + + /* Start subscribe */ + TestState state = {.options = &options, .resp3 = 1}; + redisAsyncCommand(ac,subscribe_cb,&state,"subscribe mychannel"); + + /* Make sure non-subscribe commands are handled in RESP3 */ + redisAsyncCommand(ac,integer_cb,&state,"LPUSH mylist foo"); + redisAsyncCommand(ac,integer_cb,&state,"LPUSH mylist foo"); + redisAsyncCommand(ac,integer_cb,&state,"LPUSH mylist foo"); + /* Handle an array with 3 elements as a non-subscribe command */ + redisAsyncCommand(ac,array_cb,&state,"LRANGE mylist 0 2"); + + /* Start event dispatching loop */ + test_cond(event_base_dispatch(base) == 0); + event_free(timeout); + event_base_free(base); + + /* Verify test checkpoints */ + assert(state.checkpoint == 6); +} + +/* Subscribe callback for test_command_timeout_during_pubsub: + * - a subscribe response triggers a published message + * - the published message triggers a command that times out + * - the command timeout triggers a disconnect */ +void subscribe_with_timeout_cb(redisAsyncContext *ac, void *r, void *privdata) { + redisReply *reply = r; + TestState *state = privdata; + + /* The non-clean disconnect should trigger the + * subscription callback with a NULL reply. */ + if (reply == NULL) { + state->checkpoint++; + event_base_loopbreak(base); + return; + } + + assert(reply->type == (state->resp3 ? REDIS_REPLY_PUSH : REDIS_REPLY_ARRAY) && + reply->elements == 3); + + if (strcmp(reply->element[0]->str,"subscribe") == 0) { + assert(strcmp(reply->element[1]->str,"mychannel") == 0 && + reply->element[2]->str == NULL); + publish_msg(state->options,"mychannel","Hello!"); + state->checkpoint++; + } else if (strcmp(reply->element[0]->str,"message") == 0) { + assert(strcmp(reply->element[1]->str,"mychannel") == 0 && + strcmp(reply->element[2]->str,"Hello!") == 0); + state->checkpoint++; + + /* Send a command that will trigger a timeout */ + redisAsyncCommand(ac,null_cb,state,"DEBUG SLEEP 3"); + redisAsyncCommand(ac,null_cb,state,"LPUSH mylist foo"); + } else { + printf("Unexpected pubsub command: %s\n", reply->element[0]->str); + exit(1); + } +} + +static void test_command_timeout_during_pubsub(struct config config) { + test("Command timeout during Pub/Sub: "); + /* Setup event dispatcher with a testcase timeout */ + base = event_base_new(); + struct event *timeout = evtimer_new(base,timeout_cb,NULL); + assert(timeout != NULL); + + evtimer_assign(timeout,base,timeout_cb,NULL); + struct timeval timeout_tv = {.tv_sec = 10}; + evtimer_add(timeout,&timeout_tv); + + /* Connect */ + redisOptions options = get_redis_tcp_options(config); + redisAsyncContext *ac = redisAsyncConnectWithOptions(&options); + assert(ac != NULL && ac->err == 0); + redisLibeventAttach(ac,base); + + /* Configure a command timout */ + struct timeval command_timeout = {.tv_sec = 2}; + redisAsyncSetTimeout(ac,command_timeout); + + /* Not expecting any push messages in this test */ + redisAsyncSetPushCallback(ac,unexpected_push_cb); + + /* Switch protocol */ + redisAsyncCommand(ac,NULL,NULL,"HELLO 3"); + + /* Start subscribe */ + TestState state = {.options = &options, .resp3 = 1}; + redisAsyncCommand(ac,subscribe_with_timeout_cb,&state,"subscribe mychannel"); + + /* Start event dispatching loop */ + assert(event_base_dispatch(base) == 0); + event_free(timeout); + event_base_free(base); + + /* Verify test checkpoints */ + test_cond(state.checkpoint == 5); +} + +/* Subscribe callback for test_pubsub_multiple_channels */ +void subscribe_channel_a_cb(redisAsyncContext *ac, void *r, void *privdata) { + redisReply *reply = r; + TestState *state = privdata; + + assert(reply != NULL && reply->type == REDIS_REPLY_ARRAY && + reply->elements == 3); + + if (strcmp(reply->element[0]->str,"subscribe") == 0) { + assert(strcmp(reply->element[1]->str,"A") == 0); + publish_msg(state->options,"A","Hello!"); + state->checkpoint++; + } else if (strcmp(reply->element[0]->str,"message") == 0) { + assert(strcmp(reply->element[1]->str,"A") == 0 && + strcmp(reply->element[2]->str,"Hello!") == 0); + state->checkpoint++; + + /* Unsubscribe to channels, including a channel X which we don't subscribe to */ + redisAsyncCommand(ac,unexpected_cb, + (void*)"unsubscribe should not call unexpected_cb()", + "unsubscribe B X A"); + /* Send a regular command after unsubscribing, then disconnect */ + state->disconnect = 1; + redisAsyncCommand(ac,integer_cb,state,"LPUSH mylist foo"); + } else if (strcmp(reply->element[0]->str,"unsubscribe") == 0) { + assert(strcmp(reply->element[1]->str,"A") == 0); + state->checkpoint++; + } else { + printf("Unexpected pubsub command: %s\n", reply->element[0]->str); + exit(1); + } +} + +/* Subscribe callback for test_pubsub_multiple_channels */ +void subscribe_channel_b_cb(redisAsyncContext *ac, void *r, void *privdata) { + redisReply *reply = r; + TestState *state = privdata; + + assert(reply != NULL && reply->type == REDIS_REPLY_ARRAY && + reply->elements == 3); + + if (strcmp(reply->element[0]->str,"subscribe") == 0) { + assert(strcmp(reply->element[1]->str,"B") == 0); + state->checkpoint++; + } else if (strcmp(reply->element[0]->str,"unsubscribe") == 0) { + assert(strcmp(reply->element[1]->str,"B") == 0); + state->checkpoint++; + } else { + printf("Unexpected pubsub command: %s\n", reply->element[0]->str); + exit(1); + } +} + +/* Test handling of multiple channels + * - subscribe to channel A and B + * - a published message on A triggers an unsubscribe of channel B, X and A + * where channel X is not subscribed to. + * - a command sent after unsubscribe triggers a disconnect */ +static void test_pubsub_multiple_channels(struct config config) { + test("Subscribe to multiple channels: "); + /* Setup event dispatcher with a testcase timeout */ + base = event_base_new(); + struct event *timeout = evtimer_new(base,timeout_cb,NULL); + assert(timeout != NULL); + + evtimer_assign(timeout,base,timeout_cb,NULL); + struct timeval timeout_tv = {.tv_sec = 10}; + evtimer_add(timeout,&timeout_tv); + + /* Connect */ + redisOptions options = get_redis_tcp_options(config); + redisAsyncContext *ac = redisAsyncConnectWithOptions(&options); + assert(ac != NULL && ac->err == 0); + redisLibeventAttach(ac,base); + + /* Not expecting any push messages in this test */ + redisAsyncSetPushCallback(ac,unexpected_push_cb); + + /* Start subscribing to two channels */ + TestState state = {.options = &options}; + redisAsyncCommand(ac,subscribe_channel_a_cb,&state,"subscribe A"); + redisAsyncCommand(ac,subscribe_channel_b_cb,&state,"subscribe B"); + + /* Start event dispatching loop */ + assert(event_base_dispatch(base) == 0); + event_free(timeout); + event_base_free(base); + + /* Verify test checkpoints */ + test_cond(state.checkpoint == 6); +} + +/* Command callback for test_monitor() */ +void monitor_cb(redisAsyncContext *ac, void *r, void *privdata) { + redisReply *reply = r; + TestState *state = privdata; + + /* NULL reply is received when BYE triggers a disconnect. */ + if (reply == NULL) { + event_base_loopbreak(base); + return; + } + + assert(reply != NULL && reply->type == REDIS_REPLY_STATUS); + state->checkpoint++; + + if (state->checkpoint == 1) { + /* Response from MONITOR */ + redisContext *c = redisConnectWithOptions(state->options); + assert(c != NULL); + redisReply *reply = redisCommand(c,"SET first 1"); + assert(reply->type == REDIS_REPLY_STATUS); + freeReplyObject(reply); + redisFree(c); + } else if (state->checkpoint == 2) { + /* Response for monitored command 'SET first 1' */ + assert(strstr(reply->str,"first") != NULL); + redisContext *c = redisConnectWithOptions(state->options); + assert(c != NULL); + redisReply *reply = redisCommand(c,"SET second 2"); + assert(reply->type == REDIS_REPLY_STATUS); + freeReplyObject(reply); + redisFree(c); + } else if (state->checkpoint == 3) { + /* Response for monitored command 'SET second 2' */ + assert(strstr(reply->str,"second") != NULL); + /* Send QUIT to disconnect */ + redisAsyncCommand(ac,NULL,NULL,"QUIT"); + } +} + +/* Test handling of the monitor command + * - sends MONITOR to enable monitoring. + * - sends SET commands via separate clients to be monitored. + * - sends QUIT to stop monitoring and disconnect. */ +static void test_monitor(struct config config) { + test("Enable monitoring: "); + /* Setup event dispatcher with a testcase timeout */ + base = event_base_new(); + struct event *timeout = evtimer_new(base, timeout_cb, NULL); + assert(timeout != NULL); + + evtimer_assign(timeout,base,timeout_cb,NULL); + struct timeval timeout_tv = {.tv_sec = 10}; + evtimer_add(timeout, &timeout_tv); + + /* Connect */ + redisOptions options = get_redis_tcp_options(config); + redisAsyncContext *ac = redisAsyncConnectWithOptions(&options); + assert(ac != NULL && ac->err == 0); + redisLibeventAttach(ac,base); + + /* Not expecting any push messages in this test */ + redisAsyncSetPushCallback(ac,unexpected_push_cb); + + /* Start monitor */ + TestState state = {.options = &options}; + redisAsyncCommand(ac,monitor_cb,&state,"monitor"); + + /* Start event dispatching loop */ + test_cond(event_base_dispatch(base) == 0); + event_free(timeout); + event_base_free(base); + + /* Verify test checkpoints */ + assert(state.checkpoint == 3); +} +#endif /* HIREDIS_TEST_ASYNC */ + int main(int argc, char **argv) { struct config cfg = { .tcp = { @@ -1401,6 +1999,24 @@ int main(int argc, char **argv) { } #endif +#ifdef HIREDIS_TEST_ASYNC + printf("\nTesting asynchronous API against TCP connection (%s:%d):\n", cfg.tcp.host, cfg.tcp.port); + cfg.type = CONN_TCP; + + int major; + redisContext *c = do_connect(cfg); + get_redis_version(c, &major, NULL); + disconnect(c, 0); + + test_pubsub_handling(cfg); + test_pubsub_multiple_channels(cfg); + test_monitor(cfg); + if (major >= 6) { + test_pubsub_handling_resp3(cfg); + test_command_timeout_during_pubsub(cfg); + } +#endif /* HIREDIS_TEST_ASYNC */ + if (test_inherit_fd) { printf("\nTesting against inherited fd (%s): ", cfg.unix_sock.path); if (test_unix_socket) { diff --git a/redis.conf b/redis.conf index ad37bbe33..d018c8824 100644 --- a/redis.conf +++ b/redis.conf @@ -213,7 +213,9 @@ tcp-keepalive 300 # # tls-client-key-file-pass secret -# Configure a DH parameters file to enable Diffie-Hellman (DH) key exchange: +# Configure a DH parameters file to enable Diffie-Hellman (DH) key exchange, +# required by older versions of OpenSSL (<3.0). Newer versions do not require +# this configuration and recommend against it. # # tls-dh-params-file redis.dh @@ -641,7 +643,7 @@ repl-diskless-sync-max-replicas 0 # you risk an OOM kill. repl-diskless-load disabled -# Replicas send PINGs to server in a predefined interval. It's possible to +# Master send PINGs to its replicas in a predefined interval. It's possible to # change this interval with the repl_ping_replica_period option. The default # value is 10 seconds. # @@ -1678,7 +1680,7 @@ aof-timestamp-enabled no # routing. By default this value is only shown as additional metadata in the CLUSTER SLOTS # command, but can be changed using 'cluster-preferred-endpoint-type' config. This value is # communicated along the clusterbus to all nodes, setting it to an empty string will remove -# the hostname and also propgate the removal. +# the hostname and also propagate the removal. # # cluster-announce-hostname "" diff --git a/runtest-moduleapi b/runtest-moduleapi index 8b4b108de..a3aab1f7a 100755 --- a/runtest-moduleapi +++ b/runtest-moduleapi @@ -44,6 +44,7 @@ $TCLSH tests/test_helper.tcl \ --single unit/moduleapi/aclcheck \ --single unit/moduleapi/subcommands \ --single unit/moduleapi/reply \ +--single unit/moduleapi/cmdintrospection \ --single unit/moduleapi/eventloop \ +--single unit/moduleapi/timer \ "${@}" - @@ -1531,18 +1531,15 @@ static int ACLSelectorCheckKey(aclSelector *selector, const char *key, int keyle return ACL_DENIED_KEY; } -/* Returns if a given command may possibly access channels. For this context, - * the unsubscribe commands do not have channels. */ -static int ACLDoesCommandHaveChannels(struct redisCommand *cmd) { - return (cmd->proc == publishCommand - || cmd->proc == subscribeCommand - || cmd->proc == psubscribeCommand - || cmd->proc == spublishCommand - || cmd->proc == ssubscribeCommand); -} - -/* Checks a channel against a provide list of channels. */ -static int ACLCheckChannelAgainstList(list *reference, const char *channel, int channellen, int literal) { +/* Checks a channel against a provided list of channels. The is_pattern + * argument should only be used when subscribing (not when publishing) + * and controls whether the input channel is evaluated as a channel pattern + * (like in PSUBSCRIBE) or a plain channel name (like in SUBSCRIBE). + * + * Note that a plain channel name like in PUBLISH or SUBSCRIBE can be + * matched against ACL channel patterns, but the pattern provided in PSUBSCRIBE + * can only be matched as a literal against an ACL pattern (using plain string compare). */ +static int ACLCheckChannelAgainstList(list *reference, const char *channel, int channellen, int is_pattern) { listIter li; listNode *ln; @@ -1550,8 +1547,10 @@ static int ACLCheckChannelAgainstList(list *reference, const char *channel, int while((ln = listNext(&li))) { sds pattern = listNodeValue(ln); size_t plen = sdslen(pattern); - if ((literal && !strcmp(pattern,channel)) || - (!literal && stringmatchlen(pattern,plen,channel,channellen,0))) + /* Channel patterns are matched literally against the channels in + * the list. Regular channels perform pattern matching. */ + if ((is_pattern && !strcmp(pattern,channel)) || + (!is_pattern && stringmatchlen(pattern,plen,channel,channellen,0))) { return ACL_OK; } @@ -1559,28 +1558,6 @@ static int ACLCheckChannelAgainstList(list *reference, const char *channel, int return ACL_DENIED_CHANNEL; } -/* Check if the pub/sub channels of the command can be executed - * according to the ACL channels associated with the specified selector. - * - * idx and count are the index and count of channel arguments from the - * command. The literal argument controls whether the selector's ACL channels are - * evaluated as literal values or matched as glob-like patterns. - * - * If the selector can execute the command ACL_OK is returned, otherwise - * ACL_DENIED_CHANNEL. */ -static int ACLSelectorCheckPubsubArguments(aclSelector *s, robj **argv, int idx, int count, int literal, int *idxptr) { - for (int j = idx; j < idx+count; j++) { - if (ACLCheckChannelAgainstList(s->channels, argv[j]->ptr, sdslen(argv[j]->ptr), literal != ACL_OK)) { - if (idxptr) *idxptr = j; - return ACL_DENIED_CHANNEL; - } - } - - /* If we survived all the above checks, the selector can execute the - * command. */ - return ACL_OK; -} - /* To prevent duplicate calls to getKeysResult, a cache is maintained * in between calls to the various selectors. */ typedef struct { @@ -1645,7 +1622,7 @@ static int ACLSelectorCheckCmd(aclSelector *selector, struct redisCommand *cmd, int idx = resultidx[j].pos; ret = ACLSelectorCheckKey(selector, argv[idx]->ptr, sdslen(argv[idx]->ptr), resultidx[j].flags); if (ret != ACL_OK) { - if (resultidx) *keyidxptr = resultidx[j].pos; + if (keyidxptr) *keyidxptr = resultidx[j].pos; return ret; } } @@ -1653,26 +1630,30 @@ static int ACLSelectorCheckCmd(aclSelector *selector, struct redisCommand *cmd, /* Check if the user can execute commands explicitly touching the channels * mentioned in the command arguments */ - if (!(selector->flags & SELECTOR_FLAG_ALLCHANNELS) && ACLDoesCommandHaveChannels(cmd)) { - if (cmd->proc == publishCommand || cmd->proc == spublishCommand) { - ret = ACLSelectorCheckPubsubArguments(selector,argv, 1, 1, 0, keyidxptr); - } else if (cmd->proc == subscribeCommand || cmd->proc == ssubscribeCommand) { - ret = ACLSelectorCheckPubsubArguments(selector, argv, 1, argc-1, 0, keyidxptr); - } else if (cmd->proc == psubscribeCommand) { - ret = ACLSelectorCheckPubsubArguments(selector, argv, 1, argc-1, 1, keyidxptr); - } else { - serverPanic("Encountered a command declared with channels but not handled"); - } - if (ret != ACL_OK) { - /* keyidxptr is set by ACLSelectorCheckPubsubArguments */ - return ret; + const int channel_flags = CMD_CHANNEL_PUBLISH | CMD_CHANNEL_SUBSCRIBE; + if (!(selector->flags & SELECTOR_FLAG_ALLCHANNELS) && doesCommandHaveChannelsWithFlags(cmd, channel_flags)) { + getKeysResult channels = (getKeysResult) GETKEYS_RESULT_INIT; + getChannelsFromCommand(cmd, argv, argc, &channels); + keyReference *channelref = channels.keys; + for (int j = 0; j < channels.numkeys; j++) { + int idx = channelref[j].pos; + if (!(channelref[j].flags & channel_flags)) continue; + int is_pattern = channelref[j].flags & CMD_CHANNEL_PATTERN; + int ret = ACLCheckChannelAgainstList(selector->channels, argv[idx]->ptr, sdslen(argv[idx]->ptr), is_pattern); + if (ret != ACL_OK) { + if (keyidxptr) *keyidxptr = channelref[j].pos; + getKeysFreeResult(&channels); + return ret; + } } + getKeysFreeResult(&channels); } return ACL_OK; } /* Check if the key can be accessed by the client according to - * the ACLs associated with the specified user. + * the ACLs associated with the specified user according to the + * keyspec access flags. * * If the user can access the key, ACL_OK is returned, otherwise * ACL_DENIED_KEY is returned. */ @@ -1699,7 +1680,7 @@ int ACLUserCheckKeyPerm(user *u, const char *key, int keylen, int flags) { * * If the user can access the key, ACL_OK is returned, otherwise * ACL_DENIED_CHANNEL is returned. */ -int ACLUserCheckChannelPerm(user *u, sds channel, int literal) { +int ACLUserCheckChannelPerm(user *u, sds channel, int is_pattern) { listIter li; listNode *ln; @@ -1714,7 +1695,7 @@ int ACLUserCheckChannelPerm(user *u, sds channel, int literal) { if (s->flags & SELECTOR_FLAG_ALLCHANNELS) return ACL_OK; /* Otherwise, loop over the selectors list and check each channel */ - if (ACLCheckChannelAgainstList(s->channels, channel, sdslen(channel), literal) == ACL_OK) { + if (ACLCheckChannelAgainstList(s->channels, channel, sdslen(channel), is_pattern) == ACL_OK) { return ACL_OK; } } @@ -42,11 +42,13 @@ #include <sys/param.h> void freeClientArgv(client *c); -off_t getAppendOnlyFileSize(sds filename); -off_t getBaseAndIncrAppendOnlyFilesSize(aofManifest *am); +off_t getAppendOnlyFileSize(sds filename, int *status); +off_t getBaseAndIncrAppendOnlyFilesSize(aofManifest *am, int *status); int getBaseAndIncrAppendOnlyFilesNum(aofManifest *am); int aofFileExist(char *filename); int rewriteAppendOnlyFile(char *filename); +aofManifest *aofLoadManifestFromFile(sds am_filepath); +void aofManifestFreeAndUpdate(aofManifest *am); /* ---------------------------------------------------------------------------- * AOF Manifest file implementation. @@ -226,13 +228,8 @@ sds getAofManifestAsString(aofManifest *am) { * in order to support seamless upgrades from previous versions which did not * use them. */ -#define MANIFEST_MAX_LINE 1024 void aofLoadManifestFromDisk(void) { - const char *err = NULL; - long long maxseq = 0; - server.aof_manifest = aofManifestCreate(); - if (!dirExists(server.aof_dirname)) { serverLog(LL_NOTICE, "The AOF directory %s doesn't exist", server.aof_dirname); return; @@ -247,16 +244,26 @@ void aofLoadManifestFromDisk(void) { return; } + aofManifest *am = aofLoadManifestFromFile(am_filepath); + if (am) aofManifestFreeAndUpdate(am); + sdsfree(am_name); + sdsfree(am_filepath); +} + +/* Generic manifest loading function, used in `aofLoadManifestFromDisk` and redis-check-aof tool. */ +#define MANIFEST_MAX_LINE 1024 +aofManifest *aofLoadManifestFromFile(sds am_filepath) { + const char *err = NULL; + long long maxseq = 0; + + aofManifest *am = aofManifestCreate(); FILE *fp = fopen(am_filepath, "r"); if (fp == NULL) { serverLog(LL_WARNING, "Fatal error: can't open the AOF manifest " - "file %s for reading: %s", am_name, strerror(errno)); + "file %s for reading: %s", am_filepath, strerror(errno)); exit(1); } - sdsfree(am_name); - sdsfree(am_filepath); - char buf[MANIFEST_MAX_LINE+1]; sds *argv = NULL; int argc; @@ -292,14 +299,14 @@ void aofLoadManifestFromDisk(void) { line = sdstrim(sdsnew(buf), " \t\r\n"); if (!sdslen(line)) { - err = "The AOF manifest file is invalid format"; + err = "Invalid AOF manifest file format"; goto loaderr; } argv = sdssplitargs(line, &argc); /* 'argc < 6' was done for forward compatibility. */ if (argv == NULL || argc < 6 || (argc % 2)) { - err = "The AOF manifest file is invalid format"; + err = "Invalid AOF manifest file format"; goto loaderr; } @@ -321,7 +328,7 @@ void aofLoadManifestFromDisk(void) { /* We have to make sure we load all the information. */ if (!ai->file_name || !ai->file_seq || !ai->file_type) { - err = "The AOF manifest file is invalid format"; + err = "Invalid AOF manifest file format"; goto loaderr; } @@ -329,21 +336,21 @@ void aofLoadManifestFromDisk(void) { argv = NULL; if (ai->file_type == AOF_FILE_TYPE_BASE) { - if (server.aof_manifest->base_aof_info) { + if (am->base_aof_info) { err = "Found duplicate base file information"; goto loaderr; } - server.aof_manifest->base_aof_info = ai; - server.aof_manifest->curr_base_file_seq = ai->file_seq; + am->base_aof_info = ai; + am->curr_base_file_seq = ai->file_seq; } else if (ai->file_type == AOF_FILE_TYPE_HIST) { - listAddNodeTail(server.aof_manifest->history_aof_list, ai); + listAddNodeTail(am->history_aof_list, ai); } else if (ai->file_type == AOF_FILE_TYPE_INCR) { if (ai->file_seq <= maxseq) { err = "Found a non-monotonic sequence number"; goto loaderr; } - listAddNodeTail(server.aof_manifest->incr_aof_list, ai); - server.aof_manifest->curr_incr_file_seq = ai->file_seq; + listAddNodeTail(am->incr_aof_list, ai); + am->curr_incr_file_seq = ai->file_seq; maxseq = ai->file_seq; } else { err = "Unknown AOF file type"; @@ -356,7 +363,7 @@ void aofLoadManifestFromDisk(void) { } fclose(fp); - return; + return am; loaderr: /* Sanitizer suppression: may report a false positive if we goto loaderr @@ -627,7 +634,7 @@ void aofUpgradePrepare(aofManifest *am) { server.aof_dirname, strerror(errno)); sdsfree(aof_filepath); - exit(1);; + exit(1); } sdsfree(aof_filepath); @@ -721,7 +728,7 @@ void aofOpenIfNeededOnServerStart(void) { exit(1); } - server.aof_last_incr_size = getAppendOnlyFileSize(aof_name); + server.aof_last_incr_size = getAppendOnlyFileSize(aof_name, NULL); } int aofFileExist(char *filename) { @@ -1338,26 +1345,35 @@ int loadSingleAppendOnlyFile(char *filename) { client *old_client = server.current_client; fakeClient = server.current_client = createAOFClient(); - /* Check if this AOF file has an RDB preamble. In that case we need to - * load the RDB file and later continue loading the AOF tail. */ + /* Check if the AOF file is in RDB format (it may be RDB encoded base AOF + * or old style RDB-preamble AOF). In that case we need to load the RDB file + * and later continue loading the AOF tail if it is an old style RDB-preamble AOF. */ char sig[5]; /* "REDIS" */ if (fread(sig,1,5,fp) != 5 || memcmp(sig,"REDIS",5) != 0) { - /* No RDB preamble, seek back at 0 offset. */ + /* Not in RDB format, seek back at 0 offset. */ if (fseek(fp,0,SEEK_SET) == -1) goto readerr; } else { - /* RDB preamble. Pass loading the RDB functions. */ + /* RDB format. Pass loading the RDB functions. */ rio rdb; + int old_style = !strcmp(filename, server.aof_filename); + if (old_style) + serverLog(LL_NOTICE, "Reading RDB preamble from AOF file..."); + else + serverLog(LL_NOTICE, "Reading RDB base file on AOF loading..."); - serverLog(LL_NOTICE,"Reading RDB preamble from AOF file..."); if (fseek(fp,0,SEEK_SET) == -1) goto readerr; rioInitWithFile(&rdb,fp); if (rdbLoadRio(&rdb,RDBFLAGS_AOF_PREAMBLE,NULL) != C_OK) { - serverLog(LL_WARNING,"Error reading the RDB preamble of the AOF file %s, AOF loading aborted", filename); + if (old_style) + serverLog(LL_WARNING, "Error reading the RDB preamble of the AOF file %s, AOF loading aborted", filename); + else + serverLog(LL_WARNING, "Error reading the RDB base file %s, AOF loading aborted", filename); + goto readerr; } else { loadingAbsProgress(ftello(fp)); last_progress_report_size = ftello(fp); - serverLog(LL_NOTICE,"Reading the remaining AOF tail..."); + if (old_style) serverLog(LL_NOTICE, "Reading the remaining AOF tail..."); } } @@ -1517,15 +1533,15 @@ uxeof: /* Unexpected AOF end of file. */ } } } - serverLog(LL_WARNING,"Unexpected end of file reading the append only file %s. You can: \ - 1) Make a backup of your AOF file, then use ./redis-check-aof --fix <filename>. \ - 2) Alternatively you can set the 'aof-load-truncated' configuration option to yes and restart the server.", filename); + serverLog(LL_WARNING, "Unexpected end of file reading the append only file %s. You can: " + "1) Make a backup of your AOF file, then use ./redis-check-aof --fix <filename.manifest>. " + "2) Alternatively you can set the 'aof-load-truncated' configuration option to yes and restart the server.", filename); ret = AOF_FAILED; goto cleanup; fmterr: /* Format error. */ - serverLog(LL_WARNING,"Bad file format reading the append only file %s: \ - make a backup of your AOF file, then use ./redis-check-aof --fix <filename>", filename); + serverLog(LL_WARNING, "Bad file format reading the append only file %s: " + "make a backup of your AOF file, then use ./redis-check-aof --fix <filename.manifest>", filename); ret = AOF_FAILED; /* fall through to cleanup. */ @@ -1540,7 +1556,7 @@ cleanup: /* Load the AOF files according the aofManifest pointed by am. */ int loadAppendOnlyFiles(aofManifest *am) { serverAssert(am != NULL); - int ret = C_OK; + int status, ret = C_OK; long long start; off_t total_size = 0; sds aof_name; @@ -1574,7 +1590,16 @@ int loadAppendOnlyFiles(aofManifest *am) { /* Here we calculate the total size of all BASE and INCR files in * advance, it will be set to `server.loading_total_bytes`. */ - total_size = getBaseAndIncrAppendOnlyFilesSize(am); + total_size = getBaseAndIncrAppendOnlyFilesSize(am, &status); + if (status != AOF_OK) { + /* If an AOF exists in the manifest but not on the disk, we consider this to be a fatal error. */ + if (status == AOF_NOT_EXIST) status = AOF_FAILED; + + return status; + } else if (total_size == 0) { + return AOF_EMPTY; + } + startLoading(total_size, RDBFLAGS_AOF_PREAMBLE, 0); /* Load BASE AOF if needed. */ @@ -1590,9 +1615,8 @@ int loadAppendOnlyFiles(aofManifest *am) { aof_name, (float)(ustime()-start)/1000000); } - /* If an AOF exists in the manifest but not on the disk, Or the truncated - * file is not the last file, we consider this to be a fatal error. */ - if (ret == AOF_NOT_EXIST || (ret == AOF_TRUNCATED && !last_file)) { + /* If the truncated file is not the last file, we consider this to be a fatal error. */ + if (ret == AOF_TRUNCATED && !last_file) { ret = AOF_FAILED; } @@ -1620,7 +1644,11 @@ int loadAppendOnlyFiles(aofManifest *am) { aof_name, (float)(ustime()-start)/1000000); } - if (ret == AOF_NOT_EXIST || (ret == AOF_TRUNCATED && !last_file)) { + /* We know that (at least) one of the AOF files has data (total_size > 0), + * so empty incr AOF file doesn't count as a AOF_EMPTY result */ + if (ret == AOF_EMPTY) ret = AOF_OK; + + if (ret == AOF_TRUNCATED && !last_file) { ret = AOF_FAILED; } @@ -1635,7 +1663,7 @@ int loadAppendOnlyFiles(aofManifest *am) { server.aof_fsync_offset = server.aof_current_size; cleanup: - stopLoading(ret == AOF_OK); + stopLoading(ret == AOF_OK || ret == AOF_TRUNCATED); return ret; } @@ -2007,10 +2035,14 @@ int rewriteStreamObject(rio *r, robj *key, robj *o) { /* Append XSETID after XADD, make sure lastid is correct, * in case of XDEL lastid. */ - if (!rioWriteBulkCount(r,'*',3) || + if (!rioWriteBulkCount(r,'*',7) || !rioWriteBulkString(r,"XSETID",6) || !rioWriteBulkObject(r,key) || - !rioWriteBulkStreamID(r,&s->last_id)) + !rioWriteBulkStreamID(r,&s->last_id) || + !rioWriteBulkString(r,"ENTRIESADDED",12) || + !rioWriteBulkLongLong(r,s->entries_added) || + !rioWriteBulkString(r,"MAXDELETEDID",12) || + !rioWriteBulkStreamID(r,&s->max_deleted_entry_id)) { streamIteratorStop(&si); return 0; @@ -2025,12 +2057,14 @@ int rewriteStreamObject(rio *r, robj *key, robj *o) { while(raxNext(&ri)) { streamCG *group = ri.data; /* Emit the XGROUP CREATE in order to create the group. */ - if (!rioWriteBulkCount(r,'*',5) || + if (!rioWriteBulkCount(r,'*',7) || !rioWriteBulkString(r,"XGROUP",6) || !rioWriteBulkString(r,"CREATE",6) || !rioWriteBulkObject(r,key) || !rioWriteBulkString(r,(char*)ri.key,ri.key_len) || - !rioWriteBulkStreamID(r,&group->last_id)) + !rioWriteBulkStreamID(r,&group->last_id) || + !rioWriteBulkString(r,"ENTRIESREAD",11) || + !rioWriteBulkLongLong(r,group->entries_read)) { raxStop(&ri); streamIteratorStop(&si); @@ -2332,7 +2366,7 @@ int rewriteAppendOnlyFileBackground(void) { server.aof_selected_db = -1; flushAppendOnlyFile(1); if (openNewIncrAofForAppend() != C_OK) return C_ERR; - + server.stat_aof_rewrites++; if ((childpid = redisFork(CHILD_TYPE_AOF)) == 0) { char tmpfile[256]; @@ -2388,7 +2422,10 @@ void aofRemoveTempFile(pid_t childpid) { bg_unlink(tmpfile); } -off_t getAppendOnlyFileSize(sds filename) { +/* Get size of an AOF file. + * The status argument is an optional output argument to be filled with + * one of the AOF_ status values. */ +off_t getAppendOnlyFileSize(sds filename, int *status) { struct redis_stat sb; off_t size; mstime_t latency; @@ -2396,10 +2433,12 @@ off_t getAppendOnlyFileSize(sds filename) { sds aof_filepath = makePath(server.aof_dirname, filename); latencyStartMonitor(latency); if (redis_stat(aof_filepath, &sb) == -1) { + if (status) *status = errno == ENOENT ? AOF_NOT_EXIST : AOF_OPEN_ERR; serverLog(LL_WARNING, "Unable to obtain the AOF file %s length. stat: %s", filename, strerror(errno)); size = 0; } else { + if (status) *status = AOF_OK; size = sb.st_size; } latencyEndMonitor(latency); @@ -2408,22 +2447,27 @@ off_t getAppendOnlyFileSize(sds filename) { return size; } -off_t getBaseAndIncrAppendOnlyFilesSize(aofManifest *am) { +/* Get size of all AOF files referred by the manifest (excluding history). + * The status argument is an output argument to be filled with + * one of the AOF_ status values. */ +off_t getBaseAndIncrAppendOnlyFilesSize(aofManifest *am, int *status) { off_t size = 0; - listNode *ln; listIter li; if (am->base_aof_info) { serverAssert(am->base_aof_info->file_type == AOF_FILE_TYPE_BASE); - size += getAppendOnlyFileSize(am->base_aof_info->file_name); + + size += getAppendOnlyFileSize(am->base_aof_info->file_name, status); + if (*status != AOF_OK) return 0; } listRewind(am->incr_aof_list, &li); while ((ln = listNext(&li)) != NULL) { aofInfo *ai = (aofInfo*)ln->value; serverAssert(ai->file_type == AOF_FILE_TYPE_INCR); - size += getAppendOnlyFileSize(ai->file_name); + size += getAppendOnlyFileSize(ai->file_name, status); + if (*status != AOF_OK) return 0; } return size; @@ -2497,7 +2541,7 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) { if (server.aof_fd != -1) { /* AOF enabled. */ server.aof_selected_db = -1; /* Make sure SELECT is re-issued */ - server.aof_current_size = getAppendOnlyFileSize(new_base_filename) + server.aof_last_incr_size; + server.aof_current_size = getAppendOnlyFileSize(new_base_filename, NULL) + server.aof_last_incr_size; server.aof_rewrite_base_size = server.aof_current_size; server.aof_fsync_offset = server.aof_current_size; server.aof_last_fsync = server.unixtime; diff --git a/src/blocked.c b/src/blocked.c index 0feab4a69..aa298cffb 100644 --- a/src/blocked.c +++ b/src/blocked.c @@ -108,9 +108,11 @@ void blockClient(client *c, int btype) { /* This function is called after a client has finished a blocking operation * in order to update the total command duration, log the command into * the Slow log if needed, and log the reply duration event if needed. */ -void updateStatsOnUnblock(client *c, long blocked_us, long reply_us){ +void updateStatsOnUnblock(client *c, long blocked_us, long reply_us, int had_errors){ const ustime_t total_cmd_duration = c->duration + blocked_us + reply_us; c->lastcmd->microseconds += total_cmd_duration; + if (had_errors) + c->lastcmd->failed_calls++; if (server.latency_tracking_enabled) updateCommandLatencyHistogram(&(c->lastcmd->latency_histogram), total_cmd_duration*1000); /* Log the command into the Slow log if needed. */ @@ -314,6 +316,7 @@ void serveClientsBlockedOnListKey(robj *o, readyList *rl) { * call. */ if (dstkey) incrRefCount(dstkey); + long long prev_error_replies = server.stat_total_error_replies; client *old_client = server.current_client; server.current_client = receiver; monotime replyTimer; @@ -322,7 +325,7 @@ void serveClientsBlockedOnListKey(robj *o, readyList *rl) { rl->key, dstkey, rl->db, wherefrom, whereto, &deleted); - updateStatsOnUnblock(receiver, 0, elapsedUs(replyTimer)); + updateStatsOnUnblock(receiver, 0, elapsedUs(replyTimer), server.stat_total_error_replies != prev_error_replies); unblockClient(receiver); afterCommand(receiver); server.current_client = old_client; @@ -366,6 +369,7 @@ void serveClientsBlockedOnSortedSetKey(robj *o, readyList *rl) { ? 1 : 0; int reply_nil_when_empty = use_nested_array; + long long prev_error_replies = server.stat_total_error_replies; client *old_client = server.current_client; server.current_client = receiver; monotime replyTimer; @@ -388,7 +392,7 @@ void serveClientsBlockedOnSortedSetKey(robj *o, readyList *rl) { decrRefCount(argv[1]); if (count != -1) decrRefCount(argv[2]); - updateStatsOnUnblock(receiver, 0, elapsedUs(replyTimer)); + updateStatsOnUnblock(receiver, 0, elapsedUs(replyTimer), server.stat_total_error_replies != prev_error_replies); unblockClient(receiver); afterCommand(receiver); server.current_client = old_client; @@ -421,6 +425,12 @@ void serveClientsBlockedOnStreamKey(robj *o, readyList *rl) { bkinfo *bki = dictFetchValue(receiver->bpop.keys,rl->key); streamID *gt = &bki->stream_id; + long long prev_error_replies = server.stat_total_error_replies; + client *old_client = server.current_client; + server.current_client = receiver; + monotime replyTimer; + elapsedStart(&replyTimer); + /* If we blocked in the context of a consumer * group, we need to resolve the group and update the * last ID the client is blocked for: this is needed @@ -440,8 +450,7 @@ void serveClientsBlockedOnStreamKey(robj *o, readyList *rl) { addReplyError(receiver, "-NOGROUP the consumer group this client " "was blocked on no longer exists"); - unblockClient(receiver); - continue; + goto unblock_receiver; } else { *gt = group->last_id; } @@ -470,10 +479,6 @@ void serveClientsBlockedOnStreamKey(robj *o, readyList *rl) { } } - client *old_client = server.current_client; - server.current_client = receiver; - monotime replyTimer; - elapsedStart(&replyTimer); /* Emit the two elements sub-array consisting of * the name of the stream and the data we * extracted from it. Wrapped in a single-item @@ -493,11 +498,13 @@ void serveClientsBlockedOnStreamKey(robj *o, readyList *rl) { streamReplyWithRange(receiver,s,&start,NULL, receiver->bpop.xread_count, 0, group, consumer, noack, &pi); - updateStatsOnUnblock(receiver, 0, elapsedUs(replyTimer)); /* Note that after we unblock the client, 'gt' * and other receiver->bpop stuff are no longer * valid, so we must do the setup above before - * this call. */ + * the unblockClient call. */ + +unblock_receiver: + updateStatsOnUnblock(receiver, 0, elapsedUs(replyTimer), server.stat_total_error_replies != prev_error_replies); unblockClient(receiver); afterCommand(receiver); server.current_client = old_client; @@ -545,12 +552,13 @@ void serveClientsBlockedOnKeyByModule(readyList *rl) { * different modules with different triggers to consider if a key * is ready or not. This means we can't exit the loop but need * to continue after the first failure. */ + long long prev_error_replies = server.stat_total_error_replies; client *old_client = server.current_client; server.current_client = receiver; monotime replyTimer; elapsedStart(&replyTimer); if (!moduleTryServeClientBlockedOnKey(receiver, rl->key)) continue; - updateStatsOnUnblock(receiver, 0, elapsedUs(replyTimer)); + updateStatsOnUnblock(receiver, 0, elapsedUs(replyTimer), server.stat_total_error_replies != prev_error_replies); moduleUnblockClient(receiver); afterCommand(receiver); server.current_client = old_client; diff --git a/src/call_reply.c b/src/call_reply.c index 7aa79d089..3694db55e 100644 --- a/src/call_reply.c +++ b/src/call_reply.c @@ -60,7 +60,7 @@ struct CallReply { double d; /* Reply value for double reply. */ struct CallReply *array; /* Array of sub-reply elements. used for set, array, map, and attribute */ } val; - + list *deferred_error_list; /* list of errors in sds form or NULL */ struct CallReply *attribute; /* attribute reply, NULL if not exists */ }; @@ -237,6 +237,8 @@ void freeCallReply(CallReply *rep) { freeCallReplyInternal(rep); } sdsfree(rep->original_proto); + if (rep->deferred_error_list) + listRelease(rep->deferred_error_list); zfree(rep); } @@ -488,6 +490,11 @@ int callReplyIsResp3(CallReply *rep) { return rep->flags & REPLY_FLAG_RESP3; } +/* Returns a list of errors in sds form, or NULL. */ +list *callReplyDeferredErrorList(CallReply *rep) { + return rep->deferred_error_list; +} + /* Create a new CallReply struct from the reply blob. * * The function will own the reply blob, so it must not be used or freed by @@ -496,6 +503,9 @@ int callReplyIsResp3(CallReply *rep) { * The reply blob will be freed when the returned CallReply struct is later * freed using freeCallReply(). * + * The deferred_error_list is an optional list of errors that are present + * in the reply blob, if given, this function will take ownership on it. + * * The private_data is optional and can later be accessed using * callReplyGetPrivateData(). * @@ -504,7 +514,7 @@ int callReplyIsResp3(CallReply *rep) { * DESIGNED TO HANDLE USER INPUT and using it to parse invalid replies is * unsafe. */ -CallReply *callReplyCreate(sds reply, void *private_data) { +CallReply *callReplyCreate(sds reply, list *deferred_error_list, void *private_data) { CallReply *res = zmalloc(sizeof(*res)); res->flags = REPLY_FLAG_ROOT; res->original_proto = reply; @@ -512,5 +522,6 @@ CallReply *callReplyCreate(sds reply, void *private_data) { res->proto_len = sdslen(reply); res->private_data = private_data; res->attribute = NULL; + res->deferred_error_list = deferred_error_list; return res; } diff --git a/src/call_reply.h b/src/call_reply.h index 5b07dc437..ff98f7f5a 100644 --- a/src/call_reply.h +++ b/src/call_reply.h @@ -34,7 +34,7 @@ typedef struct CallReply CallReply; -CallReply *callReplyCreate(sds reply, void *private_data); +CallReply *callReplyCreate(sds reply, list *deferred_error_list, void *private_data); int callReplyType(CallReply *rep); const char *callReplyGetString(CallReply *rep, size_t *len); long long callReplyGetLongLong(CallReply *rep); @@ -51,6 +51,7 @@ const char *callReplyGetVerbatim(CallReply *rep, size_t *len, const char **forma const char *callReplyGetProto(CallReply *rep, size_t *len); void *callReplyGetPrivateData(CallReply *rep); int callReplyIsResp3(CallReply *rep); +list *callReplyDeferredErrorList(CallReply *rep); void freeCallReply(CallReply *rep); #endif /* SRC_CALL_REPLY_H_ */ diff --git a/src/cluster.c b/src/cluster.c index d2d179e6f..32335bbf9 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -106,7 +106,7 @@ dictType clusterNodesDictType = { }; /* Cluster re-addition blacklist. This maps node IDs to the time - * we can re-add this node. The goal is to avoid readding a removed + * we can re-add this node. The goal is to avoid reading a removed * node for some time. */ dictType clusterNodesBlackListDictType = { dictSdsCaseHash, /* hash function */ @@ -243,10 +243,9 @@ int clusterLoadConfig(char *filename) { if (hostname) { *hostname = '\0'; hostname++; - zfree(n->hostname); - n->hostname = zstrdup(hostname); - } else { - n->hostname = NULL; + n->hostname = sdscpy(n->hostname, hostname); + } else if (sdslen(n->hostname) != 0) { + sdsclear(n->hostname); } /* The plaintext port for client in a TLS cluster (n->pport) is not @@ -570,20 +569,15 @@ void clusterUpdateMyselfIp(void) { /* Update the hostname for the specified node with the provided C string. */ static void updateAnnouncedHostname(clusterNode *node, char *new) { - if (!node->hostname && !new) { - return; - } - /* Previous and new hostname are the same, no need to update. */ - if (new && node->hostname && !strcmp(new, node->hostname)) { + if (new && !strcmp(new, node->hostname)) { return; } - if (node->hostname) zfree(node->hostname); if (new) { - node->hostname = zstrdup(new); - } else { - node->hostname = NULL; + node->hostname = sdscpy(node->hostname, new); + } else if (sdslen(node->hostname) != 0) { + sdsclear(node->hostname); } } @@ -959,7 +953,7 @@ clusterNode *createClusterNode(char *nodename, int flags) { node->link = NULL; node->inbound_link = NULL; memset(node->ip,0,sizeof(node->ip)); - node->hostname = NULL; + node->hostname = sdsempty(); node->port = 0; node->cport = 0; node->pport = 0; @@ -1125,7 +1119,7 @@ void freeClusterNode(clusterNode *n) { nodename = sdsnewlen(n->name, CLUSTER_NAMELEN); serverAssert(dictDelete(server.cluster->nodes,nodename) == DICT_OK); sdsfree(nodename); - zfree(n->hostname); + sdsfree(n->hostname); /* Release links and associated data structures. */ if (n->link) freeClusterLink(n->link); @@ -1947,9 +1941,9 @@ static clusterMsgPingExt *getNextPingExt(clusterMsgPingExt *ext) { * will be 8 byte padded. */ int getHostnamePingExtSize() { /* If hostname is not set, we don't send this extension */ - if (!myself->hostname) return 0; + if (sdslen(myself->hostname) == 0) return 0; - int totlen = sizeof(clusterMsgPingExt) + EIGHT_BYTE_ALIGN(strlen(myself->hostname) + 1); + int totlen = sizeof(clusterMsgPingExt) + EIGHT_BYTE_ALIGN(sdslen(myself->hostname) + 1); return totlen; } @@ -1958,19 +1952,18 @@ int getHostnamePingExtSize() { * will return the amount of bytes written. */ int writeHostnamePingExt(clusterMsgPingExt **cursor) { /* If hostname is not set, we don't send this extension */ - if (!myself->hostname) return 0; + if (sdslen(myself->hostname) == 0) return 0; /* Add the hostname information at the extension cursor */ clusterMsgPingExtHostname *ext = &(*cursor)->ext[0].hostname; - size_t hostname_len = strlen(myself->hostname); - memcpy(ext->hostname, myself->hostname, hostname_len); + memcpy(ext->hostname, myself->hostname, sdslen(myself->hostname)); uint32_t extension_size = getHostnamePingExtSize(); /* Move the write cursor */ (*cursor)->type = CLUSTERMSG_EXT_TYPE_HOSTNAME; (*cursor)->length = htonl(extension_size); /* Make sure the string is NULL terminated by adding 1 */ - *cursor = (clusterMsgPingExt *) (ext->hostname + EIGHT_BYTE_ALIGN(strlen(myself->hostname) + 1)); + *cursor = (clusterMsgPingExt *) (ext->hostname + EIGHT_BYTE_ALIGN(sdslen(myself->hostname) + 1)); return extension_size; } @@ -2975,7 +2968,7 @@ void clusterSendPing(clusterLink *link, int type) { /* Set the initial extension position */ clusterMsgPingExt *cursor = getInitialPingExt(hdr, gossipcount); /* Add in the extensions */ - if (myself->hostname) { + if (sdslen(myself->hostname) != 0) { hdr->mflags[0] |= CLUSTERMSG_FLAG0_EXT_DATA; totlen += writeHostnamePingExt(&cursor); extensions++; @@ -3959,7 +3952,8 @@ void clusterCron(void) { iteration++; /* Number of times this function was called so far. */ - updateAnnouncedHostname(myself, server.cluster_announce_hostname); + clusterUpdateMyselfHostname(); + /* The handshake timeout is the time after which a handshake node that was * not turned into a normal node is removed from the nodes. Usually it is * just the NODE_TIMEOUT value, but when NODE_TIMEOUT is too small we use @@ -4578,7 +4572,7 @@ sds clusterGenNodeDescription(clusterNode *node, int use_pport) { /* Node coordinates */ ci = sdscatlen(sdsempty(),node->name,CLUSTER_NAMELEN); - if (node->hostname) { + if (sdslen(node->hostname) != 0) { ci = sdscatfmt(ci," %s:%i@%i,%s ", node->ip, port, @@ -4804,7 +4798,7 @@ void addReplyClusterLinksDescription(client *c) { const char *getPreferredEndpoint(clusterNode *n) { switch(server.cluster_preferred_endpoint_type) { case CLUSTER_ENDPOINT_TYPE_IP: return n->ip; - case CLUSTER_ENDPOINT_TYPE_HOSTNAME: return n->hostname ? n->hostname : "?"; + case CLUSTER_ENDPOINT_TYPE_HOSTNAME: return (sdslen(n->hostname) != 0) ? n->hostname : "?"; case CLUSTER_ENDPOINT_TYPE_UNKNOWN_ENDPOINT: return ""; } return "unknown"; @@ -4898,7 +4892,7 @@ void addNodeToNodeReply(client *c, clusterNode *node) { if (server.cluster_preferred_endpoint_type == CLUSTER_ENDPOINT_TYPE_IP) { addReplyBulkCString(c, node->ip); } else if (server.cluster_preferred_endpoint_type == CLUSTER_ENDPOINT_TYPE_HOSTNAME) { - addReplyBulkCString(c, node->hostname ? node->hostname : "?"); + addReplyBulkCString(c, sdslen(node->hostname) != 0 ? node->hostname : "?"); } else if (server.cluster_preferred_endpoint_type == CLUSTER_ENDPOINT_TYPE_UNKNOWN_ENDPOINT) { addReplyNull(c); } else { @@ -4921,7 +4915,7 @@ void addNodeToNodeReply(client *c, clusterNode *node) { length++; } if (server.cluster_preferred_endpoint_type != CLUSTER_ENDPOINT_TYPE_HOSTNAME - && node->hostname) + && sdslen(node->hostname) != 0) { addReplyBulkCString(c, "hostname"); addReplyBulkCString(c, node->hostname); @@ -5032,7 +5026,7 @@ void clusterCommand(client *c) { " Reset current node (default: soft).", "SET-CONFIG-EPOCH <epoch>", " Set config epoch of current node.", -"SETSLOT <slot> (IMPORTING|MIGRATING|STABLE|NODE <node-id>)", +"SETSLOT <slot> (IMPORTING <node-id>|MIGRATING <node-id>|STABLE|NODE <node-id>)", " Set slot state.", "REPLICAS <node-id>", " Return <node-id> replicas.", @@ -5226,6 +5220,10 @@ NULL (char*)c->argv[4]->ptr); return; } + if (nodeIsSlave(n)) { + addReplyError(c,"Target node is not a master"); + return; + } /* If this hash slot was served by 'myself' before to switch * make sure there are no longer local keys for this hash slot. */ if (server.cluster->slots[slot] == myself && n != myself) { @@ -5285,7 +5283,7 @@ NULL addReplySds(c,reply); } else if (!strcasecmp(c->argv[1]->ptr,"info") && c->argc == 2) { /* CLUSTER INFO */ - char *statestr[] = {"ok","fail","needhelp"}; + char *statestr[] = {"ok","fail"}; int slots_assigned = 0, slots_ok = 0, slots_pfail = 0, slots_fail = 0; uint64_t myepoch; int j; @@ -5703,7 +5701,7 @@ int verifyDumpPayload(unsigned char *p, size_t len, uint16_t *rdbver_ptr) { if (len < 10) return C_ERR; footer = p+(len-10); - /* Verify RDB version */ + /* Set and verify RDB version. */ rdbver = (footer[1] << 8) | footer[0]; if (rdbver_ptr) { *rdbver_ptr = rdbver; diff --git a/src/cluster.h b/src/cluster.h index 465654205..314b747be 100644 --- a/src/cluster.h +++ b/src/cluster.h @@ -96,8 +96,8 @@ typedef struct clusterLink { #define CLUSTERMSG_TYPE_UPDATE 7 /* Another node slots configuration */ #define CLUSTERMSG_TYPE_MFSTART 8 /* Pause clients for manual failover */ #define CLUSTERMSG_TYPE_MODULE 9 /* Module cluster API message. */ -#define CLUSTERMSG_TYPE_COUNT 10 /* Total number of message types. */ -#define CLUSTERMSG_TYPE_PUBLISHSHARD 11 /* Pub/Sub Publish shard propagation */ +#define CLUSTERMSG_TYPE_PUBLISHSHARD 10 /* Pub/Sub Publish shard propagation */ +#define CLUSTERMSG_TYPE_COUNT 11 /* Total number of message types. */ /* Flags that a module can set in order to prevent certain Redis Cluster * features to be enabled. Useful when implementing a different distributed @@ -134,8 +134,8 @@ typedef struct clusterNode { mstime_t repl_offset_time; /* Unix time we received offset for this node */ mstime_t orphaned_time; /* Starting time of orphaned master condition */ long long repl_offset; /* Last known repl offset for this node. */ - char ip[NET_IP_STR_LEN]; /* Latest known IP address of this node */ - char *hostname; /* The known hostname for this node */ + char ip[NET_IP_STR_LEN]; /* Latest known IP address of this node */ + sds hostname; /* The known hostname for this node */ int port; /* Latest known clients port (TLS or plain). */ int pport; /* Latest known clients plaintext port. Only used if the main clients port is for TLS. */ @@ -339,8 +339,6 @@ typedef struct { * changes in clusterMsg be caught at compile time. */ -/* Avoid static_assert on non-C11 compilers. */ -#if __STDC_VERSION__ >= 201112L static_assert(offsetof(clusterMsg, sig) == 0, "unexpected field offset"); static_assert(offsetof(clusterMsg, totlen) == 4, "unexpected field offset"); static_assert(offsetof(clusterMsg, ver) == 8, "unexpected field offset"); @@ -362,7 +360,6 @@ static_assert(offsetof(clusterMsg, flags) == 2250, "unexpected field offset"); static_assert(offsetof(clusterMsg, state) == 2252, "unexpected field offset"); static_assert(offsetof(clusterMsg, mflags) == 2253, "unexpected field offset"); static_assert(offsetof(clusterMsg, data) == 2256, "unexpected field offset"); -#endif #define CLUSTERMSG_MIN_LEN (sizeof(clusterMsg)-sizeof(union clusterMsgData)) diff --git a/src/commands.c b/src/commands.c index 62291f33f..d3b2e45b3 100644 --- a/src/commands.c +++ b/src/commands.c @@ -588,9 +588,9 @@ NULL /* CLUSTER SETSLOT subcommand argument table */ struct redisCommandArg CLUSTER_SETSLOT_subcommand_Subargs[] = { -{"node-id",ARG_TYPE_INTEGER,-1,"IMPORTING",NULL,NULL,CMD_ARG_NONE}, -{"node-id",ARG_TYPE_INTEGER,-1,"MIGRATING",NULL,NULL,CMD_ARG_NONE}, -{"node-id",ARG_TYPE_INTEGER,-1,"NODE",NULL,NULL,CMD_ARG_NONE}, +{"node-id",ARG_TYPE_STRING,-1,"IMPORTING",NULL,NULL,CMD_ARG_NONE}, +{"node-id",ARG_TYPE_STRING,-1,"MIGRATING",NULL,NULL,CMD_ARG_NONE}, +{"node-id",ARG_TYPE_STRING,-1,"NODE",NULL,NULL,CMD_ARG_NONE}, {"stable",ARG_TYPE_PURE_TOKEN,-1,"STABLE",NULL,NULL,CMD_ARG_NONE}, {0} }; @@ -3092,6 +3092,12 @@ struct redisCommandArg PUBSUB_SHARDCHANNELS_Args[] = { /* PUBSUB SHARDNUMSUB tips */ #define PUBSUB_SHARDNUMSUB_tips NULL +/* PUBSUB SHARDNUMSUB argument table */ +struct redisCommandArg PUBSUB_SHARDNUMSUB_Args[] = { +{"channel",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_OPTIONAL|CMD_ARG_MULTIPLE}, +{0} +}; + /* PUBSUB command table */ struct redisCommand PUBSUB_Subcommands[] = { {"channels","List active channels","O(N) where N is the number of active channels, and assuming constant time pattern matching (relatively short channels and patterns)","2.8.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_PUBSUB,PUBSUB_CHANNELS_History,PUBSUB_CHANNELS_tips,pubsubCommand,-2,CMD_PUBSUB|CMD_LOADING|CMD_STALE,0,.args=PUBSUB_CHANNELS_Args}, @@ -3099,7 +3105,7 @@ struct redisCommand PUBSUB_Subcommands[] = { {"numpat","Get the count of unique patterns pattern subscriptions","O(1)","2.8.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_PUBSUB,PUBSUB_NUMPAT_History,PUBSUB_NUMPAT_tips,pubsubCommand,2,CMD_PUBSUB|CMD_LOADING|CMD_STALE,0}, {"numsub","Get the count of subscribers for channels","O(N) for the NUMSUB subcommand, where N is the number of requested channels","2.8.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_PUBSUB,PUBSUB_NUMSUB_History,PUBSUB_NUMSUB_tips,pubsubCommand,-2,CMD_PUBSUB|CMD_LOADING|CMD_STALE,0,.args=PUBSUB_NUMSUB_Args}, {"shardchannels","List active shard channels","O(N) where N is the number of active shard channels, and assuming constant time pattern matching (relatively short channels).","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_PUBSUB,PUBSUB_SHARDCHANNELS_History,PUBSUB_SHARDCHANNELS_tips,pubsubCommand,-2,CMD_PUBSUB|CMD_LOADING|CMD_STALE,0,.args=PUBSUB_SHARDCHANNELS_Args}, -{"shardnumsub","Get the count of subscribers for shard channels","O(N) for the SHARDNUMSUB subcommand, where N is the number of requested channels","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_PUBSUB,PUBSUB_SHARDNUMSUB_History,PUBSUB_SHARDNUMSUB_tips,pubsubCommand,-2,CMD_PUBSUB|CMD_LOADING|CMD_STALE,0}, +{"shardnumsub","Get the count of subscribers for shard channels","O(N) for the SHARDNUMSUB subcommand, where N is the number of requested channels","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_PUBSUB,PUBSUB_SHARDNUMSUB_History,PUBSUB_SHARDNUMSUB_tips,pubsubCommand,-2,CMD_PUBSUB|CMD_LOADING|CMD_STALE,0,.args=PUBSUB_SHARDNUMSUB_Args}, {0} }; @@ -3315,7 +3321,7 @@ NULL /* FUNCTION DELETE argument table */ struct redisCommandArg FUNCTION_DELETE_Args[] = { -{"function-name",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_NONE}, +{"library-name",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_NONE}, {0} }; @@ -3404,7 +3410,7 @@ struct redisCommandArg FUNCTION_LOAD_Args[] = { {"engine-name",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_NONE}, {"library-name",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_NONE}, {"replace",ARG_TYPE_PURE_TOKEN,-1,"REPLACE",NULL,NULL,CMD_ARG_OPTIONAL}, -{"library-description",ARG_TYPE_STRING,-1,"DESC",NULL,NULL,CMD_ARG_OPTIONAL}, +{"library-description",ARG_TYPE_STRING,-1,"DESCRIPTION",NULL,NULL,CMD_ARG_OPTIONAL}, {"function-code",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_NONE}, {0} }; @@ -3759,7 +3765,7 @@ struct redisCommand SCRIPT_Subcommands[] = { struct redisCommand SENTINEL_Subcommands[] = { {"ckquorum","Check for a Sentinel quorum",NULL,"2.8.4",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SENTINEL,SENTINEL_CKQUORUM_History,SENTINEL_CKQUORUM_tips,sentinelCommand,3,CMD_ADMIN|CMD_SENTINEL|CMD_ONLY_SENTINEL,0}, {"config","Configure Sentinel","O(1)","6.2.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SENTINEL,SENTINEL_CONFIG_History,SENTINEL_CONFIG_tips,sentinelCommand,-3,CMD_ADMIN|CMD_SENTINEL|CMD_ONLY_SENTINEL,0}, -{"debug",NULL,NULL,"7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SENTINEL,SENTINEL_DEBUG_History,SENTINEL_DEBUG_tips,sentinelCommand,-2,CMD_ADMIN|CMD_SENTINEL|CMD_ONLY_SENTINEL,0}, +{"debug","List or update the current configurable parameters","O(N) where N is the number of configurable parameters","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SENTINEL,SENTINEL_DEBUG_History,SENTINEL_DEBUG_tips,sentinelCommand,-2,CMD_ADMIN|CMD_SENTINEL|CMD_ONLY_SENTINEL,0}, {"failover","Force a failover",NULL,"2.8.4",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SENTINEL,SENTINEL_FAILOVER_History,SENTINEL_FAILOVER_tips,sentinelCommand,3,CMD_ADMIN|CMD_SENTINEL|CMD_ONLY_SENTINEL,0}, {"flushconfig","Rewrite configuration file","O(1)","2.8.4",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SENTINEL,SENTINEL_FLUSHCONFIG_History,SENTINEL_FLUSHCONFIG_tips,sentinelCommand,2,CMD_ADMIN|CMD_SENTINEL|CMD_ONLY_SENTINEL,0}, {"get-master-addr-by-name","Get port and address of a master","O(1)","2.8.4",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SENTINEL,SENTINEL_GET_MASTER_ADDR_BY_NAME_History,SENTINEL_GET_MASTER_ADDR_BY_NAME_tips,sentinelCommand,3,CMD_ADMIN|CMD_SENTINEL|CMD_ONLY_SENTINEL,0}, @@ -4033,6 +4039,14 @@ struct redisCommandArg COMMAND_DOCS_Args[] = { /* COMMAND GETKEYS tips */ #define COMMAND_GETKEYS_tips NULL +/********** COMMAND GETKEYSANDFLAGS ********************/ + +/* COMMAND GETKEYSANDFLAGS history */ +#define COMMAND_GETKEYSANDFLAGS_History NULL + +/* COMMAND GETKEYSANDFLAGS tips */ +#define COMMAND_GETKEYSANDFLAGS_tips NULL + /********** COMMAND HELP ********************/ /* COMMAND HELP history */ @@ -4085,6 +4099,7 @@ struct redisCommand COMMAND_Subcommands[] = { {"count","Get total number of Redis commands","O(1)","2.8.13",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SERVER,COMMAND_COUNT_History,COMMAND_COUNT_tips,commandCountCommand,2,CMD_LOADING|CMD_STALE,ACL_CATEGORY_CONNECTION}, {"docs","Get array of specific Redis command documentation","O(N) where N is the number of commands to look up","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SERVER,COMMAND_DOCS_History,COMMAND_DOCS_tips,commandDocsCommand,-2,CMD_LOADING|CMD_STALE,ACL_CATEGORY_CONNECTION,.args=COMMAND_DOCS_Args}, {"getkeys","Extract keys given a full Redis command","O(N) where N is the number of arguments to the command","2.8.13",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SERVER,COMMAND_GETKEYS_History,COMMAND_GETKEYS_tips,commandGetKeysCommand,-4,CMD_LOADING|CMD_STALE,ACL_CATEGORY_CONNECTION}, +{"getkeysandflags","Extract keys given a full Redis command","O(N) where N is the number of arguments to the command","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SERVER,COMMAND_GETKEYSANDFLAGS_History,COMMAND_GETKEYSANDFLAGS_tips,commandGetKeysAndFlagsCommand,-4,CMD_LOADING|CMD_STALE,ACL_CATEGORY_CONNECTION}, {"help","Show helpful text about the different subcommands","O(1)","5.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SERVER,COMMAND_HELP_History,COMMAND_HELP_tips,commandHelpCommand,2,CMD_LOADING|CMD_STALE,ACL_CATEGORY_CONNECTION}, {"info","Get array of specific Redis command details, or all when no argument is given.","O(N) where N is the number of commands to look up","2.8.13",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SERVER,COMMAND_INFO_History,COMMAND_INFO_tips,commandInfoCommand,-2,CMD_LOADING|CMD_STALE,ACL_CATEGORY_CONNECTION,.args=COMMAND_INFO_Args}, {"list","Get an array of Redis command names","O(N) where N is the total number of Redis commands","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SERVER,COMMAND_LIST_History,COMMAND_LIST_tips,commandListCommand,-2,CMD_LOADING|CMD_STALE,ACL_CATEGORY_CONNECTION,.args=COMMAND_LIST_Args}, @@ -4300,7 +4315,10 @@ struct redisCommandArg FLUSHDB_Args[] = { /********** INFO ********************/ /* INFO history */ -#define INFO_History NULL +commandHistory INFO_History[] = { +{"7.0.0","Added support for taking multiple section arguments."}, +{0} +}; /* INFO tips */ const char *INFO_tips[] = { @@ -4312,7 +4330,7 @@ NULL /* INFO argument table */ struct redisCommandArg INFO_Args[] = { -{"section",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_OPTIONAL}, +{"section",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_OPTIONAL|CMD_ARG_MULTIPLE}, {0} }; @@ -5913,7 +5931,10 @@ struct redisCommandArg XADD_Args[] = { /********** XAUTOCLAIM ********************/ /* XAUTOCLAIM history */ -#define XAUTOCLAIM_History NULL +commandHistory XAUTOCLAIM_History[] = { +{"7.0.0","Added an element to the reply array, containing deleted entries the command cleared from the PEL"}, +{0} +}; /* XAUTOCLAIM tips */ const char *XAUTOCLAIM_tips[] = { @@ -5977,7 +5998,10 @@ struct redisCommandArg XDEL_Args[] = { /********** XGROUP CREATE ********************/ /* XGROUP CREATE history */ -#define XGROUP_CREATE_History NULL +commandHistory XGROUP_CREATE_History[] = { +{"7.0.0","Added the `entries_read` named argument."}, +{0} +}; /* XGROUP CREATE tips */ #define XGROUP_CREATE_tips NULL @@ -5995,6 +6019,7 @@ struct redisCommandArg XGROUP_CREATE_Args[] = { {"groupname",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_NONE}, {"id",ARG_TYPE_ONEOF,-1,NULL,NULL,NULL,CMD_ARG_NONE,.subargs=XGROUP_CREATE_id_Subargs}, {"mkstream",ARG_TYPE_PURE_TOKEN,-1,"MKSTREAM",NULL,NULL,CMD_ARG_OPTIONAL}, +{"entries_read",ARG_TYPE_INTEGER,-1,"ENTRIESREAD",NULL,NULL,CMD_ARG_OPTIONAL}, {0} }; @@ -6056,7 +6081,10 @@ struct redisCommandArg XGROUP_DESTROY_Args[] = { /********** XGROUP SETID ********************/ /* XGROUP SETID history */ -#define XGROUP_SETID_History NULL +commandHistory XGROUP_SETID_History[] = { +{"7.0.0","Added the optional `entries_read` argument."}, +{0} +}; /* XGROUP SETID tips */ #define XGROUP_SETID_tips NULL @@ -6073,6 +6101,7 @@ struct redisCommandArg XGROUP_SETID_Args[] = { {"key",ARG_TYPE_KEY,0,NULL,NULL,NULL,CMD_ARG_NONE}, {"groupname",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_NONE}, {"id",ARG_TYPE_ONEOF,-1,NULL,NULL,NULL,CMD_ARG_NONE,.subargs=XGROUP_SETID_id_Subargs}, +{"entries_read",ARG_TYPE_INTEGER,-1,"ENTRIESREAD",NULL,NULL,CMD_ARG_OPTIONAL}, {0} }; @@ -6083,7 +6112,7 @@ struct redisCommand XGROUP_Subcommands[] = { {"delconsumer","Delete a consumer from a consumer group.","O(1)","5.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_STREAM,XGROUP_DELCONSUMER_History,XGROUP_DELCONSUMER_tips,xgroupCommand,5,CMD_WRITE,ACL_CATEGORY_STREAM,{{NULL,CMD_KEY_RW|CMD_KEY_DELETE,KSPEC_BS_INDEX,.bs.index={2},KSPEC_FK_RANGE,.fk.range={0,1,0}}},.args=XGROUP_DELCONSUMER_Args}, {"destroy","Destroy a consumer group.","O(N) where N is the number of entries in the group's pending entries list (PEL).","5.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_STREAM,XGROUP_DESTROY_History,XGROUP_DESTROY_tips,xgroupCommand,4,CMD_WRITE,ACL_CATEGORY_STREAM,{{NULL,CMD_KEY_RW|CMD_KEY_DELETE,KSPEC_BS_INDEX,.bs.index={2},KSPEC_FK_RANGE,.fk.range={0,1,0}}},.args=XGROUP_DESTROY_Args}, {"help","Show helpful text about the different subcommands","O(1)","5.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_STREAM,XGROUP_HELP_History,XGROUP_HELP_tips,xgroupCommand,2,CMD_LOADING|CMD_STALE,ACL_CATEGORY_STREAM}, -{"setid","Set a consumer group to an arbitrary last delivered ID value.","O(1)","5.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_STREAM,XGROUP_SETID_History,XGROUP_SETID_tips,xgroupCommand,5,CMD_WRITE,ACL_CATEGORY_STREAM,{{NULL,CMD_KEY_RW|CMD_KEY_UPDATE,KSPEC_BS_INDEX,.bs.index={2},KSPEC_FK_RANGE,.fk.range={0,1,0}}},.args=XGROUP_SETID_Args}, +{"setid","Set a consumer group to an arbitrary last delivered ID value.","O(1)","5.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_STREAM,XGROUP_SETID_History,XGROUP_SETID_tips,xgroupCommand,-5,CMD_WRITE,ACL_CATEGORY_STREAM,{{NULL,CMD_KEY_RW|CMD_KEY_UPDATE,KSPEC_BS_INDEX,.bs.index={2},KSPEC_FK_RANGE,.fk.range={0,1,0}}},.args=XGROUP_SETID_Args}, {0} }; @@ -6116,7 +6145,10 @@ struct redisCommandArg XINFO_CONSUMERS_Args[] = { /********** XINFO GROUPS ********************/ /* XINFO GROUPS history */ -#define XINFO_GROUPS_History NULL +commandHistory XINFO_GROUPS_History[] = { +{"7.0.0","Added the `entries-read` and `lag` fields"}, +{0} +}; /* XINFO GROUPS tips */ #define XINFO_GROUPS_tips NULL @@ -6138,7 +6170,10 @@ struct redisCommandArg XINFO_GROUPS_Args[] = { /********** XINFO STREAM ********************/ /* XINFO STREAM history */ -#define XINFO_STREAM_History NULL +commandHistory XINFO_STREAM_History[] = { +{"7.0.0","Added the `max-deleted-entry-id`, `entries-added`, `recorded-first-entry-id`, `entries-read` and `lag` fields"}, +{0} +}; /* XINFO STREAM tips */ #define XINFO_STREAM_tips NULL @@ -6317,7 +6352,10 @@ struct redisCommandArg XREVRANGE_Args[] = { /********** XSETID ********************/ /* XSETID history */ -#define XSETID_History NULL +commandHistory XSETID_History[] = { +{"7.0.0","Added the `entries_added` and `max_deleted_entry_id` arguments."}, +{0} +}; /* XSETID tips */ #define XSETID_tips NULL @@ -6326,6 +6364,8 @@ struct redisCommandArg XREVRANGE_Args[] = { struct redisCommandArg XSETID_Args[] = { {"key",ARG_TYPE_KEY,0,NULL,NULL,NULL,CMD_ARG_NONE}, {"last-id",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_NONE}, +{"entries_added",ARG_TYPE_INTEGER,-1,"ENTRIESADDED",NULL,NULL,CMD_ARG_OPTIONAL}, +{"max_deleted_entry_id",ARG_TYPE_STRING,-1,"MAXDELETEDID",NULL,NULL,CMD_ARG_OPTIONAL}, {0} }; @@ -6922,18 +6962,18 @@ struct redisCommand redisCommandTable[] = { {"publish","Post a message to a channel","O(N+M) where N is the number of clients subscribed to the receiving channel and M is the total number of subscribed patterns (by any client).","2.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_PUBSUB,PUBLISH_History,PUBLISH_tips,publishCommand,3,CMD_PUBSUB|CMD_LOADING|CMD_STALE|CMD_FAST|CMD_MAY_REPLICATE|CMD_SENTINEL,0,.args=PUBLISH_Args}, {"pubsub","A container for Pub/Sub commands","Depends on subcommand.","2.8.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_PUBSUB,PUBSUB_History,PUBSUB_tips,NULL,-2,0,0,.subcommands=PUBSUB_Subcommands}, {"punsubscribe","Stop listening for messages posted to channels matching the given patterns","O(N+M) where N is the number of patterns the client is already subscribed and M is the number of total patterns subscribed in the system (by any client).","2.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_PUBSUB,PUNSUBSCRIBE_History,PUNSUBSCRIBE_tips,punsubscribeCommand,-1,CMD_PUBSUB|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_SENTINEL,0,.args=PUNSUBSCRIBE_Args}, -{"spublish","Post a message to a shard channel","O(N) where N is the number of clients subscribed to the receiving shard channel.","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_PUBSUB,SPUBLISH_History,SPUBLISH_tips,spublishCommand,3,CMD_PUBSUB|CMD_LOADING|CMD_STALE|CMD_FAST|CMD_MAY_REPLICATE,0,{{NULL,CMD_KEY_CHANNEL,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}}},.args=SPUBLISH_Args}, -{"ssubscribe","Listen for messages published to the given shard channels","O(N) where N is the number of shard channels to subscribe to.","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_PUBSUB,SSUBSCRIBE_History,SSUBSCRIBE_tips,ssubscribeCommand,-2,CMD_PUBSUB|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE,0,{{NULL,CMD_KEY_CHANNEL,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={-1,1,0}}},.args=SSUBSCRIBE_Args}, +{"spublish","Post a message to a shard channel","O(N) where N is the number of clients subscribed to the receiving shard channel.","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_PUBSUB,SPUBLISH_History,SPUBLISH_tips,spublishCommand,3,CMD_PUBSUB|CMD_LOADING|CMD_STALE|CMD_FAST|CMD_MAY_REPLICATE,0,{{NULL,CMD_KEY_NOT_KEY,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}}},.args=SPUBLISH_Args}, +{"ssubscribe","Listen for messages published to the given shard channels","O(N) where N is the number of shard channels to subscribe to.","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_PUBSUB,SSUBSCRIBE_History,SSUBSCRIBE_tips,ssubscribeCommand,-2,CMD_PUBSUB|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE,0,{{NULL,CMD_KEY_NOT_KEY,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={-1,1,0}}},.args=SSUBSCRIBE_Args}, {"subscribe","Listen for messages published to the given channels","O(N) where N is the number of channels to subscribe to.","2.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_PUBSUB,SUBSCRIBE_History,SUBSCRIBE_tips,subscribeCommand,-2,CMD_PUBSUB|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_SENTINEL,0,.args=SUBSCRIBE_Args}, -{"sunsubscribe","Stop listening for messages posted to the given shard channels","O(N) where N is the number of clients already subscribed to a channel.","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_PUBSUB,SUNSUBSCRIBE_History,SUNSUBSCRIBE_tips,sunsubscribeCommand,-1,CMD_PUBSUB|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE,0,{{NULL,CMD_KEY_CHANNEL,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={-1,1,0}}},.args=SUNSUBSCRIBE_Args}, +{"sunsubscribe","Stop listening for messages posted to the given shard channels","O(N) where N is the number of clients already subscribed to a channel.","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_PUBSUB,SUNSUBSCRIBE_History,SUNSUBSCRIBE_tips,sunsubscribeCommand,-1,CMD_PUBSUB|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE,0,{{NULL,CMD_KEY_NOT_KEY,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={-1,1,0}}},.args=SUNSUBSCRIBE_Args}, {"unsubscribe","Stop listening for messages posted to the given channels","O(N) where N is the number of clients already subscribed to a channel.","2.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_PUBSUB,UNSUBSCRIBE_History,UNSUBSCRIBE_tips,unsubscribeCommand,-1,CMD_PUBSUB|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_SENTINEL,0,.args=UNSUBSCRIBE_Args}, /* scripting */ {"eval","Execute a Lua script server side","Depends on the script that is executed.","2.6.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SCRIPTING,EVAL_History,EVAL_tips,evalCommand,-3,CMD_NOSCRIPT|CMD_SKIP_MONITOR|CMD_MAY_REPLICATE|CMD_NO_MANDATORY_KEYS|CMD_STALE,ACL_CATEGORY_SCRIPTING,{{"We cannot tell how the keys will be used so we assume the worst, RW and UPDATE",CMD_KEY_RW|CMD_KEY_ACCESS|CMD_KEY_UPDATE,KSPEC_BS_INDEX,.bs.index={2},KSPEC_FK_KEYNUM,.fk.keynum={0,1,1}}},evalGetKeys,.args=EVAL_Args}, {"evalsha","Execute a Lua script server side","Depends on the script that is executed.","2.6.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SCRIPTING,EVALSHA_History,EVALSHA_tips,evalShaCommand,-3,CMD_NOSCRIPT|CMD_SKIP_MONITOR|CMD_MAY_REPLICATE|CMD_NO_MANDATORY_KEYS|CMD_STALE,ACL_CATEGORY_SCRIPTING,{{NULL,CMD_KEY_RW|CMD_KEY_ACCESS|CMD_KEY_UPDATE,KSPEC_BS_INDEX,.bs.index={2},KSPEC_FK_KEYNUM,.fk.keynum={0,1,1}}},evalGetKeys,.args=EVALSHA_Args}, {"evalsha_ro","Execute a read-only Lua script server side","Depends on the script that is executed.","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SCRIPTING,EVALSHA_RO_History,EVALSHA_RO_tips,evalShaRoCommand,-3,CMD_NOSCRIPT|CMD_SKIP_MONITOR|CMD_NO_MANDATORY_KEYS|CMD_STALE,ACL_CATEGORY_SCRIPTING,{{NULL,CMD_KEY_RO|CMD_KEY_ACCESS,KSPEC_BS_INDEX,.bs.index={2},KSPEC_FK_KEYNUM,.fk.keynum={0,1,1}}},evalGetKeys,.args=EVALSHA_RO_Args}, {"eval_ro","Execute a read-only Lua script server side","Depends on the script that is executed.","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SCRIPTING,EVAL_RO_History,EVAL_RO_tips,evalRoCommand,-3,CMD_NOSCRIPT|CMD_SKIP_MONITOR|CMD_NO_MANDATORY_KEYS|CMD_STALE,ACL_CATEGORY_SCRIPTING,{{"We cannot tell how the keys will be used so we assume the worst, RO and ACCESS",CMD_KEY_RO|CMD_KEY_ACCESS,KSPEC_BS_INDEX,.bs.index={2},KSPEC_FK_KEYNUM,.fk.keynum={0,1,1}}},evalGetKeys,.args=EVAL_RO_Args}, -{"fcall","PATCH__TBD__38__","PATCH__TBD__37__","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SCRIPTING,FCALL_History,FCALL_tips,fcallCommand,-3,CMD_NOSCRIPT|CMD_SKIP_MONITOR|CMD_MAY_REPLICATE|CMD_NO_MANDATORY_KEYS|CMD_STALE,ACL_CATEGORY_SCRIPTING,{{"We cannot tell how the keys will be used so we assume the worst, RW and UPDATE",CMD_KEY_RW|CMD_KEY_ACCESS|CMD_KEY_UPDATE,KSPEC_BS_INDEX,.bs.index={2},KSPEC_FK_KEYNUM,.fk.keynum={0,1,1}}},functionGetKeys,.args=FCALL_Args}, -{"fcall_ro","PATCH__TBD__7__","PATCH__TBD__6__","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SCRIPTING,FCALL_RO_History,FCALL_RO_tips,fcallroCommand,-3,CMD_NOSCRIPT|CMD_SKIP_MONITOR|CMD_NO_MANDATORY_KEYS|CMD_STALE,ACL_CATEGORY_SCRIPTING,{{"We cannot tell how the keys will be used so we assume the worst, RO and ACCESS",CMD_KEY_RO|CMD_KEY_ACCESS,KSPEC_BS_INDEX,.bs.index={2},KSPEC_FK_KEYNUM,.fk.keynum={0,1,1}}},functionGetKeys,.args=FCALL_RO_Args}, +{"fcall","Invoke a function","Depends on the function that is executed.","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SCRIPTING,FCALL_History,FCALL_tips,fcallCommand,-3,CMD_NOSCRIPT|CMD_SKIP_MONITOR|CMD_MAY_REPLICATE|CMD_NO_MANDATORY_KEYS|CMD_STALE,ACL_CATEGORY_SCRIPTING,{{"We cannot tell how the keys will be used so we assume the worst, RW and UPDATE",CMD_KEY_RW|CMD_KEY_ACCESS|CMD_KEY_UPDATE,KSPEC_BS_INDEX,.bs.index={2},KSPEC_FK_KEYNUM,.fk.keynum={0,1,1}}},functionGetKeys,.args=FCALL_Args}, +{"fcall_ro","Invoke a read-only function","Depends on the function that is executed.","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SCRIPTING,FCALL_RO_History,FCALL_RO_tips,fcallroCommand,-3,CMD_NOSCRIPT|CMD_SKIP_MONITOR|CMD_NO_MANDATORY_KEYS|CMD_STALE,ACL_CATEGORY_SCRIPTING,{{"We cannot tell how the keys will be used so we assume the worst, RO and ACCESS",CMD_KEY_RO|CMD_KEY_ACCESS,KSPEC_BS_INDEX,.bs.index={2},KSPEC_FK_KEYNUM,.fk.keynum={0,1,1}}},functionGetKeys,.args=FCALL_RO_Args}, {"function","A container for function commands","Depends on subcommand.","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SCRIPTING,FUNCTION_History,FUNCTION_tips,NULL,-2,0,0,.subcommands=FUNCTION_Subcommands}, {"script","A container for Lua scripts management commands","Depends on subcommand.","2.6.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SCRIPTING,SCRIPT_History,SCRIPT_tips,NULL,-2,0,0,.subcommands=SCRIPT_Subcommands}, /* sentinel */ @@ -6957,13 +6997,13 @@ struct redisCommand redisCommandTable[] = { {"module","A container for module commands","Depends on subcommand.","4.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SERVER,MODULE_History,MODULE_tips,NULL,-2,0,0,.subcommands=MODULE_Subcommands}, {"monitor","Listen for all requests received by the server in real time",NULL,"1.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SERVER,MONITOR_History,MONITOR_tips,monitorCommand,1,CMD_ADMIN|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE,0}, {"psync","Internal command used for replication",NULL,"2.8.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SERVER,PSYNC_History,PSYNC_tips,syncCommand,-3,CMD_NO_ASYNC_LOADING|CMD_ADMIN|CMD_NO_MULTI|CMD_NOSCRIPT,0,.args=PSYNC_Args}, -{"replconf","An internal command for configuring the replication stream","O(1)","3.0.0",CMD_DOC_SYSCMD,NULL,NULL,COMMAND_GROUP_SERVER,REPLCONF_History,REPLCONF_tips,replconfCommand,-1,CMD_ADMIN|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE,0}, -{"replicaof","Make the server a replica of another instance, or promote it as master.","O(1)","5.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SERVER,REPLICAOF_History,REPLICAOF_tips,replicaofCommand,3,CMD_NO_ASYNC_LOADING|CMD_ADMIN|CMD_ALLOW_BUSY|CMD_NOSCRIPT|CMD_STALE,0,.args=REPLICAOF_Args}, +{"replconf","An internal command for configuring the replication stream","O(1)","3.0.0",CMD_DOC_SYSCMD,NULL,NULL,COMMAND_GROUP_SERVER,REPLCONF_History,REPLCONF_tips,replconfCommand,-1,CMD_ADMIN|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_ALLOW_BUSY,0}, +{"replicaof","Make the server a replica of another instance, or promote it as master.","O(1)","5.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SERVER,REPLICAOF_History,REPLICAOF_tips,replicaofCommand,3,CMD_NO_ASYNC_LOADING|CMD_ADMIN|CMD_NOSCRIPT|CMD_STALE,0,.args=REPLICAOF_Args}, {"restore-asking","An internal command for migrating keys in a cluster","O(1) to create the new key and additional O(N*M) to reconstruct the serialized value, where N is the number of Redis objects composing the value and M their average size. For small string values the time complexity is thus O(1)+O(1*M) where M is small, so simply O(1). However for sorted set values the complexity is O(N*M*log(N)) because inserting values into sorted sets is O(log(N)).","3.0.0",CMD_DOC_SYSCMD,NULL,NULL,COMMAND_GROUP_SERVER,RESTORE_ASKING_History,RESTORE_ASKING_tips,restoreCommand,-4,CMD_WRITE|CMD_DENYOOM|CMD_ASKING,ACL_CATEGORY_KEYSPACE|ACL_CATEGORY_DANGEROUS,{{NULL,CMD_KEY_OW|CMD_KEY_UPDATE,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}}}}, {"role","Return the role of the instance in the context of replication","O(1)","2.8.12",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SERVER,ROLE_History,ROLE_tips,roleCommand,1,CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_FAST|CMD_SENTINEL,ACL_CATEGORY_ADMIN|ACL_CATEGORY_DANGEROUS}, {"save","Synchronously save the dataset to disk","O(N) where N is the total number of keys in all databases","1.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SERVER,SAVE_History,SAVE_tips,saveCommand,1,CMD_NO_ASYNC_LOADING|CMD_ADMIN|CMD_NOSCRIPT|CMD_NO_MULTI,0}, {"shutdown","Synchronously save the dataset to disk and then shut down the server","O(N) when saving, where N is the total number of keys in all databases when saving data, otherwise O(1)","1.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SERVER,SHUTDOWN_History,SHUTDOWN_tips,shutdownCommand,-1,CMD_ADMIN|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_NO_MULTI|CMD_SENTINEL|CMD_ALLOW_BUSY,0,.args=SHUTDOWN_Args}, -{"slaveof","Make the server a replica of another instance, or promote it as master. Deprecated starting with Redis 5. Use REPLICAOF instead.","O(1)","1.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SERVER,SLAVEOF_History,SLAVEOF_tips,replicaofCommand,3,CMD_NO_ASYNC_LOADING|CMD_ADMIN|CMD_NOSCRIPT|CMD_STALE,0,.args=SLAVEOF_Args}, +{"slaveof","Make the server a replica of another instance, or promote it as master.","O(1)","1.0.0",CMD_DOC_DEPRECATED,"`REPLICAOF`","5.0.0",COMMAND_GROUP_SERVER,SLAVEOF_History,SLAVEOF_tips,replicaofCommand,3,CMD_NO_ASYNC_LOADING|CMD_ADMIN|CMD_NOSCRIPT|CMD_STALE,0,.args=SLAVEOF_Args}, {"slowlog","A container for slow log commands","Depends on subcommand.","2.2.12",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SERVER,SLOWLOG_History,SLOWLOG_tips,NULL,-2,0,0,.subcommands=SLOWLOG_Subcommands}, {"swapdb","Swaps two Redis databases","O(N) where N is the count of clients watching or blocking on keys from both databases.","4.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SERVER,SWAPDB_History,SWAPDB_tips,swapdbCommand,3,CMD_WRITE|CMD_FAST,ACL_CATEGORY_KEYSPACE|ACL_CATEGORY_DANGEROUS,.args=SWAPDB_Args}, {"sync","Internal command used for replication",NULL,"1.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SERVER,SYNC_History,SYNC_tips,syncCommand,1,CMD_NO_ASYNC_LOADING|CMD_ADMIN|CMD_NO_MULTI|CMD_NOSCRIPT,0}, @@ -7036,7 +7076,7 @@ struct redisCommand redisCommandTable[] = { {"xread","Return never seen elements in multiple streams, with IDs greater than the ones reported by the caller for each stream. Can block.","For each stream mentioned: O(N) with N being the number of elements being returned, it means that XREAD-ing with a fixed COUNT is O(1). Note that when the BLOCK option is used, XADD will pay O(M) time in order to serve the M clients blocked on the stream getting new data.","5.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_STREAM,XREAD_History,XREAD_tips,xreadCommand,-4,CMD_BLOCKING|CMD_READONLY|CMD_BLOCKING,ACL_CATEGORY_STREAM,{{NULL,CMD_KEY_RO|CMD_KEY_ACCESS,KSPEC_BS_KEYWORD,.bs.keyword={"STREAMS",1},KSPEC_FK_RANGE,.fk.range={-1,1,2}}},xreadGetKeys,.args=XREAD_Args}, {"xreadgroup","Return new entries from a stream using a consumer group, or access the history of the pending entries for a given consumer. Can block.","For each stream mentioned: O(M) with M being the number of elements returned. If M is constant (e.g. always asking for the first 10 elements with COUNT), you can consider it O(1). On the other side when XREADGROUP blocks, XADD will pay the O(N) time in order to serve the N clients blocked on the stream getting new data.","5.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_STREAM,XREADGROUP_History,XREADGROUP_tips,xreadCommand,-7,CMD_BLOCKING|CMD_WRITE,ACL_CATEGORY_STREAM,{{NULL,CMD_KEY_RO|CMD_KEY_ACCESS,KSPEC_BS_KEYWORD,.bs.keyword={"STREAMS",4},KSPEC_FK_RANGE,.fk.range={-1,1,2}}},xreadGetKeys,.args=XREADGROUP_Args}, {"xrevrange","Return a range of elements in a stream, with IDs matching the specified IDs interval, in reverse order (from greater to smaller IDs) compared to XRANGE","O(N) with N being the number of elements returned. If N is constant (e.g. always asking for the first 10 elements with COUNT), you can consider it O(1).","5.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_STREAM,XREVRANGE_History,XREVRANGE_tips,xrevrangeCommand,-4,CMD_READONLY,ACL_CATEGORY_STREAM,{{NULL,CMD_KEY_RO|CMD_KEY_ACCESS,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}}},.args=XREVRANGE_Args}, -{"xsetid","An internal command for replicating stream values","O(1)","5.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_STREAM,XSETID_History,XSETID_tips,xsetidCommand,3,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_STREAM,{{NULL,CMD_KEY_RW|CMD_KEY_UPDATE,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}}},.args=XSETID_Args}, +{"xsetid","An internal command for replicating stream values","O(1)","5.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_STREAM,XSETID_History,XSETID_tips,xsetidCommand,-3,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_STREAM,{{NULL,CMD_KEY_RW|CMD_KEY_UPDATE,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}}},.args=XSETID_Args}, {"xtrim","Trims the stream to (approximately if '~' is passed) a certain size","O(N), with N being the number of evicted entries. Constant times are very small however, since entries are organized in macro nodes containing multiple entries that can be released with a single deallocation.","5.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_STREAM,XTRIM_History,XTRIM_tips,xtrimCommand,-4,CMD_WRITE,ACL_CATEGORY_STREAM,{{NULL,CMD_KEY_RW|CMD_KEY_DELETE,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}}},.args=XTRIM_Args}, /* string */ {"append","Append a value to a key","O(1). The amortized time complexity is O(1) assuming the appended value is small and the already present value is of any size, since the dynamic string library used by Redis will double the free space available on every reallocation.","2.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_STRING,APPEND_History,APPEND_tips,appendCommand,3,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_STRING,{{NULL,CMD_KEY_RW|CMD_KEY_INSERT,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}}},.args=APPEND_Args}, diff --git a/src/commands/cluster-setslot.json b/src/commands/cluster-setslot.json index dc9af6138..5d1aa45fc 100644 --- a/src/commands/cluster-setslot.json +++ b/src/commands/cluster-setslot.json @@ -26,17 +26,17 @@ "arguments": [ { "name": "node-id", - "type": "integer", + "type": "string", "token": "IMPORTING" }, { "name": "node-id", - "type": "integer", + "type": "string", "token": "MIGRATING" }, { "name": "node-id", - "type": "integer", + "type": "string", "token": "NODE" }, { diff --git a/src/commands/command-getkeysandflags.json b/src/commands/command-getkeysandflags.json new file mode 100644 index 000000000..1ac8e990d --- /dev/null +++ b/src/commands/command-getkeysandflags.json @@ -0,0 +1,18 @@ +{ + "GETKEYSANDFLAGS": { + "summary": "Extract keys given a full Redis command", + "complexity": "O(N) where N is the number of arguments to the command", + "group": "server", + "since": "7.0.0", + "arity": -4, + "container": "COMMAND", + "function": "commandGetKeysAndFlagsCommand", + "command_flags": [ + "LOADING", + "STALE" + ], + "acl_categories": [ + "CONNECTION" + ] + } +} diff --git a/src/commands/fcall.json b/src/commands/fcall.json index 45fad3c9b..27b7b4e35 100644 --- a/src/commands/fcall.json +++ b/src/commands/fcall.json @@ -1,7 +1,7 @@ { "FCALL": { - "summary": "PATCH__TBD__38__", - "complexity": "PATCH__TBD__37__", + "summary": "Invoke a function", + "complexity": "Depends on the function that is executed.", "group": "scripting", "since": "7.0.0", "arity": -3, diff --git a/src/commands/fcall_ro.json b/src/commands/fcall_ro.json index d1caf5f65..092915700 100644 --- a/src/commands/fcall_ro.json +++ b/src/commands/fcall_ro.json @@ -1,7 +1,7 @@ { "FCALL_RO": { - "summary": "PATCH__TBD__7__", - "complexity": "PATCH__TBD__6__", + "summary": "Invoke a read-only function", + "complexity": "Depends on the function that is executed.", "group": "scripting", "since": "7.0.0", "arity": -3, diff --git a/src/commands/function-delete.json b/src/commands/function-delete.json index 2cfb4068c..01dc78ba4 100644 --- a/src/commands/function-delete.json +++ b/src/commands/function-delete.json @@ -20,7 +20,7 @@ ], "arguments": [ { - "name": "function-name", + "name": "library-name", "type": "string" } ] diff --git a/src/commands/function-load.json b/src/commands/function-load.json index df756ceab..0a363e328 100644 --- a/src/commands/function-load.json +++ b/src/commands/function-load.json @@ -37,7 +37,7 @@ { "name": "library-description", "type": "string", - "token": "DESC", + "token": "DESCRIPTION", "optional": true }, { diff --git a/src/commands/info.json b/src/commands/info.json index 48720f9c3..612294d34 100644 --- a/src/commands/info.json +++ b/src/commands/info.json @@ -6,6 +6,12 @@ "since": "1.0.0", "arity": -1, "function": "infoCommand", + "history": [ + [ + "7.0.0", + "Added support for taking multiple section arguments." + ] + ], "command_flags": [ "LOADING", "STALE", @@ -23,6 +29,7 @@ { "name": "section", "type": "string", + "multiple": true, "optional": true } ] diff --git a/src/commands/pubsub-shardnumsub.json b/src/commands/pubsub-shardnumsub.json index 167132b31..55f5101ac 100644 --- a/src/commands/pubsub-shardnumsub.json +++ b/src/commands/pubsub-shardnumsub.json @@ -11,6 +11,14 @@ "PUBSUB", "LOADING", "STALE" + ], + "arguments": [ + { + "name": "channel", + "type": "string", + "optional": true, + "multiple": true + } ] } } diff --git a/src/commands/replconf.json b/src/commands/replconf.json index e880aa929..630b62136 100644 --- a/src/commands/replconf.json +++ b/src/commands/replconf.json @@ -13,7 +13,8 @@ "ADMIN", "NOSCRIPT", "LOADING", - "STALE" + "STALE", + "ALLOW_BUSY" ] } } diff --git a/src/commands/replicaof.json b/src/commands/replicaof.json index 90ec59fed..805b81e4c 100644 --- a/src/commands/replicaof.json +++ b/src/commands/replicaof.json @@ -9,7 +9,6 @@ "command_flags": [ "NO_ASYNC_LOADING", "ADMIN", - "ALLOW_BUSY", "NOSCRIPT", "STALE" ], diff --git a/src/commands/sentinel-debug.json b/src/commands/sentinel-debug.json index ed771c4f7..44c6bec9b 100644 --- a/src/commands/sentinel-debug.json +++ b/src/commands/sentinel-debug.json @@ -1,5 +1,7 @@ { "DEBUG": { + "summary": "List or update the current configurable parameters", + "complexity": "O(N) where N is the number of configurable parameters", "group": "sentinel", "since": "7.0.0", "arity": -2, diff --git a/src/commands/slaveof.json b/src/commands/slaveof.json index f9db6afbb..271eb2d1b 100644 --- a/src/commands/slaveof.json +++ b/src/commands/slaveof.json @@ -1,11 +1,16 @@ { "SLAVEOF": { - "summary": "Make the server a replica of another instance, or promote it as master. Deprecated starting with Redis 5. Use REPLICAOF instead.", + "summary": "Make the server a replica of another instance, or promote it as master.", "complexity": "O(1)", "group": "server", "since": "1.0.0", "arity": 3, "function": "replicaofCommand", + "deprecated_since": "5.0.0", + "replaced_by": "`REPLICAOF`", + "doc_flags": [ + "DEPRECATED" + ], "command_flags": [ "NO_ASYNC_LOADING", "ADMIN", diff --git a/src/commands/spublish.json b/src/commands/spublish.json index e6c4fb062..2cbcdc19a 100644 --- a/src/commands/spublish.json +++ b/src/commands/spublish.json @@ -26,7 +26,7 @@ "key_specs": [ { "flags": [ - "CHANNEL" + "NOT_KEY" ], "begin_search": { "index": { diff --git a/src/commands/ssubscribe.json b/src/commands/ssubscribe.json index dd1070046..eb570ea53 100644 --- a/src/commands/ssubscribe.json +++ b/src/commands/ssubscribe.json @@ -22,7 +22,7 @@ "key_specs": [ { "flags": [ - "CHANNEL" + "NOT_KEY" ], "begin_search": { "index": { diff --git a/src/commands/sunsubscribe.json b/src/commands/sunsubscribe.json index f7271825e..481415490 100644 --- a/src/commands/sunsubscribe.json +++ b/src/commands/sunsubscribe.json @@ -23,7 +23,7 @@ "key_specs": [ { "flags": [ - "CHANNEL" + "NOT_KEY" ], "begin_search": { "index": { diff --git a/src/commands/xautoclaim.json b/src/commands/xautoclaim.json index b951eac29..726bf38fe 100644 --- a/src/commands/xautoclaim.json +++ b/src/commands/xautoclaim.json @@ -6,6 +6,12 @@ "since": "6.2.0", "arity": -6, "function": "xautoclaimCommand", + "history": [ + [ + "7.0.0", + "Added an element to the reply array, containing deleted entries the command cleared from the PEL" + ] + ], "command_flags": [ "WRITE", "FAST" diff --git a/src/commands/xgroup-create.json b/src/commands/xgroup-create.json index a14c0e577..2b1ee03b4 100644 --- a/src/commands/xgroup-create.json +++ b/src/commands/xgroup-create.json @@ -7,6 +7,12 @@ "arity": -5, "container": "XGROUP", "function": "xgroupCommand", + "history": [ + [ + "7.0.0", + "Added the `entries_read` named argument." + ] + ], "command_flags": [ "WRITE", "DENYOOM" @@ -64,6 +70,12 @@ "name": "mkstream", "type": "pure-token", "optional": true + }, + { + "token": "ENTRIESREAD", + "name": "entries_read", + "type": "integer", + "optional": true } ] } diff --git a/src/commands/xgroup-setid.json b/src/commands/xgroup-setid.json index 5c2ffcf19..af4b83c19 100644 --- a/src/commands/xgroup-setid.json +++ b/src/commands/xgroup-setid.json @@ -4,9 +4,15 @@ "complexity": "O(1)", "group": "stream", "since": "5.0.0", - "arity": 5, + "arity": -5, "container": "XGROUP", "function": "xgroupCommand", + "history": [ + [ + "7.0.0", + "Added the optional `entries_read` argument." + ] + ], "command_flags": [ "WRITE" ], @@ -57,6 +63,12 @@ "token": "$" } ] + }, + { + "name": "entries_read", + "token": "ENTRIESREAD", + "type": "integer", + "optional": true } ] } diff --git a/src/commands/xinfo-groups.json b/src/commands/xinfo-groups.json index 546d2030e..e9b61ba06 100644 --- a/src/commands/xinfo-groups.json +++ b/src/commands/xinfo-groups.json @@ -6,6 +6,12 @@ "since": "5.0.0", "arity": 3, "container": "XINFO", + "history": [ + [ + "7.0.0", + "Added the `entries-read` and `lag` fields" + ] + ], "function": "xinfoCommand", "command_flags": [ "READONLY" diff --git a/src/commands/xinfo-stream.json b/src/commands/xinfo-stream.json index 43ae9bc8b..5b7d9ad57 100644 --- a/src/commands/xinfo-stream.json +++ b/src/commands/xinfo-stream.json @@ -6,6 +6,12 @@ "since": "5.0.0", "arity": -3, "container": "XINFO", + "history": [ + [ + "7.0.0", + "Added the `max-deleted-entry-id`, `entries-added`, `recorded-first-entry-id`, `entries-read` and `lag` fields" + ] + ], "function": "xinfoCommand", "command_flags": [ "READONLY" diff --git a/src/commands/xsetid.json b/src/commands/xsetid.json index 8faa2c7e9..7654784e1 100644 --- a/src/commands/xsetid.json +++ b/src/commands/xsetid.json @@ -4,8 +4,14 @@ "complexity": "O(1)", "group": "stream", "since": "5.0.0", - "arity": 3, + "arity": -3, "function": "xsetidCommand", + "history": [ + [ + "7.0.0", + "Added the `entries_added` and `max_deleted_entry_id` arguments." + ] + ], "command_flags": [ "WRITE", "DENYOOM", @@ -43,6 +49,18 @@ { "name": "last-id", "type": "string" + }, + { + "name": "entries_added", + "token": "ENTRIESADDED", + "type": "integer", + "optional": true + }, + { + "name": "max_deleted_entry_id", + "token": "MAXDELETEDID", + "type": "string", + "optional": true } ] } diff --git a/src/connection.c b/src/connection.c index 3a17d983d..11fc4ba28 100644 --- a/src/connection.c +++ b/src/connection.c @@ -178,6 +178,21 @@ static int connSocketWrite(connection *conn, const void *data, size_t data_len) return ret; } +static int connSocketWritev(connection *conn, const struct iovec *iov, int iovcnt) { + int ret = writev(conn->fd, iov, iovcnt); + if (ret < 0 && errno != EAGAIN) { + conn->last_errno = errno; + + /* Don't overwrite the state of a connection that is not already + * connected, not to mess with handler callbacks. + */ + if (errno != EINTR && conn->state == CONN_STATE_CONNECTED) + conn->state = CONN_STATE_ERROR; + } + + return ret; +} + static int connSocketRead(connection *conn, void *buf, size_t buf_len) { int ret = read(conn->fd, buf, buf_len); if (!ret) { @@ -349,6 +364,7 @@ ConnectionType CT_Socket = { .ae_handler = connSocketEventHandler, .close = connSocketClose, .write = connSocketWrite, + .writev = connSocketWritev, .read = connSocketRead, .accept = connSocketAccept, .connect = connSocketConnect, diff --git a/src/connection.h b/src/connection.h index 07c1d4dd8..dad2e2fd6 100644 --- a/src/connection.h +++ b/src/connection.h @@ -32,6 +32,7 @@ #define __REDIS_CONNECTION_H #include <errno.h> +#include <sys/uio.h> #define CONN_INFO_LEN 32 @@ -59,6 +60,7 @@ typedef struct ConnectionType { void (*ae_handler)(struct aeEventLoop *el, int fd, void *clientData, int mask); int (*connect)(struct connection *conn, const char *addr, int port, const char *source_addr, ConnectionCallbackFunc connect_handler); int (*write)(struct connection *conn, const void *data, size_t data_len); + int (*writev)(struct connection *conn, const struct iovec *iov, int iovcnt); int (*read)(struct connection *conn, void *buf, size_t buf_len); void (*close)(struct connection *conn); int (*accept)(struct connection *conn, ConnectionCallbackFunc accept_handler); @@ -142,6 +144,18 @@ static inline int connWrite(connection *conn, const void *data, size_t data_len) return conn->type->write(conn, data, data_len); } +/* Gather output data from the iovcnt buffers specified by the members of the iov + * array: iov[0], iov[1], ..., iov[iovcnt-1] and write to connection, behaves the same as writev(3). + * + * Like writev(3), a short write is possible. A -1 return indicates an error. + * + * The caller should NOT rely on errno. Testing for an EAGAIN-like condition, use + * connGetState() to see if the connection state is still CONN_STATE_CONNECTED. + */ +static inline int connWritev(connection *conn, const struct iovec *iov, int iovcnt) { + return conn->type->writev(conn, iov, iovcnt); +} + /* Read from the connection, behaves the same as read(2). * * Like read(2), a short read is possible. A return value of 0 will indicate the @@ -83,16 +83,16 @@ robj *lookupKey(redisDb *db, robj *key, int flags) { robj *val = NULL; if (de) { val = dictGetVal(de); - int force_delete_expired = flags & LOOKUP_WRITE; - if (force_delete_expired) { - /* Forcing deletion of expired keys on a replica makes the replica - * inconsistent with the master. The reason it's allowed for write - * commands is to make writable replicas behave consistently. It - * shall not be used in readonly commands. Modules are accepted so - * that we don't break old modules. */ - client *c = server.in_script ? scriptGetClient() : server.current_client; - serverAssert(!c || !c->cmd || (c->cmd->flags & (CMD_WRITE|CMD_MODULE))); - } + /* Forcing deletion of expired keys on a replica makes the replica + * inconsistent with the master. We forbid it on readonly replicas, but + * we have to allow it on writable replicas to make write commands + * behave consistently. + * + * It's possible that the WRITE flag is set even during a readonly + * command, since the command may trigger events that cause modules to + * perform additional writes. */ + int is_ro_replica = server.masterhost && server.repl_slave_ro; + int force_delete_expired = flags & LOOKUP_WRITE && !is_ro_replica; if (expireIfNeeded(db, key, force_delete_expired)) { /* The key is no longer valid. */ val = NULL; @@ -1340,6 +1340,11 @@ int dbSwapDatabases(int id1, int id2) { redisDb aux = server.db[id1]; redisDb *db1 = &server.db[id1], *db2 = &server.db[id2]; + /* Swapdb should make transaction fail if there is any + * client watching keys */ + touchAllWatchedKeysInDb(db1, db2); + touchAllWatchedKeysInDb(db2, db1); + /* Swap hash tables. Note that we don't swap blocking_keys, * ready_keys and watched_keys, since we want clients to * remain in the same DB they were. */ @@ -1361,14 +1366,9 @@ int dbSwapDatabases(int id1, int id2) { * However normally we only do this check for efficiency reasons * in dbAdd() when a list is created. So here we need to rescan * the list of clients blocked on lists and signal lists as ready - * if needed. - * - * Also the swapdb should make transaction fail if there is any - * client watching keys */ + * if needed. */ scanDatabaseForReadyLists(db1); - touchAllWatchedKeysInDb(db1, db2); scanDatabaseForReadyLists(db2); - touchAllWatchedKeysInDb(db2, db1); return C_OK; } @@ -1387,6 +1387,10 @@ void swapMainDbWithTempDb(redisDb *tempDb) { redisDb aux = server.db[i]; redisDb *activedb = &server.db[i], *newdb = &tempDb[i]; + /* Swapping databases should make transaction fail if there is any + * client watching keys. */ + touchAllWatchedKeysInDb(activedb, newdb); + /* Swap hash tables. Note that we don't swap blocking_keys, * ready_keys and watched_keys, since clients * remain in the same DB they were. */ @@ -1408,12 +1412,8 @@ void swapMainDbWithTempDb(redisDb *tempDb) { * However normally we only do this check for efficiency reasons * in dbAdd() when a list is created. So here we need to rescan * the list of clients blocked on lists and signal lists as ready - * if needed. - * - * Also the swapdb should make transaction fail if there is any - * client watching keys. */ + * if needed. */ scanDatabaseForReadyLists(activedb); - touchAllWatchedKeysInDb(activedb, newdb); } trackingInvalidateKeysOnFlush(1); @@ -1692,7 +1692,7 @@ int64_t getAllKeySpecsFlags(struct redisCommand *cmd, int inv) { /* Fetch the keys based of the provided key specs. Returns the number of keys found, or -1 on error. * There are several flags that can be used to modify how this function finds keys in a command. * - * GET_KEYSPEC_INCLUDE_CHANNELS: Return channels as if they were keys. + * GET_KEYSPEC_INCLUDE_NOT_KEYS: Return 'fake' keys as if they were keys. * GET_KEYSPEC_RETURN_PARTIAL: Skips invalid and incomplete keyspecs but returns the keys * found in other valid keyspecs. */ @@ -1703,8 +1703,8 @@ int getKeysUsingKeySpecs(struct redisCommand *cmd, robj **argv, int argc, int se for (j = 0; j < cmd->key_specs_num; j++) { keySpec *spec = cmd->key_specs + j; serverAssert(spec->begin_search_type != KSPEC_BS_INVALID); - /* Skip specs that represent channels instead of keys */ - if ((spec->flags & CMD_KEY_CHANNEL) && !(search_flags & GET_KEYSPEC_INCLUDE_CHANNELS)) { + /* Skip specs that represent 'fake' keys */ + if ((spec->flags & CMD_KEY_NOT_KEY) && !(search_flags & GET_KEYSPEC_INCLUDE_NOT_KEYS)) { continue; } @@ -1821,31 +1821,123 @@ invalid_spec: * associated with how Redis will access the key. * * 'cmd' must be point to the corresponding entry into the redisCommand - * table, according to the command name in argv[0]. - * - * This function uses the command's key specs, which contain the key-spec flags, - * (e.g. RO / RW) and only resorts to the command-specific helper function if - * any of the keys-specs are marked as INCOMPLETE. */ + * table, according to the command name in argv[0]. */ int getKeysFromCommandWithSpecs(struct redisCommand *cmd, robj **argv, int argc, int search_flags, getKeysResult *result) { - if (cmd->flags & CMD_MODULE_GETKEYS) { + /* The command has at least one key-spec not marked as NOT_KEY */ + int has_keyspec = (getAllKeySpecsFlags(cmd, 1) & CMD_KEY_NOT_KEY); + /* The command has at least one key-spec marked as VARIABLE_FLAGS */ + int has_varflags = (getAllKeySpecsFlags(cmd, 0) & CMD_KEY_VARIABLE_FLAGS); + + /* Flags indicating that we have a getkeys callback */ + int has_module_getkeys = cmd->flags & CMD_MODULE_GETKEYS; + int has_native_getkeys = !(cmd->flags & CMD_MODULE) && cmd->getkeys_proc; + + /* The key-spec that's auto generated by RM_CreateCommand sets VARIABLE_FLAGS since no flags are given. + * If the module provides getkeys callback, we'll prefer it, but if it didn't, we'll use key-spec anyway. */ + if ((cmd->flags & CMD_MODULE) && has_varflags && !has_module_getkeys) + has_varflags = 0; + + /* We prefer key-specs if there are any, and their flags are reliable. */ + if (has_keyspec && !has_varflags) { + int ret = getKeysUsingKeySpecs(cmd,argv,argc,search_flags,result); + if (ret >= 0) + return ret; + /* If the specs returned with an error (probably an INVALID or INCOMPLETE spec), + * fallback to the callback method. */ + } + + /* Resort to getkeys callback methods. */ + if (has_module_getkeys) return moduleGetCommandKeysViaAPI(cmd,argv,argc,result); - } else { - if (!(getAllKeySpecsFlags(cmd, 0) & CMD_KEY_VARIABLE_FLAGS)) { - int ret = getKeysUsingKeySpecs(cmd,argv,argc,search_flags,result); - if (ret >= 0) - return ret; - } - if (!(cmd->flags & CMD_MODULE) && cmd->getkeys_proc) - return cmd->getkeys_proc(cmd,argv,argc,result); - return 0; - } + + /* We use native getkeys as a last resort, since not all these native getkeys provide + * flags properly (only the ones that correspond to INVALID, INCOMPLETE or VARIABLE_FLAGS do.*/ + if (has_native_getkeys) + return cmd->getkeys_proc(cmd,argv,argc,result); + return 0; } /* This function returns a sanity check if the command may have keys. */ int doesCommandHaveKeys(struct redisCommand *cmd) { return (!(cmd->flags & CMD_MODULE) && cmd->getkeys_proc) || /* has getkeys_proc (non modules) */ (cmd->flags & CMD_MODULE_GETKEYS) || /* module with GETKEYS */ - (getAllKeySpecsFlags(cmd, 1) & CMD_KEY_CHANNEL); /* has at least one key-spec not marked as CHANNEL */ + (getAllKeySpecsFlags(cmd, 1) & CMD_KEY_NOT_KEY); /* has at least one key-spec not marked as NOT_KEY */ +} + +/* A simplified channel spec table that contains all of the redis commands + * and which channels they have and how they are accessed. */ +typedef struct ChannelSpecs { + redisCommandProc *proc; /* Command procedure to match against */ + uint64_t flags; /* CMD_CHANNEL_* flags for this command */ + int start; /* The initial position of the first channel */ + int count; /* The number of channels, or -1 if all remaining + * arguments are channels. */ +} ChannelSpecs; + +ChannelSpecs commands_with_channels[] = { + {subscribeCommand, CMD_CHANNEL_SUBSCRIBE, 1, -1}, + {ssubscribeCommand, CMD_CHANNEL_SUBSCRIBE, 1, -1}, + {unsubscribeCommand, CMD_CHANNEL_UNSUBSCRIBE, 1, -1}, + {sunsubscribeCommand, CMD_CHANNEL_UNSUBSCRIBE, 1, -1}, + {psubscribeCommand, CMD_CHANNEL_PATTERN | CMD_CHANNEL_SUBSCRIBE, 1, -1}, + {punsubscribeCommand, CMD_CHANNEL_PATTERN | CMD_CHANNEL_UNSUBSCRIBE, 1, -1}, + {publishCommand, CMD_CHANNEL_PUBLISH, 1, 1}, + {spublishCommand, CMD_CHANNEL_PUBLISH, 1, 1}, + {NULL,0} /* Terminator. */ +}; + +/* Returns 1 if the command may access any channels matched by the flags + * argument. */ +int doesCommandHaveChannelsWithFlags(struct redisCommand *cmd, int flags) { + /* If a module declares get channels, we are just going to assume + * has channels. This API is allowed to return false positives. */ + if (cmd->flags & CMD_MODULE_GETCHANNELS) { + return 1; + } + for (ChannelSpecs *spec = commands_with_channels; spec->proc != NULL; spec += 1) { + if (cmd->proc == spec->proc) { + return !!(spec->flags & flags); + } + } + return 0; +} + +/* Return all the arguments that are channels in the command passed via argc / argv. + * This function behaves similar to getKeysFromCommandWithSpecs, but with channels + * instead of keys. + * + * The command returns the positions of all the channel arguments inside the array, + * so the actual return value is a heap allocated array of integers. The + * length of the array is returned by reference into *numkeys. + * + * Along with the position, this command also returns the flags that are + * associated with how Redis will access the channel. + * + * 'cmd' must be point to the corresponding entry into the redisCommand + * table, according to the command name in argv[0]. */ +int getChannelsFromCommand(struct redisCommand *cmd, robj **argv, int argc, getKeysResult *result) { + keyReference *keys; + /* If a module declares get channels, use that. */ + if (cmd->flags & CMD_MODULE_GETCHANNELS) { + return moduleGetCommandChannelsViaAPI(cmd, argv, argc, result); + } + /* Otherwise check the channel spec table */ + for (ChannelSpecs *spec = commands_with_channels; spec != NULL; spec += 1) { + if (cmd->proc == spec->proc) { + int start = spec->start; + int stop = (spec->count == -1) ? argc : start + spec->count; + if (stop > argc) stop = argc; + int count = 0; + keys = getKeysPrepareResult(result, stop - start); + for (int i = start; i < stop; i++ ) { + keys[count].pos = i; + keys[count++].flags = spec->flags; + } + result->numkeys = count; + return count; + } + } + return 0; } /* The base case is to use the keys position as given in the command table diff --git a/src/debug.c b/src/debug.c index 2da2c5d50..b95daaa36 100644 --- a/src/debug.c +++ b/src/debug.c @@ -482,6 +482,10 @@ void debugCommand(client *c) { " Show low level client eviction pools info (maxmemory-clients).", "PAUSE-CRON <0|1>", " Stop periodic cron job processing.", +"REPLYBUFFER-PEAK-RESET-TIME <NEVER||RESET|time>", +" Sets the time (in milliseconds) to wait between client reply buffer peak resets.", +" In case NEVER is provided the last observed peak will never be reset", +" In case RESET is provided the peak reset time will be restored to the default value", NULL }; addReplyHelp(c, help); @@ -825,7 +829,7 @@ NULL int memerr; unsigned long long sz = memtoull((const char *)c->argv[2]->ptr, &memerr); if (memerr || !quicklistisSetPackedThreshold(sz)) { - addReplyError(c, "argument must be a memory value bigger then 1 and smaller than 4gb"); + addReplyError(c, "argument must be a memory value bigger than 1 and smaller than 4gb"); } else { addReply(c,shared.ok); } @@ -921,12 +925,12 @@ NULL addReplyStatus(c,"Apparently Redis did not crash: test passed"); } else if (!strcasecmp(c->argv[1]->ptr,"set-disable-deny-scripts") && c->argc == 3) { - server.script_disable_deny_script = atoi(c->argv[2]->ptr);; + server.script_disable_deny_script = atoi(c->argv[2]->ptr); addReply(c,shared.ok); } else if (!strcasecmp(c->argv[1]->ptr,"config-rewrite-force-all") && c->argc == 2) { if (rewriteConfig(server.configfile, 1) == -1) - addReplyError(c, "CONFIG-REWRITE-FORCE-ALL failed"); + addReplyErrorFormat(c, "CONFIG-REWRITE-FORCE-ALL failed: %s", strerror(errno)); else addReply(c, shared.ok); } else if(!strcasecmp(c->argv[1]->ptr,"client-eviction") && c->argc == 2) { @@ -958,6 +962,16 @@ NULL { server.pause_cron = atoi(c->argv[2]->ptr); addReply(c,shared.ok); + } else if (!strcasecmp(c->argv[1]->ptr,"replybuffer-peak-reset-time") && c->argc == 3 ) { + if (!strcasecmp(c->argv[2]->ptr, "never")) { + server.reply_buffer_peak_reset_time = -1; + } else if(!strcasecmp(c->argv[2]->ptr, "reset")) { + server.reply_buffer_peak_reset_time = REPLY_BUFFER_DEFAULT_PEAK_RESET_TIME; + } else { + if (getLongFromObjectOrReply(c, c->argv[2], &server.reply_buffer_peak_reset_time, NULL) != C_OK) + return; + } + addReply(c, shared.ok); } else { addReplySubcommandSyntaxError(c); return; @@ -1681,13 +1695,19 @@ void logStackTrace(void *eip, int uplevel) { void logServerInfo(void) { sds infostring, clients; serverLogRaw(LL_WARNING|LL_RAW, "\n------ INFO OUTPUT ------\n"); - infostring = genRedisInfoString("all"); + int all = 0, everything = 0; + robj *argv[1]; + argv[0] = createStringObject("all", strlen("all")); + dict *section_dict = genInfoSectionDict(argv, 1, NULL, &all, &everything); + infostring = genRedisInfoString(section_dict, all, everything); serverLogRaw(LL_WARNING|LL_RAW, infostring); serverLogRaw(LL_WARNING|LL_RAW, "\n------ CLIENT LIST OUTPUT ------\n"); clients = getAllClientsInfoString(-1); serverLogRaw(LL_WARNING|LL_RAW, clients); sdsfree(infostring); sdsfree(clients); + releaseInfoSectionDict(section_dict); + decrRefCount(argv[0]); } /* Log certain config values, which can be used for debuggin */ @@ -1723,10 +1743,10 @@ void logCurrentClient(void) { sdsfree(client); for (j = 0; j < cc->argc; j++) { robj *decoded; - decoded = getDecodedObject(cc->argv[j]); - serverLog(LL_WARNING|LL_RAW,"argv[%d]: '%s'\n", j, - (char*)decoded->ptr); + sds repr = sdscatrepr(sdsempty(),decoded->ptr, min(sdslen(decoded->ptr), 128)); + serverLog(LL_WARNING|LL_RAW,"argv[%d]: '%s'\n", j, (char*)repr); + sdsfree(repr); decrRefCount(decoded); } /* Check if the first argument, usually a key, is found inside the @@ -1764,7 +1784,10 @@ int memtest_test_linux_anonymous_maps(void) { if (!fd) return 0; fp = fopen("/proc/self/maps","r"); - if (!fp) return 0; + if (!fp) { + closeDirectLogFiledes(fd); + return 0; + } while(fgets(line,sizeof(line),fp) != NULL) { char *start, *end, *p = line; diff --git a/src/debugmacro.h b/src/debugmacro.h index 58e6577e5..dcd79a33f 100644 --- a/src/debugmacro.h +++ b/src/debugmacro.h @@ -30,6 +30,9 @@ * POSSIBILITY OF SUCH DAMAGE. */ +#ifndef _REDIS_DEBUGMACRO_H_ +#define _REDIS_DEBUGMACRO_H_ + #include <stdio.h> #define D(...) \ do { \ @@ -39,3 +42,5 @@ fprintf(fp,"\n"); \ fclose(fp); \ } while (0) + +#endif /* _REDIS_DEBUGMACRO_H_ */ diff --git a/src/defrag.c b/src/defrag.c index 16b1a4d2d..d4983c6d5 100644 --- a/src/defrag.c +++ b/src/defrag.c @@ -128,6 +128,27 @@ robj *activeDefragStringOb(robj* ob, long *defragged) { return ret; } +/* Defrag helper for lua scripts + * + * returns NULL in case the allocation wasn't moved. + * when it returns a non-null value, the old pointer was already released + * and should NOT be accessed. */ +luaScript *activeDefragLuaScript(luaScript *script, long *defragged) { + luaScript *ret = NULL; + + /* try to defrag script struct */ + if ((ret = activeDefragAlloc(script))) { + script = ret; + (*defragged)++; + } + + /* try to defrag actual script object */ + robj *ob = activeDefragStringOb(script->body, defragged); + if (ob) script->body = ob; + + return ret; +} + /* Defrag helper for dictEntries to be used during dict iteration (called on * each step). Returns a stat of how many pointers were moved. */ long dictIterDefragEntry(dictIterator *iter) { @@ -256,6 +277,7 @@ long activeDefragZsetEntry(zset *zs, dictEntry *de) { #define DEFRAG_SDS_DICT_VAL_IS_SDS 1 #define DEFRAG_SDS_DICT_VAL_IS_STROB 2 #define DEFRAG_SDS_DICT_VAL_VOID_PTR 3 +#define DEFRAG_SDS_DICT_VAL_LUA_SCRIPT 4 /* Defrag a dict with sds key and optional value (either ptr, sds or robj string) */ long activeDefragSdsDict(dict* d, int val_type) { @@ -280,6 +302,10 @@ long activeDefragSdsDict(dict* d, int val_type) { void *newptr, *ptr = dictGetVal(de); if ((newptr = activeDefragAlloc(ptr))) de->v.val = newptr, defragged++; + } else if (val_type == DEFRAG_SDS_DICT_VAL_LUA_SCRIPT) { + void *newptr, *ptr = dictGetVal(de); + if ((newptr = activeDefragLuaScript(ptr, &defragged))) + de->v.val = newptr; } defragged += dictIterDefragEntry(di); } @@ -939,7 +965,7 @@ long defragOtherGlobals() { /* there are many more pointers to defrag (e.g. client argv, output / aof buffers, etc. * but we assume most of these are short lived, we only need to defrag allocations * that remain static for a long time */ - defragged += activeDefragSdsDict(evalScriptsDict(), DEFRAG_SDS_DICT_VAL_IS_STROB); + defragged += activeDefragSdsDict(evalScriptsDict(), DEFRAG_SDS_DICT_VAL_LUA_SCRIPT); defragged += moduleDefragGlobals(); return defragged; } @@ -1130,7 +1156,7 @@ void activeDefragCycle(void) { /* Move on to next database, and stop if we reached the last one. */ if (++current_db >= server.dbnum) { /* defrag other items not part of the db / keys */ - defragOtherGlobals(); + server.stat_active_defrag_hits += defragOtherGlobals(); long long now = ustime(); size_t frag_bytes; diff --git a/src/eval.c b/src/eval.c index 41dc3b611..1a9437a09 100644 --- a/src/eval.c +++ b/src/eval.c @@ -47,11 +47,6 @@ void ldbEnable(client *c); void evalGenericCommandWithDebugging(client *c, int evalsha); sds ldbCatStackValue(sds s, lua_State *lua, int idx); -typedef struct luaScript { - uint64_t flags; - robj *body; -} luaScript; - static void dictLuaScriptDestructor(dict *d, void *val) { UNUSED(d); if (val == NULL) return; /* Lazy freeing will set value to NULL. */ @@ -63,7 +58,7 @@ static uint64_t dictStrCaseHash(const void *key) { return dictGenCaseHashFunction((unsigned char*)key, strlen((char*)key)); } -/* server.lua_scripts sha (as sds string) -> scripts (as robj) cache. */ +/* server.lua_scripts sha (as sds string) -> scripts (as luaScript) cache. */ dictType shaScriptObjectDictType = { dictStrCaseHash, /* hash function */ NULL, /* key dup */ @@ -246,11 +241,14 @@ void scriptingInit(int setup) { " if i and i.what == 'C' then\n" " i = dbg.getinfo(3,'nSl')\n" " end\n" + " if type(err) ~= 'table' then\n" + " err = {err='ERR' .. tostring(err)}" + " end" " if i then\n" - " return i.source .. ':' .. i.currentline .. ': ' .. err\n" - " else\n" - " return err\n" - " end\n" + " err['source'] = i.source\n" + " err['line'] = i.currentline\n" + " end" + " return err\n" "end\n"; luaL_loadbuffer(lua,errh_func,strlen(errh_func),"@err_handler_def"); lua_pcall(lua,0,0,0); @@ -392,7 +390,7 @@ sds luaCreateFunction(client *c, robj *body) { if (luaL_loadbuffer(lctx.lua,funcdef,sdslen(funcdef),"@user_script")) { if (c != NULL) { addReplyErrorFormat(c, - "Error compiling script (new function): %s\n", + "Error compiling script (new function): %s", lua_tostring(lctx.lua,-1)); } lua_pop(lctx.lua,1); @@ -403,7 +401,7 @@ sds luaCreateFunction(client *c, robj *body) { if (lua_pcall(lctx.lua,0,0,0)) { if (c != NULL) { - addReplyErrorFormat(c,"Error running script (new function): %s\n", + addReplyErrorFormat(c,"Error running script (new function): %s", lua_tostring(lctx.lua,-1)); } lua_pop(lctx.lua,1); @@ -1479,8 +1477,8 @@ int ldbRepl(lua_State *lua) { while((argv = ldbReplParseCommand(&argc, &err)) == NULL) { char buf[1024]; if (err) { - lua_pushstring(lua, err); - lua_error(lua); + luaPushError(lua, err); + luaError(lua); } int nread = connRead(ldb.conn,buf,sizeof(buf)); if (nread <= 0) { @@ -1497,8 +1495,8 @@ int ldbRepl(lua_State *lua) { if (sdslen(ldb.cbuf) > 1<<20) { sdsfree(ldb.cbuf); ldb.cbuf = sdsempty(); - lua_pushstring(lua, "max client buffer reached"); - lua_error(lua); + luaPushError(lua, "max client buffer reached"); + luaError(lua); } } @@ -1558,8 +1556,8 @@ ldbLog(sdsnew(" next line of code.")); ldbEval(lua,argv,argc); ldbSendLogs(); } else if (!strcasecmp(argv[0],"a") || !strcasecmp(argv[0],"abort")) { - lua_pushstring(lua, "script aborted for user request"); - lua_error(lua); + luaPushError(lua, "script aborted for user request"); + luaError(lua); } else if (argc > 1 && (!strcasecmp(argv[0],"r") || !strcasecmp(argv[0],"redis"))) { ldbRedis(lua,argv,argc); @@ -1640,8 +1638,8 @@ void luaLdbLineHook(lua_State *lua, lua_Debug *ar) { /* If the client closed the connection and we have a timeout * connection, let's kill the script otherwise the process * will remain blocked indefinitely. */ - lua_pushstring(lua, "timeout during Lua debugging with client closing connection"); - lua_error(lua); + luaPushError(lua, "timeout during Lua debugging with client closing connection"); + luaError(lua); } rctx->start_time = getMonotonicUs(); rctx->snapshot_time = mstime(); diff --git a/src/function_lua.c b/src/function_lua.c index 3dbc8419e..8f21a1721 100644 --- a/src/function_lua.c +++ b/src/function_lua.c @@ -86,8 +86,8 @@ static void luaEngineLoadHook(lua_State *lua, lua_Debug *ar) { if (duration > LOAD_TIMEOUT_MS) { lua_sethook(lua, luaEngineLoadHook, LUA_MASKLINE, 0); - lua_pushstring(lua,"FUNCTION LOAD timeout"); - lua_error(lua); + luaPushError(lua,"FUNCTION LOAD timeout"); + luaError(lua); } } @@ -151,10 +151,13 @@ static int luaEngineCreate(void *engine_ctx, functionLibInfo *li, sds blob, sds lua_sethook(lua,luaEngineLoadHook,LUA_MASKCOUNT,100000); /* Run the compiled code to allow it to register functions */ if (lua_pcall(lua,0,0,0)) { - *err = sdscatprintf(sdsempty(), "Error registering functions: %s", lua_tostring(lua, -1)); + errorInfo err_info = {0}; + luaExtractErrorInformation(lua, &err_info); + *err = sdscatprintf(sdsempty(), "Error registering functions: %s", err_info.msg); lua_pop(lua, 2); /* pops the error and globals table */ lua_sethook(lua,NULL,0,0); /* Disable hook */ luaSaveOnRegistry(lua, REGISTRY_LOAD_CTX_NAME, NULL); + luaErrorInformationDiscard(&err_info); return C_ERR; } lua_sethook(lua,NULL,0,0); /* Disable hook */ @@ -429,11 +432,11 @@ static int luaRegisterFunction(lua_State *lua) { loadCtx *load_ctx = luaGetFromRegistry(lua, REGISTRY_LOAD_CTX_NAME); if (!load_ctx) { luaPushError(lua, "redis.register_function can only be called on FUNCTION LOAD command"); - return luaRaiseError(lua); + return luaError(lua); } if (luaRegisterFunctionReadArgs(lua, ®ister_f_args) != C_OK) { - return luaRaiseError(lua); + return luaError(lua); } sds err = NULL; @@ -441,7 +444,7 @@ static int luaRegisterFunction(lua_State *lua) { luaRegisterFunctionArgsDispose(lua, ®ister_f_args); luaPushError(lua, err); sdsfree(err); - return luaRaiseError(lua); + return luaError(lua); } return 0; @@ -475,11 +478,14 @@ int luaEngineInitEngine() { " if i and i.what == 'C' then\n" " i = dbg.getinfo(3,'nSl')\n" " end\n" + " if type(err) ~= 'table' then\n" + " err = {err='ERR' .. tostring(err)}" + " end" " if i then\n" - " return i.source .. ':' .. i.currentline .. ': ' .. err\n" - " else\n" - " return err\n" - " end\n" + " err['source'] = i.source\n" + " err['line'] = i.currentline\n" + " end" + " return err\n" "end\n" "return error_handler"; luaL_loadbuffer(lua_engine_ctx->lua, errh_func, strlen(errh_func), "@err_handler_def"); diff --git a/src/functions.c b/src/functions.c index 57fb7ec24..739d178aa 100644 --- a/src/functions.c +++ b/src/functions.c @@ -808,7 +808,7 @@ void functionFlushCommand(client *c) { void functionHelpCommand(client *c) { const char *help[] = { -"LOAD <ENGINE NAME> <LIBRARY NAME> [REPLACE] [DESC <LIBRARY DESCRIPTION>] <LIBRARY CODE>", +"LOAD <ENGINE NAME> <LIBRARY NAME> [REPLACE] [DESCRIPTION <LIBRARY DESCRIPTION>] <LIBRARY CODE>", " Create a new library with the given library name and code.", "DELETE <LIBRARY NAME>", " Delete the given library.", diff --git a/src/geohash.h b/src/geohash.h index 8fa324afd..4befb9303 100644 --- a/src/geohash.h +++ b/src/geohash.h @@ -34,7 +34,6 @@ #include <stddef.h> #include <stdint.h> -#include <stdint.h> #if defined(__cplusplus) extern "C" { diff --git a/src/geohash_helper.c b/src/geohash_helper.c index ec4dbd23a..a9224a1dd 100644 --- a/src/geohash_helper.c +++ b/src/geohash_helper.c @@ -91,8 +91,8 @@ uint8_t geohashEstimateStepsByRadius(double range_meters, double lat) { * \-----------------/ -------- \-----------------/ * \ / / \ \ / * \ (long,lat) / / (long,lat) \ \ (long,lat) / - * \ / / \ / \ - * --------- /----------------\ /--------------\ + * \ / / \ / \ + * --------- /----------------\ /---------------\ * Northern Hemisphere Southern Hemisphere Around the equator */ int geohashBoundingBox(GeoShape *shape, double *bounds) { @@ -164,14 +164,14 @@ GeoHashRadius geohashCalculateAreasByShapeWGS84(GeoShape *shape) { geohashDecode(long_range, lat_range, neighbors.east, &east); geohashDecode(long_range, lat_range, neighbors.west, &west); - if (geohashGetDistance(longitude,latitude,longitude,north.latitude.max) - < radius_meters) decrease_step = 1; - if (geohashGetDistance(longitude,latitude,longitude,south.latitude.min) - < radius_meters) decrease_step = 1; - if (geohashGetDistance(longitude,latitude,east.longitude.max,latitude) - < radius_meters) decrease_step = 1; - if (geohashGetDistance(longitude,latitude,west.longitude.min,latitude) - < radius_meters) decrease_step = 1; + if (north.latitude.max < max_lat) + decrease_step = 1; + if (south.latitude.min > min_lat) + decrease_step = 1; + if (east.longitude.max < max_lon) + decrease_step = 1; + if (west.longitude.min > min_lon) + decrease_step = 1; } if (steps > 1 && decrease_step) { diff --git a/src/help.h b/src/help.h index 2bceea2a4..edeb9b26e 100644 --- a/src/help.h +++ b/src/help.h @@ -1,4 +1,4 @@ -/* Automatically generated by ./utils/generate-command-help.rb, do not edit. */ +/* Automatically generated by utils/generate-command-help.rb, do not edit. */ #ifndef __REDIS_HELP_H #define __REDIS_HELP_H @@ -429,6 +429,11 @@ struct commandHelp { "Extract keys given a full Redis command", 9, "2.8.13" }, + { "COMMAND GETKEYSANDFLAGS", + "", + "Extract keys given a full Redis command", + 9, + "7.0.0" }, { "COMMAND HELP", "", "Show helpful text about the different subcommands", @@ -571,12 +576,12 @@ struct commandHelp { "6.2.0" }, { "FCALL", "function numkeys key [key ...] arg [arg ...]", - "PATCH__TBD__38__", + "Invoke a function", 10, "7.0.0" }, { "FCALL_RO", "function numkeys key [key ...] arg [arg ...]", - "PATCH__TBD__7__", + "Invoke a read-only function", 10, "7.0.0" }, { "FLUSHALL", @@ -595,7 +600,7 @@ struct commandHelp { 10, "7.0.0" }, { "FUNCTION DELETE", - "function-name", + "library-name", "Delete a function by name", 10, "7.0.0" }, @@ -625,7 +630,7 @@ struct commandHelp { 10, "7.0.0" }, { "FUNCTION LOAD", - "engine-name library-name [REPLACE] [DESC library-description] function-code", + "engine-name library-name [REPLACE] [DESCRIPTION library-description] function-code", "Create a function with the given arguments (name, code, description)", 10, "7.0.0" }, @@ -820,7 +825,7 @@ struct commandHelp { 1, "2.6.0" }, { "INFO", - "[section]", + "[section [section ...]]", "Get information and statistics about the server", 9, "1.0.0" }, @@ -1180,7 +1185,7 @@ struct commandHelp { 6, "7.0.0" }, { "PUBSUB SHARDNUMSUB", - "", + "[channel [channel ...]]", "Get the count of subscribers for shard channels", 6, "7.0.0" }, @@ -1391,7 +1396,7 @@ struct commandHelp { "1.0.0" }, { "SLAVEOF", "host port", - "Make the server a replica of another instance, or promote it as master. Deprecated starting with Redis 5. Use REPLICAOF instead.", + "Make the server a replica of another instance, or promote it as master.", 9, "1.0.0" }, { "SLOWLOG", @@ -1590,7 +1595,7 @@ struct commandHelp { 14, "5.0.0" }, { "XGROUP CREATE", - "key groupname id|$ [MKSTREAM]", + "key groupname id|$ [MKSTREAM] [ENTRIESREAD entries_read]", "Create a consumer group.", 14, "5.0.0" }, @@ -1615,7 +1620,7 @@ struct commandHelp { 14, "5.0.0" }, { "XGROUP SETID", - "key groupname id|$", + "key groupname id|$ [ENTRIESREAD entries_read]", "Set a consumer group to an arbitrary last delivered ID value.", 14, "5.0.0" }, @@ -1675,7 +1680,7 @@ struct commandHelp { 14, "5.0.0" }, { "XSETID", - "key last-id", + "key last-id [ENTRIESADDED entries_added] [MAXDELETEDID max_deleted_entry_id]", "An internal command for replicating stream values", 14, "5.0.0" }, diff --git a/src/module.c b/src/module.c index 8c44422c6..7130139a6 100644 --- a/src/module.c +++ b/src/module.c @@ -70,7 +70,7 @@ typedef struct RedisModuleInfoCtx { struct RedisModule *module; - const char *requested_section; + dict *requested_sections; sds info; /* info string we collected so far */ int sections; /* number of sections we collected so far */ int in_section; /* indication if we're in an active section or not */ @@ -154,7 +154,8 @@ struct RedisModuleCtx { gets called for clients blocked on keys. */ - /* Used if there is the REDISMODULE_CTX_KEYS_POS_REQUEST flag set. */ + /* Used if there is the REDISMODULE_CTX_KEYS_POS_REQUEST or + * REDISMODULE_CTX_CHANNEL_POS_REQUEST flag set. */ getKeysResult *keys_result; struct RedisModulePoolAllocBlock *pa_head; @@ -173,6 +174,7 @@ typedef struct RedisModuleCtx RedisModuleCtx; when the context is destroyed */ #define REDISMODULE_CTX_NEW_CLIENT (1<<7) /* Free client object when the context is destroyed */ +#define REDISMODULE_CTX_CHANNELS_POS_REQUEST (1<<8) /* This represents a Redis key opened with RM_OpenKey(). */ struct RedisModuleKey { @@ -390,6 +392,7 @@ typedef struct RedisModuleKeyOptCtx { In most cases, only 'from_dbid' is valid, but in callbacks such as `copy2`, 'from_dbid' and 'to_dbid' are both valid. */ } RedisModuleKeyOptCtx; + /* -------------------------------------------------------------------------- * Prototypes * -------------------------------------------------------------------------- */ @@ -404,6 +407,16 @@ static void moduleInitKeyTypeSpecific(RedisModuleKey *key); void RM_FreeDict(RedisModuleCtx *ctx, RedisModuleDict *d); void RM_FreeServerInfo(RedisModuleCtx *ctx, RedisModuleServerInfoData *data); +/* Helpers for RM_SetCommandInfo. */ +static int moduleValidateCommandInfo(const RedisModuleCommandInfo *info); +static int64_t moduleConvertKeySpecsFlags(int64_t flags, int from_api); +static int moduleValidateCommandArgs(RedisModuleCommandArg *args, + const RedisModuleCommandInfoVersion *version); +static struct redisCommandArg *moduleCopyCommandArgs(RedisModuleCommandArg *args, + const RedisModuleCommandInfoVersion *version); +static redisCommandArgType moduleConvertArgType(RedisModuleCommandArgType type, int *error); +static int moduleConvertArgFlags(int flags); + /* -------------------------------------------------------------------------- * ## Heap allocation raw functions * @@ -770,6 +783,25 @@ int moduleGetCommandKeysViaAPI(struct redisCommand *cmd, robj **argv, int argc, return result->numkeys; } +/* This function returns the list of channels, with the same interface as + * moduleGetCommandKeysViaAPI, for modules that declare "getchannels-api" + * during registration. Unlike keys, this is the only way to declare channels. */ +int moduleGetCommandChannelsViaAPI(struct redisCommand *cmd, robj **argv, int argc, getKeysResult *result) { + RedisModuleCommand *cp = (void*)(unsigned long)cmd->getkeys_proc; + RedisModuleCtx ctx; + moduleCreateContext(&ctx, cp->module, REDISMODULE_CTX_CHANNELS_POS_REQUEST); + + /* Initialize getKeysResult */ + getKeysPrepareResult(result, MAX_KEYS_BUFFER); + ctx.keys_result = result; + + cp->func(&ctx,(void**)argv,argc); + /* We currently always use the array allocated by RM_RM_ChannelAtPosWithFlags() and don't try + * to optimize for the pre-allocated buffer. */ + moduleFreeContext(&ctx); + return result->numkeys; +} + /* -------------------------------------------------------------------------- * ## Commands API * @@ -789,17 +821,23 @@ int RM_IsKeysPositionRequest(RedisModuleCtx *ctx) { * keys, since it was flagged as "getkeys-api" during the registration, * the command implementation checks for this special call using the * RedisModule_IsKeysPositionRequest() API and uses this function in - * order to report keys, like in the following example: + * order to report keys. + * + * The supported flags are the ones used by RM_SetCommandInfo, see REDISMODULE_CMD_KEY_*. + * + * + * The following is an example of how it could be used: * * if (RedisModule_IsKeysPositionRequest(ctx)) { - * RedisModule_KeyAtPos(ctx,1); - * RedisModule_KeyAtPos(ctx,2); + * RedisModule_KeyAtPosWithFlags(ctx, 2, REDISMODULE_CMD_KEY_RO | REDISMODULE_CMD_KEY_ACCESS); + * RedisModule_KeyAtPosWithFlags(ctx, 1, REDISMODULE_CMD_KEY_RW | REDISMODULE_CMD_KEY_UPDATE | REDISMODULE_CMD_KEY_ACCESS); * } * - * Note: in the example below the get keys API would not be needed since - * keys are at fixed positions. This interface is only used for commands - * with a more complex structure. */ -void RM_KeyAtPos(RedisModuleCtx *ctx, int pos) { + * Note: in the example above the get keys API could have been handled by key-specs (preferred). + * Implementing the getkeys-api is required only when is it not possible to declare key-specs that cover all keys. + * + */ +void RM_KeyAtPosWithFlags(RedisModuleCtx *ctx, int pos, int flags) { if (!(ctx->flags & REDISMODULE_CTX_KEYS_POS_REQUEST) || !ctx->keys_result) return; if (pos <= 0) return; @@ -811,7 +849,74 @@ void RM_KeyAtPos(RedisModuleCtx *ctx, int pos) { getKeysPrepareResult(res, newsize); } - res->keys[res->numkeys++].pos = pos; + res->keys[res->numkeys].pos = pos; + res->keys[res->numkeys].flags = moduleConvertKeySpecsFlags(flags, 1); + res->numkeys++; +} + +/* This API existed before RM_KeyAtPosWithFlags was added, now deprecated and + * can be used for compatibility with older versions, before key-specs and flags + * were introduced. */ +void RM_KeyAtPos(RedisModuleCtx *ctx, int pos) { + /* Default flags require full access */ + int flags = moduleConvertKeySpecsFlags(CMD_KEY_FULL_ACCESS, 0); + RM_KeyAtPosWithFlags(ctx, pos, flags); +} + +/* Return non-zero if a module command, that was declared with the + * flag "getchannels-api", is called in a special way to get the channel positions + * and not to get executed. Otherwise zero is returned. */ +int RM_IsChannelsPositionRequest(RedisModuleCtx *ctx) { + return (ctx->flags & REDISMODULE_CTX_CHANNELS_POS_REQUEST) != 0; +} + +/* When a module command is called in order to obtain the position of + * channels, since it was flagged as "getchannels-api" during the + * registration, the command implementation checks for this special call + * using the RedisModule_IsChannelsPositionRequest() API and uses this + * function in order to report the channels. + * + * The supported flags are: + * * REDISMODULE_CMD_CHANNEL_SUBSCRIBE: This command will subscribe to the channel. + * * REDISMODULE_CMD_CHANNEL_UNSUBSCRIBE: This command will unsubscribe from this channel. + * * REDISMODULE_CMD_CHANNEL_PUBLISH: This command will publish to this channel. + * * REDISMODULE_CMD_CHANNEL_PATTERN: Instead of acting on a specific channel, will act on any + * channel specified by the pattern. This is the same access + * used by the PSUBSCRIBE and PUNSUBSCRIBE commands available + * in Redis. Not intended to be used with PUBLISH permissions. + * + * The following is an example of how it could be used: + * + * if (RedisModule_IsChannelsPositionRequest(ctx)) { + * RedisModule_ChannelAtPosWithFlags(ctx, 1, REDISMODULE_CMD_CHANNEL_SUBSCRIBE | REDISMODULE_CMD_CHANNEL_PATTERN); + * RedisModule_ChannelAtPosWithFlags(ctx, 1, REDISMODULE_CMD_CHANNEL_PUBLISH); + * } + * + * Note: One usage of declaring channels is for evaluating ACL permissions. In this context, + * unsubscribing is always allowed, so commands will only be checked against subscribe and + * publish permissions. This is preferred over using RM_ACLCheckChannelPermissions, since + * it allows the ACLs to be checked before the command is executed. */ +void RM_ChannelAtPosWithFlags(RedisModuleCtx *ctx, int pos, int flags) { + if (!(ctx->flags & REDISMODULE_CTX_CHANNELS_POS_REQUEST) || !ctx->keys_result) return; + if (pos <= 0) return; + + getKeysResult *res = ctx->keys_result; + + /* Check overflow */ + if (res->numkeys == res->size) { + int newsize = res->size + (res->size > 8192 ? 8192 : res->size); + getKeysPrepareResult(res, newsize); + } + + int new_flags = 0; + if (flags & REDISMODULE_CMD_CHANNEL_SUBSCRIBE) new_flags |= CMD_CHANNEL_SUBSCRIBE; + if (flags & REDISMODULE_CMD_CHANNEL_UNSUBSCRIBE) new_flags |= CMD_CHANNEL_UNSUBSCRIBE; + if (flags & REDISMODULE_CMD_CHANNEL_PUBLISH) new_flags |= CMD_CHANNEL_PUBLISH; + if (flags & REDISMODULE_CMD_CHANNEL_PATTERN) new_flags |= CMD_CHANNEL_PATTERN; + + res->keys[res->numkeys].pos = pos; + res->keys[res->numkeys].flags = new_flags; + res->numkeys++; } /* Helper for RM_CreateCommand(). Turns a string representing command @@ -840,6 +945,7 @@ int64_t commandFlagsFromString(char *s) { else if (!strcasecmp(t,"no-auth")) flags |= CMD_NO_AUTH; else if (!strcasecmp(t,"may-replicate")) flags |= CMD_MAY_REPLICATE; else if (!strcasecmp(t,"getkeys-api")) flags |= CMD_MODULE_GETKEYS; + else if (!strcasecmp(t,"getchannels-api")) flags |= CMD_MODULE_GETCHANNELS; else if (!strcasecmp(t,"no-cluster")) flags |= CMD_MODULE_NO_CLUSTER; else if (!strcasecmp(t,"no-mandatory-keys")) flags |= CMD_NO_MANDATORY_KEYS; else if (!strcasecmp(t,"allow-busy")) flags |= CMD_ALLOW_BUSY; @@ -850,33 +956,6 @@ int64_t commandFlagsFromString(char *s) { return flags; } -/* Helper for RM_CreateCommand(). Turns a string representing keys spec - * flags into the keys spec flags used by the Redis core. - * - * It returns the set of flags, or -1 if unknown flags are found. */ -int64_t commandKeySpecsFlagsFromString(const char *s) { - int count, j; - int64_t flags = 0; - sds *tokens = sdssplitlen(s,strlen(s)," ",1,&count); - for (j = 0; j < count; j++) { - char *t = tokens[j]; - if (!strcasecmp(t,"RO")) flags |= CMD_KEY_RO; - else if (!strcasecmp(t,"RW")) flags |= CMD_KEY_RW; - else if (!strcasecmp(t,"OW")) flags |= CMD_KEY_OW; - else if (!strcasecmp(t,"RM")) flags |= CMD_KEY_RM; - else if (!strcasecmp(t,"access")) flags |= CMD_KEY_ACCESS; - else if (!strcasecmp(t,"insert")) flags |= CMD_KEY_INSERT; - else if (!strcasecmp(t,"update")) flags |= CMD_KEY_UPDATE; - else if (!strcasecmp(t,"delete")) flags |= CMD_KEY_DELETE; - else if (!strcasecmp(t,"channel")) flags |= CMD_KEY_CHANNEL; - else if (!strcasecmp(t,"incomplete")) flags |= CMD_KEY_INCOMPLETE; - else break; - } - sdsfreesplitres(tokens,count); - if (j != count) return -1; /* Some token not processed correctly. */ - return flags; -} - RedisModuleCommand *moduleCreateCommandProxy(struct RedisModule *module, sds declared_name, sds fullname, RedisModuleCmdFunc cmdfunc, int64_t flags, int firstkey, int lastkey, int keystep); /* Register a new command in the Redis server, that will be handled by @@ -946,6 +1025,8 @@ RedisModuleCommand *moduleCreateCommandProxy(struct RedisModule *module, sds dec * * **"allow-busy"**: Permit the command while the server is blocked either by * a script or by a slow module command, see * RM_Yield. + * * **"getchannels-api"**: The command implements the interface to return + * the arguments that are channels. * * The last three parameters specify which arguments of the new command are * Redis keys. See https://redis.io/commands/command for more information. @@ -965,9 +1046,7 @@ RedisModuleCommand *moduleCreateCommandProxy(struct RedisModule *module, sds dec * NOTE: The scheme described above serves a limited purpose and can * only be used to find keys that exist at constant indices. * For non-trivial key arguments, you may pass 0,0,0 and use - * RedisModule_AddCommandKeySpec (see documentation). - * - */ + * RedisModule_SetCommandInfo to set key specs using a more advanced scheme. */ int RM_CreateCommand(RedisModuleCtx *ctx, const char *name, RedisModuleCmdFunc cmdfunc, const char *strflags, int firstkey, int lastkey, int keystep) { int64_t flags = strflags ? commandFlagsFromString((char*)strflags) : 0; if (flags == -1) return REDISMODULE_ERR; @@ -1020,7 +1099,7 @@ RedisModuleCommand *moduleCreateCommandProxy(struct RedisModule *module, sds dec cp->rediscmd->key_specs = cp->rediscmd->key_specs_static; if (firstkey != 0) { cp->rediscmd->key_specs_num = 1; - cp->rediscmd->key_specs[0].flags = 0; + cp->rediscmd->key_specs[0].flags = CMD_KEY_FULL_ACCESS | CMD_KEY_VARIABLE_FLAGS; cp->rediscmd->key_specs[0].begin_search_type = KSPEC_BS_INDEX; cp->rediscmd->key_specs[0].bs.index.pos = firstkey; cp->rediscmd->key_specs[0].find_keys_type = KSPEC_FK_RANGE; @@ -1121,216 +1200,735 @@ int RM_CreateSubcommand(RedisModuleCommand *parent, const char *name, RedisModul return REDISMODULE_OK; } -/* Return `struct RedisModule *` as `void *` to avoid exposing it outside of module.c. */ -void *moduleGetHandleByName(char *modulename) { - return dictFetchValue(modules,modulename); -} +/* Accessors of array elements of structs where the element size is stored + * separately in the version struct. */ +static RedisModuleCommandHistoryEntry * +moduleCmdHistoryEntryAt(const RedisModuleCommandInfoVersion *version, + RedisModuleCommandHistoryEntry *entries, int index) { + off_t offset = index * version->sizeof_historyentry; + return (RedisModuleCommandHistoryEntry *)((char *)(entries) + offset); +} +static RedisModuleCommandKeySpec * +moduleCmdKeySpecAt(const RedisModuleCommandInfoVersion *version, + RedisModuleCommandKeySpec *keyspecs, int index) { + off_t offset = index * version->sizeof_keyspec; + return (RedisModuleCommandKeySpec *)((char *)(keyspecs) + offset); +} +static RedisModuleCommandArg * +moduleCmdArgAt(const RedisModuleCommandInfoVersion *version, + const RedisModuleCommandArg *args, int index) { + off_t offset = index * version->sizeof_arg; + return (RedisModuleCommandArg *)((char *)(args) + offset); +} + +/* Set additional command information. + * + * Affects the output of `COMMAND`, `COMMAND INFO` and `COMMAND DOCS`, Cluster, + * ACL and is used to filter commands with the wrong number of arguments before + * the call reaches the module code. + * + * This function can be called after creating a command using RM_CreateCommand + * and fetching the command pointer using RM_GetCommand. The information can + * only be set once for each command and has the following structure: + * + * typedef struct RedisModuleCommandInfo { + * const RedisModuleCommandInfoVersion *version; + * const char *summary; + * const char *complexity; + * const char *since; + * RedisModuleCommandHistoryEntry *history; + * const char *tips; + * int arity; + * RedisModuleCommandKeySpec *key_specs; + * RedisModuleCommandArg *args; + * } RedisModuleCommandInfo; + * + * All fields except `version` are optional. Explanation of the fields: + * + * - `version`: This field enables compatibility with different Redis versions. + * Always set this field to REDISMODULE_COMMAND_INFO_VERSION. + * + * - `summary`: A short description of the command (optional). + * + * - `complexity`: Complexity description (optional). + * + * - `since`: The version where the command was introduced (optional). + * Note: The version specified should be the module's, not Redis version. + * + * - `history`: An array of RedisModuleCommandHistoryEntry (optional), which is + * a struct with the following fields: + * + * const char *since; + * const char *changes; + * + * `since` is a version string and `changes` is a string describing the + * changes. The array is terminated by a zeroed entry, i.e. an entry with + * both strings set to NULL. + * + * - `tips`: A string of space-separated tips regarding this command, meant for + * clients and proxies. See https://redis.io/topics/command-tips. + * + * - `arity`: Number of arguments, including the command name itself. A positive + * number specifies an exact number of arguments and a negative number + * specifies a minimum number of arguments, so use -N to say >= N. Redis + * validates a call before passing it to a module, so this can replace an + * arity check inside the module command implementation. A value of 0 (or an + * omitted arity field) is equivalent to -2 if the command has sub commands + * and -1 otherwise. + * + * - `key_specs`: An array of RedisModuleCommandKeySpec, terminated by an + * element memset to zero. This is a scheme that tries to describe the + * positions of key arguments better than the old RM_CreateCommand arguments + * `firstkey`, `lastkey`, `keystep` and is needed if those three are not + * enough to describe the key positions. There are two steps to retrieve key + * positions: *begin search* (BS) in which index should find the first key and + * *find keys* (FK) which, relative to the output of BS, describes how can we + * will which arguments are keys. Additionally, there are key specific flags. + * + * Key-specs cause the triplet (firstkey, lastkey, keystep) given in + * RM_CreateCommand to be recomputed, but it is still useful to provide + * these three parameters in RM_CreateCommand, to better support old Redis + * versions where RM_SetCommandInfo is not available. + * + * Note that key-specs don't fully replace the "getkeys-api" (see + * RM_CreateCommand, RM_IsKeysPositionRequest and RM_KeyAtPosWithFlags) so + * it may be a good idea to supply both key-specs and implement the + * getkeys-api. + * + * A key-spec has the following structure: + * + * typedef struct RedisModuleCommandKeySpec { + * const char *notes; + * uint64_t flags; + * RedisModuleKeySpecBeginSearchType begin_search_type; + * union { + * struct { + * int pos; + * } index; + * struct { + * const char *keyword; + * int startfrom; + * } keyword; + * } bs; + * RedisModuleKeySpecFindKeysType find_keys_type; + * union { + * struct { + * int lastkey; + * int keystep; + * int limit; + * } range; + * struct { + * int keynumidx; + * int firstkey; + * int keystep; + * } keynum; + * } fk; + * } RedisModuleCommandKeySpec; + * + * Explanation of the fields of RedisModuleCommandKeySpec: + * + * * `notes`: Optional notes or clarifications about this key spec. + * + * * `flags`: A bitwise or of key-spec flags described below. + * + * * `begin_search_type`: This describes how the first key is discovered. + * There are two ways to determine the first key: + * + * * `REDISMODULE_KSPEC_BS_UNKNOWN`: There is no way to tell where the + * key args start. + * * `REDISMODULE_KSPEC_BS_INDEX`: Key args start at a constant index. + * * `REDISMODULE_KSPEC_BS_KEYWORD`: Key args start just after a + * specific keyword. + * + * * `bs`: This is a union in which the `index` or `keyword` branch is used + * depending on the value of the `begin_search_type` field. + * + * * `bs.index.pos`: The index from which we start the search for keys. + * (`REDISMODULE_KSPEC_BS_INDEX` only.) + * + * * `bs.keyword.keyword`: The keyword (string) that indicates the + * beginning of key arguments. (`REDISMODULE_KSPEC_BS_KEYWORD` only.) + * + * * `bs.keyword.startfrom`: An index in argv from which to start + * searching. Can be negative, which means start search from the end, + * in reverse. Example: -2 means to start in reverse from the + * penultimate argument. (`REDISMODULE_KSPEC_BS_KEYWORD` only.) + * + * * `find_keys_type`: After the "begin search", this describes which + * arguments are keys. The strategies are: + * + * * `REDISMODULE_KSPEC_BS_UNKNOWN`: There is no way to tell where the + * key args are located. + * * `REDISMODULE_KSPEC_FK_RANGE`: Keys end at a specific index (or + * relative to the last argument). + * * `REDISMODULE_KSPEC_FK_KEYNUM`: There's an argument that contains + * the number of key args somewhere before the keys themselves. + * + * `find_keys_type` and `fk` can be omitted if this keyspec describes + * exactly one key. + * + * * `fk`: This is a union in which the `range` or `keynum` branch is used + * depending on the value of the `find_keys_type` field. + * + * * `fk.range` (for `REDISMODULE_KSPEC_FK_RANGE`): A struct with the + * following fields: + * + * * `lastkey`: Index of the last key relative to the result of the + * begin search step. Can be negative, in which case it's not + * relative. -1 indicates the last argument, -2 one before the + * last and so on. + * + * * `keystep`: How many arguments should we skip after finding a + * key, in order to find the next one? + * + * * `limit`: If `lastkey` is -1, we use `limit` to stop the search + * by a factor. 0 and 1 mean no limit. 2 means 1/2 of the + * remaining args, 3 means 1/3, and so on. + * + * * `fk.keynum` (for `REDISMODULE_KSPEC_FK_KEYNUM`): A struct with the + * following fields: + * + * * `keynumidx`: Index of the argument containing the number of + * keys to come, relative to the result of the begin search step. + * + * * `firstkey`: Index of the fist key relative to the result of the + * begin search step. (Usually it's just after `keynumidx`, in + * which case it should be set to `keynumidx + 1`.) + * + * * `keystep`: How many argumentss should we skip after finding a + * key, in order to find the next one? + * + * Key-spec flags: + * + * The first four refer to what the command actually does with the *value or + * metadata of the key*, and not necessarily the user data or how it affects + * it. Each key-spec may must have exactly one of these. Any operation + * that's not distinctly deletion, overwrite or read-only would be marked as + * RW. + * + * * `REDISMODULE_CMD_KEY_RO`: Read-Only. Reads the value of the key, but + * doesn't necessarily return it. + * + * * `REDISMODULE_CMD_KEY_RW`: Read-Write. Modifies the data stored in the + * value of the key or its metadata. + * + * * `REDISMODULE_CMD_KEY_OW`: Overwrite. Overwrites the data stored in the + * value of the key. + * + * * `REDISMODULE_CMD_KEY_RM`: Deletes the key. + * + * The next four refer to *user data inside the value of the key*, not the + * metadata like LRU, type, cardinality. It refers to the logical operation + * on the user's data (actual input strings or TTL), being + * used/returned/copied/changed. It doesn't refer to modification or + * returning of metadata (like type, count, presence of data). ACCESS can be + * combined with one of the write operations INSERT, DELETE or UPDATE. Any + * write that's not an INSERT or a DELETE would be UPDATE. + * + * * `REDISMODULE_CMD_KEY_ACCESS`: Returns, copies or uses the user data + * from the value of the key. + * + * * `REDISMODULE_CMD_KEY_UPDATE`: Updates data to the value, new value may + * depend on the old value. + * + * * `REDISMODULE_CMD_KEY_INSERT`: Adds data to the value with no chance of + * modification or deletion of existing data. + * + * * `REDISMODULE_CMD_KEY_DELETE`: Explicitly deletes some content from the + * value of the key. + * + * Other flags: + * + * * `REDISMODULE_CMD_KEY_NOT_KEY`: The key is not actually a key, but + * should be routed in cluster mode as if it was a key. + * + * * `REDISMODULE_CMD_KEY_INCOMPLETE`: The keyspec might not point out all + * the keys it should cover. + * + * * `REDISMODULE_CMD_KEY_VARIABLE_FLAGS`: Some keys might have different + * flags depending on arguments. + * + * - `args`: An array of RedisModuleCommandArg, terminated by an element memset + * to zero. RedisModuleCommandArg is a structure with at the fields described + * below. + * + * typedef struct RedisModuleCommandArg { + * const char *name; + * RedisModuleCommandArgType type; + * int key_spec_index; + * const char *token; + * const char *summary; + * const char *since; + * int flags; + * struct RedisModuleCommandArg *subargs; + * } RedisModuleCommandArg; + * + * Explanation of the fields: + * + * * `name`: Name of the argument. + * + * * `type`: The type of the argument. See below for details. The types + * `REDISMODULE_ARG_TYPE_ONEOF` and `REDISMODULE_ARG_TYPE_BLOCK` require + * an argument to have sub-arguments, i.e. `subargs`. + * + * * `key_spec_index`: If the `type` is `REDISMODULE_ARG_TYPE_KEY` you must + * provide the index of the key-spec associated with this argument. See + * `key_specs` above. If the argument is not a key, you may specify -1. + * + * * `token`: The token preceding the argument (optional). Example: the + * argument `seconds` in `SET` has a token `EX`. If the argument consists + * of only a token (for example `NX` in `SET`) the type should be + * `REDISMODULE_ARG_TYPE_PURE_TOKEN` and `value` should be NULL. + * + * * `summary`: A short description of the argument (optional). + * + * * `since`: The first version which included this argument (optional). + * + * * `flags`: A bitwise or of the macros `REDISMODULE_CMD_ARG_*`. See below. + * + * * `value`: The display-value of the argument. This string is what should + * be displayed when creating the command syntax from the output of + * `COMMAND`. If `token` is not NULL, it should also be displayed. + * + * Explanation of `RedisModuleCommandArgType`: + * + * * `REDISMODULE_ARG_TYPE_STRING`: String argument. + * * `REDISMODULE_ARG_TYPE_INTEGER`: Integer argument. + * * `REDISMODULE_ARG_TYPE_DOUBLE`: Double-precision float argument. + * * `REDISMODULE_ARG_TYPE_KEY`: String argument representing a keyname. + * * `REDISMODULE_ARG_TYPE_PATTERN`: String, but regex pattern. + * * `REDISMODULE_ARG_TYPE_UNIX_TIME`: Integer, but Unix timestamp. + * * `REDISMODULE_ARG_TYPE_PURE_TOKEN`: Argument doesn't have a placeholder. + * It's just a token without a value. Example: the `KEEPTTL` option of the + * `SET` command. + * * `REDISMODULE_ARG_TYPE_ONEOF`: Used when the user can choose only one of + * a few sub-arguments. Requires `subargs`. Example: the `NX` and `XX` + * options of `SET`. + * * `REDISMODULE_ARG_TYPE_BLOCK`: Used when one wants to group together + * several sub-arguments, usually to apply something on all of them, like + * making the entire group "optional". Requires `subargs`. Example: the + * `LIMIT offset count` parameters in `ZRANGE`. + * + * Explanation of the command argument flags: + * + * * `REDISMODULE_CMD_ARG_OPTIONAL`: The argument is optional (like GET in + * the SET command). + * * `REDISMODULE_CMD_ARG_MULTIPLE`: The argument may repeat itself (like + * key in DEL). + * * `REDISMODULE_CMD_ARG_MULTIPLE_TOKEN`: The argument may repeat itself, + * and so does its token (like `GET pattern` in SORT). + * + * On success REDISMODULE_OK is returned. On error REDISMODULE_ERR is returned + * and `errno` is set to EINVAL if invalid info was provided or EEXIST if info + * has already been set. If the info is invalid, a warning is logged explaining + * which part of the info is invalid and why. */ +int RM_SetCommandInfo(RedisModuleCommand *command, const RedisModuleCommandInfo *info) { + if (!moduleValidateCommandInfo(info)) { + errno = EINVAL; + return REDISMODULE_ERR; + } -/* Returns 1 if `cmd` is a command of the module `modulename`. 0 otherwise. */ -int moduleIsModuleCommand(void *module_handle, struct redisCommand *cmd) { - if (cmd->proc != RedisModuleCommandDispatcher) - return 0; - if (module_handle == NULL) - return 0; - RedisModuleCommand *cp = (void*)(unsigned long)cmd->getkeys_proc; - return (cp->module == module_handle); -} + struct redisCommand *cmd = command->rediscmd; -void extendKeySpecsIfNeeded(struct redisCommand *cmd) { - /* We extend even if key_specs_num == key_specs_max because - * this function is called prior to adding a new spec */ - if (cmd->key_specs_num < cmd->key_specs_max) - return; + /* Check if any info has already been set. Overwriting info involves freeing + * the old info, which is not implemented. */ + if (cmd->summary || cmd->complexity || cmd->since || cmd->history || + cmd->tips || cmd->args || + !(cmd->key_specs_num == 0 || + /* Allow key spec populated from legacy (first,last,step) to exist. */ + (cmd->key_specs_num == 1 && cmd->key_specs == cmd->key_specs_static && + cmd->key_specs[0].begin_search_type == KSPEC_BS_INDEX && + cmd->key_specs[0].find_keys_type == KSPEC_FK_RANGE))) { + errno = EEXIST; + return REDISMODULE_ERR; + } - cmd->key_specs_max++; + if (info->summary) cmd->summary = zstrdup(info->summary); + if (info->complexity) cmd->complexity = zstrdup(info->complexity); + if (info->since) cmd->since = zstrdup(info->since); - if (cmd->key_specs == cmd->key_specs_static) { - cmd->key_specs = zmalloc(sizeof(keySpec) * cmd->key_specs_max); - memcpy(cmd->key_specs, cmd->key_specs_static, sizeof(keySpec) * cmd->key_specs_num); - } else { - cmd->key_specs = zrealloc(cmd->key_specs, sizeof(keySpec) * cmd->key_specs_max); + const RedisModuleCommandInfoVersion *version = info->version; + if (info->history) { + size_t count = 0; + while (moduleCmdHistoryEntryAt(version, info->history, count)->since) + count++; + serverAssert(count < SIZE_MAX / sizeof(commandHistory)); + cmd->history = zmalloc(sizeof(commandHistory) * (count + 1)); + for (size_t j = 0; j < count; j++) { + RedisModuleCommandHistoryEntry *entry = + moduleCmdHistoryEntryAt(version, info->history, j); + cmd->history[j].since = zstrdup(entry->since); + cmd->history[j].changes = zstrdup(entry->changes); + } + cmd->history[count].since = NULL; + cmd->history[count].changes = NULL; + cmd->num_history = count; + } + + if (info->tips) { + int count; + sds *tokens = sdssplitlen(info->tips, strlen(info->tips), " ", 1, &count); + if (tokens) { + cmd->tips = zmalloc(sizeof(char *) * (count + 1)); + for (int j = 0; j < count; j++) { + cmd->tips[j] = zstrdup(tokens[j]); + } + cmd->tips[count] = NULL; + cmd->num_tips = count; + sdsfreesplitres(tokens, count); + } } -} -int moduleAddCommandKeySpec(RedisModuleCommand *command, const char *specflags, int *index) { - int64_t flags = specflags ? commandKeySpecsFlagsFromString(specflags) : 0; - if (flags == -1) - return REDISMODULE_ERR; + if (info->arity) cmd->arity = info->arity; - struct redisCommand *cmd = command->rediscmd; + if (info->key_specs) { + /* Count and allocate the key specs. */ + size_t count = 0; + while (moduleCmdKeySpecAt(version, info->key_specs, count)->begin_search_type) + count++; + serverAssert(count < INT_MAX); + if (count <= STATIC_KEY_SPECS_NUM) { + cmd->key_specs_max = STATIC_KEY_SPECS_NUM; + cmd->key_specs = cmd->key_specs_static; + } else { + cmd->key_specs_max = count; + cmd->key_specs = zmalloc(sizeof(keySpec) * count); + } - extendKeySpecsIfNeeded(cmd); + /* Copy the contents of the RedisModuleCommandKeySpec array. */ + cmd->key_specs_num = count; + for (size_t j = 0; j < count; j++) { + RedisModuleCommandKeySpec *spec = + moduleCmdKeySpecAt(version, info->key_specs, j); + cmd->key_specs[j].notes = spec->notes ? zstrdup(spec->notes) : NULL; + cmd->key_specs[j].flags = moduleConvertKeySpecsFlags(spec->flags, 1); + switch (spec->begin_search_type) { + case REDISMODULE_KSPEC_BS_UNKNOWN: + cmd->key_specs[j].begin_search_type = KSPEC_BS_UNKNOWN; + break; + case REDISMODULE_KSPEC_BS_INDEX: + cmd->key_specs[j].begin_search_type = KSPEC_BS_INDEX; + cmd->key_specs[j].bs.index.pos = spec->bs.index.pos; + break; + case REDISMODULE_KSPEC_BS_KEYWORD: + cmd->key_specs[j].begin_search_type = KSPEC_BS_KEYWORD; + cmd->key_specs[j].bs.keyword.keyword = zstrdup(spec->bs.keyword.keyword); + cmd->key_specs[j].bs.keyword.startfrom = spec->bs.keyword.startfrom; + break; + default: + /* Can't happen; stopped in moduleValidateCommandInfo(). */ + serverPanic("Unknown begin_search_type"); + } - *index = cmd->key_specs_num; - cmd->key_specs[cmd->key_specs_num].begin_search_type = KSPEC_BS_INVALID; - cmd->key_specs[cmd->key_specs_num].find_keys_type = KSPEC_FK_INVALID; - cmd->key_specs[cmd->key_specs_num].flags = flags; - cmd->key_specs_num++; - return REDISMODULE_OK; -} + switch (spec->find_keys_type) { + case REDISMODULE_KSPEC_FK_OMITTED: + /* Omitted field is shorthand to say that it's a single key. */ + cmd->key_specs[j].find_keys_type = KSPEC_FK_RANGE; + cmd->key_specs[j].fk.range.lastkey = 0; + cmd->key_specs[j].fk.range.keystep = 1; + cmd->key_specs[j].fk.range.limit = 0; + break; + case REDISMODULE_KSPEC_FK_UNKNOWN: + cmd->key_specs[j].find_keys_type = KSPEC_FK_UNKNOWN; + break; + case REDISMODULE_KSPEC_FK_RANGE: + cmd->key_specs[j].find_keys_type = KSPEC_FK_RANGE; + cmd->key_specs[j].fk.range.lastkey = spec->fk.range.lastkey; + cmd->key_specs[j].fk.range.keystep = spec->fk.range.keystep; + cmd->key_specs[j].fk.range.limit = spec->fk.range.limit; + break; + case REDISMODULE_KSPEC_FK_KEYNUM: + cmd->key_specs[j].find_keys_type = KSPEC_FK_KEYNUM; + cmd->key_specs[j].fk.keynum.keynumidx = spec->fk.keynum.keynumidx; + cmd->key_specs[j].fk.keynum.firstkey = spec->fk.keynum.firstkey; + cmd->key_specs[j].fk.keynum.keystep = spec->fk.keynum.keystep; + break; + default: + /* Can't happen; stopped in moduleValidateCommandInfo(). */ + serverPanic("Unknown find_keys_type"); + } + } -int moduleSetCommandKeySpecBeginSearch(RedisModuleCommand *command, int index, keySpec *spec) { - struct redisCommand *cmd = command->rediscmd; + /* Update the legacy (first,last,step) spec used by the COMMAND command, + * by trying to "glue" consecutive range key specs. */ + populateCommandLegacyRangeSpec(cmd); + populateCommandMovableKeys(cmd); + } - if (index >= cmd->key_specs_num) - return REDISMODULE_ERR; + if (info->args) { + cmd->args = moduleCopyCommandArgs(info->args, version); + /* Populate arg.num_args with the number of subargs, recursively */ + cmd->num_args = populateArgsStructure(cmd->args); + } - cmd->key_specs[index].begin_search_type = spec->begin_search_type; - cmd->key_specs[index].bs = spec->bs; + /* Fields added in future versions to be added here, under conditions like + * `if (info->version >= 2) { access version 2 fields here }` */ return REDISMODULE_OK; } -int moduleSetCommandKeySpecFindKeys(RedisModuleCommand *command, int index, keySpec *spec) { - struct redisCommand *cmd = command->rediscmd; +/* Returns 1 if v is a power of two, 0 otherwise. */ +static inline int isPowerOfTwo(uint64_t v) { + return v && !(v & (v - 1)); +} - if (index >= cmd->key_specs_num) - return REDISMODULE_ERR; +/* Returns 1 if the command info is valid and 0 otherwise. */ +static int moduleValidateCommandInfo(const RedisModuleCommandInfo *info) { + const RedisModuleCommandInfoVersion *version = info->version; + if (!version) { + serverLog(LL_WARNING, "Invalid command info: version missing"); + return 0; + } + + /* No validation for the fields summary, complexity, since, tips (strings or + * NULL) and arity (any integer). */ + + /* History: If since is set, changes must also be set. */ + if (info->history) { + for (size_t j = 0; + moduleCmdHistoryEntryAt(version, info->history, j)->since; + j++) + { + if (!moduleCmdHistoryEntryAt(version, info->history, j)->changes) { + serverLog(LL_WARNING, "Invalid command info: history[%zd].changes missing", j); + return 0; + } + } + } - cmd->key_specs[index].find_keys_type = spec->find_keys_type; - cmd->key_specs[index].fk = spec->fk; + /* Key specs. */ + if (info->key_specs) { + for (size_t j = 0; + moduleCmdKeySpecAt(version, info->key_specs, j)->begin_search_type; + j++) + { + RedisModuleCommandKeySpec *spec = + moduleCmdKeySpecAt(version, info->key_specs, j); + if (j >= INT_MAX) { + serverLog(LL_WARNING, "Invalid command info: Too many key specs"); + return 0; /* redisCommand.key_specs_num is an int. */ + } - /* Refresh legacy range */ - populateCommandLegacyRangeSpec(cmd); - /* Refresh movablekeys flag */ - populateCommandMovableKeys(cmd); + /* Flags. Exactly one flag in a group is set if and only if the + * masked bits is a power of two. */ + uint64_t key_flags = + REDISMODULE_CMD_KEY_RO | REDISMODULE_CMD_KEY_RW | + REDISMODULE_CMD_KEY_OW | REDISMODULE_CMD_KEY_RM; + uint64_t write_flags = + REDISMODULE_CMD_KEY_INSERT | REDISMODULE_CMD_KEY_DELETE | + REDISMODULE_CMD_KEY_UPDATE; + if (!isPowerOfTwo(spec->flags & key_flags)) { + serverLog(LL_WARNING, + "Invalid command info: key_specs[%zd].flags: " + "Exactly one of the flags RO, RW, OW, RM reqired", j); + return 0; + } + if ((spec->flags & write_flags) != 0 && + !isPowerOfTwo(spec->flags & write_flags)) + { + serverLog(LL_WARNING, + "Invalid command info: key_specs[%zd].flags: " + "INSERT, DELETE and UPDATE are mutually exclusive", j); + return 0; + } - return REDISMODULE_OK; -} + switch (spec->begin_search_type) { + case REDISMODULE_KSPEC_BS_UNKNOWN: break; + case REDISMODULE_KSPEC_BS_INDEX: break; + case REDISMODULE_KSPEC_BS_KEYWORD: + if (spec->bs.keyword.keyword == NULL) { + serverLog(LL_WARNING, + "Invalid command info: key_specs[%zd].bs.keyword.keyword " + "required when begin_search_type is KEYWORD", j); + return 0; + } + break; + default: + serverLog(LL_WARNING, + "Invalid command info: key_specs[%zd].begin_search_type: " + "Invalid value %d", j, spec->begin_search_type); + return 0; + } -/* **The key spec API is not officially released and it is going to be changed - * in Redis 7.0. It has been disabled temporarily.** - * - * Key specs is a scheme that tries to describe the location - * of key arguments better than the old [first,last,step] scheme - * which is limited and doesn't fit many commands. - * - * This information is used by ACL, Cluster and the `COMMAND` command. - * - * There are two steps to retrieve the key arguments: - * - * - `begin_search` (BS): in which index should we start seacrhing for keys? - * - `find_keys` (FK): relative to the output of BS, how can we will which args are keys? - * - * There are two types of BS: - * - * - `index`: key args start at a constant index - * - `keyword`: key args start just after a specific keyword - * - * There are two kinds of FK: - * - * - `range`: keys end at a specific index (or relative to the last argument) - * - `keynum`: there's an arg that contains the number of key args somewhere before the keys themselves - * - * This function adds a new key spec to a command, returning a unique id in `spec_id`. - * The caller must then call one of the RedisModule_SetCommandKeySpecBeginSearch* APIs - * followed by one of the RedisModule_SetCommandKeySpecFindKeys* APIs. - * - * It should be called just after RedisModule_CreateCommand. - * - * Example: - * - * if (RedisModule_CreateCommand(ctx,"kspec.smove",kspec_legacy,"",0,0,0) == REDISMODULE_ERR) - * return REDISMODULE_ERR; - * - * if (RedisModule_AddCommandKeySpec(ctx,"kspec.smove","RW access delete",&spec_id) == REDISMODULE_ERR) - * return REDISMODULE_ERR; - * if (RedisModule_SetCommandKeySpecBeginSearchIndex(ctx,"kspec.smove",spec_id,1) == REDISMODULE_ERR) - * return REDISMODULE_ERR; - * if (RedisModule_SetCommandKeySpecFindKeysRange(ctx,"kspec.smove",spec_id,0,1,0) == REDISMODULE_ERR) - * return REDISMODULE_ERR; - * - * if (RedisModule_AddCommandKeySpec(ctx,"kspec.smove","RW insert",&spec_id) == REDISMODULE_ERR) - * return REDISMODULE_ERR; - * if (RedisModule_SetCommandKeySpecBeginSearchIndex(ctx,"kspec.smove",spec_id,2) == REDISMODULE_ERR) - * return REDISMODULE_ERR; - * if (RedisModule_SetCommandKeySpecFindKeysRange(ctx,"kspec.smove",spec_id,0,1,0) == REDISMODULE_ERR) - * return REDISMODULE_ERR; - * - * It is also possible to use this API on subcommands (See RedisModule_CreateSubcommand). - * The name of the subcommand should be the name of the parent command + "|" + name of subcommand. - * - * Example: - * - * RedisModule_AddCommandKeySpec(ctx,"module.object|encoding","RO",&spec_id) - * - * Returns REDISMODULE_OK on success - */ -int RM_AddCommandKeySpec(RedisModuleCommand *command, const char *specflags, int *spec_id) { - return moduleAddCommandKeySpec(command, specflags, spec_id); -} + /* Validate find_keys_type. */ + switch (spec->find_keys_type) { + case REDISMODULE_KSPEC_FK_OMITTED: break; /* short for RANGE {0,1,0} */ + case REDISMODULE_KSPEC_FK_UNKNOWN: break; + case REDISMODULE_KSPEC_FK_RANGE: break; + case REDISMODULE_KSPEC_FK_KEYNUM: break; + default: + serverLog(LL_WARNING, + "Invalid command info: key_specs[%zd].find_keys_type: " + "Invalid value %d", j, spec->find_keys_type); + return 0; + } + } + } -/* Set a "index" key arguments spec to a command (begin_search step). - * See RedisModule_AddCommandKeySpec's doc. - * - * - `index`: The index from which we start the search for keys - * - * Returns REDISMODULE_OK */ -int RM_SetCommandKeySpecBeginSearchIndex(RedisModuleCommand *command, int spec_id, int index) { - keySpec spec; - spec.begin_search_type = KSPEC_BS_INDEX; - spec.bs.index.pos = index; + /* Args, subargs (recursive) */ + return moduleValidateCommandArgs(info->args, version); +} + +/* When from_api is true, converts from REDISMODULE_CMD_KEY_* flags to CMD_KEY_* flags. + * When from_api is false, converts from CMD_KEY_* flags to REDISMODULE_CMD_KEY_* flags. */ +static int64_t moduleConvertKeySpecsFlags(int64_t flags, int from_api) { + int64_t out = 0; + int64_t map[][2] = { + {REDISMODULE_CMD_KEY_RO, CMD_KEY_RO}, + {REDISMODULE_CMD_KEY_RW, CMD_KEY_RW}, + {REDISMODULE_CMD_KEY_OW, CMD_KEY_OW}, + {REDISMODULE_CMD_KEY_RM, CMD_KEY_RM}, + {REDISMODULE_CMD_KEY_ACCESS, CMD_KEY_ACCESS}, + {REDISMODULE_CMD_KEY_INSERT, CMD_KEY_INSERT}, + {REDISMODULE_CMD_KEY_UPDATE, CMD_KEY_UPDATE}, + {REDISMODULE_CMD_KEY_DELETE, CMD_KEY_DELETE}, + {REDISMODULE_CMD_KEY_NOT_KEY, CMD_KEY_NOT_KEY}, + {REDISMODULE_CMD_KEY_INCOMPLETE, CMD_KEY_INCOMPLETE}, + {REDISMODULE_CMD_KEY_VARIABLE_FLAGS, CMD_KEY_VARIABLE_FLAGS}, + {0,0}}; + + int from_idx = from_api ? 0 : 1, to_idx = !from_idx; + for (int i=0; map[i][0]; i++) + if (flags & map[i][from_idx]) out |= map[i][to_idx]; + return out; +} + +/* Validates an array of RedisModuleCommandArg. Returns 1 if it's valid and 0 if + * it's invalid. */ +static int moduleValidateCommandArgs(RedisModuleCommandArg *args, + const RedisModuleCommandInfoVersion *version) { + if (args == NULL) return 1; /* Missing args is OK. */ + for (size_t j = 0; moduleCmdArgAt(version, args, j)->name != NULL; j++) { + RedisModuleCommandArg *arg = moduleCmdArgAt(version, args, j); + int arg_type_error = 0; + moduleConvertArgType(arg->type, &arg_type_error); + if (arg_type_error) { + serverLog(LL_WARNING, + "Invalid command info: Argument \"%s\": Undefined type %d", + arg->name, arg->type); + return 0; + } + if (arg->type == REDISMODULE_ARG_TYPE_PURE_TOKEN && !arg->token) { + serverLog(LL_WARNING, + "Invalid command info: Argument \"%s\": " + "token required when type is PURE_TOKEN", args[j].name); + return 0; + } - return moduleSetCommandKeySpecBeginSearch(command, spec_id, &spec); -} + if (arg->type == REDISMODULE_ARG_TYPE_KEY) { + if (arg->key_spec_index < 0) { + serverLog(LL_WARNING, + "Invalid command info: Argument \"%s\": " + "key_spec_index required when type is KEY", + arg->name); + return 0; + } + } else if (arg->key_spec_index != -1 && arg->key_spec_index != 0) { + /* 0 is allowed for convenience, to allow it to be omitted in + * compound struct literals on the form `.field = value`. */ + serverLog(LL_WARNING, + "Invalid command info: Argument \"%s\": " + "key_spec_index specified but type isn't KEY", + arg->name); + return 0; + } -/* Set a "keyword" key arguments spec to a command (begin_search step). - * See RedisModule_AddCommandKeySpec's doc. - * - * - `keyword`: The keyword that indicates the beginning of key args - * - `startfrom`: An index in argv from which to start searching. - * Can be negative, which means start search from the end, in reverse - * (Example: -2 means to start in reverse from the panultimate arg) - * - * Returns REDISMODULE_OK */ -int RM_SetCommandKeySpecBeginSearchKeyword(RedisModuleCommand *command, int spec_id, const char *keyword, int startfrom) { - keySpec spec; - spec.begin_search_type = KSPEC_BS_KEYWORD; - spec.bs.keyword.keyword = keyword; - spec.bs.keyword.startfrom = startfrom; + if (arg->flags & ~(_REDISMODULE_CMD_ARG_NEXT - 1)) { + serverLog(LL_WARNING, + "Invalid command info: Argument \"%s\": Invalid flags", + arg->name); + return 0; + } - return moduleSetCommandKeySpecBeginSearch(command, spec_id, &spec); + if (arg->type == REDISMODULE_ARG_TYPE_ONEOF || + arg->type == REDISMODULE_ARG_TYPE_BLOCK) + { + if (arg->subargs == NULL) { + serverLog(LL_WARNING, + "Invalid command info: Argument \"%s\": " + "subargs required when type is ONEOF or BLOCK", + arg->name); + return 0; + } + if (!moduleValidateCommandArgs(arg->subargs, version)) return 0; + } else { + if (arg->subargs != NULL) { + serverLog(LL_WARNING, + "Invalid command info: Argument \"%s\": " + "subargs specified but type isn't ONEOF nor BLOCK", + arg->name); + return 0; + } + } + } + return 1; } -/* Set a "range" key arguments spec to a command (find_keys step). - * See RedisModule_AddCommandKeySpec's doc. - * - * - `lastkey`: Relative index (to the result of the begin_search step) where the last key is. - * Can be negative, in which case it's not relative. -1 indicating till the last argument, - * -2 one before the last and so on. - * - `keystep`: How many args should we skip after finding a key, in order to find the next one. - * - `limit`: If lastkey is -1, we use limit to stop the search by a factor. 0 and 1 mean no limit. - * 2 means 1/2 of the remaining args, 3 means 1/3, and so on. - * - * Returns REDISMODULE_OK */ -int RM_SetCommandKeySpecFindKeysRange(RedisModuleCommand *command, int spec_id, int lastkey, int keystep, int limit) { - keySpec spec; - spec.find_keys_type = KSPEC_FK_RANGE; - spec.fk.range.lastkey = lastkey; - spec.fk.range.keystep = keystep; - spec.fk.range.limit = limit; +/* Converts an array of RedisModuleCommandArg into a freshly allocated array of + * struct redisCommandArg. */ +static struct redisCommandArg *moduleCopyCommandArgs(RedisModuleCommandArg *args, + const RedisModuleCommandInfoVersion *version) { + size_t count = 0; + while (moduleCmdArgAt(version, args, count)->name) count++; + serverAssert(count < SIZE_MAX / sizeof(struct redisCommandArg)); + struct redisCommandArg *realargs = zcalloc((count+1) * sizeof(redisCommandArg)); + + for (size_t j = 0; j < count; j++) { + RedisModuleCommandArg *arg = moduleCmdArgAt(version, args, j); + realargs[j].name = zstrdup(arg->name); + realargs[j].type = moduleConvertArgType(arg->type, NULL); + if (arg->type == REDISMODULE_ARG_TYPE_KEY) + realargs[j].key_spec_index = arg->key_spec_index; + else + realargs[j].key_spec_index = -1; + if (arg->token) realargs[j].token = zstrdup(arg->token); + if (arg->summary) realargs[j].summary = zstrdup(arg->summary); + if (arg->since) realargs[j].since = zstrdup(arg->since); + realargs[j].flags = moduleConvertArgFlags(arg->flags); + if (arg->subargs) realargs[j].subargs = moduleCopyCommandArgs(arg->subargs, version); + } + return realargs; +} + +static redisCommandArgType moduleConvertArgType(RedisModuleCommandArgType type, int *error) { + if (error) *error = 0; + switch (type) { + case REDISMODULE_ARG_TYPE_STRING: return ARG_TYPE_STRING; + case REDISMODULE_ARG_TYPE_INTEGER: return ARG_TYPE_INTEGER; + case REDISMODULE_ARG_TYPE_DOUBLE: return ARG_TYPE_DOUBLE; + case REDISMODULE_ARG_TYPE_KEY: return ARG_TYPE_KEY; + case REDISMODULE_ARG_TYPE_PATTERN: return ARG_TYPE_PATTERN; + case REDISMODULE_ARG_TYPE_UNIX_TIME: return ARG_TYPE_UNIX_TIME; + case REDISMODULE_ARG_TYPE_PURE_TOKEN: return ARG_TYPE_PURE_TOKEN; + case REDISMODULE_ARG_TYPE_ONEOF: return ARG_TYPE_ONEOF; + case REDISMODULE_ARG_TYPE_BLOCK: return ARG_TYPE_BLOCK; + default: + if (error) *error = 1; + return -1; + } +} - return moduleSetCommandKeySpecFindKeys(command, spec_id, &spec); +static int moduleConvertArgFlags(int flags) { + int realflags = 0; + if (flags & REDISMODULE_CMD_ARG_OPTIONAL) realflags |= CMD_ARG_OPTIONAL; + if (flags & REDISMODULE_CMD_ARG_MULTIPLE) realflags |= CMD_ARG_MULTIPLE; + if (flags & REDISMODULE_CMD_ARG_MULTIPLE_TOKEN) realflags |= CMD_ARG_MULTIPLE_TOKEN; + return realflags; } -/* Set a "keynum" key arguments spec to a command (find_keys step). - * See RedisModule_AddCommandKeySpec's doc. - * - * - `keynumidx`: Relative index (to the result of the begin_search step) where the arguments that - * contains the number of keys is. - * - `firstkey`: Relative index (to the result of the begin_search step) where the first key is - * found (Usually it's just after keynumidx, so it should be keynumidx+1) - * - `keystep`: How many args should we skip after finding a key, in order to find the next one. - * - * Returns REDISMODULE_OK */ -int RM_SetCommandKeySpecFindKeysKeynum(RedisModuleCommand *command, int spec_id, int keynumidx, int firstkey, int keystep) { - keySpec spec; - spec.find_keys_type = KSPEC_FK_KEYNUM; - spec.fk.keynum.keynumidx = keynumidx; - spec.fk.keynum.firstkey = firstkey; - spec.fk.keynum.keystep = keystep; +/* Return `struct RedisModule *` as `void *` to avoid exposing it outside of module.c. */ +void *moduleGetHandleByName(char *modulename) { + return dictFetchValue(modules,modulename); +} - return moduleSetCommandKeySpecFindKeys(command, spec_id, &spec); +/* Returns 1 if `cmd` is a command of the module `modulename`. 0 otherwise. */ +int moduleIsModuleCommand(void *module_handle, struct redisCommand *cmd) { + if (cmd->proc != RedisModuleCommandDispatcher) + return 0; + if (module_handle == NULL) + return 0; + RedisModuleCommand *cp = (void*)(unsigned long)cmd->getkeys_proc; + return (cp->module == module_handle); } /* -------------------------------------------------------------------------- @@ -2399,6 +2997,15 @@ int RM_ReplyWithCallReply(RedisModuleCtx *ctx, RedisModuleCallReply *reply) { size_t proto_len; const char *proto = callReplyGetProto(reply, &proto_len); addReplyProto(c, proto, proto_len); + /* Propagate the error list from that reply to the other client, to do some + * post error reply handling, like statistics. + * Note that if the original reply had an array with errors, and the module + * replied with just a portion of the original reply, and not the entire + * reply, the errors are currently not propagated and the errors stats + * will not get propagated. */ + list *errors = callReplyDeferredErrorList(reply); + if (errors) + deferredAfterErrorReply(c, errors); return REDISMODULE_OK; } @@ -5051,7 +5658,7 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch errno = ENOENT; goto cleanup; } - c->cmd = c->lastcmd = cmd; + c->cmd = c->lastcmd = c->realcmd = cmd; /* Basic arity checks. */ if ((cmd->arity > 0 && cmd->arity != argc) || (argc < -cmd->arity)) { @@ -5135,7 +5742,8 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch proto = sdscatlen(proto,o->buf,o->used); listDelNode(c->reply,listFirst(c->reply)); } - reply = callReplyCreate(proto, ctx); + reply = callReplyCreate(proto, c->deferred_reply_errors, ctx); + c->deferred_reply_errors = NULL; /* now the responsibility of the reply object. */ autoMemoryAdd(ctx,REDISMODULE_AM_REPLY,reply); cleanup: @@ -6572,6 +7180,7 @@ void moduleHandleBlockedClients(void) { * was blocked on keys (RM_BlockClientOnKeys()), because we already * called such callback in moduleTryServeClientBlockedOnKey() when * the key was signaled as ready. */ + long long prev_error_replies = server.stat_total_error_replies; uint64_t reply_us = 0; if (c && !bc->blocked_on_keys && bc->reply_callback) { RedisModuleCtx ctx; @@ -6586,13 +7195,6 @@ void moduleHandleBlockedClients(void) { reply_us = elapsedUs(replyTimer); moduleFreeContext(&ctx); } - /* Update stats now that we've finished the blocking operation. - * This needs to be out of the reply callback above given that a - * module might not define any callback and still do blocking ops. - */ - if (c && !bc->blocked_on_keys) { - updateStatsOnUnblock(c, bc->background_duration, reply_us); - } /* Free privdata if any. */ if (bc->privdata && bc->free_privdata) { @@ -6613,6 +7215,14 @@ void moduleHandleBlockedClients(void) { moduleReleaseTempClient(bc->reply_client); moduleReleaseTempClient(bc->thread_safe_ctx_client); + /* Update stats now that we've finished the blocking operation. + * This needs to be out of the reply callback above given that a + * module might not define any callback and still do blocking ops. + */ + if (c && !bc->blocked_on_keys) { + updateStatsOnUnblock(c, bc->background_duration, reply_us, server.stat_total_error_replies != prev_error_replies); + } + if (c != NULL) { /* Before unblocking the client, set the disconnect callback * to NULL, because if we reached this point, the client was @@ -6670,10 +7280,11 @@ void moduleBlockedClientTimedOut(client *c) { ctx.client = bc->client; ctx.blocked_client = bc; ctx.blocked_privdata = bc->privdata; + long long prev_error_replies = server.stat_total_error_replies; bc->timeout_callback(&ctx,(void**)c->argv,c->argc); moduleFreeContext(&ctx); if (!bc->blocked_on_keys) { - updateStatsOnUnblock(c, bc->background_duration, 0); + updateStatsOnUnblock(c, bc->background_duration, 0, server.stat_total_error_replies != prev_error_replies); } /* For timeout events, we do not want to call the disconnect callback, * because the blocked client will be automatically disconnected in @@ -7427,6 +8038,24 @@ int RM_GetTimerInfo(RedisModuleCtx *ctx, RedisModuleTimerID id, uint64_t *remain return REDISMODULE_OK; } +/* Query timers to see if any timer belongs to the module. + * Return 1 if any timer was found, otherwise 0 would be returned. */ +int moduleHoldsTimer(struct RedisModule *module) { + raxIterator iter; + int found = 0; + raxStart(&iter,Timers); + raxSeek(&iter,"^",NULL,0); + while (raxNext(&iter)) { + RedisModuleTimer *timer = iter.data; + if (timer->module == module) { + found = 1; + break; + } + } + raxStop(&iter); + return found; +} + /* -------------------------------------------------------------------------- * ## Modules EventLoop API * --------------------------------------------------------------------------*/ @@ -7808,28 +8437,34 @@ int RM_ACLCheckCommandPermissions(RedisModuleUser *user, RedisModuleString **arg return REDISMODULE_OK; } -/* Check if the key can be accessed by the user, according to the ACLs associated with it - * and the flags used. The supported flags are: - * - * REDISMODULE_KEY_PERMISSION_READ: Can the module read data from the key. - * REDISMODULE_KEY_PERMISSION_WRITE: Can the module write data to the key. +/* Check if the key can be accessed by the user according to the ACLs attached to the user + * and the flags representing the key access. The flags are the same that are used in the + * keyspec for logical operations. These flags are documented in RedisModule_SetCommandInfo as + * the REDISMODULE_CMD_KEY_ACCESS, REDISMODULE_CMD_KEY_UPDATE, REDISMODULE_CMD_KEY_INSERT, + * and REDISMODULE_CMD_KEY_DELETE flags. + * + * If no flags are supplied, the user is still required to have some access to the key for + * this command to return successfully. * - * On success a REDISMODULE_OK is returned, otherwise - * REDISMODULE_ERR is returned and errno is set to the following values: + * If the user is able to access the key then REDISMODULE_OK is returned, otherwise + * REDISMODULE_ERR is returned and errno is set to one of the following values: * * * EINVAL: The provided flags are invalid. * * EACCESS: The user does not have permission to access the key. */ int RM_ACLCheckKeyPermissions(RedisModuleUser *user, RedisModuleString *key, int flags) { - int acl_flags = 0; - if (flags & REDISMODULE_KEY_PERMISSION_READ) acl_flags |= ACL_READ_PERMISSION; - if (flags & REDISMODULE_KEY_PERMISSION_WRITE) acl_flags |= ACL_WRITE_PERMISSION; - if (!acl_flags || ((flags & REDISMODULE_KEY_PERMISSION_ALL) != flags)) { + const int allow_mask = (REDISMODULE_CMD_KEY_ACCESS + | REDISMODULE_CMD_KEY_INSERT + | REDISMODULE_CMD_KEY_DELETE + | REDISMODULE_CMD_KEY_UPDATE); + + if ((flags & allow_mask) != flags) { errno = EINVAL; return REDISMODULE_ERR; } - if (ACLUserCheckKeyPerm(user->user, key->ptr, sdslen(key->ptr), acl_flags) != ACL_OK) { + int keyspec_flags = moduleConvertKeySpecsFlags(flags, 0); + if (ACLUserCheckKeyPerm(user->user, key->ptr, sdslen(key->ptr), keyspec_flags) != ACL_OK) { errno = EACCES; return REDISMODULE_ERR; } @@ -7837,14 +8472,34 @@ int RM_ACLCheckKeyPermissions(RedisModuleUser *user, RedisModuleString *key, int return REDISMODULE_OK; } -/* Check if the pubsub channel can be accessed by the user, according to the ACLs associated with it. - * Glob-style pattern matching is employed, unless the literal flag is - * set. +/* Check if the pubsub channel can be accessed by the user based off of the given + * access flags. See RM_ChannelAtPosWithFlags for more information about the + * possible flags that can be passed in. * - * If the user can access the pubsub channel, REDISMODULE_OK is returned, otherwise - * REDISMODULE_ERR is returned. */ -int RM_ACLCheckChannelPermissions(RedisModuleUser *user, RedisModuleString *ch, int literal) { - if (ACLUserCheckChannelPerm(user->user, ch->ptr, literal) != ACL_OK) + * If the user is able to acecss the pubsub channel then REDISMODULE_OK is returned, otherwise + * REDISMODULE_ERR is returned and errno is set to one of the following values: + * + * * EINVAL: The provided flags are invalid. + * * EACCESS: The user does not have permission to access the pubsub channel. + */ +int RM_ACLCheckChannelPermissions(RedisModuleUser *user, RedisModuleString *ch, int flags) { + const int allow_mask = (REDISMODULE_CMD_CHANNEL_PUBLISH + | REDISMODULE_CMD_CHANNEL_SUBSCRIBE + | REDISMODULE_CMD_CHANNEL_UNSUBSCRIBE + | REDISMODULE_CMD_CHANNEL_PATTERN); + + if ((flags & allow_mask) != flags) { + errno = EINVAL; + return REDISMODULE_ERR; + } + + /* Unsubscribe permissions are currently always allowed. */ + if (flags & REDISMODULE_CMD_CHANNEL_UNSUBSCRIBE){ + return REDISMODULE_OK; + } + + int is_pattern = flags & REDISMODULE_CMD_CHANNEL_PATTERN; + if (ACLUserCheckChannelPerm(user->user, ch->ptr, is_pattern) != ACL_OK) return REDISMODULE_ERR; return REDISMODULE_OK; @@ -8254,9 +8909,10 @@ int RM_InfoAddSection(RedisModuleInfoCtx *ctx, const char *name) { * 1) no section was requested (emit all) * 2) the module name was requested (emit all) * 3) this specific section was requested. */ - if (ctx->requested_section) { - if (strcasecmp(ctx->requested_section, full_name) && - strcasecmp(ctx->requested_section, ctx->module->name)) { + if (ctx->requested_sections) { + if ((!full_name || !dictFind(ctx->requested_sections, full_name)) && + (!dictFind(ctx->requested_sections, ctx->module->name))) + { sdsfree(full_name); ctx->in_section = 0; return REDISMODULE_ERR; @@ -8405,7 +9061,7 @@ int RM_RegisterInfoFunc(RedisModuleCtx *ctx, RedisModuleInfoFunc cb) { return REDISMODULE_OK; } -sds modulesCollectInfo(sds info, const char *section, int for_crash_report, int sections) { +sds modulesCollectInfo(sds info, dict *sections_dict, int for_crash_report, int sections) { dictIterator *di = dictGetIterator(modules); dictEntry *de; @@ -8413,7 +9069,7 @@ sds modulesCollectInfo(sds info, const char *section, int for_crash_report, int struct RedisModule *module = dictGetVal(de); if (!module->info_cb) continue; - RedisModuleInfoCtx info_ctx = {module, section, info, sections, 0, 0}; + RedisModuleInfoCtx info_ctx = {module, sections_dict, info, sections, 0, 0}; module->info_cb(&info_ctx, for_crash_report); /* Implicitly end dicts (no way to handle errors, and we must add the newline). */ if (info_ctx.in_dict_field) @@ -8435,7 +9091,11 @@ RedisModuleServerInfoData *RM_GetServerInfo(RedisModuleCtx *ctx, const char *sec struct RedisModuleServerInfoData *d = zmalloc(sizeof(*d)); d->rax = raxNew(); if (ctx != NULL) autoMemoryAdd(ctx,REDISMODULE_AM_INFO,d); - sds info = genRedisInfoString(section); + int all = 0, everything = 0; + robj *argv[1]; + argv[0] = section ? createStringObject(section, strlen(section)) : NULL; + dict *section_dict = genInfoSectionDict(argv, section ? 1 : 0, NULL, &all, &everything); + sds info = genRedisInfoString(section_dict, all, everything); int totlines, i; sds *lines = sdssplitlen(info, sdslen(info), "\r\n", 2, &totlines); for(i=0; i<totlines; i++) { @@ -8451,6 +9111,8 @@ RedisModuleServerInfoData *RM_GetServerInfo(RedisModuleCtx *ctx, const char *sec } sdsfree(info); sdsfreesplitres(lines,totlines); + releaseInfoSectionDict(section_dict); + if(argv[0]) decrRefCount(argv[0]); return d; } @@ -9995,17 +10657,23 @@ int moduleFreeCommand(struct RedisModule *module, struct redisCommand *cmd) { return C_ERR; /* Free everything except cmd->fullname and cmd itself. */ + for (int j = 0; j < cmd->key_specs_num; j++) { + if (cmd->key_specs[j].notes) + zfree((char *)cmd->key_specs[j].notes); + if (cmd->key_specs[j].begin_search_type == KSPEC_BS_KEYWORD) + zfree((char *)cmd->key_specs[j].bs.keyword.keyword); + } if (cmd->key_specs != cmd->key_specs_static) zfree(cmd->key_specs); for (int j = 0; cmd->tips && cmd->tips[j]; j++) - sdsfree((sds)cmd->tips[j]); + zfree((char *)cmd->tips[j]); for (int j = 0; cmd->history && cmd->history[j].since; j++) { - sdsfree((sds)cmd->history[j].since); - sdsfree((sds)cmd->history[j].changes); + zfree((char *)cmd->history[j].since); + zfree((char *)cmd->history[j].changes); } - sdsfree((sds)cmd->summary); - sdsfree((sds)cmd->since); - sdsfree((sds)cmd->complexity); + zfree((char *)cmd->summary); + zfree((char *)cmd->since); + zfree((char *)cmd->complexity); if (cmd->latency_histogram) { hdr_close(cmd->latency_histogram); cmd->latency_histogram = NULL; @@ -10125,6 +10793,7 @@ int moduleLoad(const char *path, void **module_argv, int module_argc) { * * EBUSY: The module exports a new data type and can only be reloaded. * * EPERM: The module exports APIs which are used by other module. * * EAGAIN: The module has blocked clients. + * * EINPROGRESS: The module holds timer not fired. * * ECANCELED: Unload module error. */ int moduleUnload(sds name) { struct RedisModule *module = dictFetchValue(modules,name); @@ -10141,6 +10810,9 @@ int moduleUnload(sds name) { } else if (module->blocked_clients) { errno = EAGAIN; return C_ERR; + } else if (moduleHoldsTimer(module)) { + errno = EINPROGRESS; + return C_ERR; } /* Give module a chance to clean up. */ @@ -10255,6 +10927,8 @@ sds genModulesInfoStringRenderModuleOptions(struct RedisModule *module) { output = sdscat(output,"handle-io-errors|"); if (module->options & REDISMODULE_OPTIONS_HANDLE_REPL_ASYNC_LOAD) output = sdscat(output,"handle-repl-async-load|"); + if (module->options & REDISMODULE_OPTION_NO_IMPLICIT_SIGNAL_MODIFIED) + output = sdscat(output,"no-implicit-signal-modified|"); output = sdstrim(output,"|"); output = sdscat(output,"]"); return output; @@ -10345,6 +11019,10 @@ NULL errmsg = "the module has blocked clients. " "Please wait them unblocked and try again"; break; + case EINPROGRESS: + errmsg = "the module holds timer that is not fired. " + "Please stop the timer or wait until it fires."; + break; default: errmsg = "operation not possible."; break; @@ -10511,6 +11189,10 @@ int RM_ModuleTypeReplaceValue(RedisModuleKey *key, moduleType *mt, void *new_val * contains the indexes of all key name arguments. This function is * essentially a more efficient way to do `COMMAND GETKEYS`. * + * The out_flags argument is optional, and can be set to NULL. + * When provided it is filled with REDISMODULE_CMD_KEY_ flags in matching + * indexes with the key indexes of the returned array. + * * A NULL return value indicates the specified command has no keys, or * an error condition. Error conditions are indicated by setting errno * as follows: @@ -10520,9 +11202,10 @@ int RM_ModuleTypeReplaceValue(RedisModuleKey *key, moduleType *mt, void *new_val * * NOTE: The returned array is not a Redis Module object so it does not * get automatically freed even when auto-memory is used. The caller - * must explicitly call RM_Free() to free it. + * must explicitly call RM_Free() to free it, same as the out_flags pointer if + * used. */ -int *RM_GetCommandKeys(RedisModuleCtx *ctx, RedisModuleString **argv, int argc, int *num_keys) { +int *RM_GetCommandKeysWithFlags(RedisModuleCtx *ctx, RedisModuleString **argv, int argc, int *num_keys, int **out_flags) { UNUSED(ctx); struct redisCommand *cmd; int *res = NULL; @@ -10557,13 +11240,22 @@ int *RM_GetCommandKeys(RedisModuleCtx *ctx, RedisModuleString **argv, int argc, /* The return value here expects an array of key positions */ unsigned long int size = sizeof(int) * result.numkeys; res = zmalloc(size); + if (out_flags) + *out_flags = zmalloc(size); for (int i = 0; i < result.numkeys; i++) { res[i] = result.keys[i].pos; + if (out_flags) + (*out_flags)[i] = moduleConvertKeySpecsFlags(result.keys[i].flags, 0); } return res; } +/* Identinal to RM_GetCommandKeysWithFlags when flags are not needed. */ +int *RM_GetCommandKeys(RedisModuleCtx *ctx, RedisModuleString **argv, int argc, int *num_keys) { + return RM_GetCommandKeysWithFlags(ctx, argv, argc, num_keys, NULL); +} + /* Return the name of the command currently running */ const char *RM_GetCurrentCommandName(RedisModuleCtx *ctx) { if (!ctx || !ctx->client || !ctx->client->cmd) @@ -10803,6 +11495,7 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(CreateCommand); REGISTER_API(GetCommand); REGISTER_API(CreateSubcommand); + REGISTER_API(SetCommandInfo); REGISTER_API(SetModuleAttribs); REGISTER_API(IsModuleNameBusy); REGISTER_API(WrongArity); @@ -10913,6 +11606,9 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(StreamTrimByID); REGISTER_API(IsKeysPositionRequest); REGISTER_API(KeyAtPos); + REGISTER_API(KeyAtPosWithFlags); + REGISTER_API(IsChannelsPositionRequest); + REGISTER_API(ChannelAtPosWithFlags); REGISTER_API(GetClientId); REGISTER_API(GetClientUserNameById); REGISTER_API(GetContextFlags); @@ -11090,6 +11786,7 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(GetServerVersion); REGISTER_API(GetClientCertificate); REGISTER_API(GetCommandKeys); + REGISTER_API(GetCommandKeysWithFlags); REGISTER_API(GetCurrentCommandName); REGISTER_API(GetTypeMethodVersion); REGISTER_API(RegisterDefragFunc); @@ -11098,13 +11795,6 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(DefragShouldStop); REGISTER_API(DefragCursorSet); REGISTER_API(DefragCursorGet); -#ifdef INCLUDE_UNRELEASED_KEYSPEC_API - REGISTER_API(AddCommandKeySpec); - REGISTER_API(SetCommandKeySpecBeginSearchIndex); - REGISTER_API(SetCommandKeySpecBeginSearchKeyword); - REGISTER_API(SetCommandKeySpecFindKeysRange); - REGISTER_API(SetCommandKeySpecFindKeysKeynum); -#endif REGISTER_API(EventLoopAdd); REGISTER_API(EventLoopDel); REGISTER_API(EventLoopAddOneShot); diff --git a/src/modules/Makefile b/src/modules/Makefile index 3db19e79a..c4bc7eb1a 100644 --- a/src/modules/Makefile +++ b/src/modules/Makefile @@ -11,6 +11,13 @@ else SHOBJ_LDFLAGS ?= -bundle -undefined dynamic_lookup endif +# OS X 11.x doesn't have /usr/lib/libSystem.dylib and needs an explicit setting. +ifeq ($(uname_S),Darwin) +ifeq ("$(wildcard /usr/lib/libSystem.dylib)","") +LIBS = -L /Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/lib -lsystem +endif +endif + .SUFFIXES: .c .so .xo .o all: helloworld.so hellotype.so helloblock.so hellocluster.so hellotimer.so hellodict.so hellohook.so helloacl.so diff --git a/src/modules/hellocluster.c b/src/modules/hellocluster.c index 8f822e31e..a6508f837 100644 --- a/src/modules/hellocluster.c +++ b/src/modules/hellocluster.c @@ -76,7 +76,7 @@ int ListCommand_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int void PingReceiver(RedisModuleCtx *ctx, const char *sender_id, uint8_t type, const unsigned char *payload, uint32_t len) { RedisModule_Log(ctx,"notice","PING (type %d) RECEIVED from %.*s: '%.*s'", type,REDISMODULE_NODE_ID_LEN,sender_id,(int)len, payload); - RedisModule_SendClusterMessage(ctx,NULL,MSGTYPE_PONG,(unsigned char*)"Ohi!",4); + RedisModule_SendClusterMessage(ctx,NULL,MSGTYPE_PONG,"Ohi!",4); RedisModuleCallReply *reply = RedisModule_Call(ctx, "INCR", "c", "pings_received"); RedisModule_FreeCallReply(reply); } diff --git a/src/multi.c b/src/multi.c index a98195672..11f33f48f 100644 --- a/src/multi.c +++ b/src/multi.c @@ -189,7 +189,7 @@ void execCommand(client *c) { c->argc = c->mstate.commands[j].argc; c->argv = c->mstate.commands[j].argv; c->argv_len = c->mstate.commands[j].argv_len; - c->cmd = c->mstate.commands[j].cmd; + c->cmd = c->realcmd = c->mstate.commands[j].cmd; /* ACL permissions are also checked at the time of execution in case * they were changed after the commands were queued. */ @@ -240,7 +240,7 @@ void execCommand(client *c) { c->argv = orig_argv; c->argv_len = orig_argv_len; c->argc = orig_argc; - c->cmd = orig_cmd; + c->cmd = c->realcmd = orig_cmd; discardTransaction(c); server.in_exec = 0; @@ -257,10 +257,13 @@ void execCommand(client *c) { /* In the client->watched_keys list we need to use watchedKey structures * as in order to identify a key in Redis we need both the key name and the - * DB */ + * DB. This struct is also referenced from db->watched_keys dict, where the + * values are lists of watchedKey pointers. */ typedef struct watchedKey { robj *key; redisDb *db; + client *client; + unsigned expired:1; /* Flag that we're watching an already expired key. */ } watchedKey; /* Watch for the specified key */ @@ -284,13 +287,15 @@ void watchForKey(client *c, robj *key) { dictAdd(c->db->watched_keys,key,clients); incrRefCount(key); } - listAddNodeTail(clients,c); /* Add the new key to the list of keys watched by this client */ wk = zmalloc(sizeof(*wk)); wk->key = key; + wk->client = c; wk->db = c->db; + wk->expired = keyIsExpired(c->db, key); incrRefCount(key); listAddNodeTail(c->watched_keys,wk); + listAddNodeTail(clients,wk); } /* Unwatch all the keys watched by this client. To clean the EXEC dirty @@ -305,12 +310,12 @@ void unwatchAllKeys(client *c) { list *clients; watchedKey *wk; - /* Lookup the watched key -> clients list and remove the client + /* Lookup the watched key -> clients list and remove the client's wk * from the list */ wk = listNodeValue(ln); clients = dictFetchValue(wk->db->watched_keys, wk->key); serverAssertWithInfo(c,NULL,clients != NULL); - listDelNode(clients,listSearchKey(clients,c)); + listDelNode(clients,listSearchKey(clients,wk)); /* Kill the entry at all if this was the only client */ if (listLength(clients) == 0) dictDelete(wk->db->watched_keys, wk->key); @@ -321,8 +326,8 @@ void unwatchAllKeys(client *c) { } } -/* iterates over the watched_keys list and - * look for an expired key . */ +/* Iterates over the watched_keys list and looks for an expired key. Keys which + * were expired already when WATCH was called are ignored. */ int isWatchedKeyExpired(client *c) { listIter li; listNode *ln; @@ -331,6 +336,7 @@ int isWatchedKeyExpired(client *c) { listRewind(c->watched_keys,&li); while ((ln = listNext(&li))) { wk = listNodeValue(ln); + if (wk->expired) continue; /* was expired when WATCH was called */ if (keyIsExpired(wk->db, wk->key)) return 1; } @@ -352,13 +358,31 @@ void touchWatchedKey(redisDb *db, robj *key) { /* Check if we are already watching for this key */ listRewind(clients,&li); while((ln = listNext(&li))) { - client *c = listNodeValue(ln); + watchedKey *wk = listNodeValue(ln); + client *c = wk->client; + + if (wk->expired) { + /* The key was already expired when WATCH was called. */ + if (db == wk->db && + equalStringObjects(key, wk->key) && + dictFind(db->dict, key->ptr) == NULL) + { + /* Already expired key is deleted, so logically no change. Clear + * the flag. Deleted keys are not flagged as expired. */ + wk->expired = 0; + goto skip_client; + } + break; + } c->flags |= CLIENT_DIRTY_CAS; /* As the client is marked as dirty, there is no point in getting here * again in case that key (or others) are modified again (or keep the * memory overhead till EXEC). */ unwatchAllKeys(c); + + skip_client: + continue; } } @@ -379,14 +403,31 @@ void touchAllWatchedKeysInDb(redisDb *emptied, redisDb *replaced_with) { dictIterator *di = dictGetSafeIterator(emptied->watched_keys); while((de = dictNext(di)) != NULL) { robj *key = dictGetKey(de); - if (dictFind(emptied->dict, key->ptr) || + int exists_in_emptied = dictFind(emptied->dict, key->ptr) != NULL; + if (exists_in_emptied || (replaced_with && dictFind(replaced_with->dict, key->ptr))) { list *clients = dictGetVal(de); if (!clients) continue; listRewind(clients,&li); while((ln = listNext(&li))) { - client *c = listNodeValue(ln); + watchedKey *wk = listNodeValue(ln); + if (wk->expired) { + if (!replaced_with || !dictFind(replaced_with->dict, key->ptr)) { + /* Expired key now deleted. No logical change. Clear the + * flag. Deleted keys are not flagged as expired. */ + wk->expired = 0; + continue; + } else if (keyIsExpired(replaced_with, key)) { + /* Expired key remains expired. */ + continue; + } + } else if (!exists_in_emptied && keyIsExpired(replaced_with, key)) { + /* Non-existing key is replaced with an expired key. */ + wk->expired = 1; + continue; + } + client *c = wk->client; c->flags |= CLIENT_DIRTY_CAS; /* As the client is marked as dirty, there is no point in getting here * again for others keys (or keep the memory overhead till EXEC). */ diff --git a/src/networking.c b/src/networking.c index 05001c564..b05d02b1b 100644 --- a/src/networking.c +++ b/src/networking.c @@ -131,7 +131,7 @@ client *createClient(connection *conn) { connSetReadHandler(conn, readQueryFromClient); connSetPrivateData(conn, c); } - + c->buf = zmalloc(PROTO_REPLY_CHUNK_BYTES); selectDb(c,0); uint64_t client_id; atomicGetIncr(server.next_client_id, client_id, 1); @@ -140,7 +140,9 @@ client *createClient(connection *conn) { c->conn = conn; c->name = NULL; c->bufpos = 0; - c->buf_usable_size = zmalloc_usable_size(c)-offsetof(client,buf); + c->buf_usable_size = zmalloc_usable_size(c->buf); + c->buf_peak = c->buf_usable_size; + c->buf_peak_last_reset_time = server.unixtime; c->ref_repl_buf_node = NULL; c->ref_block_pos = 0; c->qb_pos = 0; @@ -154,7 +156,7 @@ client *createClient(connection *conn) { c->argv_len_sum = 0; c->original_argc = 0; c->original_argv = NULL; - c->cmd = c->lastcmd = NULL; + c->cmd = c->lastcmd = c->realcmd = NULL; c->multibulklen = 0; c->bulklen = -1; c->sentlen = 0; @@ -173,6 +175,7 @@ client *createClient(connection *conn) { c->slave_capa = SLAVE_CAPA_NONE; c->slave_req = SLAVE_REQ_NONE; c->reply = listCreate(); + c->deferred_reply_errors = NULL; c->reply_bytes = 0; c->obuf_soft_limit_reached_time = 0; listSetFreeMethod(c->reply,freeClientReplyValue); @@ -313,6 +316,9 @@ size_t _addReplyToBuffer(client *c, const char *s, size_t len) { size_t reply_len = len > available ? available : len; memcpy(c->buf+c->bufpos,s,reply_len); c->bufpos+=reply_len; + /* We update the buffer peak after appending the reply to the buffer */ + if(c->buf_peak < (size_t)c->bufpos) + c->buf_peak = (size_t)c->bufpos; return reply_len; } @@ -437,24 +443,46 @@ void addReplyErrorLength(client *c, const char *s, size_t len) { addReplyProto(c,"\r\n",2); } -/* Do some actions after an error reply was sent (Log if needed, updates stats, etc.) */ -void afterErrorReply(client *c, const char *s, size_t len) { - /* Increment the global error counter */ - server.stat_total_error_replies++; - /* Increment the error stats - * If the string already starts with "-..." then the error prefix - * is provided by the caller ( we limit the search to 32 chars). Otherwise we use "-ERR". */ - if (s[0] != '-') { - incrementErrorCount("ERR", 3); - } else { - char *spaceloc = memchr(s, ' ', len < 32 ? len : 32); - if (spaceloc) { - const size_t errEndPos = (size_t)(spaceloc - s); - incrementErrorCount(s+1, errEndPos-1); - } else { - /* Fallback to ERR if we can't retrieve the error prefix */ +/* Do some actions after an error reply was sent (Log if needed, updates stats, etc.) + * Possible flags: + * * ERR_REPLY_FLAG_NO_STATS_UPDATE - indicate not to update any error stats. */ +void afterErrorReply(client *c, const char *s, size_t len, int flags) { + /* Module clients fall into two categories: + * Calls to RM_Call, in which case the error isn't being returned to a client, so should not be counted. + * Module thread safe context calls to RM_ReplyWithError, which will be added to a real client by the main thread later. */ + if (c->flags & CLIENT_MODULE) { + if (!c->deferred_reply_errors) { + c->deferred_reply_errors = listCreate(); + listSetFreeMethod(c->deferred_reply_errors, (void (*)(void*))sdsfree); + } + listAddNodeTail(c->deferred_reply_errors, sdsnewlen(s, len)); + return; + } + + if (!(flags & ERR_REPLY_FLAG_NO_STATS_UPDATE)) { + /* Increment the global error counter */ + server.stat_total_error_replies++; + /* Increment the error stats + * If the string already starts with "-..." then the error prefix + * is provided by the caller ( we limit the search to 32 chars). Otherwise we use "-ERR". */ + if (s[0] != '-') { incrementErrorCount("ERR", 3); + } else { + char *spaceloc = memchr(s, ' ', len < 32 ? len : 32); + if (spaceloc) { + const size_t errEndPos = (size_t)(spaceloc - s); + incrementErrorCount(s+1, errEndPos-1); + } else { + /* Fallback to ERR if we can't retrieve the error prefix */ + incrementErrorCount("ERR", 3); + } } + } else { + /* stat_total_error_replies will not be updated, which means that + * the cmd stats will not be updated as well, we still want this command + * to be counted as failed so we update it here. We update c->realcmd in + * case c->cmd was changed (like in GEOADD). */ + c->realcmd->failed_calls++; } /* Sometimes it could be normal that a slave replies to a master with @@ -500,7 +528,7 @@ void afterErrorReply(client *c, const char *s, size_t len) { * Unlike addReplyErrorSds and others alike which rely on addReplyErrorLength. */ void addReplyErrorObject(client *c, robj *err) { addReply(c, err); - afterErrorReply(c, err->ptr, sdslen(err->ptr)-2); /* Ignore trailing \r\n */ + afterErrorReply(c, err->ptr, sdslen(err->ptr)-2, 0); /* Ignore trailing \r\n */ } /* Sends either a reply or an error reply by checking the first char. @@ -521,34 +549,57 @@ void addReplyOrErrorObject(client *c, robj *reply) { /* See addReplyErrorLength for expectations from the input string. */ void addReplyError(client *c, const char *err) { addReplyErrorLength(c,err,strlen(err)); - afterErrorReply(c,err,strlen(err)); + afterErrorReply(c,err,strlen(err),0); +} + +/* Add error reply to the given client. + * Supported flags: + * * ERR_REPLY_FLAG_NO_STATS_UPDATE - indicate not to perform any error stats updates */ +void addReplyErrorSdsEx(client *c, sds err, int flags) { + addReplyErrorLength(c,err,sdslen(err)); + afterErrorReply(c,err,sdslen(err),flags); + sdsfree(err); } /* See addReplyErrorLength for expectations from the input string. */ /* As a side effect the SDS string is freed. */ void addReplyErrorSds(client *c, sds err) { - addReplyErrorLength(c,err,sdslen(err)); - afterErrorReply(c,err,sdslen(err)); - sdsfree(err); + addReplyErrorSdsEx(c, err, 0); } -/* See addReplyErrorLength for expectations from the formatted string. - * The formatted string is safe to contain \r and \n anywhere. */ -void addReplyErrorFormat(client *c, const char *fmt, ...) { - va_list ap; - va_start(ap,fmt); - sds s = sdscatvprintf(sdsempty(),fmt,ap); - va_end(ap); +/* Internal function used by addReplyErrorFormat and addReplyErrorFormatEx. + * Refer to afterErrorReply for more information about the flags. */ +static void addReplyErrorFormatInternal(client *c, int flags, const char *fmt, va_list ap) { + va_list cpy; + va_copy(cpy,ap); + sds s = sdscatvprintf(sdsempty(),fmt,cpy); + va_end(cpy); /* Trim any newlines at the end (ones will be added by addReplyErrorLength) */ s = sdstrim(s, "\r\n"); /* Make sure there are no newlines in the middle of the string, otherwise * invalid protocol is emitted. */ s = sdsmapchars(s, "\r\n", " ", 2); addReplyErrorLength(c,s,sdslen(s)); - afterErrorReply(c,s,sdslen(s)); + afterErrorReply(c,s,sdslen(s),flags); sdsfree(s); } +void addReplyErrorFormatEx(client *c, int flags, const char *fmt, ...) { + va_list ap; + va_start(ap,fmt); + addReplyErrorFormatInternal(c, flags, fmt, ap); + va_end(ap); +} + +/* See addReplyErrorLength for expectations from the formatted string. + * The formatted string is safe to contain \r and \n anywhere. */ +void addReplyErrorFormat(client *c, const char *fmt, ...) { + va_list ap; + va_start(ap,fmt); + addReplyErrorFormatInternal(c, 0, fmt, ap); + va_end(ap); +} + void addReplyErrorArity(client *c) { addReplyErrorFormat(c, "wrong number of arguments for '%s' command", c->cmd->fullname); @@ -696,6 +747,24 @@ void setDeferredAggregateLen(client *c, void *node, long length, char prefix) { * we return NULL in addReplyDeferredLen() */ if (node == NULL) return; + /* Things like *2\r\n, %3\r\n or ~4\r\n are emitted very often by the protocol + * so we have a few shared objects to use if the integer is small + * like it is most of the times. */ + const size_t hdr_len = OBJ_SHARED_HDR_STRLEN(length); + const int opt_hdr = length < OBJ_SHARED_BULKHDR_LEN; + if (prefix == '*' && opt_hdr) { + setDeferredReply(c, node, shared.mbulkhdr[length]->ptr, hdr_len); + return; + } + if (prefix == '%' && opt_hdr) { + setDeferredReply(c, node, shared.maphdr[length]->ptr, hdr_len); + return; + } + if (prefix == '~' && opt_hdr) { + setDeferredReply(c, node, shared.sethdr[length]->ptr, hdr_len); + return; + } + char lenstr[128]; size_t lenstr_len = sprintf(lenstr, "%c%ld\r\n", prefix, length); setDeferredReply(c, node, lenstr, lenstr_len); @@ -788,11 +857,19 @@ void addReplyLongLongWithPrefix(client *c, long long ll, char prefix) { /* Things like $3\r\n or *2\r\n are emitted very often by the protocol * so we have a few shared objects to use if the integer is small * like it is most of the times. */ - if (prefix == '*' && ll < OBJ_SHARED_BULKHDR_LEN && ll >= 0) { - addReply(c,shared.mbulkhdr[ll]); + const int opt_hdr = ll < OBJ_SHARED_BULKHDR_LEN && ll >= 0; + const size_t hdr_len = OBJ_SHARED_HDR_STRLEN(ll); + if (prefix == '*' && opt_hdr) { + addReplyProto(c,shared.mbulkhdr[ll]->ptr,hdr_len); + return; + } else if (prefix == '$' && opt_hdr) { + addReplyProto(c,shared.bulkhdr[ll]->ptr,hdr_len); + return; + } else if (prefix == '%' && opt_hdr) { + addReplyProto(c,shared.maphdr[ll]->ptr,hdr_len); return; - } else if (prefix == '$' && ll < OBJ_SHARED_BULKHDR_LEN && ll >= 0) { - addReply(c,shared.bulkhdr[ll]); + } else if (prefix == '~' && opt_hdr) { + addReplyProto(c,shared.sethdr[ll]->ptr,hdr_len); return; } @@ -1024,10 +1101,28 @@ void AddReplyFromClient(client *dst, client *src) { src->reply_bytes = 0; src->bufpos = 0; + if (src->deferred_reply_errors) { + deferredAfterErrorReply(dst, src->deferred_reply_errors); + listRelease(src->deferred_reply_errors); + src->deferred_reply_errors = NULL; + } + /* Check output buffer limits */ closeClientOnOutputBufferLimitReached(dst, 1); } +/* Append the listed errors to the server error statistics. the input + * list is not modified and remains the responsibility of the caller. */ +void deferredAfterErrorReply(client *c, list *errors) { + listIter li; + listNode *ln; + listRewind(errors,&li); + while((ln = listNext(&li))) { + sds err = ln->value; + afterErrorReply(c, err, sdslen(err), 0); + } +} + /* Logically copy 'src' replica client buffers info to 'dst' replica. * Basically increase referenced buffer block node reference count. */ void copyReplicaOutputBuffer(client *dst, client *src) { @@ -1494,9 +1589,12 @@ void freeClient(client *c) { /* Free data structures. */ listRelease(c->reply); + zfree(c->buf); freeReplicaReferencedReplBuffer(c); freeClientArgv(c); freeClientOriginalArgv(c); + if (c->deferred_reply_errors) + listRelease(c->deferred_reply_errors); /* Unlink the client: this will close the socket, remove the I/O * handlers, and remove references of the client from different @@ -1658,10 +1756,82 @@ client *lookupClientByID(uint64_t id) { return (c == raxNotFound) ? NULL : c; } +/* This function should be called from _writeToClient when the reply list is not empty, + * it gathers the scattered buffers from reply list and sends them away with connWritev. + * If we write successfully, it returns C_OK, otherwise, C_ERR is returned, + * and 'nwritten' is an output parameter, it means how many bytes server write + * to client. */ +static int _writevToClient(client *c, ssize_t *nwritten) { + struct iovec iov[IOV_MAX]; + int iovcnt = 0; + size_t iov_bytes_len = 0; + /* If the static reply buffer is not empty, + * add it to the iov array for writev() as well. */ + if (c->bufpos > 0) { + iov[iovcnt].iov_base = c->buf + c->sentlen; + iov[iovcnt].iov_len = c->bufpos - c->sentlen; + iov_bytes_len += iov[iovcnt++].iov_len; + } + /* The first node of reply list might be incomplete from the last call, + * thus it needs to be calibrated to get the actual data address and length. */ + size_t offset = c->bufpos > 0 ? 0 : c->sentlen; + listIter iter; + listNode *next; + clientReplyBlock *o; + listRewind(c->reply, &iter); + while ((next = listNext(&iter)) && iovcnt < IOV_MAX && iov_bytes_len < NET_MAX_WRITES_PER_EVENT) { + o = listNodeValue(next); + if (o->used == 0) { /* empty node, just release it and skip. */ + c->reply_bytes -= o->size; + listDelNode(c->reply, next); + offset = 0; + continue; + } + + iov[iovcnt].iov_base = o->buf + offset; + iov[iovcnt].iov_len = o->used - offset; + iov_bytes_len += iov[iovcnt++].iov_len; + offset = 0; + } + if (iovcnt == 0) return C_OK; + *nwritten = connWritev(c->conn, iov, iovcnt); + if (*nwritten <= 0) return C_ERR; + + /* Locate the new node which has leftover data and + * release all nodes in front of it. */ + ssize_t remaining = *nwritten; + if (c->bufpos > 0) { /* deal with static reply buffer first. */ + int buf_len = c->bufpos - c->sentlen; + c->sentlen += remaining; + /* If the buffer was sent, set bufpos to zero to continue with + * the remainder of the reply. */ + if (remaining >= buf_len) { + c->bufpos = 0; + c->sentlen = 0; + } + remaining -= buf_len; + } + listRewind(c->reply, &iter); + while (remaining > 0) { + next = listNext(&iter); + o = listNodeValue(next); + if (remaining < (ssize_t)(o->used - c->sentlen)) { + c->sentlen += remaining; + break; + } + remaining -= (ssize_t)(o->used - c->sentlen); + c->reply_bytes -= o->size; + listDelNode(c->reply, next); + c->sentlen = 0; + } + + return C_OK; +} + /* This function does actual writing output buffers to different types of * clients, it is called by writeToClient. - * If we write successfully, it return C_OK, otherwise, C_ERR is returned, - * And 'nwritten' is a output parameter, it means how many bytes server write + * If we write successfully, it returns C_OK, otherwise, C_ERR is returned, + * and 'nwritten' is an output parameter, it means how many bytes server write * to client. */ int _writeToClient(client *c, ssize_t *nwritten) { *nwritten = 0; @@ -1690,8 +1860,18 @@ int _writeToClient(client *c, ssize_t *nwritten) { return C_OK; } - if (c->bufpos > 0) { - *nwritten = connWrite(c->conn,c->buf+c->sentlen,c->bufpos-c->sentlen); + /* When the reply list is not empty, it's better to use writev to save us some + * system calls and TCP packets. */ + if (listLength(c->reply) > 0) { + int ret = _writevToClient(c, nwritten); + if (ret != C_OK) return ret; + + /* If there are no longer objects in the list, we expect + * the count of reply bytes to be exactly zero. */ + if (listLength(c->reply) == 0) + serverAssert(c->reply_bytes == 0); + } else if (c->bufpos > 0) { + *nwritten = connWrite(c->conn, c->buf + c->sentlen, c->bufpos - c->sentlen); if (*nwritten <= 0) return C_ERR; c->sentlen += *nwritten; @@ -1701,31 +1881,8 @@ int _writeToClient(client *c, ssize_t *nwritten) { c->bufpos = 0; c->sentlen = 0; } - } else { - clientReplyBlock *o = listNodeValue(listFirst(c->reply)); - size_t objlen = o->used; - - if (objlen == 0) { - c->reply_bytes -= o->size; - listDelNode(c->reply,listFirst(c->reply)); - return C_OK; - } - - *nwritten = connWrite(c->conn, o->buf + c->sentlen, objlen - c->sentlen); - if (*nwritten <= 0) return C_ERR; - c->sentlen += *nwritten; + } - /* If we fully sent the object on head go to the next one */ - if (c->sentlen == objlen) { - c->reply_bytes -= o->size; - listDelNode(c->reply,listFirst(c->reply)); - c->sentlen = 0; - /* If there are no longer objects in the list, we expect - * the count of reply bytes to be exactly zero. */ - if (listLength(c->reply) == 0) - serverAssert(c->reply_bytes == 0); - } - } return C_OK; } @@ -1863,6 +2020,10 @@ void resetClient(client *c) { c->multibulklen = 0; c->bulklen = -1; + if (c->deferred_reply_errors) + listRelease(c->deferred_reply_errors); + c->deferred_reply_errors = NULL; + /* We clear the ASKING flag as well if we are not inside a MULTI, and * if what we just executed is not the ASKING command itself. */ if (!(c->flags & CLIENT_MULTI) && prevcmd != askingCommand) @@ -2556,7 +2717,7 @@ sds catClientInfoString(sds s, client *client) { } sds ret = sdscatfmt(s, - "id=%U addr=%s laddr=%s %s name=%s age=%I idle=%I flags=%s db=%i sub=%i psub=%i multi=%i qbuf=%U qbuf-free=%U argv-mem=%U multi-mem=%U obl=%U oll=%U omem=%U tot-mem=%U events=%s cmd=%s user=%s redir=%I resp=%i", + "id=%U addr=%s laddr=%s %s name=%s age=%I idle=%I flags=%s db=%i sub=%i psub=%i multi=%i qbuf=%U qbuf-free=%U argv-mem=%U multi-mem=%U rbs=%U rbp=%U obl=%U oll=%U omem=%U tot-mem=%U events=%s cmd=%s user=%s redir=%I resp=%i", (unsigned long long) client->id, getClientPeerId(client), getClientSockname(client), @@ -2573,6 +2734,8 @@ sds catClientInfoString(sds s, client *client) { (unsigned long long) sdsavail(client->querybuf), (unsigned long long) client->argv_len_sum, (unsigned long long) client->mstate.argv_len_sums, + (unsigned long long) client->buf_usable_size, + (unsigned long long) client->buf_peak, (unsigned long long) client->bufpos, (unsigned long long) listLength(client->reply) + used_blocks_of_repl_buf, (unsigned long long) obufmem, /* should not include client->buf since we want to see 0 for static clients. */ @@ -2919,6 +3082,7 @@ NULL else replyToBlockedClientTimedOut(target); unblockClient(target); + updateStatsOnUnblock(target, 0, 0, 1); addReply(c,shared.cone); } else { addReply(c,shared.czero); @@ -3414,6 +3578,7 @@ size_t getClientMemoryUsage(client *c, size_t *output_buffer_mem_usage) { *output_buffer_mem_usage = mem; mem += sdsZmallocSize(c->querybuf); mem += zmalloc_size(c); + mem += c->buf_usable_size; /* For efficiency (less work keeping track of the argv memory), it doesn't include the used memory * i.e. unused sds space and internal fragmentation, just the string length. but this is enough to * spot problematic clients. */ @@ -692,7 +692,7 @@ int rdbSaveObjectType(rio *rdb, robj *o) { else serverPanic("Unknown hash encoding"); case OBJ_STREAM: - return rdbSaveType(rdb,RDB_TYPE_STREAM_LISTPACKS); + return rdbSaveType(rdb,RDB_TYPE_STREAM_LISTPACKS_2); case OBJ_MODULE: return rdbSaveType(rdb,RDB_TYPE_MODULE_2); default: @@ -986,6 +986,19 @@ ssize_t rdbSaveObject(rio *rdb, robj *o, robj *key, int dbid) { nwritten += n; if ((n = rdbSaveLen(rdb,s->last_id.seq)) == -1) return -1; nwritten += n; + /* Save the first entry ID. */ + if ((n = rdbSaveLen(rdb,s->first_id.ms)) == -1) return -1; + nwritten += n; + if ((n = rdbSaveLen(rdb,s->first_id.seq)) == -1) return -1; + nwritten += n; + /* Save the maximal tombstone ID. */ + if ((n = rdbSaveLen(rdb,s->max_deleted_entry_id.ms)) == -1) return -1; + nwritten += n; + if ((n = rdbSaveLen(rdb,s->max_deleted_entry_id.seq)) == -1) return -1; + nwritten += n; + /* Save the offset. */ + if ((n = rdbSaveLen(rdb,s->entries_added)) == -1) return -1; + nwritten += n; /* The consumer groups and their clients are part of the stream * type, so serialize every consumer group. */ @@ -1020,6 +1033,13 @@ ssize_t rdbSaveObject(rio *rdb, robj *o, robj *key, int dbid) { return -1; } nwritten += n; + + /* Save the group's logical reads counter. */ + if ((n = rdbSaveLen(rdb,cg->entries_read)) == -1) { + raxStop(&ri); + return -1; + } + nwritten += n; /* Save the global PEL. */ if ((n = rdbSaveStreamPEL(rdb,cg->pel,1)) == -1) { @@ -1151,8 +1171,9 @@ ssize_t rdbSaveAuxFieldStrInt(rio *rdb, char *key, long long val) { /* Save a few default AUX fields with information about the RDB generated. */ int rdbSaveInfoAuxFields(rio *rdb, int rdbflags, rdbSaveInfo *rsi) { + UNUSED(rdbflags); int redis_bits = (sizeof(void*) == 8) ? 64 : 32; - int aof_preamble = (rdbflags & RDBFLAGS_AOF_PREAMBLE) != 0; + int aof_base = (rdbflags & RDBFLAGS_AOF_PREAMBLE) != 0; /* Add a few fields about the state when the RDB was created. */ if (rdbSaveAuxFieldStrStr(rdb,"redis-ver",REDIS_VERSION) == -1) return -1; @@ -1169,7 +1190,7 @@ int rdbSaveInfoAuxFields(rio *rdb, int rdbflags, rdbSaveInfo *rsi) { if (rdbSaveAuxFieldStrInt(rdb,"repl-offset",server.master_repl_offset) == -1) return -1; } - if (rdbSaveAuxFieldStrInt(rdb,"aof-preamble",aof_preamble) == -1) return -1; + if (rdbSaveAuxFieldStrInt(rdb, "aof-base", aof_base) == -1) return -1; return 1; } @@ -1470,6 +1491,7 @@ int rdbSaveBackground(int req, char *filename, rdbSaveInfo *rsi) { pid_t childpid; if (hasActiveChildProcess()) return C_ERR; + server.stat_rdb_saves++; server.dirty_before_bgsave = server.dirty; server.lastbgsave_try = time(NULL); @@ -2319,7 +2341,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) { rdbReportCorruptRDB("Unknown RDB encoding type %d",rdbtype); break; } - } else if (rdbtype == RDB_TYPE_STREAM_LISTPACKS) { + } else if (rdbtype == RDB_TYPE_STREAM_LISTPACKS || rdbtype == RDB_TYPE_STREAM_LISTPACKS_2) { o = createStreamObject(); stream *s = o->ptr; uint64_t listpacks = rdbLoadLen(rdb,NULL); @@ -2395,6 +2417,30 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) { /* Load the last entry ID. */ s->last_id.ms = rdbLoadLen(rdb,NULL); s->last_id.seq = rdbLoadLen(rdb,NULL); + + if (rdbtype == RDB_TYPE_STREAM_LISTPACKS_2) { + /* Load the first entry ID. */ + s->first_id.ms = rdbLoadLen(rdb,NULL); + s->first_id.seq = rdbLoadLen(rdb,NULL); + + /* Load the maximal deleted entry ID. */ + s->max_deleted_entry_id.ms = rdbLoadLen(rdb,NULL); + s->max_deleted_entry_id.seq = rdbLoadLen(rdb,NULL); + + /* Load the offset. */ + s->entries_added = rdbLoadLen(rdb,NULL); + } else { + /* During migration the offset can be initialized to the stream's + * length. At this point, we also don't care about tombstones + * because CG offsets will be later initialized as well. */ + s->max_deleted_entry_id.ms = 0; + s->max_deleted_entry_id.seq = 0; + s->entries_added = s->length; + + /* Since the rax is already loaded, we can find the first entry's + * ID. */ + streamGetEdgeID(s,1,1,&s->first_id); + } if (rioGetReadError(rdb)) { rdbReportReadError("Stream object metadata loading failed."); @@ -2430,8 +2476,22 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) { decrRefCount(o); return NULL; } + + /* Load group offset. */ + uint64_t cg_offset; + if (rdbtype == RDB_TYPE_STREAM_LISTPACKS_2) { + cg_offset = rdbLoadLen(rdb,NULL); + if (rioGetReadError(rdb)) { + rdbReportReadError("Stream cgroup offset loading failed."); + sdsfree(cgname); + decrRefCount(o); + return NULL; + } + } else { + cg_offset = streamEstimateDistanceFromFirstEverEntry(s,&cg_id); + } - streamCG *cgroup = streamCreateCG(s,cgname,sdslen(cgname),&cg_id); + streamCG *cgroup = streamCreateCG(s,cgname,sdslen(cgname),&cg_id,cg_offset); if (cgroup == NULL) { rdbReportCorruptRDB("Duplicated consumer group name %s", cgname); @@ -2962,6 +3022,9 @@ int rdbLoadRioWithLoadingCtx(rio *rdb, int rdbflags, rdbSaveInfo *rsi, rdbLoadin } else if (!strcasecmp(auxkey->ptr,"aof-preamble")) { long long haspreamble = strtoll(auxval->ptr,NULL,10); if (haspreamble) serverLog(LL_NOTICE,"RDB has an AOF tail"); + } else if (!strcasecmp(auxkey->ptr, "aof-base")) { + long long isbase = strtoll(auxval->ptr, NULL, 10); + if (isbase) serverLog(LL_NOTICE, "RDB is base AOF"); } else if (!strcasecmp(auxkey->ptr,"redis-bits")) { /* Just ignored. */ } else { @@ -3049,9 +3112,9 @@ int rdbLoadRioWithLoadingCtx(rio *rdb, int rdbflags, rdbSaveInfo *rsi, rdbLoadin * received from the master. In the latter case, the master is * responsible for key expiry. If we would expire keys here, the * snapshot taken by the master may not be reflected on the slave. - * Similarly if the RDB is the preamble of an AOF file, we want to - * load all the keys as they are, since the log of operations later - * assume to work in an exact keyspace state. */ + * Similarly, if the base AOF is RDB format, we want to load all + * the keys they are, since the log of operations in the incr AOF + * is assumed to work in the exact keyspace state. */ if (val == NULL) { /* Since we used to have bug that could lead to empty keys * (See #8453), we rather not fail when empty key is encountered @@ -94,10 +94,11 @@ #define RDB_TYPE_HASH_LISTPACK 16 #define RDB_TYPE_ZSET_LISTPACK 17 #define RDB_TYPE_LIST_QUICKLIST_2 18 +#define RDB_TYPE_STREAM_LISTPACKS_2 19 /* NOTE: WHEN ADDING NEW RDB TYPE, UPDATE rdbIsObjectType() BELOW */ /* Test if a type is an object type. */ -#define rdbIsObjectType(t) ((t >= 0 && t <= 7) || (t >= 9 && t <= 18)) +#define rdbIsObjectType(t) ((t >= 0 && t <= 7) || (t >= 9 && t <= 19)) /* Special RDB opcodes (saved/loaded with rdbSaveType/rdbLoadType). */ #define RDB_OPCODE_FUNCTION 246 /* engine data */ diff --git a/src/redis-benchmark.c b/src/redis-benchmark.c index 86421e33f..64893ac37 100644 --- a/src/redis-benchmark.c +++ b/src/redis-benchmark.c @@ -125,6 +125,7 @@ static struct config { int enable_tracking; pthread_mutex_t liveclients_mutex; pthread_mutex_t is_updating_slots_mutex; + int resp3; /* use RESP3 */ } config; typedef struct _client { @@ -632,6 +633,9 @@ static void writeHandler(aeEventLoop *el, int fd, void *privdata, int mask) { fprintf(stderr, "Error writing to the server: %s\n", strerror(errno)); freeClient(c); return; + } else if (nwritten > 0) { + c->written += nwritten; + return; } } else { aeDeleteFileEvent(el,c->context->fd,AE_WRITABLE); @@ -748,6 +752,15 @@ static client createClient(char *cmd, size_t len, client from, int thread_id) { (int)sdslen(config.input_dbnumstr),config.input_dbnumstr); c->prefix_pending++; } + + if (config.resp3) { + char *buf = NULL; + int len = redisFormatCommand(&buf, "HELLO 3"); + c->obuf = sdscatlen(c->obuf, buf, len); + free(buf); + c->prefix_pending++; + } + c->prefixlen = sdslen(c->obuf); /* Append the request itself. */ if (from) { @@ -1440,6 +1453,8 @@ int parseOptions(int argc, char **argv) { } else if (!strcmp(argv[i],"-u") && !lastarg) { parseRedisUri(argv[++i],"redis-benchmark",&config.conn_info,&config.tls); config.input_dbnumstr = sdsfromlonglong(config.conn_info.input_dbnum); + } else if (!strcmp(argv[i],"-3")) { + config.resp3 = 1; } else if (!strcmp(argv[i],"-d")) { if (lastarg) goto invalid; config.datasize = atoi(argv[++i]); @@ -1566,6 +1581,7 @@ usage: " -n <requests> Total number of requests (default 100000)\n" " -d <size> Data size of SET/GET value in bytes (default 3)\n" " --dbnum <db> SELECT the specified db number (default 0)\n" +" -3 Start session in RESP3 protocol mode.\n" " --threads <num> Enable multi-thread mode.\n" " --cluster Enable cluster mode.\n" " If the command is supplied on the command line in cluster\n" @@ -1739,6 +1755,7 @@ int main(int argc, char **argv) { config.is_updating_slots = 0; config.slots_last_update = 0; config.enable_tracking = 0; + config.resp3 = 0; i = parseOptions(argc,argv); argc -= i; diff --git a/src/redis-check-aof.c b/src/redis-check-aof.c index 01f42ec1b..a3da79dd4 100644 --- a/src/redis-check-aof.c +++ b/src/redis-check-aof.c @@ -30,6 +30,24 @@ #include "server.h" #include <sys/stat.h> +#include <sys/types.h> +#include <regex.h> +#include <libgen.h> + +#define AOF_CHECK_OK 0 +#define AOF_CHECK_EMPTY 1 +#define AOF_CHECK_TRUNCATED 2 +#define AOF_CHECK_TIMESTAMP_TRUNCATED 3 + +typedef enum { + AOF_RESP, + AOF_RDB_PREAMBLE, + AOF_MULTI_PART, +} input_file_type; + +aofManifest *aofManifestCreate(void); +void aofManifestFree(aofManifest *am); +aofManifest *aofLoadManifestFromFile(sds am_filepath); #define ERROR(...) { \ char __buf[1024]; \ @@ -51,47 +69,6 @@ int consumeNewline(char *buf) { return 1; } -int readAnnotations(FILE *fp) { - char buf[AOF_ANNOTATION_LINE_MAX_LEN]; - while (1) { - epos = ftello(fp); - if (fgets(buf, sizeof(buf), fp) == NULL) { - return 0; - } - if (buf[0] == '#') { - if (to_timestamp && strncmp(buf, "#TS:", 4) == 0) { - time_t ts = strtol(buf+4, NULL, 10); - if (ts <= to_timestamp) continue; - if (epos == 0) { - printf("AOF has nothing before timestamp %ld, " - "aborting...\n", to_timestamp); - fclose(fp); - exit(1); - } - /* Truncate remaining AOF if exceeding 'to_timestamp' */ - if (ftruncate(fileno(fp), epos) == -1) { - printf("Failed to truncate AOF to timestamp %ld\n", - to_timestamp); - exit(1); - } else { - printf("Successfully truncated AOF to timestamp %ld\n", - to_timestamp); - fclose(fp); - exit(0); - } - } - continue; - } else { - if (fseek(fp, -(ftello(fp)-epos), SEEK_CUR) == -1) { - ERROR("Fseek error: %s", strerror(errno)); - return 0; - } - return 1; - } - } - return 1; -} - int readLong(FILE *fp, char prefix, long *target) { char buf[128], *eptr; epos = ftello(fp); @@ -133,9 +110,13 @@ int readString(FILE *fp, char** target) { len += 2; *target = (char*)zmalloc(len); if (!readBytes(fp,*target,len)) { + zfree(*target); + *target = NULL; return 0; } if (!consumeNewline(*target+len-2)) { + zfree(*target); + *target = NULL; return 0; } (*target)[len-2] = '\0'; @@ -146,156 +127,454 @@ int readArgc(FILE *fp, long *target) { return readLong(fp,'*',target); } -off_t process(FILE *fp) { +/* Used to decode a RESP record in the AOF file to obtain the original + * redis command, and also check whether the command is MULTI/EXEC. If the + * command is MULTI, the parameter out_multi will be incremented by one, and + * if the command is EXEC, the parameter out_multi will be decremented + * by one. The parameter out_multi will be used by the upper caller to determine + * whether the AOF file contains unclosed transactions. + **/ +int processRESP(FILE *fp, char *filename, int *out_multi) { long argc; - off_t pos = 0; - int i, multi = 0; char *str; - while(1) { - if (!multi) pos = ftello(fp); - if (!readAnnotations(fp)) break; - if (!readArgc(fp, &argc)) break; - - for (i = 0; i < argc; i++) { - if (!readString(fp,&str)) break; - if (i == 0) { - if (strcasecmp(str, "multi") == 0) { - if (multi++) { - ERROR("Unexpected MULTI"); - break; - } - } else if (strcasecmp(str, "exec") == 0) { - if (--multi) { - ERROR("Unexpected EXEC"); - break; - } + if (!readArgc(fp, &argc)) return 0; + + for (int i = 0; i < argc; i++) { + if (!readString(fp, &str)) return 0; + if (i == 0) { + if (strcasecmp(str, "multi") == 0) { + if ((*out_multi)++) { + ERROR("Unexpected MULTI in AOF %s", filename); + zfree(str); + return 0; + } + } else if (strcasecmp(str, "exec") == 0) { + if (--(*out_multi)) { + ERROR("Unexpected EXEC in AOF %s", filename); + zfree(str); + return 0; } } - zfree(str); - } - - /* Stop if the loop did not finish */ - if (i < argc) { - if (str) zfree(str); - break; } + zfree(str); } - if (feof(fp) && multi && strlen(error) == 0) { - ERROR("Reached EOF before reading EXEC for MULTI"); - } - if (strlen(error) > 0) { - printf("%s\n", error); - } - return pos; + return 1; } -int redis_check_aof_main(int argc, char **argv) { - char *filename; - int fix = 0; +/* Used to parse an annotation in the AOF file, the annotation starts with '#' + * in AOF. Currently AOF only contains timestamp annotations, but this function + * can easily be extended to handle other annotations. + * + * The processing rule of time annotation is that once the timestamp is found to + * be greater than 'to_timestamp', the AOF after the annotation is truncated. + * Note that in Multi Part AOF, this truncation is only allowed when the last_file + * parameter is 1. + **/ +int processAnnotations(FILE *fp, char *filename, int last_file) { + char buf[AOF_ANNOTATION_LINE_MAX_LEN]; - if (argc < 2) { - goto invalid_args; - } else if (argc == 2) { - filename = argv[1]; - } else if (argc == 3) { - if (!strcmp(argv[1],"--fix")) { - filename = argv[2]; - fix = 1; - } else { - goto invalid_args; + epos = ftello(fp); + if (fgets(buf, sizeof(buf), fp) == NULL) { + printf("Failed to read annotations from AOF %s, aborting...\n", filename); + exit(1); + } + + if (to_timestamp && strncmp(buf, "#TS:", 4) == 0) { + char *endptr; + errno = 0; + time_t ts = strtol(buf+4, &endptr, 10); + if (errno != 0 || *endptr != '\r') { + printf("Invalid timestamp annotation\n"); + exit(1); } - } else if (argc == 4) { - if (!strcmp(argv[1], "--truncate-to-timestamp")) { - to_timestamp = strtol(argv[2],NULL,10); - filename = argv[3]; + if (ts <= to_timestamp) return 1; + if (epos == 0) { + printf("AOF %s has nothing before timestamp %ld, " + "aborting...\n", filename, to_timestamp); + exit(1); + } + if (!last_file) { + printf("Failed to truncate AOF %s to timestamp %ld to offset %ld because it is not the last file.\n", + filename, to_timestamp, (long int)epos); + printf("If you insist, please delete all files after this file according to the manifest " + "file and delete the corresponding records in manifest file manually. Then re-run redis-check-aof.\n"); + exit(1); + } + /* Truncate remaining AOF if exceeding 'to_timestamp' */ + if (ftruncate(fileno(fp), epos) == -1) { + printf("Failed to truncate AOF %s to timestamp %ld\n", + filename, to_timestamp); + exit(1); } else { - goto invalid_args; + return 0; } - } else { - goto invalid_args; } + return 1; +} + +/* Used to check the validity of a single AOF file. The AOF file can be: + * 1. Old-style AOF + * 2. Old-style RDB-preamble AOF + * 3. BASE or INCR in Multi Part AOF + * */ +int checkSingleAof(char *aof_filename, char *aof_filepath, int last_file, int fix, int preamble) { + off_t pos = 0, diff; + int multi = 0; + char buf[2]; - FILE *fp = fopen(filename,"r+"); + FILE *fp = fopen(aof_filepath, "r+"); if (fp == NULL) { - printf("Cannot open file: %s\n", filename); + printf("Cannot open file %s: %s, aborting...\n", aof_filepath, strerror(errno)); exit(1); } struct redis_stat sb; if (redis_fstat(fileno(fp),&sb) == -1) { - printf("Cannot stat file: %s\n", filename); + printf("Cannot stat file: %s, aborting...\n", aof_filename); exit(1); } off_t size = sb.st_size; if (size == 0) { - printf("Empty file: %s\n", filename); - exit(1); + return AOF_CHECK_EMPTY; } - /* This AOF file may have an RDB preamble. Check this to start, and if this - * is the case, start processing the RDB part. */ - if (size >= 8) { /* There must be at least room for the RDB header. */ - char sig[5]; - int has_preamble = fread(sig,sizeof(sig),1,fp) == 1 && - memcmp(sig,"REDIS",sizeof(sig)) == 0; - rewind(fp); - if (has_preamble) { - printf("The AOF appears to start with an RDB preamble.\n" - "Checking the RDB preamble to start:\n"); - if (redis_check_rdb_main(argc,argv,fp) == C_ERR) { - printf("RDB preamble of AOF file is not sane, aborting.\n"); - exit(1); - } else { - printf("RDB preamble is OK, proceeding with AOF tail...\n"); + if (preamble) { + char *argv[2] = {NULL, aof_filename}; + if (redis_check_rdb_main(2, argv, fp) == C_ERR) { + printf("RDB preamble of AOF file is not sane, aborting.\n"); + exit(1); + } else { + printf("RDB preamble is OK, proceeding with AOF tail...\n"); + } + } + + while(1) { + if (!multi) pos = ftello(fp); + if (fgets(buf, sizeof(buf), fp) == NULL) { + if (feof(fp)) { + break; + } + printf("Failed to read from AOF %s, aborting...\n", aof_filename); + exit(1); + } + + if (fseek(fp, -1, SEEK_CUR) == -1) { + printf("Failed to fseek in AOF %s: %s", aof_filename, strerror(errno)); + exit(1); + } + + if (buf[0] == '#') { + if (!processAnnotations(fp, aof_filepath, last_file)) { + fclose(fp); + return AOF_CHECK_TIMESTAMP_TRUNCATED; } + } else if (buf[0] == '*'){ + if (!processRESP(fp, aof_filepath, &multi)) break; + } else { + printf("AOF %s format error\n", aof_filename); + break; } } - off_t pos = process(fp); - off_t diff = size-pos; + if (feof(fp) && multi && strlen(error) == 0) { + ERROR("Reached EOF before reading EXEC for MULTI"); + } + + if (strlen(error) > 0) { + printf("%s\n", error); + } + + diff = size-pos; /* In truncate-to-timestamp mode, just exit if there is nothing to truncate. */ if (diff == 0 && to_timestamp) { - printf("Truncate nothing in AOF to timestamp %ld\n", to_timestamp); + printf("Truncate nothing in AOF %s to timestamp %ld\n", aof_filename, to_timestamp); fclose(fp); - exit(0); + return AOF_CHECK_OK; } - printf("AOF analyzed: size=%lld, ok_up_to=%lld, ok_up_to_line=%lld, diff=%lld\n", - (long long) size, (long long) pos, line, (long long) diff); + printf("AOF analyzed: filename=%s, size=%lld, ok_up_to=%lld, ok_up_to_line=%lld, diff=%lld\n", + aof_filename, (long long) size, (long long) pos, line, (long long) diff); if (diff > 0) { if (fix) { + if (!last_file) { + printf("Failed to truncate AOF %s because it is not the last file\n", aof_filename); + exit(1); + } + char buf[2]; - printf("This will shrink the AOF from %lld bytes, with %lld bytes, to %lld bytes\n",(long long)size,(long long)diff,(long long)pos); + printf("This will shrink the AOF %s from %lld bytes, with %lld bytes, to %lld bytes\n", + aof_filename, (long long)size, (long long)diff, (long long)pos); printf("Continue? [y/N]: "); - if (fgets(buf,sizeof(buf),stdin) == NULL || - strncasecmp(buf,"y",1) != 0) { - printf("Aborting...\n"); - exit(1); + if (fgets(buf, sizeof(buf), stdin) == NULL || strncasecmp(buf, "y", 1) != 0) { + printf("Aborting...\n"); + exit(1); } if (ftruncate(fileno(fp), pos) == -1) { - printf("Failed to truncate AOF\n"); + printf("Failed to truncate AOF %s\n", aof_filename); exit(1); } else { - printf("Successfully truncated AOF\n"); + fclose(fp); + return AOF_CHECK_TRUNCATED; } } else { - printf("AOF is not valid. " - "Use the --fix option to try fixing it.\n"); + printf("AOF %s is not valid. Use the --fix option to try fixing it.\n", aof_filename); exit(1); } - } else { - printf("AOF is valid\n"); + } + fclose(fp); + return AOF_CHECK_OK; +} + +/* Used to determine whether the file is a RDB file. These two possibilities: + * 1. The file is an old style RDB-preamble AOF + * 2. The file is a BASE AOF in Multi Part AOF + * */ +int fileIsRDB(char *filepath) { + FILE *fp = fopen(filepath, "r"); + if (fp == NULL) { + printf("Cannot open file %s: %s\n", filepath, strerror(errno)); + exit(1); + } + + struct redis_stat sb; + if (redis_fstat(fileno(fp), &sb) == -1) { + printf("Cannot stat file: %s\n", filepath); + exit(1); + } + + off_t size = sb.st_size; + if (size == 0) { + fclose(fp); + return 0; + } + + if (size >= 8) { /* There must be at least room for the RDB header. */ + char sig[5]; + int rdb_file = fread(sig, sizeof(sig), 1, fp) == 1 && + memcmp(sig, "REDIS", sizeof(sig)) == 0; + if (rdb_file) { + fclose(fp); + return 1; + } + } + + fclose(fp); + return 0; +} + +/* Used to determine whether the file is a manifest file. */ +#define MANIFEST_MAX_LINE 1024 +int fileIsManifest(char *filepath) { + int is_manifest = 0; + FILE *fp = fopen(filepath, "r"); + if (fp == NULL) { + printf("Cannot open file %s: %s\n", filepath, strerror(errno)); + exit(1); + } + + struct redis_stat sb; + if (redis_fstat(fileno(fp), &sb) == -1) { + printf("Cannot stat file: %s\n", filepath); + exit(1); + } + + off_t size = sb.st_size; + if (size == 0) { + fclose(fp); + return 0; + } + + char buf[MANIFEST_MAX_LINE+1]; + while (1) { + if (fgets(buf, MANIFEST_MAX_LINE+1, fp) == NULL) { + if (feof(fp)) { + break; + } else { + printf("Cannot read file: %s\n", filepath); + exit(1); + } + } + + /* Skip comments lines */ + if (buf[0] == '#') { + continue; + } else if (!memcmp(buf, "file", strlen("file"))) { + is_manifest = 1; + } } fclose(fp); + return is_manifest; +} + +/* Get the format of the file to be checked. It can be: + * AOF_RESP: Old-style AOF + * AOF_RDB_PREAMBLE: Old-style RDB-preamble AOF + * AOF_MULTI_PART: manifest in Multi Part AOF + * + * redis-check-aof tool will automatically perform different + * verification logic according to different file formats. + * */ +input_file_type getInputFileType(char *filepath) { + if (fileIsManifest(filepath)) { + return AOF_MULTI_PART; + } else if (fileIsRDB(filepath)) { + return AOF_RDB_PREAMBLE; + } else { + return AOF_RESP; + } +} + +/* Check if Multi Part AOF is valid. It will check the BASE file and INCR files + * at once according to the manifest instructions (this is somewhat similar to + * redis' AOF loading). + * + * When the verification is successful, we can guarantee: + * 1. The manifest file format is valid + * 2. Both BASE AOF and INCR AOFs format are valid + * 3. No BASE or INCR AOFs files are missing + * + * Note that in Multi Part AOF, we only allow truncation for the last AOF file. + * */ +void checkMultiPartAof(char *dirpath, char *manifest_filepath, int fix) { + int total_num = 0, aof_num = 0, last_file; + int ret; + + printf("Start checking Multi Part AOF\n"); + aofManifest *am = aofLoadManifestFromFile(manifest_filepath); + + if (am->base_aof_info) total_num++; + if (am->incr_aof_list) total_num += listLength(am->incr_aof_list); + + if (am->base_aof_info) { + sds aof_filename = am->base_aof_info->file_name; + sds aof_filepath = makePath(dirpath, aof_filename); + last_file = ++aof_num == total_num; + int aof_preable = fileIsRDB(aof_filepath); + + printf("Start to check BASE AOF (%s format).\n", aof_preable ? "RDB":"RESP"); + ret = checkSingleAof(aof_filename, aof_filepath, last_file, fix, aof_preable); + if (ret == AOF_CHECK_OK) { + printf("BASE AOF %s is valid\n", aof_filename); + } else if (ret == AOF_CHECK_EMPTY) { + printf("BASE AOF %s is empty\n", aof_filename); + } else if (ret == AOF_CHECK_TIMESTAMP_TRUNCATED) { + printf("Successfully truncated AOF %s to timestamp %ld\n", + aof_filename, to_timestamp); + } else if (ret == AOF_CHECK_TRUNCATED) { + printf("Successfully truncated AOF %s\n", aof_filename); + } + sdsfree(aof_filepath); + } + + if (listLength(am->incr_aof_list)) { + listNode *ln; + listIter li; + + printf("Start to check INCR files.\n"); + listRewind(am->incr_aof_list, &li); + while ((ln = listNext(&li)) != NULL) { + aofInfo *ai = (aofInfo*)ln->value; + sds aof_filename = (char*)ai->file_name; + sds aof_filepath = makePath(dirpath, aof_filename); + last_file = ++aof_num == total_num; + ret = checkSingleAof(aof_filename, aof_filepath, last_file, fix, 0); + if (ret == AOF_CHECK_OK) { + printf("INCR AOF %s is valid\n", aof_filename); + } else if (ret == AOF_CHECK_EMPTY) { + printf("INCR AOF %s is empty\n", aof_filename); + } else if (ret == AOF_CHECK_TIMESTAMP_TRUNCATED) { + printf("Successfully truncated AOF %s to timestamp %ld\n", + aof_filename, to_timestamp); + } else if (ret == AOF_CHECK_TRUNCATED) { + printf("Successfully truncated AOF %s\n", aof_filename); + } + sdsfree(aof_filepath); + } + } + + aofManifestFree(am); + printf("All AOF files and manifest are valid\n"); +} + +/* Check if old style AOF is valid. Internally, it will identify whether + * the AOF is in RDB-preamble format, and will eventually call `checkSingleAof` + * to do the check. */ +void checkOldStyleAof(char *filepath, int fix, int preamble) { + printf("Start checking Old-Style AOF\n"); + int ret = checkSingleAof(filepath, filepath, 1, fix, preamble); + if (ret == AOF_CHECK_OK) { + printf("AOF %s is valid\n", filepath); + } else if (ret == AOF_CHECK_EMPTY) { + printf("AOF %s is empty\n", filepath); + } else if (ret == AOF_CHECK_TIMESTAMP_TRUNCATED) { + printf("Successfully truncated AOF %s to timestamp %ld\n", + filepath, to_timestamp); + } else if (ret == AOF_CHECK_TRUNCATED) { + printf("Successfully truncated AOF %s\n", filepath); + } +} + +int redis_check_aof_main(int argc, char **argv) { + char *filepath; + char temp_filepath[PATH_MAX + 1]; + char *dirpath; + int fix = 0; + + if (argc < 2) { + goto invalid_args; + } else if (argc == 2) { + filepath = argv[1]; + } else if (argc == 3) { + if (!strcmp(argv[1], "--fix")) { + filepath = argv[2]; + fix = 1; + } else { + goto invalid_args; + } + } else if (argc == 4) { + if (!strcmp(argv[1], "--truncate-to-timestamp")) { + char *endptr; + errno = 0; + to_timestamp = strtol(argv[2], &endptr, 10); + if (errno != 0 || *endptr != '\0') { + printf("Invalid timestamp, aborting...\n"); + exit(1); + } + filepath = argv[3]; + } else { + goto invalid_args; + } + } else { + goto invalid_args; + } + + /* In the glibc implementation dirname may modify their argument. */ + memcpy(temp_filepath, filepath, strlen(filepath) + 1); + dirpath = dirname(temp_filepath); + + /* Select the corresponding verification method according to the input file type. */ + input_file_type type = getInputFileType(filepath); + switch (type) { + case AOF_MULTI_PART: + checkMultiPartAof(dirpath, filepath, fix); + break; + case AOF_RESP: + checkOldStyleAof(filepath, fix, 0); + break; + case AOF_RDB_PREAMBLE: + checkOldStyleAof(filepath, fix, 1); + break; + } + exit(0); invalid_args: - printf("Usage: %s [--fix|--truncate-to-timestamp $timestamp] <file.aof>\n", - argv[0]); + printf("Usage: %s [--fix|--truncate-to-timestamp $timestamp] <file.manifest|file.aof>\n", + argv[0]); exit(1); } diff --git a/src/redis-cli.c b/src/redis-cli.c index 31a2973c7..bbbe6d6ec 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -58,7 +58,7 @@ #include "adlist.h" #include "zmalloc.h" #include "linenoise.h" -#include "help.h" +#include "help.h" /* Used for backwards-compatibility with pre-7.0 servers that don't support COMMAND DOCS. */ #include "anet.h" #include "ae.h" #include "cli_common.h" @@ -167,13 +167,21 @@ int *spectrum_palette; int spectrum_palette_size; /* Dict Helpers */ - static uint64_t dictSdsHash(const void *key); static int dictSdsKeyCompare(dict *d, const void *key1, const void *key2); static void dictSdsDestructor(dict *d, void *val); static void dictListDestructor(dict *d, void *val); +/* Command documentation info used for help output */ +struct commandDocs { + char *name; + char *params; /* A string describing the syntax of the command arguments. */ + char *summary; + char *group; + char *since; +}; + /* Cluster Manager Command Info */ typedef struct clusterManagerCommand { char *name; @@ -398,11 +406,11 @@ typedef struct { sds full; /* Only used for help on commands */ - struct commandHelp *org; + struct commandDocs org; } helpEntry; -static helpEntry *helpEntries; -static int helpEntriesLen; +static helpEntry *helpEntries = NULL; +static int helpEntriesLen = 0; static sds cliVersion(void) { sds version; @@ -418,7 +426,8 @@ static sds cliVersion(void) { return version; } -static void cliInitHelp(void) { +/* For backwards compatibility with pre-7.0 servers. Initializes command help. */ +static void cliOldInitHelp(void) { int commandslen = sizeof(commandHelp)/sizeof(struct commandHelp); int groupslen = sizeof(commandGroups)/sizeof(char*); int i, len, pos = 0; @@ -433,7 +442,11 @@ static void cliInitHelp(void) { tmp.argv[0] = sdscatprintf(sdsempty(),"@%s",commandGroups[i]); tmp.full = tmp.argv[0]; tmp.type = CLI_HELP_GROUP; - tmp.org = NULL; + tmp.org.name = NULL; + tmp.org.params = NULL; + tmp.org.summary = NULL; + tmp.org.since = NULL; + tmp.org.group = NULL; helpEntries[pos++] = tmp; } @@ -441,17 +454,22 @@ static void cliInitHelp(void) { tmp.argv = sdssplitargs(commandHelp[i].name,&tmp.argc); tmp.full = sdsnew(commandHelp[i].name); tmp.type = CLI_HELP_COMMAND; - tmp.org = &commandHelp[i]; + tmp.org.name = commandHelp[i].name; + tmp.org.params = commandHelp[i].params; + tmp.org.summary = commandHelp[i].summary; + tmp.org.since = commandHelp[i].since; + tmp.org.group = commandGroups[commandHelp[i].group]; helpEntries[pos++] = tmp; } } -/* cliInitHelp() setups the helpEntries array with the command and group +/* For backwards compatibility with pre-7.0 servers. + * cliOldInitHelp() setups the helpEntries array with the command and group * names from the help.h file. However the Redis instance we are connecting * to may support more commands, so this function integrates the previous * entries with additional entries obtained using the COMMAND command * available in recent versions of Redis. */ -static void cliIntegrateHelp(void) { +static void cliOldIntegrateHelp(void) { if (cliConnect(CC_QUIET) == REDIS_ERR) return; redisReply *reply = redisCommand(context, "COMMAND"); @@ -486,33 +504,334 @@ static void cliIntegrateHelp(void) { new->type = CLI_HELP_COMMAND; sdstoupper(new->argv[0]); - struct commandHelp *ch = zmalloc(sizeof(*ch)); - ch->name = new->argv[0]; - ch->params = sdsempty(); + new->org.name = new->argv[0]; + new->org.params = sdsempty(); int args = llabs(entry->element[1]->integer); args--; /* Remove the command name itself. */ if (entry->element[3]->integer == 1) { - ch->params = sdscat(ch->params,"key "); + new->org.params = sdscat(new->org.params,"key "); args--; } - while(args-- > 0) ch->params = sdscat(ch->params,"arg "); + while(args-- > 0) new->org.params = sdscat(new->org.params,"arg "); if (entry->element[1]->integer < 0) - ch->params = sdscat(ch->params,"...options..."); - ch->summary = "Help not available"; - ch->group = 0; - ch->since = "not known"; - new->org = ch; + new->org.params = sdscat(new->org.params,"...options..."); + new->org.summary = "Help not available"; + new->org.since = "Not known"; + new->org.group = commandGroups[0]; } freeReplyObject(reply); } +/* Concatenate a string to an sds string, but if it's empty substitute double quote marks. */ +static sds sdscat_orempty(sds params, char *value) { + if (value[0] == '\0') { + return sdscat(params, "\"\""); + } + return sdscat(params, value); +} + +static sds cliAddArgument(sds params, redisReply *argMap); + +/* Concatenate a list of arguments to the parameter string, separated by a separator string. */ +static sds cliConcatArguments(sds params, redisReply *arguments, char *separator) { + for (size_t j = 0; j < arguments->elements; j++) { + params = cliAddArgument(params, arguments->element[j]); + if (j != arguments->elements - 1) { + params = sdscat(params, separator); + } + } + return params; +} + +/* Add an argument to the parameter string. */ +static sds cliAddArgument(sds params, redisReply *argMap) { + char *name = NULL; + char *type = NULL; + int optional = 0; + int multiple = 0; + int multipleToken = 0; + redisReply *arguments = NULL; + sds tokenPart = sdsempty(); + sds repeatPart = sdsempty(); + + /* First read the fields describing the argument. */ + if (argMap->type != REDIS_REPLY_MAP && argMap->type != REDIS_REPLY_ARRAY) { + return params; + } + for (size_t i = 0; i < argMap->elements; i += 2) { + assert(argMap->element[i]->type == REDIS_REPLY_STRING); + char *key = argMap->element[i]->str; + if (!strcmp(key, "name")) { + assert(argMap->element[i + 1]->type == REDIS_REPLY_STRING); + name = argMap->element[i + 1]->str; + } else if (!strcmp(key, "token")) { + assert(argMap->element[i + 1]->type == REDIS_REPLY_STRING); + char *token = argMap->element[i + 1]->str; + tokenPart = sdscat_orempty(tokenPart, token); + } else if (!strcmp(key, "type")) { + assert(argMap->element[i + 1]->type == REDIS_REPLY_STRING); + type = argMap->element[i + 1]->str; + } else if (!strcmp(key, "arguments")) { + arguments = argMap->element[i + 1]; + } else if (!strcmp(key, "flags")) { + redisReply *flags = argMap->element[i + 1]; + assert(flags->type == REDIS_REPLY_SET || flags->type == REDIS_REPLY_ARRAY); + for (size_t j = 0; j < flags->elements; j++) { + assert(flags->element[j]->type == REDIS_REPLY_STATUS); + char *flag = flags->element[j]->str; + if (!strcmp(flag, "optional")) { + optional = 1; + } else if (!strcmp(flag, "multiple")) { + multiple = 1; + } else if (!strcmp(flag, "multiple_token")) { + multipleToken = 1; + } + } + } + } + + /* Then build the "repeating part" of the argument string. */ + if (!strcmp(type, "key") || + !strcmp(type, "string") || + !strcmp(type, "integer") || + !strcmp(type, "double") || + !strcmp(type, "pattern") || + !strcmp(type, "unix-time") || + !strcmp(type, "token")) + { + repeatPart = sdscat_orempty(repeatPart, name); + } else if (!strcmp(type, "oneof")) { + repeatPart = cliConcatArguments(repeatPart, arguments, "|"); + } else if (!strcmp(type, "block")) { + repeatPart = cliConcatArguments(repeatPart, arguments, " "); + } else if (strcmp(type, "pure-token") != 0) { + fprintf(stderr, "Unknown type '%s' set for argument '%s'\n", type, name); + } + + /* Finally, build the parameter string. */ + if (tokenPart[0] != '\0' && strcmp(type, "pure-token") != 0) { + tokenPart = sdscat(tokenPart, " "); + } + if (optional) { + params = sdscat(params, "["); + } + params = sdscat(params, tokenPart); + params = sdscat(params, repeatPart); + if (multiple) { + params = sdscat(params, " ["); + if (multipleToken) { + params = sdscat(params, tokenPart); + } + params = sdscat(params, repeatPart); + params = sdscat(params, " ...]"); + } + if (optional) { + params = sdscat(params, "]"); + } + sdsfree(tokenPart); + sdsfree(repeatPart); + return params; +} + +/* Fill in the fields of a help entry for the command/subcommand name. */ +static void cliFillInCommandHelpEntry(helpEntry *help, char *cmdname, char *subcommandname) { + help->argc = subcommandname ? 2 : 1; + help->argv = zmalloc(sizeof(sds) * help->argc); + help->argv[0] = sdsnew(cmdname); + sdstoupper(help->argv[0]); + if (subcommandname) { + /* Subcommand name is two words separated by a pipe character. */ + help->argv[1] = sdsnew(strchr(subcommandname, '|') + 1); + sdstoupper(help->argv[1]); + } + sds fullname = sdsnew(help->argv[0]); + if (subcommandname) { + fullname = sdscat(fullname, " "); + fullname = sdscat(fullname, help->argv[1]); + } + help->full = fullname; + help->type = CLI_HELP_COMMAND; + + help->org.name = help->full; + help->org.params = sdsempty(); + help->org.since = NULL; +} + +/* Initialize a command help entry for the command/subcommand described in 'specs'. + * 'next' points to the next help entry to be filled in. + * 'groups' is a set of command group names to be filled in. + * Returns a pointer to the next available position in the help entries table. + * If the command has subcommands, this is called recursively for the subcommands. + */ +static helpEntry *cliInitCommandHelpEntry(char *cmdname, char *subcommandname, + helpEntry *next, redisReply *specs, + dict *groups) { + helpEntry *help = next++; + cliFillInCommandHelpEntry(help, cmdname, subcommandname); + + assert(specs->type == REDIS_REPLY_MAP || specs->type == REDIS_REPLY_ARRAY); + for (size_t j = 0; j < specs->elements; j += 2) { + assert(specs->element[j]->type == REDIS_REPLY_STRING); + char *key = specs->element[j]->str; + if (!strcmp(key, "summary")) { + redisReply *reply = specs->element[j + 1]; + assert(reply->type == REDIS_REPLY_STRING); + help->org.summary = sdsnew(reply->str); + } else if (!strcmp(key, "since")) { + redisReply *reply = specs->element[j + 1]; + assert(reply->type == REDIS_REPLY_STRING); + help->org.since = sdsnew(reply->str); + } else if (!strcmp(key, "group")) { + redisReply *reply = specs->element[j + 1]; + assert(reply->type == REDIS_REPLY_STRING); + help->org.group = sdsnew(reply->str); + sds group = sdsdup(help->org.group); + if (dictAdd(groups, group, NULL) != DICT_OK) { + sdsfree(group); + } + } else if (!strcmp(key, "arguments")) { + redisReply *args = specs->element[j + 1]; + assert(args->type == REDIS_REPLY_ARRAY); + help->org.params = cliConcatArguments(help->org.params, args, " "); + } else if (!strcmp(key, "subcommands")) { + redisReply *subcommands = specs->element[j + 1]; + assert(subcommands->type == REDIS_REPLY_MAP || subcommands->type == REDIS_REPLY_ARRAY); + for (size_t i = 0; i < subcommands->elements; i += 2) { + assert(subcommands->element[i]->type == REDIS_REPLY_STRING); + char *subcommandname = subcommands->element[i]->str; + redisReply *subcommand = subcommands->element[i + 1]; + assert(subcommand->type == REDIS_REPLY_MAP || subcommand->type == REDIS_REPLY_ARRAY); + next = cliInitCommandHelpEntry(cmdname, subcommandname, next, subcommand, groups); + } + } + } + return next; +} + +/* Returns the total number of commands and subcommands in the command docs table. */ +static size_t cliCountCommands(redisReply* commandTable) { + size_t numCommands = commandTable->elements / 2; + + /* The command docs table maps command names to a map of their specs. */ + for (size_t i = 0; i < commandTable->elements; i += 2) { + assert(commandTable->element[i]->type == REDIS_REPLY_STRING); /* Command name. */ + assert(commandTable->element[i + 1]->type == REDIS_REPLY_MAP || + commandTable->element[i + 1]->type == REDIS_REPLY_ARRAY); + redisReply *map = commandTable->element[i + 1]; + for (size_t j = 0; j < map->elements; j += 2) { + assert(map->element[j]->type == REDIS_REPLY_STRING); + char *key = map->element[j]->str; + if (!strcmp(key, "subcommands")) { + redisReply *subcommands = map->element[j + 1]; + assert(subcommands->type == REDIS_REPLY_MAP || subcommands->type == REDIS_REPLY_ARRAY); + numCommands += subcommands->elements / 2; + } + } + } + return numCommands; +} + +/* Comparator for sorting help table entries. */ +int helpEntryCompare(const void *entry1, const void *entry2) { + helpEntry *i1 = (helpEntry *)entry1; + helpEntry *i2 = (helpEntry *)entry2; + return strcmp(i1->full, i2->full); +} + +/* Initializes command help entries for command groups. + * Called after the command help entries have already been filled in. + * Extends the help table with new entries for the command groups. + */ +void cliInitGroupHelpEntries(dict *groups) { + dictIterator *iter = dictGetIterator(groups); + dictEntry *entry; + helpEntry tmp; + + int numGroups = dictSize(groups); + int pos = helpEntriesLen; + helpEntriesLen += numGroups; + helpEntries = zrealloc(helpEntries, sizeof(helpEntry)*helpEntriesLen); + + for (entry = dictNext(iter); entry != NULL; entry = dictNext(iter)) { + tmp.argc = 1; + tmp.argv = zmalloc(sizeof(sds)); + tmp.argv[0] = sdscatprintf(sdsempty(),"@%s",(char *)entry->key); + tmp.full = tmp.argv[0]; + tmp.type = CLI_HELP_GROUP; + tmp.org.name = NULL; + tmp.org.params = NULL; + tmp.org.summary = NULL; + tmp.org.since = NULL; + tmp.org.group = NULL; + helpEntries[pos++] = tmp; + } + dictReleaseIterator(iter); +} + +/* Initializes help entries for all commands in the COMMAND DOCS reply. */ +void cliInitCommandHelpEntries(redisReply *commandTable, dict *groups) { + helpEntry *next = helpEntries; + for (size_t i = 0; i < commandTable->elements; i += 2) { + assert(commandTable->element[i]->type == REDIS_REPLY_STRING); + char *cmdname = commandTable->element[i]->str; + + assert(commandTable->element[i + 1]->type == REDIS_REPLY_MAP || + commandTable->element[i + 1]->type == REDIS_REPLY_ARRAY); + redisReply *cmdspecs = commandTable->element[i + 1]; + next = cliInitCommandHelpEntry(cmdname, NULL, next, cmdspecs, groups); + } +} + +/* cliInitHelp() sets up the helpEntries array with the command and group + * names and command descriptions obtained using the COMMAND DOCS command. + */ +static void cliInitHelp(void) { + /* Dict type for a set of strings, used to collect names of command groups. */ + dictType groupsdt = { + dictSdsHash, /* hash function */ + NULL, /* key dup */ + NULL, /* val dup */ + dictSdsKeyCompare, /* key compare */ + dictSdsDestructor, /* key destructor */ + NULL, /* val destructor */ + NULL /* allow to expand */ + }; + redisReply *commandTable; + dict *groups; + + if (cliConnect(CC_QUIET) == REDIS_ERR) return; + commandTable = redisCommand(context, "COMMAND DOCS"); + if (commandTable == NULL || commandTable->type == REDIS_REPLY_ERROR) { + /* New COMMAND DOCS subcommand not supported - generate help from old help.h data instead. */ + freeReplyObject(commandTable); + cliOldInitHelp(); + cliOldIntegrateHelp(); + return; + }; + if (commandTable->type != REDIS_REPLY_MAP && commandTable->type != REDIS_REPLY_ARRAY) return; + + /* Scan the array reported by COMMAND DOCS and fill in the entries */ + helpEntriesLen = cliCountCommands(commandTable); + helpEntries = zmalloc(sizeof(helpEntry)*helpEntriesLen); + + groups = dictCreate(&groupsdt); + cliInitCommandHelpEntries(commandTable, groups); + cliInitGroupHelpEntries(groups); + + qsort(helpEntries, helpEntriesLen, sizeof(helpEntry), helpEntryCompare); + freeReplyObject(commandTable); + dictRelease(groups); +} + /* Output command help to stdout. */ -static void cliOutputCommandHelp(struct commandHelp *help, int group) { +static void cliOutputCommandHelp(struct commandDocs *help, int group) { printf("\r\n \x1b[1m%s\x1b[0m \x1b[90m%s\x1b[0m\r\n", help->name, help->params); printf(" \x1b[33msummary:\x1b[0m %s\r\n", help->summary); - printf(" \x1b[33msince:\x1b[0m %s\r\n", help->since); + if (help->since != NULL) { + printf(" \x1b[33msince:\x1b[0m %s\r\n", help->since); + } if (group) { - printf(" \x1b[33mgroup:\x1b[0m %s\r\n", commandGroups[help->group]); + printf(" \x1b[33mgroup:\x1b[0m %s\r\n", help->group); } } @@ -538,22 +857,16 @@ static void cliOutputGenericHelp(void) { /* Output all command help, filtering by group or command name. */ static void cliOutputHelp(int argc, char **argv) { - int i, j, len; - int group = -1; + int i, j; + char *group = NULL; helpEntry *entry; - struct commandHelp *help; + struct commandDocs *help; if (argc == 0) { cliOutputGenericHelp(); return; } else if (argc > 0 && argv[0][0] == '@') { - len = sizeof(commandGroups)/sizeof(char*); - for (i = 0; i < len; i++) { - if (strcasecmp(argv[0]+1,commandGroups[i]) == 0) { - group = i; - break; - } - } + group = argv[0]+1; } assert(argc > 0); @@ -561,8 +874,8 @@ static void cliOutputHelp(int argc, char **argv) { entry = &helpEntries[i]; if (entry->type != CLI_HELP_COMMAND) continue; - help = entry->org; - if (group == -1) { + help = &entry->org; + if (group == NULL) { /* Compare all arguments */ if (argc <= entry->argc) { for (j = 0; j < argc; j++) { @@ -572,10 +885,8 @@ static void cliOutputHelp(int argc, char **argv) { cliOutputCommandHelp(help,1); } } - } else { - if (group == help->group) { - cliOutputCommandHelp(help,0); - } + } else if (strcasecmp(group, help->group) == 0) { + cliOutputCommandHelp(help,0); } } printf("\r\n"); @@ -649,7 +960,7 @@ static char *hintsCallback(const char *buf, int *color, int *bold) { if (entry) { *color = 90; *bold = 0; - sds hint = sdsnew(entry->org->params); + sds hint = sdsnew(entry->org.params); /* Remove arguments from the returned hint to show only the * ones the user did not yet type. */ @@ -2229,10 +2540,8 @@ static void repl(void) { int argc; sds *argv; - /* Initialize the help and, if possible, use the COMMAND command in order - * to retrieve missing entries. */ + /* Initialize the help using the results of the COMMAND command. */ cliInitHelp(); - cliIntegrateHelp(); config.interactive = 1; linenoiseSetMultiLine(1); diff --git a/src/redismodule.h b/src/redismodule.h index 3255fcc30..79ce2c697 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -236,14 +236,16 @@ typedef uint64_t RedisModuleTimerID; /* Declare that the module can handle errors with RedisModule_SetModuleOptions. */ #define REDISMODULE_OPTIONS_HANDLE_IO_ERRORS (1<<0) -/* Declare that the module can handle diskless async replication with RedisModule_SetModuleOptions. */ -#define REDISMODULE_OPTIONS_HANDLE_REPL_ASYNC_LOAD (1<<1) - /* When set, Redis will not call RedisModule_SignalModifiedKey(), implicitly in * RedisModule_CloseKey, and the module needs to do that when manually when keys * are modified from the user's sperspective, to invalidate WATCH. */ #define REDISMODULE_OPTION_NO_IMPLICIT_SIGNAL_MODIFIED (1<<1) +/* Declare that the module can handle diskless async replication with RedisModule_SetModuleOptions. */ +#define REDISMODULE_OPTIONS_HANDLE_REPL_ASYNC_LOAD (1<<2) + +/* Definitions for RedisModule_SetCommandInfo. */ + typedef enum { REDISMODULE_ARG_TYPE_STRING, REDISMODULE_ARG_TYPE_INTEGER, @@ -260,12 +262,146 @@ typedef enum { #define REDISMODULE_CMD_ARG_OPTIONAL (1<<0) /* The argument is optional (like GET in SET command) */ #define REDISMODULE_CMD_ARG_MULTIPLE (1<<1) /* The argument may repeat itself (like key in DEL) */ #define REDISMODULE_CMD_ARG_MULTIPLE_TOKEN (1<<2) /* The argument may repeat itself, and so does its token (like `GET pattern` in SORT) */ +#define _REDISMODULE_CMD_ARG_NEXT (1<<3) -/* Redis ACL key permission flags, which specify which permissions a module - * needs on a key. */ -#define REDISMODULE_KEY_PERMISSION_READ (1<<0) -#define REDISMODULE_KEY_PERMISSION_WRITE (1<<1) -#define REDISMODULE_KEY_PERMISSION_ALL (REDISMODULE_KEY_PERMISSION_READ | REDISMODULE_KEY_PERMISSION_WRITE) +typedef enum { + REDISMODULE_KSPEC_BS_INVALID = 0, /* Must be zero. An implicitly value of + * zero is provided when the field is + * absent in a struct literal. */ + REDISMODULE_KSPEC_BS_UNKNOWN, + REDISMODULE_KSPEC_BS_INDEX, + REDISMODULE_KSPEC_BS_KEYWORD +} RedisModuleKeySpecBeginSearchType; + +typedef enum { + REDISMODULE_KSPEC_FK_OMITTED = 0, /* Used when the field is absent in a + * struct literal. Don't use this value + * explicitly. */ + REDISMODULE_KSPEC_FK_UNKNOWN, + REDISMODULE_KSPEC_FK_RANGE, + REDISMODULE_KSPEC_FK_KEYNUM +} RedisModuleKeySpecFindKeysType; + +/* Key-spec flags. For details, see the documentation of + * RedisModule_SetCommandInfo and the key-spec flags in server.h. */ +#define REDISMODULE_CMD_KEY_RO (1ULL<<0) +#define REDISMODULE_CMD_KEY_RW (1ULL<<1) +#define REDISMODULE_CMD_KEY_OW (1ULL<<2) +#define REDISMODULE_CMD_KEY_RM (1ULL<<3) +#define REDISMODULE_CMD_KEY_ACCESS (1ULL<<4) +#define REDISMODULE_CMD_KEY_UPDATE (1ULL<<5) +#define REDISMODULE_CMD_KEY_INSERT (1ULL<<6) +#define REDISMODULE_CMD_KEY_DELETE (1ULL<<7) +#define REDISMODULE_CMD_KEY_NOT_KEY (1ULL<<8) +#define REDISMODULE_CMD_KEY_INCOMPLETE (1ULL<<9) +#define REDISMODULE_CMD_KEY_VARIABLE_FLAGS (1ULL<<10) + +/* Channel flags, for details see the documentation of + * RedisModule_ChannelAtPosWithFlags. */ +#define REDISMODULE_CMD_CHANNEL_PATTERN (1ULL<<0) +#define REDISMODULE_CMD_CHANNEL_PUBLISH (1ULL<<1) +#define REDISMODULE_CMD_CHANNEL_SUBSCRIBE (1ULL<<2) +#define REDISMODULE_CMD_CHANNEL_UNSUBSCRIBE (1ULL<<3) + +typedef struct RedisModuleCommandArg { + const char *name; + RedisModuleCommandArgType type; + int key_spec_index; /* If type is KEY, this is a zero-based index of + * the key_spec in the command. For other types, + * you may specify -1. */ + const char *token; /* If type is PURE_TOKEN, this is the token. */ + const char *summary; + const char *since; + int flags; /* The REDISMODULE_CMD_ARG_* macros. */ + struct RedisModuleCommandArg *subargs; +} RedisModuleCommandArg; + +typedef struct { + const char *since; + const char *changes; +} RedisModuleCommandHistoryEntry; + +typedef struct { + const char *notes; + uint64_t flags; /* REDISMODULE_CMD_KEY_* macros. */ + RedisModuleKeySpecBeginSearchType begin_search_type; + union { + struct { + /* The index from which we start the search for keys */ + int pos; + } index; + struct { + /* The keyword that indicates the beginning of key args */ + const char *keyword; + /* An index in argv from which to start searching. + * Can be negative, which means start search from the end, in reverse + * (Example: -2 means to start in reverse from the panultimate arg) */ + int startfrom; + } keyword; + } bs; + RedisModuleKeySpecFindKeysType find_keys_type; + union { + struct { + /* Index of the last key relative to the result of the begin search + * step. Can be negative, in which case it's not relative. -1 + * indicating till the last argument, -2 one before the last and so + * on. */ + int lastkey; + /* How many args should we skip after finding a key, in order to + * find the next one. */ + int keystep; + /* If lastkey is -1, we use limit to stop the search by a factor. 0 + * and 1 mean no limit. 2 means 1/2 of the remaining args, 3 means + * 1/3, and so on. */ + int limit; + } range; + struct { + /* Index of the argument containing the number of keys to come + * relative to the result of the begin search step */ + int keynumidx; + /* Index of the fist key. (Usually it's just after keynumidx, in + * which case it should be set to keynumidx + 1.) */ + int firstkey; + /* How many args should we skip after finding a key, in order to + * find the next one, relative to the result of the begin search + * step. */ + int keystep; + } keynum; + } fk; +} RedisModuleCommandKeySpec; + +typedef struct { + int version; + size_t sizeof_historyentry; + size_t sizeof_keyspec; + size_t sizeof_arg; +} RedisModuleCommandInfoVersion; + +static const RedisModuleCommandInfoVersion RedisModule_CurrentCommandInfoVersion = { + .version = 1, + .sizeof_historyentry = sizeof(RedisModuleCommandHistoryEntry), + .sizeof_keyspec = sizeof(RedisModuleCommandKeySpec), + .sizeof_arg = sizeof(RedisModuleCommandArg) +}; + +#define REDISMODULE_COMMAND_INFO_VERSION (&RedisModule_CurrentCommandInfoVersion) + +typedef struct { + /* Always set version to REDISMODULE_COMMAND_INFO_VERSION */ + const RedisModuleCommandInfoVersion *version; + /* Version 1 fields (added in Redis 7.0.0) */ + const char *summary; /* Summary of the command */ + const char *complexity; /* Complexity description */ + const char *since; /* Debut module version of the command */ + RedisModuleCommandHistoryEntry *history; /* History */ + /* A string of space-separated tips meant for clients/proxies regarding this + * command */ + const char *tips; + /* Number of arguments, it is possible to use -N to say >= N */ + int arity; + RedisModuleCommandKeySpec *key_specs; + RedisModuleCommandArg *args; +} RedisModuleCommandInfo; /* Eventloop definitions. */ #define REDISMODULE_EVENTLOOP_READABLE 1 @@ -623,7 +759,6 @@ typedef struct RedisModuleScanCursor RedisModuleScanCursor; typedef struct RedisModuleDefragCtx RedisModuleDefragCtx; typedef struct RedisModuleUser RedisModuleUser; typedef struct RedisModuleKeyOptCtx RedisModuleKeyOptCtx; -typedef struct RedisModuleCommandArg RedisModuleCommandArg; typedef int (*RedisModuleCmdFunc)(RedisModuleCtx *ctx, RedisModuleString **argv, int argc); typedef void (*RedisModuleDisconnectFunc)(RedisModuleCtx *ctx, RedisModuleBlockedClient *bc); @@ -697,6 +832,7 @@ REDISMODULE_API int (*RedisModule_GetApi)(const char *, void *) REDISMODULE_ATTR REDISMODULE_API int (*RedisModule_CreateCommand)(RedisModuleCtx *ctx, const char *name, RedisModuleCmdFunc cmdfunc, const char *strflags, int firstkey, int lastkey, int keystep) REDISMODULE_ATTR; REDISMODULE_API RedisModuleCommand *(*RedisModule_GetCommand)(RedisModuleCtx *ctx, const char *name) REDISMODULE_ATTR; REDISMODULE_API int (*RedisModule_CreateSubcommand)(RedisModuleCommand *parent, const char *name, RedisModuleCmdFunc cmdfunc, const char *strflags, int firstkey, int lastkey, int keystep) REDISMODULE_ATTR; +REDISMODULE_API int (*RedisModule_SetCommandInfo)(RedisModuleCommand *command, const RedisModuleCommandInfo *info) REDISMODULE_ATTR; REDISMODULE_API void (*RedisModule_SetModuleAttribs)(RedisModuleCtx *ctx, const char *name, int ver, int apiver) REDISMODULE_ATTR; REDISMODULE_API int (*RedisModule_IsModuleNameBusy)(const char *name) REDISMODULE_ATTR; REDISMODULE_API int (*RedisModule_WrongArity)(RedisModuleCtx *ctx) REDISMODULE_ATTR; @@ -810,6 +946,9 @@ REDISMODULE_API long long (*RedisModule_StreamTrimByLength)(RedisModuleKey *key, REDISMODULE_API long long (*RedisModule_StreamTrimByID)(RedisModuleKey *key, int flags, RedisModuleStreamID *id) REDISMODULE_ATTR; REDISMODULE_API int (*RedisModule_IsKeysPositionRequest)(RedisModuleCtx *ctx) REDISMODULE_ATTR; REDISMODULE_API void (*RedisModule_KeyAtPos)(RedisModuleCtx *ctx, int pos) REDISMODULE_ATTR; +REDISMODULE_API void (*RedisModule_KeyAtPosWithFlags)(RedisModuleCtx *ctx, int pos, int flags) REDISMODULE_ATTR; +REDISMODULE_API int (*RedisModule_IsChannelsPositionRequest)(RedisModuleCtx *ctx) REDISMODULE_ATTR; +REDISMODULE_API void (*RedisModule_ChannelAtPosWithFlags)(RedisModuleCtx *ctx, int pos, int flags) REDISMODULE_ATTR; REDISMODULE_API unsigned long long (*RedisModule_GetClientId)(RedisModuleCtx *ctx) REDISMODULE_ATTR; REDISMODULE_API RedisModuleString * (*RedisModule_GetClientUserNameById)(RedisModuleCtx *ctx, uint64_t id) REDISMODULE_ATTR; REDISMODULE_API int (*RedisModule_GetClientInfoById)(void *ci, uint64_t id) REDISMODULE_ATTR; @@ -924,14 +1063,6 @@ REDISMODULE_API int (*RedisModule_GetKeyspaceNotificationFlagsAll)() REDISMODULE REDISMODULE_API int (*RedisModule_IsSubEventSupported)(RedisModuleEvent event, uint64_t subevent) REDISMODULE_ATTR; REDISMODULE_API int (*RedisModule_GetServerVersion)() REDISMODULE_ATTR; REDISMODULE_API int (*RedisModule_GetTypeMethodVersion)() REDISMODULE_ATTR; -#ifdef INCLUDE_UNRELEASED_KEYSPEC_API -REDISMODULE_API int (*RedisModule_AddCommandKeySpec)(RedisModuleCommand *command, const char *specflags, int *spec_id) REDISMODULE_ATTR; -REDISMODULE_API int (*RedisModule_SetCommandKeySpecBeginSearchIndex)(RedisModuleCommand *command, int spec_id, int index) REDISMODULE_ATTR; -REDISMODULE_API int (*RedisModule_SetCommandKeySpecBeginSearchKeyword)(RedisModuleCommand *command, int spec_id, const char *keyword, int startfrom) REDISMODULE_ATTR; -REDISMODULE_API int (*RedisModule_SetCommandKeySpecFindKeysRange)(RedisModuleCommand *command, int spec_id, int lastkey, int keystep, int limit) REDISMODULE_ATTR; -REDISMODULE_API int (*RedisModule_SetCommandKeySpecFindKeysKeynum)(RedisModuleCommand *command, int spec_id, int keynumidx, int firstkey, int keystep) REDISMODULE_ATTR; -#endif - REDISMODULE_API void (*RedisModule_Yield)(RedisModuleCtx *ctx, int flags, const char *busy_reply) REDISMODULE_ATTR; REDISMODULE_API RedisModuleBlockedClient * (*RedisModule_BlockClient)(RedisModuleCtx *ctx, RedisModuleCmdFunc reply_callback, RedisModuleCmdFunc timeout_callback, void (*free_privdata)(RedisModuleCtx*,void*), long long timeout_ms) REDISMODULE_ATTR; REDISMODULE_API int (*RedisModule_UnblockClient)(RedisModuleBlockedClient *bc, void *privdata) REDISMODULE_ATTR; @@ -995,6 +1126,7 @@ REDISMODULE_API int (*RedisModule_AuthenticateClientWithUser)(RedisModuleCtx *ct REDISMODULE_API int (*RedisModule_DeauthenticateAndCloseClient)(RedisModuleCtx *ctx, uint64_t client_id) REDISMODULE_ATTR; REDISMODULE_API RedisModuleString * (*RedisModule_GetClientCertificate)(RedisModuleCtx *ctx, uint64_t id) REDISMODULE_ATTR; REDISMODULE_API int *(*RedisModule_GetCommandKeys)(RedisModuleCtx *ctx, RedisModuleString **argv, int argc, int *num_keys) REDISMODULE_ATTR; +REDISMODULE_API int *(*RedisModule_GetCommandKeysWithFlags)(RedisModuleCtx *ctx, RedisModuleString **argv, int argc, int *num_keys, int **out_flags) REDISMODULE_ATTR; REDISMODULE_API const char *(*RedisModule_GetCurrentCommandName)(RedisModuleCtx *ctx) REDISMODULE_ATTR; REDISMODULE_API int (*RedisModule_RegisterDefragFunc)(RedisModuleCtx *ctx, RedisModuleDefragFunc func) REDISMODULE_ATTR; REDISMODULE_API void *(*RedisModule_DefragAlloc)(RedisModuleDefragCtx *ctx, void *ptr) REDISMODULE_ATTR; @@ -1023,6 +1155,7 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(CreateCommand); REDISMODULE_GET_API(GetCommand); REDISMODULE_GET_API(CreateSubcommand); + REDISMODULE_GET_API(SetCommandInfo); REDISMODULE_GET_API(SetModuleAttribs); REDISMODULE_GET_API(IsModuleNameBusy); REDISMODULE_GET_API(WrongArity); @@ -1136,6 +1269,9 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(StreamTrimByID); REDISMODULE_GET_API(IsKeysPositionRequest); REDISMODULE_GET_API(KeyAtPos); + REDISMODULE_GET_API(KeyAtPosWithFlags); + REDISMODULE_GET_API(IsChannelsPositionRequest); + REDISMODULE_GET_API(ChannelAtPosWithFlags); REDISMODULE_GET_API(GetClientId); REDISMODULE_GET_API(GetClientUserNameById); REDISMODULE_GET_API(GetContextFlags); @@ -1250,13 +1386,6 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(IsSubEventSupported); REDISMODULE_GET_API(GetServerVersion); REDISMODULE_GET_API(GetTypeMethodVersion); -#ifdef INCLUDE_UNRELEASED_KEYSPEC_API - REDISMODULE_GET_API(AddCommandKeySpec); - REDISMODULE_GET_API(SetCommandKeySpecBeginSearchIndex); - REDISMODULE_GET_API(SetCommandKeySpecBeginSearchKeyword); - REDISMODULE_GET_API(SetCommandKeySpecFindKeysRange); - REDISMODULE_GET_API(SetCommandKeySpecFindKeysKeynum); -#endif REDISMODULE_GET_API(Yield); REDISMODULE_GET_API(GetThreadSafeContext); REDISMODULE_GET_API(GetDetachedThreadSafeContext); @@ -1320,6 +1449,7 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(AuthenticateClientWithUser); REDISMODULE_GET_API(GetClientCertificate); REDISMODULE_GET_API(GetCommandKeys); + REDISMODULE_GET_API(GetCommandKeysWithFlags); REDISMODULE_GET_API(GetCurrentCommandName); REDISMODULE_GET_API(RegisterDefragFunc); REDISMODULE_GET_API(DefragAlloc); @@ -1347,7 +1477,6 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int /* Things only defined for the modules core, not exported to modules * including this file. */ #define RedisModuleString robj -#define RedisModuleCommandArg redisCommandArg #endif /* REDISMODULE_CORE */ #endif /* REDISMODULE_H */ diff --git a/src/replication.c b/src/replication.c index e387a8fd4..9c2d110b0 100644 --- a/src/replication.c +++ b/src/replication.c @@ -705,18 +705,12 @@ int replicationSetupSlaveForFullResync(client *slave, long long offset) { * * On success return C_OK, otherwise C_ERR is returned and we proceed * with the usual full resync. */ -int masterTryPartialResynchronization(client *c) { - long long psync_offset, psync_len; +int masterTryPartialResynchronization(client *c, long long psync_offset) { + long long psync_len; char *master_replid = c->argv[1]->ptr; char buf[128]; int buflen; - /* Parse the replication offset asked by the slave. Go to full sync - * on parse error: this should never happen but we try to handle - * it in a robust way compared to aborting. */ - if (getLongLongFromObjectOrReply(c,c->argv[2],&psync_offset,NULL) != - C_OK) goto need_full_resync; - /* Is the replication ID of this master the same advertised by the wannabe * slave via PSYNC? If the replication ID changed this master has a * different replication history, and there is no way to continue. @@ -977,7 +971,14 @@ void syncCommand(client *c) { * So the slave knows the new replid and offset to try a PSYNC later * if the connection with the master is lost. */ if (!strcasecmp(c->argv[0]->ptr,"psync")) { - if (masterTryPartialResynchronization(c) == C_OK) { + long long psync_offset; + if (getLongLongFromObjectOrReply(c, c->argv[2], &psync_offset, NULL) != C_OK) { + serverLog(LL_WARNING, "Replica %s asks for synchronization but with a wrong offset", + replicationGetSlaveName(c)); + return; + } + + if (masterTryPartialResynchronization(c, psync_offset) == C_OK) { server.stat_sync_partial_ok++; return; /* No full resync needed, return. */ } else { @@ -1545,6 +1546,9 @@ void updateSlavesWaitingBgsave(int bgsaveerr, int type) { listNode *ln; listIter li; + /* Note: there's a chance we got here from within the REPLCONF ACK command + * so we must avoid using freeClient, otherwise we'll crash on our way up. */ + listRewind(server.slaves,&li); while((ln = listNext(&li))) { client *slave = ln->value; @@ -1553,7 +1557,7 @@ void updateSlavesWaitingBgsave(int bgsaveerr, int type) { struct redis_stat buf; if (bgsaveerr != C_OK) { - freeClient(slave); + freeClientAsync(slave); serverLog(LL_WARNING,"SYNC failed. BGSAVE child returned an error"); continue; } @@ -1597,7 +1601,7 @@ void updateSlavesWaitingBgsave(int bgsaveerr, int type) { } else { if ((slave->repldbfd = open(server.rdb_filename,O_RDONLY)) == -1 || redis_fstat(slave->repldbfd,&buf) == -1) { - freeClient(slave); + freeClientAsync(slave); serverLog(LL_WARNING,"SYNC failed. Can't open/stat DB after BGSAVE: %s", strerror(errno)); continue; } @@ -1609,7 +1613,7 @@ void updateSlavesWaitingBgsave(int bgsaveerr, int type) { connSetWriteHandler(slave->conn,NULL); if (connSetWriteHandler(slave->conn,sendBulkToSlave) == C_ERR) { - freeClient(slave); + freeClientAsync(slave); continue; } } @@ -2716,7 +2720,7 @@ void syncWithMaster(connection *conn) { return; } - /* If reached this point, we should be in REPL_STATE_RECEIVE_PSYNC. */ + /* If reached this point, we should be in REPL_STATE_RECEIVE_PSYNC_REPLY. */ if (server.repl_state != REPL_STATE_RECEIVE_PSYNC_REPLY) { serverLog(LL_WARNING,"syncWithMaster(): state machine error, " "state should be RECEIVE_PSYNC but is %d", diff --git a/src/script.c b/src/script.c index 14a64b961..d78d9fd6b 100644 --- a/src/script.c +++ b/src/script.c @@ -146,6 +146,7 @@ int scriptPrepareForRun(scriptRunCtx *run_ctx, client *engine_client, client *ca return C_ERR; } + /* Deny writes if we're unale to persist. */ int deny_write_type = writeCommandsDeniedByDiskError(); if (deny_write_type != DISK_ERROR_TYPE_NONE && server.masterhost == NULL) { if (deny_write_type == DISK_ERROR_TYPE_RDB) @@ -164,6 +165,17 @@ int scriptPrepareForRun(scriptRunCtx *run_ctx, client *engine_client, client *ca addReplyError(caller, "Can not execute a script with write flag using *_ro command."); return C_ERR; } + + /* Don't accept write commands if there are not enough good slaves and + * user configured the min-slaves-to-write option. */ + if (server.masterhost == NULL && + server.repl_min_slaves_max_lag && + server.repl_min_slaves_to_write && + server.repl_good_slaves_count < server.repl_min_slaves_to_write) + { + addReplyErrorObject(caller, shared.noreplicaserr); + return C_ERR; + } } } else { /* Special handling for backwards compatibility (no shebang eval[sha]) mode */ @@ -359,6 +371,19 @@ static int scriptVerifyWriteCommandAllow(scriptRunCtx *run_ctx, char **err) { return C_ERR; } + /* Don't accept write commands if there are not enough good slaves and + * user configured the min-slaves-to-write option. Note this only reachable + * for Eval scripts that didn't declare flags, see the other check in + * scriptPrepareForRun */ + if (server.masterhost == NULL && + server.repl_min_slaves_max_lag && + server.repl_min_slaves_to_write && + server.repl_good_slaves_count < server.repl_min_slaves_to_write) + { + *err = sdsdup(shared.noreplicaserr->ptr); + return C_ERR; + } + return C_OK; } @@ -480,32 +505,31 @@ void scriptCall(scriptRunCtx *run_ctx, robj* *argv, int argc, sds *err) { argc = c->argc; struct redisCommand *cmd = lookupCommand(argv, argc); + c->cmd = c->lastcmd = c->realcmd = cmd; if (scriptVerifyCommandArity(cmd, argc, err) != C_OK) { - return; + goto error; } - c->cmd = c->lastcmd = cmd; - /* There are commands that are not allowed inside scripts. */ if (!server.script_disable_deny_script && (cmd->flags & CMD_NOSCRIPT)) { *err = sdsnew("This Redis command is not allowed from script"); - return; + goto error; } if (scriptVerifyAllowStale(c, err) != C_OK) { - return; + goto error; } if (scriptVerifyACL(c, err) != C_OK) { - return; + goto error; } if (scriptVerifyWriteCommandAllow(run_ctx, err) != C_OK) { - return; + goto error; } if (scriptVerifyOOM(run_ctx, err) != C_OK) { - return; + goto error; } if (cmd->flags & CMD_WRITE) { @@ -514,7 +538,7 @@ void scriptCall(scriptRunCtx *run_ctx, robj* *argv, int argc, sds *err) { } if (scriptVerifyClusterState(c, run_ctx->original_client, err) != C_OK) { - return; + goto error; } int call_flags = CMD_CALL_SLOWLOG | CMD_CALL_STATS; @@ -526,6 +550,11 @@ void scriptCall(scriptRunCtx *run_ctx, robj* *argv, int argc, sds *err) { } call(c, call_flags); serverAssert((c->flags & CLIENT_BLOCKED) == 0); + return; + +error: + afterErrorReply(c, *err, sdslen(*err), 0); + incrCommandStatsOnError(cmd, ERROR_COMMAND_REJECTED); } /* Returns the time when the script invocation started */ diff --git a/src/script_lua.c b/src/script_lua.c index d7332cf86..9a08a7e47 100644 --- a/src/script_lua.c +++ b/src/script_lua.c @@ -238,9 +238,12 @@ static void redisProtocolToLuaType_Error(void *ctx, const char *str, size_t len, * to push elements to the stack. On failure, exit with panic. */ serverPanic("lua stack limit reach when parsing redis.call reply"); } - lua_newtable(lua); - lua_pushstring(lua,"err"); - lua_pushlstring(lua,str,len); + sds err_msg = sdscatlen(sdsnew("-"), str, len); + luaPushErrorBuff(lua,err_msg); + /* push a field indicate to ignore updating the stats on this error + * because it was already updated when executing the command. */ + lua_pushstring(lua,"ignore_error_stats_update"); + lua_pushboolean(lua, true); lua_settable(lua,-3); } @@ -428,40 +431,66 @@ static void redisProtocolToLuaType_Double(void *ctx, double d, const char *proto /* This function is used in order to push an error on the Lua stack in the * format used by redis.pcall to return errors, which is a lua table - * with a single "err" field set to the error string. Note that this - * table is never a valid reply by proper commands, since the returned - * tables are otherwise always indexed by integers, never by strings. */ -void luaPushError(lua_State *lua, char *error) { - lua_Debug dbg; + * with an "err" field set to the error string including the error code. + * Note that this table is never a valid reply by proper commands, + * since the returned tables are otherwise always indexed by integers, never by strings. + * + * The function takes ownership on the given err_buffer. */ +void luaPushErrorBuff(lua_State *lua, sds err_buffer) { + sds msg; + sds error_code; /* If debugging is active and in step mode, log errors resulting from * Redis commands. */ if (ldbIsEnabled()) { - ldbLog(sdscatprintf(sdsempty(),"<error> %s",error)); + ldbLog(sdscatprintf(sdsempty(),"<error> %s",err_buffer)); + } + + /* There are two possible formats for the received `error` string: + * 1) "-CODE msg": in this case we remove the leading '-' since we don't store it as part of the lua error format. + * 2) "msg": in this case we prepend a generic 'ERR' code since all error statuses need some error code. + * We support format (1) so this function can reuse the error messages used in other places in redis. + * We support format (2) so it'll be easy to pass descriptive errors to this function without worrying about format. + */ + if (err_buffer[0] == '-') { + /* derive error code from the message */ + char *err_msg = strstr(err_buffer, " "); + if (!err_msg) { + msg = sdsnew(err_buffer+1); + error_code = sdsnew("ERR"); + } else { + *err_msg = '\0'; + msg = sdsnew(err_msg+1); + error_code = sdsnew(err_buffer + 1); + } + sdsfree(err_buffer); + } else { + msg = err_buffer; + error_code = sdsnew("ERR"); } + /* Trim newline at end of string. If we reuse the ready-made Redis error objects (case 1 above) then we might + * have a newline that needs to be trimmed. In any case the lua Redis error table shouldn't end with a newline. */ + msg = sdstrim(msg, "\r\n"); + sds final_msg = sdscatfmt(error_code, " %s", msg); lua_newtable(lua); lua_pushstring(lua,"err"); - - /* Attempt to figure out where this function was called, if possible */ - if(lua_getstack(lua, 1, &dbg) && lua_getinfo(lua, "nSl", &dbg)) { - sds msg = sdscatprintf(sdsempty(), "%s: %d: %s", - dbg.source, dbg.currentline, error); - lua_pushstring(lua, msg); - sdsfree(msg); - } else { - lua_pushstring(lua, error); - } + lua_pushstring(lua, final_msg); lua_settable(lua,-3); + + sdsfree(msg); + sdsfree(final_msg); +} + +void luaPushError(lua_State *lua, const char *error) { + luaPushErrorBuff(lua, sdsnew(error)); } /* In case the error set into the Lua stack by luaPushError() was generated * by the non-error-trapping version of redis.pcall(), which is redis.call(), * this function will raise the Lua error so that the execution of the * script will be halted. */ -int luaRaiseError(lua_State *lua) { - lua_pushstring(lua,"err"); - lua_gettable(lua,-2); +int luaError(lua_State *lua) { return lua_error(lua); } @@ -511,8 +540,15 @@ static void luaReplyToRedisReply(client *c, client* script_client, lua_State *lu lua_gettable(lua,-2); t = lua_type(lua,-1); if (t == LUA_TSTRING) { - addReplyErrorFormat(c,"-%s",lua_tostring(lua,-1)); - lua_pop(lua,2); + lua_pop(lua, 1); /* pop the error message, we will use luaExtractErrorInformation to get error information */ + errorInfo err_info = {0}; + luaExtractErrorInformation(lua, &err_info); + addReplyErrorFormatEx(c, + err_info.ignore_err_stats_update? ERR_REPLY_FLAG_NO_STATS_UPDATE: 0, + "-%s", + err_info.msg); + luaErrorInformationDiscard(&err_info); + lua_pop(lua,1); /* pop the result table */ return; } lua_pop(lua,1); /* Discard field name pushed before. */ @@ -655,55 +691,19 @@ static void luaReplyToRedisReply(client *c, client* script_client, lua_State *lu * Lua redis.* functions implementations. * ------------------------------------------------------------------------- */ -#define LUA_CMD_OBJCACHE_SIZE 32 -#define LUA_CMD_OBJCACHE_MAX_LEN 64 -static int luaRedisGenericCommand(lua_State *lua, int raise_error) { - int j, argc = lua_gettop(lua); - scriptRunCtx* rctx = luaGetFromRegistry(lua, REGISTRY_RUN_CTX_NAME); - if (!rctx) { - luaPushError(lua, "redis.call/pcall can only be called inside a script invocation"); - return luaRaiseError(lua); - } - sds err = NULL; - client* c = rctx->c; - sds reply; - - /* Cached across calls. */ - static robj **argv = NULL; - static int argv_size = 0; - static robj *cached_objects[LUA_CMD_OBJCACHE_SIZE]; - static size_t cached_objects_len[LUA_CMD_OBJCACHE_SIZE]; - static int inuse = 0; /* Recursive calls detection. */ - - /* By using Lua debug hooks it is possible to trigger a recursive call - * to luaRedisGenericCommand(), which normally should never happen. - * To make this function reentrant is futile and makes it slower, but - * we should at least detect such a misuse, and abort. */ - if (inuse) { - char *recursion_warning = - "luaRedisGenericCommand() recursive call detected. " - "Are you doing funny stuff with Lua debug hooks?"; - serverLog(LL_WARNING,"%s",recursion_warning); - luaPushError(lua,recursion_warning); - return 1; - } - inuse++; - +static robj **luaArgsToRedisArgv(lua_State *lua, int *argc) { + int j; /* Require at least one argument */ - if (argc == 0) { - luaPushError(lua, - "Please specify at least one argument for redis.call()"); - inuse--; - return raise_error ? luaRaiseError(lua) : 1; + *argc = lua_gettop(lua); + if (*argc == 0) { + luaPushError(lua, "Please specify at least one argument for this redis lib call"); + return NULL; } /* Build the arguments vector */ - if (argv_size < argc) { - argv = zrealloc(argv,sizeof(robj*)*argc); - argv_size = argc; - } + robj **argv = zcalloc(sizeof(robj*) * *argc); - for (j = 0; j < argc; j++) { + for (j = 0; j < *argc; j++) { char *obj_s; size_t obj_len; char dbuf[64]; @@ -720,38 +720,62 @@ static int luaRedisGenericCommand(lua_State *lua, int raise_error) { if (obj_s == NULL) break; /* Not a string. */ } - /* Try to use a cached object. */ - if (j < LUA_CMD_OBJCACHE_SIZE && cached_objects[j] && - cached_objects_len[j] >= obj_len) - { - sds s = cached_objects[j]->ptr; - argv[j] = cached_objects[j]; - cached_objects[j] = NULL; - memcpy(s,obj_s,obj_len+1); - sdssetlen(s, obj_len); - } else { - argv[j] = createStringObject(obj_s, obj_len); - } + argv[j] = createStringObject(obj_s, obj_len); } + /* Pop all arguments from the stack, we do not need them anymore + * and this way we guaranty we will have room on the stack for the result. */ + lua_pop(lua, *argc); + /* Check if one of the arguments passed by the Lua script * is not a string or an integer (lua_isstring() return true for * integers as well). */ - if (j != argc) { + if (j != *argc) { j--; while (j >= 0) { decrRefCount(argv[j]); j--; } - luaPushError(lua, - "Lua redis() command arguments must be strings or integers"); - inuse--; - return raise_error ? luaRaiseError(lua) : 1; + zfree(argv); + luaPushError(lua, "Lua redis lib command arguments must be strings or integers"); + return NULL; } - /* Pop all arguments from the stack, we do not need them anymore - * and this way we guaranty we will have room on the stack for the result. */ - lua_pop(lua, argc); + return argv; +} + +static int luaRedisGenericCommand(lua_State *lua, int raise_error) { + int j; + scriptRunCtx* rctx = luaGetFromRegistry(lua, REGISTRY_RUN_CTX_NAME); + if (!rctx) { + luaPushError(lua, "redis.call/pcall can only be called inside a script invocation"); + return luaError(lua); + } + sds err = NULL; + client* c = rctx->c; + sds reply; + + int argc; + robj **argv = luaArgsToRedisArgv(lua, &argc); + if (argv == NULL) { + return raise_error ? luaError(lua) : 1; + } + + static int inuse = 0; /* Recursive calls detection. */ + + /* By using Lua debug hooks it is possible to trigger a recursive call + * to luaRedisGenericCommand(), which normally should never happen. + * To make this function reentrant is futile and makes it slower, but + * we should at least detect such a misuse, and abort. */ + if (inuse) { + char *recursion_warning = + "luaRedisGenericCommand() recursive call detected. " + "Are you doing funny stuff with Lua debug hooks?"; + serverLog(LL_WARNING,"%s",recursion_warning); + luaPushError(lua,recursion_warning); + return 1; + } + inuse++; /* Log the command if debugging is active. */ if (ldbIsEnabled()) { @@ -769,11 +793,15 @@ static int luaRedisGenericCommand(lua_State *lua, int raise_error) { ldbLog(cmdlog); } - scriptCall(rctx, argv, argc, &err); if (err) { luaPushError(lua, err); sdsfree(err); + /* push a field indicate to ignore updating the stats on this error + * because it was already updated when executing the command. */ + lua_pushstring(lua,"ignore_error_stats_update"); + lua_pushboolean(lua, true); + lua_settable(lua,-3); goto cleanup; } @@ -810,48 +838,48 @@ static int luaRedisGenericCommand(lua_State *lua, int raise_error) { cleanup: /* Clean up. Command code may have changed argv/argc so we use the * argv/argc of the client instead of the local variables. */ - for (j = 0; j < c->argc; j++) { - robj *o = c->argv[j]; - - /* Try to cache the object in the cached_objects array. - * The object must be small, SDS-encoded, and with refcount = 1 - * (we must be the only owner) for us to cache it. */ - if (j < LUA_CMD_OBJCACHE_SIZE && - o->refcount == 1 && - (o->encoding == OBJ_ENCODING_RAW || - o->encoding == OBJ_ENCODING_EMBSTR) && - sdslen(o->ptr) <= LUA_CMD_OBJCACHE_MAX_LEN) - { - sds s = o->ptr; - if (cached_objects[j]) decrRefCount(cached_objects[j]); - cached_objects[j] = o; - cached_objects_len[j] = sdsalloc(s); - } else { - decrRefCount(o); - } - } - - if (c->argv != argv) { - zfree(c->argv); - argv = NULL; - argv_size = 0; - } - + freeClientArgv(c); c->user = NULL; - c->argv = NULL; - c->argc = 0; + inuse--; if (raise_error) { /* If we are here we should have an error in the stack, in the * form of a table with an "err" field. Extract the string to * return the plain error. */ - inuse--; - return luaRaiseError(lua); + return luaError(lua); } - inuse--; return 1; } +/* Our implementation to lua pcall. + * We need this implementation for backward + * comparability with older Redis versions. + * + * On Redis 7, the error object is a table, + * compare to older version where the error + * object is a string. To keep backward + * comparability we catch the table object + * and just return the error message. */ +static int luaRedisPcall(lua_State *lua) { + int argc = lua_gettop(lua); + lua_pushboolean(lua, 1); /* result place holder */ + lua_insert(lua, 1); + if (lua_pcall(lua, argc - 1, LUA_MULTRET, 0)) { + /* Error */ + lua_remove(lua, 1); /* remove the result place holder, now we have room for at least one element */ + if (lua_istable(lua, -1)) { + lua_getfield(lua, -1, "err"); + if (lua_isstring(lua, -1)) { + lua_replace(lua, -2); /* replace the error message with the table */ + } + } + lua_pushboolean(lua, 0); /* push result */ + lua_insert(lua, 1); + } + return lua_gettop(lua); + +} + /* redis.call() */ static int luaRedisCallCommand(lua_State *lua) { return luaRedisGenericCommand(lua,1); @@ -871,8 +899,8 @@ static int luaRedisSha1hexCommand(lua_State *lua) { char *s; if (argc != 1) { - lua_pushstring(lua, "wrong number of arguments"); - return lua_error(lua); + luaPushError(lua, "wrong number of arguments"); + return luaError(lua); } s = (char*)lua_tolstring(lua,1,&len); @@ -903,7 +931,21 @@ static int luaRedisReturnSingleFieldTable(lua_State *lua, char *field) { /* redis.error_reply() */ static int luaRedisErrorReplyCommand(lua_State *lua) { - return luaRedisReturnSingleFieldTable(lua,"err"); + if (lua_gettop(lua) != 1 || lua_type(lua,-1) != LUA_TSTRING) { + luaPushError(lua, "wrong number or type of arguments"); + return 1; + } + + /* add '-' if not exists */ + const char *err = lua_tostring(lua, -1); + sds err_buff = NULL; + if (err[0] != '-') { + err_buff = sdscatfmt(sdsempty(), "-%s", err); + } else { + err_buff = sdsnew(err); + } + luaPushErrorBuff(lua, err_buff); + return 1; } /* redis.status_reply() */ @@ -920,25 +962,65 @@ static int luaRedisSetReplCommand(lua_State *lua) { scriptRunCtx* rctx = luaGetFromRegistry(lua, REGISTRY_RUN_CTX_NAME); if (!rctx) { - lua_pushstring(lua, "redis.set_repl can only be called inside a script invocation"); - return lua_error(lua); + luaPushError(lua, "redis.set_repl can only be called inside a script invocation"); + return luaError(lua); } if (argc != 1) { - lua_pushstring(lua, "redis.set_repl() requires two arguments."); - return lua_error(lua); + luaPushError(lua, "redis.set_repl() requires two arguments."); + return luaError(lua); } flags = lua_tonumber(lua,-1); if ((flags & ~(PROPAGATE_AOF|PROPAGATE_REPL)) != 0) { - lua_pushstring(lua, "Invalid replication flags. Use REPL_AOF, REPL_REPLICA, REPL_ALL or REPL_NONE."); - return lua_error(lua); + luaPushError(lua, "Invalid replication flags. Use REPL_AOF, REPL_REPLICA, REPL_ALL or REPL_NONE."); + return luaError(lua); } scriptSetRepl(rctx, flags); return 0; } +/* redis.acl_check_cmd() + * + * Checks ACL permissions for given command for the current user. */ +static int luaRedisAclCheckCmdPermissionsCommand(lua_State *lua) { + scriptRunCtx* rctx = luaGetFromRegistry(lua, REGISTRY_RUN_CTX_NAME); + if (!rctx) { + luaPushError(lua, "redis.acl_check_cmd can only be called inside a script invocation"); + return luaError(lua); + } + int raise_error = 0; + + int argc; + robj **argv = luaArgsToRedisArgv(lua, &argc); + + /* Require at least one argument */ + if (argv == NULL) return luaError(lua); + + /* Find command */ + struct redisCommand *cmd; + if ((cmd = lookupCommand(argv, argc)) == NULL) { + luaPushError(lua, "Invalid command passed to redis.acl_check_cmd()"); + raise_error = 1; + } else { + int keyidxptr; + if (ACLCheckAllUserCommandPerm(rctx->original_client->user, cmd, argv, argc, &keyidxptr) != ACL_OK) { + lua_pushboolean(lua, 0); + } else { + lua_pushboolean(lua, 1); + } + } + + while (argc--) decrRefCount(argv[argc]); + zfree(argv); + if (raise_error) + return luaError(lua); + else + return 1; +} + + /* redis.log() */ static int luaLogCommand(lua_State *lua) { int j, argc = lua_gettop(lua); @@ -946,16 +1028,16 @@ static int luaLogCommand(lua_State *lua) { sds log; if (argc < 2) { - lua_pushstring(lua, "redis.log() requires two arguments or more."); - return lua_error(lua); + luaPushError(lua, "redis.log() requires two arguments or more."); + return luaError(lua); } else if (!lua_isnumber(lua,-argc)) { - lua_pushstring(lua, "First argument must be a number (log level)."); - return lua_error(lua); + luaPushError(lua, "First argument must be a number (log level)."); + return luaError(lua); } level = lua_tonumber(lua,-argc); if (level < LL_DEBUG || level > LL_WARNING) { - lua_pushstring(lua, "Invalid debug level."); - return lua_error(lua); + luaPushError(lua, "Invalid debug level."); + return luaError(lua); } if (level < server.verbosity) return 0; @@ -980,20 +1062,20 @@ static int luaLogCommand(lua_State *lua) { static int luaSetResp(lua_State *lua) { scriptRunCtx* rctx = luaGetFromRegistry(lua, REGISTRY_RUN_CTX_NAME); if (!rctx) { - lua_pushstring(lua, "redis.setresp can only be called inside a script invocation"); - return lua_error(lua); + luaPushError(lua, "redis.setresp can only be called inside a script invocation"); + return luaError(lua); } int argc = lua_gettop(lua); if (argc != 1) { - lua_pushstring(lua, "redis.setresp() requires one argument."); - return lua_error(lua); + luaPushError(lua, "redis.setresp() requires one argument."); + return luaError(lua); } int resp = lua_tonumber(lua,-argc); if (resp != 2 && resp != 3) { - lua_pushstring(lua, "RESP version must be 2 or 3."); - return lua_error(lua); + luaPushError(lua, "RESP version must be 2 or 3."); + return luaError(lua); } scriptSetResp(rctx, resp); return 0; @@ -1193,6 +1275,9 @@ void luaRegisterRedisAPI(lua_State* lua) { luaLoadLibraries(lua); luaRemoveUnsupportedFunctions(lua); + lua_pushcfunction(lua,luaRedisPcall); + lua_setglobal(lua, "pcall"); + /* Register the redis commands table and fields */ lua_newtable(lua); @@ -1251,8 +1336,13 @@ void luaRegisterRedisAPI(lua_State* lua) { lua_pushstring(lua,"REPL_ALL"); lua_pushnumber(lua,PROPAGATE_AOF|PROPAGATE_REPL); + lua_settable(lua,-3); + /* redis.acl_check_cmd */ + lua_pushstring(lua,"acl_check_cmd"); + lua_pushcfunction(lua,luaRedisAclCheckCmdPermissionsCommand); lua_settable(lua,-3); + /* Finally set the table as 'redis' global var. */ lua_setglobal(lua,REDIS_API_NAME); @@ -1348,11 +1438,50 @@ static void luaMaskCountHook(lua_State *lua, lua_Debug *ar) { */ lua_sethook(lua, luaMaskCountHook, LUA_MASKLINE, 0); - lua_pushstring(lua,"Script killed by user with SCRIPT KILL..."); - lua_error(lua); + luaPushError(lua,"Script killed by user with SCRIPT KILL..."); + luaError(lua); } } +void luaErrorInformationDiscard(errorInfo *err_info) { + if (err_info->msg) sdsfree(err_info->msg); + if (err_info->source) sdsfree(err_info->source); + if (err_info->line) sdsfree(err_info->line); +} + +void luaExtractErrorInformation(lua_State *lua, errorInfo *err_info) { + if (lua_isstring(lua, -1)) { + err_info->msg = sdscatfmt(sdsempty(), "ERR %s", lua_tostring(lua, -1)); + err_info->line = NULL; + err_info->source = NULL; + err_info->ignore_err_stats_update = 0; + } + + lua_getfield(lua, -1, "err"); + if (lua_isstring(lua, -1)) { + err_info->msg = sdsnew(lua_tostring(lua, -1)); + } + lua_pop(lua, 1); + + lua_getfield(lua, -1, "source"); + if (lua_isstring(lua, -1)) { + err_info->source = sdsnew(lua_tostring(lua, -1)); + } + lua_pop(lua, 1); + + lua_getfield(lua, -1, "line"); + if (lua_isstring(lua, -1)) { + err_info->line = sdsnew(lua_tostring(lua, -1)); + } + lua_pop(lua, 1); + + lua_getfield(lua, -1, "ignore_error_stats_update"); + if (lua_isboolean(lua, -1)) { + err_info->ignore_err_stats_update = lua_toboolean(lua, -1); + } + lua_pop(lua, 1); +} + void luaCallFunction(scriptRunCtx* run_ctx, lua_State *lua, robj** keys, size_t nkeys, robj** args, size_t nargs, int debug_enabled) { client* c = run_ctx->original_client; int delhook = 0; @@ -1410,9 +1539,28 @@ void luaCallFunction(scriptRunCtx* run_ctx, lua_State *lua, robj** keys, size_t } if (err) { - addReplyErrorFormat(c,"Error running script (call to %s): %s\n", - run_ctx->funcname, lua_tostring(lua,-1)); - lua_pop(lua,1); /* Consume the Lua reply and remove error handler. */ + /* Error object is a table of the following format: + * {err='<error msg>', source='<source file>', line=<line>} + * We can construct the error message from this information */ + if (!lua_istable(lua, -1)) { + /* Should not happened, and we should considered assert it */ + addReplyErrorFormat(c,"Error running script (call to %s)\n", run_ctx->funcname); + } else { + errorInfo err_info = {0}; + sds final_msg = sdsempty(); + luaExtractErrorInformation(lua, &err_info); + final_msg = sdscatfmt(final_msg, "-%s", + err_info.msg); + if (err_info.line && err_info.source) { + final_msg = sdscatfmt(final_msg, " script: %s, on %s:%s.", + run_ctx->funcname, + err_info.source, + err_info.line); + } + addReplyErrorSdsEx(c, final_msg, err_info.ignore_err_stats_update? ERR_REPLY_FLAG_NO_STATS_UPDATE : 0); + luaErrorInformationDiscard(&err_info); + } + lua_pop(lua,1); /* Consume the Lua error */ } else { /* On success convert the Lua return value into Redis protocol, and * send it to * the client. */ diff --git a/src/script_lua.h b/src/script_lua.h index ac13178ca..5a4533784 100644 --- a/src/script_lua.h +++ b/src/script_lua.h @@ -58,6 +58,13 @@ #define REGISTRY_SET_GLOBALS_PROTECTION_NAME "__GLOBAL_PROTECTION__" #define REDIS_API_NAME "redis" +typedef struct errorInfo { + sds msg; + sds source; + sds line; + int ignore_err_stats_update; +}errorInfo; + void luaRegisterRedisAPI(lua_State* lua); sds luaGetStringSds(lua_State *lua, int index); void luaEnableGlobalsProtection(lua_State *lua, int is_eval); @@ -65,11 +72,14 @@ void luaRegisterGlobalProtectionFunction(lua_State *lua); void luaSetGlobalProtection(lua_State *lua); void luaRegisterLogFunction(lua_State* lua); void luaRegisterVersion(lua_State* lua); -void luaPushError(lua_State *lua, char *error); -int luaRaiseError(lua_State *lua); +void luaPushErrorBuff(lua_State *lua, sds err_buff); +void luaPushError(lua_State *lua, const char *error); +int luaError(lua_State *lua); void luaSaveOnRegistry(lua_State* lua, const char* name, void* ptr); void* luaGetFromRegistry(lua_State* lua, const char* name); void luaCallFunction(scriptRunCtx* r_ctx, lua_State *lua, robj** keys, size_t nkeys, robj** args, size_t nargs, int debug_enabled); +void luaExtractErrorInformation(lua_State *lua, errorInfo *err_info); +void luaErrorInformationDiscard(errorInfo *err_info); unsigned long luaMemory(lua_State *lua); diff --git a/src/sentinel.c b/src/sentinel.c index f65e29876..eb37e5ede 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -1190,10 +1190,6 @@ int sentinelUpdateSentinelAddressInAllMasters(sentinelRedisInstance *ri) { if (match->link->pc != NULL) instanceLinkCloseConnection(match->link,match->link->pc); - /* Remove any sentinel with port number set to 0 */ - if (match->addr->port == 0) - dictDelete(master->sentinels,match->name); - if (match == ri) continue; /* Address already updated for it. */ /* Update the address of the matching Sentinel by copying the address @@ -2281,6 +2277,16 @@ werr: return C_ERR; } +/* Call sentinelFlushConfig() produce a success/error reply to the + * calling client. + */ +static void sentinelFlushConfigAndReply(client *c) { + if (sentinelFlushConfig() == C_ERR) + addReplyError(c, "Failed to save config file. Check server logs."); + else + addReply(c, shared.ok); +} + /* ====================== hiredis connection handling ======================= */ /* Send the AUTH command with the specified master password if needed. @@ -2868,9 +2874,22 @@ void sentinelProcessHelloMessage(char *hello, int hello_len) { getSentinelRedisInstanceByAddrAndRunID( master->sentinels, token[0],port,NULL); if (other) { + /* If there is already other sentinel with same address (but + * different runid) then remove the old one across all masters */ sentinelEvent(LL_NOTICE,"+sentinel-invalid-addr",other,"%@"); - other->addr->port = 0; /* It means: invalid address. */ - sentinelUpdateSentinelAddressInAllMasters(other); + dictIterator *di; + dictEntry *de; + + /* Keep a copy of runid. 'other' about to be deleted in loop. */ + sds runid_obsolete = sdsnew(other->runid); + + di = dictGetIterator(sentinel.masters); + while((de = dictNext(di)) != NULL) { + sentinelRedisInstance *master = dictGetVal(de); + removeMatchingSentinelFromMaster(master, runid_obsolete); + } + dictReleaseIterator(di); + sdsfree(runid_obsolete); } } @@ -3174,8 +3193,7 @@ void sentinelConfigSetCommand(client *c) { return; } - sentinelFlushConfig(); - addReply(c, shared.ok); + sentinelFlushConfigAndReply(c); /* Drop Sentinel connections to initiate a reconnect if needed. */ if (drop_conns) @@ -3445,7 +3463,6 @@ void addReplySentinelRedisInstance(client *c, sentinelRedisInstance *ri) { } void sentinelSetDebugConfigParameters(client *c){ - int j; int badarg = 0; /* Bad argument position for error reporting. */ char *option; @@ -3464,7 +3481,7 @@ void sentinelSetDebugConfigParameters(client *c){ goto badfmt; } sentinel_info_period = ll; - + } else if (!strcasecmp(option,"ping-period") && moreargs > 0) { /* ping-period <milliseconds> */ robj *o = c->argv[++j]; @@ -3473,7 +3490,7 @@ void sentinelSetDebugConfigParameters(client *c){ goto badfmt; } sentinel_ping_period = ll; - + } else if (!strcasecmp(option,"ask-period") && moreargs > 0) { /* ask-period <milliseconds> */ robj *o = c->argv[++j]; @@ -3482,7 +3499,7 @@ void sentinelSetDebugConfigParameters(client *c){ goto badfmt; } sentinel_ask_period = ll; - + } else if (!strcasecmp(option,"publish-period") && moreargs > 0) { /* publish-period <milliseconds> */ robj *o = c->argv[++j]; @@ -3491,8 +3508,8 @@ void sentinelSetDebugConfigParameters(client *c){ goto badfmt; } sentinel_publish_period = ll; - - }else if (!strcasecmp(option,"default-down-after") && moreargs > 0) { + + } else if (!strcasecmp(option,"default-down-after") && moreargs > 0) { /* default-down-after <milliseconds> */ robj *o = c->argv[++j]; if (getLongLongFromObject(o,&ll) == C_ERR || ll <= 0) { @@ -3500,7 +3517,7 @@ void sentinelSetDebugConfigParameters(client *c){ goto badfmt; } sentinel_default_down_after = ll; - + } else if (!strcasecmp(option,"tilt-trigger") && moreargs > 0) { /* tilt-trigger <milliseconds> */ robj *o = c->argv[++j]; @@ -3509,7 +3526,7 @@ void sentinelSetDebugConfigParameters(client *c){ goto badfmt; } sentinel_tilt_trigger = ll; - + } else if (!strcasecmp(option,"tilt-period") && moreargs > 0) { /* tilt-period <milliseconds> */ robj *o = c->argv[++j]; @@ -3518,7 +3535,7 @@ void sentinelSetDebugConfigParameters(client *c){ goto badfmt; } sentinel_tilt_period = ll; - + } else if (!strcasecmp(option,"slave-reconf-timeout") && moreargs > 0) { /* slave-reconf-timeout <milliseconds> */ robj *o = c->argv[++j]; @@ -3527,7 +3544,7 @@ void sentinelSetDebugConfigParameters(client *c){ goto badfmt; } sentinel_slave_reconf_timeout = ll; - + } else if (!strcasecmp(option,"min-link-reconnect-period") && moreargs > 0) { /* min-link-reconnect-period <milliseconds> */ robj *o = c->argv[++j]; @@ -3536,7 +3553,7 @@ void sentinelSetDebugConfigParameters(client *c){ goto badfmt; } sentinel_min_link_reconnect_period = ll; - + } else if (!strcasecmp(option,"default-failover-timeout") && moreargs > 0) { /* default-failover-timeout <milliseconds> */ robj *o = c->argv[++j]; @@ -3545,7 +3562,7 @@ void sentinelSetDebugConfigParameters(client *c){ goto badfmt; } sentinel_default_failover_timeout = ll; - + } else if (!strcasecmp(option,"election-timeout") && moreargs > 0) { /* election-timeout <milliseconds> */ robj *o = c->argv[++j]; @@ -3554,7 +3571,7 @@ void sentinelSetDebugConfigParameters(client *c){ goto badfmt; } sentinel_election_timeout = ll; - + } else if (!strcasecmp(option,"script-max-runtime") && moreargs > 0) { /* script-max-runtime <milliseconds> */ robj *o = c->argv[++j]; @@ -3563,7 +3580,7 @@ void sentinelSetDebugConfigParameters(client *c){ goto badfmt; } sentinel_script_max_runtime = ll; - + } else if (!strcasecmp(option,"script-retry-delay") && moreargs > 0) { /* script-retry-delay <milliseconds> */ robj *o = c->argv[++j]; @@ -3572,27 +3589,25 @@ void sentinelSetDebugConfigParameters(client *c){ goto badfmt; } sentinel_script_retry_delay = ll; - + } else { addReplyErrorFormat(c,"Unknown option or number of arguments for " - "SENTINEL SET '%s'", option); + "SENTINEL DEBUG '%s'", option); + return; } - } addReply(c,shared.ok); return; badfmt: /* Bad format errors */ - addReplyErrorFormat(c,"Invalid argument '%s' for SENTINEL SET '%s'", + addReplyErrorFormat(c,"Invalid argument '%s' for SENTINEL DEBUG '%s'", (char*)c->argv[badarg]->ptr,option); return; } - void addReplySentinelDebugInfo(client *c) { - void *mbl; int fields = 0; @@ -3621,7 +3636,7 @@ void addReplySentinelDebugInfo(client *c) { addReplyBulkCString(c,"DEFAULT-FAILOVER-TIMEOUT"); addReplyBulkLongLong(c,sentinel_default_failover_timeout); fields++; - + addReplyBulkCString(c,"TILT-TRIGGER"); addReplyBulkLongLong(c,sentinel_tilt_trigger); fields++; @@ -3726,8 +3741,9 @@ void sentinelCommand(client *c) { " Set a global Sentinel configuration parameter.", "CONFIG GET <param>", " Get global Sentinel configuration parameter.", -"DEBUG", +"DEBUG [<param> <value> ...]", " Show a list of configurable time parameters and their values (milliseconds).", +" Or update current configurable parameters values (one or more).", "GET-MASTER-ADDR-BY-NAME <master-name>", " Return the ip and port number of the master with that name.", "FAILOVER <master-name>", @@ -3929,14 +3945,12 @@ NULL if (ri == NULL) { addReplyError(c,sentinelCheckCreateInstanceErrors(SRI_MASTER)); } else { - sentinelFlushConfig(); + sentinelFlushConfigAndReply(c); sentinelEvent(LL_WARNING,"+monitor",ri,"%@ quorum %d",ri->quorum); - addReply(c,shared.ok); } } else if (!strcasecmp(c->argv[1]->ptr,"flushconfig")) { if (c->argc != 2) goto numargserr; - sentinelFlushConfig(); - addReply(c,shared.ok); + sentinelFlushConfigAndReply(c); return; } else if (!strcasecmp(c->argv[1]->ptr,"remove")) { /* SENTINEL REMOVE <name> */ @@ -3947,8 +3961,7 @@ NULL == NULL) return; sentinelEvent(LL_WARNING,"-monitor",ri,"%@"); dictDelete(sentinel.masters,c->argv[2]->ptr); - sentinelFlushConfig(); - addReply(c,shared.ok); + sentinelFlushConfigAndReply(c); } else if (!strcasecmp(c->argv[1]->ptr,"ckquorum")) { /* SENTINEL CKQUORUM <name> */ sentinelRedisInstance *ri; @@ -4092,46 +4105,51 @@ numargserr: addReplyErrorArity(c); } -#define info_section_from_redis(section_name) do { \ - if (defsections || allsections || !strcasecmp(section,section_name)) { \ - sds redissection; \ - if (sections++) info = sdscat(info,"\r\n"); \ - redissection = genRedisInfoString(section_name); \ - info = sdscatlen(info,redissection,sdslen(redissection)); \ - sdsfree(redissection); \ - } \ -} while(0) +void addInfoSectionsToDict(dict *section_dict, char **sections); /* SENTINEL INFO [section] */ void sentinelInfoCommand(client *c) { - if (c->argc > 2) { - addReplyErrorObject(c,shared.syntaxerr); - return; - } + char *sentinel_sections[] = {"server", "clients", "cpu", "stats", "sentinel", NULL}; + int sec_all = 0, sec_everything = 0; + static dict *cached_all_info_sections = NULL; - int defsections = 0, allsections = 0; - char *section = c->argc == 2 ? c->argv[1]->ptr : NULL; - if (section) { - allsections = !strcasecmp(section,"all"); - defsections = !strcasecmp(section,"default"); - } else { - defsections = 1; - } + /* Get requested section list. */ + dict *sections_dict = genInfoSectionDict(c->argv+1, c->argc-1, sentinel_sections, &sec_all, &sec_everything); - int sections = 0; - sds info = sdsempty(); + /* Purge unsupported sections from the requested ones. */ + dictEntry *de; + dictIterator *di = dictGetSafeIterator(sections_dict); + while((de = dictNext(di)) != NULL) { + int i; + sds sec = dictGetKey(de); + for (i=0; sentinel_sections[i]; i++) + if (!strcasecmp(sentinel_sections[i], sec)) + break; + /* section not found? remove it */ + if (!sentinel_sections[i]) + dictDelete(sections_dict, sec); + } + dictReleaseIterator(di); - info_section_from_redis("server"); - info_section_from_redis("clients"); - info_section_from_redis("cpu"); - info_section_from_redis("stats"); + /* Insert explicit all sections (don't pass these vars to genRedisInfoString) */ + if (sec_all || sec_everything) { + releaseInfoSectionDict(sections_dict); + /* We cache this dict as an optimization. */ + if (!cached_all_info_sections) { + cached_all_info_sections = dictCreate(&stringSetDictType); + addInfoSectionsToDict(cached_all_info_sections, sentinel_sections); + } + sections_dict = cached_all_info_sections; + } - if (defsections || allsections || !strcasecmp(section,"sentinel")) { + sds info = genRedisInfoString(sections_dict, 0, 0); + if (sec_all || (dictFind(sections_dict, "sentinel") != NULL)) { dictIterator *di; dictEntry *de; int master_id = 0; - if (sections++) info = sdscat(info,"\r\n"); + if (sdslen(info) != 0) + info = sdscat(info,"\r\n"); info = sdscatprintf(info, "# Sentinel\r\n" "sentinel_masters:%lu\r\n" @@ -4164,7 +4182,8 @@ void sentinelInfoCommand(client *c) { } dictReleaseIterator(di); } - + if (sections_dict != cached_all_info_sections) + releaseInfoSectionDict(sections_dict); addReplyBulkSds(c, info); } @@ -4348,22 +4367,16 @@ void sentinelSetCommand(client *c) { break; } } - if (changes && sentinelFlushConfig() == C_ERR) { - addReplyErrorFormat(c,"Failed to save Sentinel new configuration on disk"); - return; - } - addReply(c,shared.ok); + if (changes) sentinelFlushConfigAndReply(c); return; badfmt: /* Bad format errors */ addReplyErrorFormat(c,"Invalid argument '%s' for SENTINEL SET '%s'", (char*)c->argv[badarg]->ptr,option); seterr: - if (changes && sentinelFlushConfig() == C_ERR) { - addReplyErrorFormat(c,"Failed to save Sentinel new configuration on disk"); - return; - } - return; + /* TODO: Handle the case of both bad input and save error, possibly handling + * SENTINEL SET atomically. */ + if (changes) sentinelFlushConfig(); } /* Our fake PUBLISH command: it is actually useful only to receive hello messages diff --git a/src/server.c b/src/server.c index 565cd9467..00c279837 100644 --- a/src/server.c +++ b/src/server.c @@ -289,6 +289,33 @@ uint64_t dictSdsCaseHash(const void *key) { return dictGenCaseHashFunction((unsigned char*)key, sdslen((char*)key)); } +/* Dict hash function for null terminated string */ +uint64_t distCStrHash(const void *key) { + return dictGenHashFunction((unsigned char*)key, strlen((char*)key)); +} + +/* Dict hash function for null terminated string */ +uint64_t distCStrCaseHash(const void *key) { + return dictGenCaseHashFunction((unsigned char*)key, strlen((char*)key)); +} + +/* Dict compare function for null terminated string */ +int distCStrKeyCompare(dict *d, const void *key1, const void *key2) { + int l1,l2; + UNUSED(d); + + l1 = strlen((char*)key1); + l2 = strlen((char*)key2); + if (l1 != l2) return 0; + return memcmp(key1, key2, l1) == 0; +} + +/* Dict case insensitive compare function for null terminated string */ +int distCStrKeyCaseCompare(dict *d, const void *key1, const void *key2) { + UNUSED(d); + return strcasecmp(key1, key2) == 0; +} + int dictEncObjKeyCompare(dict *d, const void *key1, const void *key2) { robj *o1 = (robj*) key1, *o2 = (robj*) key2; @@ -487,14 +514,13 @@ dictType migrateCacheDictType = { NULL /* allow to expand */ }; -/* Replication cached script dict (server.repl_scriptcache_dict). - * Keys are sds SHA1 strings, while values are not used at all in the current - * implementation. */ -dictType replScriptCacheDictType = { - dictSdsCaseHash, /* hash function */ +/* Dict for for case-insensitive search using null terminated C strings. + * The keys stored in dict are sds though. */ +dictType stringSetDictType = { + distCStrCaseHash, /* hash function */ NULL, /* key dup */ NULL, /* val dup */ - dictSdsKeyCaseCompare, /* key compare */ + distCStrKeyCaseCompare, /* key compare */ dictSdsDestructor, /* key destructor */ NULL, /* val destructor */ NULL /* allow to expand */ @@ -680,6 +706,51 @@ int clientsCronResizeQueryBuffer(client *c) { return 0; } +/* The client output buffer can be adjusted to better fit the memory requirements. + * + * the logic is: + * in case the last observed peak size of the buffer equals the buffer size - we double the size + * in case the last observed peak size of the buffer is less than half the buffer size - we shrink by half. + * The buffer peak will be reset back to the buffer position every server.reply_buffer_peak_reset_time milliseconds + * The function always returns 0 as it never terminates the client. */ +int clientsCronResizeOutputBuffer(client *c, mstime_t now_ms) { + + size_t new_buffer_size = 0; + char *oldbuf = NULL; + const size_t buffer_target_shrink_size = c->buf_usable_size/2; + const size_t buffer_target_expand_size = c->buf_usable_size*2; + + if (buffer_target_shrink_size >= PROTO_REPLY_MIN_BYTES && + c->buf_peak < buffer_target_shrink_size ) + { + new_buffer_size = max(PROTO_REPLY_MIN_BYTES,c->buf_peak+1); + server.stat_reply_buffer_shrinks++; + } else if (buffer_target_expand_size < PROTO_REPLY_CHUNK_BYTES*2 && + c->buf_peak == c->buf_usable_size) + { + new_buffer_size = min(PROTO_REPLY_CHUNK_BYTES,buffer_target_expand_size); + server.stat_reply_buffer_expands++; + } + + /* reset the peak value each server.reply_buffer_peak_reset_time seconds. in case the client will be idle + * it will start to shrink. + */ + if (server.reply_buffer_peak_reset_time >=0 && + now_ms - c->buf_peak_last_reset_time >= server.reply_buffer_peak_reset_time) + { + c->buf_peak = c->bufpos; + c->buf_peak_last_reset_time = now_ms; + } + + if (new_buffer_size) { + oldbuf = c->buf; + c->buf = zmalloc_usable(new_buffer_size, &c->buf_usable_size); + memcpy(c->buf,oldbuf,c->bufpos); + zfree(oldbuf); + } + return 0; +} + /* This function is used in order to track clients using the biggest amount * of memory in the latest few seconds. This way we can provide such information * in the INFO output (clients section), without having to do an O(N) scan for @@ -873,6 +944,8 @@ void clientsCron(void) { * terminated. */ if (clientsCronHandleTimeout(c,now)) continue; if (clientsCronResizeQueryBuffer(c)) continue; + if (clientsCronResizeOutputBuffer(c,now)) continue; + if (clientsCronTrackExpansiveClients(c, curr_peak_mem_usage_slot)) continue; /* Iterating all the clients in getMemoryOverheadData() is too slow and @@ -1234,7 +1307,7 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { } } /* Just for the sake of defensive programming, to avoid forgetting to - * call this function when need. */ + * call this function when needed. */ updateDictResizePolicy(); @@ -1694,6 +1767,7 @@ void createSharedObjects(void) { shared.retrycount = createStringObject("RETRYCOUNT",10); shared.force = createStringObject("FORCE",5); shared.justid = createStringObject("JUSTID",6); + shared.entriesread = createStringObject("ENTRIESREAD",11); shared.lastid = createStringObject("LASTID",6); shared.default_username = createStringObject("default",7); shared.ping = createStringObject("ping",4); @@ -1717,6 +1791,10 @@ void createSharedObjects(void) { sdscatprintf(sdsempty(),"*%d\r\n",j)); shared.bulkhdr[j] = createObject(OBJ_STRING, sdscatprintf(sdsempty(),"$%d\r\n",j)); + shared.maphdr[j] = createObject(OBJ_STRING, + sdscatprintf(sdsempty(),"%%%d\r\n",j)); + shared.sethdr[j] = createObject(OBJ_STRING, + sdscatprintf(sdsempty(),"~%d\r\n",j)); } /* The following two shared objects, minstring and maxstring, are not * actually used for their value but as a special object meaning @@ -1891,7 +1969,7 @@ int restartServer(int flags, mstime_t delay) { rewriteConfig(server.configfile, 0) == -1) { serverLog(LL_WARNING,"Can't restart: configuration rewrite process " - "failed"); + "failed: %s", strerror(errno)); return C_ERR; } @@ -2256,12 +2334,16 @@ void resetServerStats(void) { memset(server.inst_metric[j].samples,0, sizeof(server.inst_metric[j].samples)); } + server.stat_aof_rewrites = 0; + server.stat_rdb_saves = 0; atomicSet(server.stat_net_input_bytes, 0); atomicSet(server.stat_net_output_bytes, 0); server.stat_unexpected_error_replies = 0; server.stat_total_error_replies = 0; server.stat_dump_payload_sanitizations = 0; server.aof_delayed_fsync = 0; + server.stat_reply_buffer_shrinks = 0; + server.stat_reply_buffer_expands = 0; lazyfreeResetStats(); } @@ -2322,6 +2404,7 @@ void initServer(void) { server.blocking_op_nesting = 0; server.thp_enabled = 0; server.cluster_drop_packet_filter = -1; + server.reply_buffer_peak_reset_time = REPLY_BUFFER_DEFAULT_PEAK_RESET_TIME; resetReplicationBuffer(); if ((server.tls_port || server.tls_replication || server.tls_cluster) @@ -2631,12 +2714,14 @@ void setImplicitACLCategories(struct redisCommand *c) { c->acl_categories |= ACL_CATEGORY_SLOW; } -/* Recursively populate the args structure and return the number of args. */ +/* Recursively populate the args structure (setting num_args to the number of + * subargs) and return the number of args. */ int populateArgsStructure(struct redisCommandArg *args) { if (!args) return 0; int count = 0; while (args->name) { + serverAssert(count < INT_MAX); args->num_args = populateArgsStructure(args->subargs); count++; args++; @@ -3050,6 +3135,34 @@ void propagatePendingCommands() { redisOpArrayFree(&server.also_propagate); } +/* Increment the command failure counters (either rejected_calls or failed_calls). + * The decision which counter to increment is done using the flags argument, options are: + * * ERROR_COMMAND_REJECTED - update rejected_calls + * * ERROR_COMMAND_FAILED - update failed_calls + * + * The function also reset the prev_err_count to make sure we will not count the same error + * twice, its possible to pass a NULL cmd value to indicate that the error was counted elsewhere. + * + * The function returns true if stats was updated and false if not. */ +int incrCommandStatsOnError(struct redisCommand *cmd, int flags) { + /* hold the prev error count captured on the last command execution */ + static long long prev_err_count = 0; + int res = 0; + if (cmd) { + if ((server.stat_total_error_replies - prev_err_count) > 0) { + if (flags & ERROR_COMMAND_REJECTED) { + cmd->rejected_calls++; + res = 1; + } else if (flags & ERROR_COMMAND_FAILED) { + cmd->failed_calls++; + res = 1; + } + } + } + prev_err_count = server.stat_total_error_replies; + return res; +} + /* Call() is the core of Redis execution of a command. * * The following flags can be passed: @@ -3091,8 +3204,7 @@ void call(client *c, int flags) { long long dirty; monotime call_timer; uint64_t client_old_flags = c->flags; - struct redisCommand *real_cmd = c->cmd; - static long long prev_err_count; + struct redisCommand *real_cmd = c->realcmd; /* Initialization: clear the flags that must be set by the command on * demand, and initialize the array for additional commands propagation. */ @@ -3113,7 +3225,7 @@ void call(client *c, int flags) { /* Call the command. */ dirty = server.dirty; - prev_err_count = server.stat_total_error_replies; + incrCommandStatsOnError(NULL, 0); /* Update cache time, in case we have nested calls we want to * update only on the first call*/ @@ -3131,11 +3243,13 @@ void call(client *c, int flags) { server.in_nested_call--; - /* Update failed command calls if required. - * We leverage a static variable (prev_err_count) to retain - * the counter across nested function calls and avoid logging - * the same error twice. */ - if ((server.stat_total_error_replies - prev_err_count) > 0) { + /* Update failed command calls if required. */ + + if (!incrCommandStatsOnError(real_cmd, ERROR_COMMAND_FAILED) && c->deferred_reply_errors) { + /* When call is used from a module client, error stats, and total_error_replies + * isn't updated since these errors, if handled by the module, are internal, + * and not reflected to users. however, the commandstats does show these calls + * (made by RM_Call), so it should log if they failed or succeeded. */ real_cmd->failed_calls++; } @@ -3257,7 +3371,6 @@ void call(client *c, int flags) { server.fixed_time_expire--; server.stat_numcommands++; - prev_err_count = server.stat_total_error_replies; /* Record peak memory after each command and before the eviction that runs * before the next command. */ @@ -3393,7 +3506,7 @@ int processCommand(client *c) { /* Now lookup the command and check ASAP about trivial error conditions * such as wrong arity, bad command name and so forth. */ - c->cmd = c->lastcmd = lookupCommand(c->argv,c->argc); + c->cmd = c->lastcmd = c->realcmd = lookupCommand(c->argv,c->argc); if (!c->cmd) { if (isContainerCommandBySds(c->argv[0]->ptr)) { /* If we can't find the command but argv[0] by itself is a command @@ -4146,8 +4259,9 @@ void addReplyFlagsForKeyArgs(client *c, uint64_t flags) { {CMD_KEY_UPDATE, "update"}, {CMD_KEY_INSERT, "insert"}, {CMD_KEY_DELETE, "delete"}, - {CMD_KEY_CHANNEL, "channel"}, + {CMD_KEY_NOT_KEY, "not_key"}, {CMD_KEY_INCOMPLETE, "incomplete"}, + {CMD_KEY_VARIABLE_FLAGS, "variable_flags"}, {0,NULL} }; addReplyCommandFlags(c, flags, docFlagNames); @@ -4435,7 +4549,9 @@ void addReplyCommandInfo(client *c, struct redisCommand *cmd) { /* Output the representation of a Redis command. Used by the COMMAND DOCS. */ void addReplyCommandDocs(client *c, struct redisCommand *cmd) { /* Count our reply len so we don't have to use deferred reply. */ - long maplen = 3; + long maplen = 1; + if (cmd->summary) maplen++; + if (cmd->since) maplen++; if (cmd->complexity) maplen++; if (cmd->doc_flags) maplen++; if (cmd->deprecated_since) maplen++; @@ -4445,12 +4561,16 @@ void addReplyCommandDocs(client *c, struct redisCommand *cmd) { if (cmd->subcommands_dict) maplen++; addReplyMapLen(c, maplen); - addReplyBulkCString(c, "summary"); - addReplyBulkCString(c, cmd->summary); - - addReplyBulkCString(c, "since"); - addReplyBulkCString(c, cmd->since); + if (cmd->summary) { + addReplyBulkCString(c, "summary"); + addReplyBulkCString(c, cmd->summary); + } + if (cmd->since) { + addReplyBulkCString(c, "since"); + addReplyBulkCString(c, cmd->since); + } + /* Always have the group, for module commands the group is always "module". */ addReplyBulkCString(c, "group"); addReplyBulkCString(c, COMMAND_GROUP_STR[cmd->group]); @@ -4484,10 +4604,8 @@ void addReplyCommandDocs(client *c, struct redisCommand *cmd) { } } -/* Helper for COMMAND(S) command - * - * COMMAND(S) GETKEYS cmd arg1 arg2 ... */ -void getKeysSubcommand(client *c) { +/* Helper for COMMAND GETKEYS and GETKEYSANDFLAGS */ +void getKeysSubcommandImpl(client *c, int with_flags) { struct redisCommand *cmd = lookupCommand(c->argv+2,c->argc-2); getKeysResult result = GETKEYS_RESULT_INIT; int j; @@ -4505,7 +4623,7 @@ void getKeysSubcommand(client *c) { return; } - if (!getKeysFromCommand(cmd,c->argv+2,c->argc-2,&result)) { + if (!getKeysFromCommandWithSpecs(cmd,c->argv+2,c->argc-2,GET_KEYSPEC_DEFAULT,&result)) { if (cmd->flags & CMD_NO_MANDATORY_KEYS) { addReplyArrayLen(c,0); } else { @@ -4513,11 +4631,29 @@ void getKeysSubcommand(client *c) { } } else { addReplyArrayLen(c,result.numkeys); - for (j = 0; j < result.numkeys; j++) addReplyBulk(c,c->argv[result.keys[j].pos+2]); + for (j = 0; j < result.numkeys; j++) { + if (!with_flags) { + addReplyBulk(c,c->argv[result.keys[j].pos+2]); + } else { + addReplyArrayLen(c,2); + addReplyBulk(c,c->argv[result.keys[j].pos+2]); + addReplyFlagsForKeyArgs(c,result.keys[j].flags); + } + } } getKeysFreeResult(&result); } +/* COMMAND GETKEYSANDFLAGS cmd arg1 arg2 ... */ +void commandGetKeysAndFlagsCommand(client *c) { + getKeysSubcommandImpl(c, 1); +} + +/* COMMAND GETKEYS cmd arg1 arg2 ... */ +void getKeysSubcommand(client *c) { + getKeysSubcommandImpl(c, 0); +} + /* COMMAND (no args) */ void commandCommand(client *c) { dictIterator *di; @@ -4734,6 +4870,8 @@ void commandHelpCommand(client *c) { " commands are returned.", "GETKEYS <full-command>", " Return the keys from a full Redis command.", +"GETKEYSANDFLAGS <full-command>", +" Return the keys and the access flags from a full Redis command.", NULL }; @@ -4867,25 +5005,78 @@ sds genRedisInfoStringLatencyStats(sds info, dict *commands) { return info; } +/* Takes a null terminated sections list, and adds them to the dict. */ +void addInfoSectionsToDict(dict *section_dict, char **sections) { + while (*sections) { + sds section = sdsnew(*sections); + if (dictAdd(section_dict, section, NULL)==DICT_ERR) + sdsfree(section); + sections++; + } +} + +/* Cached copy of the default sections, as an optimization. */ +static dict *cached_default_info_sections = NULL; + +void releaseInfoSectionDict(dict *sec) { + if (sec != cached_default_info_sections) + dictRelease(sec); +} + +/* Create a dictionary with unique section names to be used by genRedisInfoString. + * 'argv' and 'argc' are list of arguments for INFO. + * 'defaults' is an optional null terminated list of default sections. + * 'out_all' and 'out_everything' are optional. + * The resulting dictionary should be released with releaseInfoSectionDict. */ +dict *genInfoSectionDict(robj **argv, int argc, char **defaults, int *out_all, int *out_everything) { + char *default_sections[] = { + "server", "clients", "memory", "persistence", "stats", "replication", + "cpu", "module_list", "errorstats", "cluster", "keyspace", NULL}; + if (!defaults) + defaults = default_sections; + + if (argc == 0) { + /* In this case we know the dict is not gonna be modified, so we cache + * it as an optimization for a common case. */ + if (cached_default_info_sections) + return cached_default_info_sections; + cached_default_info_sections = dictCreate(&stringSetDictType); + dictExpand(cached_default_info_sections, 16); + addInfoSectionsToDict(cached_default_info_sections, defaults); + return cached_default_info_sections; + } + + dict *section_dict = dictCreate(&stringSetDictType); + dictExpand(section_dict, min(argc,16)); + for (int i = 0; i < argc; i++) { + if (!strcasecmp(argv[i]->ptr,"default")) { + addInfoSectionsToDict(section_dict, defaults); + } else if (!strcasecmp(argv[i]->ptr,"all")) { + if (out_all) *out_all = 1; + } else if (!strcasecmp(argv[i]->ptr,"everything")) { + if (out_everything) *out_everything = 1; + if (out_all) *out_all = 1; + } else { + sds section = sdsnew(argv[i]->ptr); + if (dictAdd(section_dict, section, NULL) != DICT_OK) + sdsfree(section); + } + } + return section_dict; +} + /* Create the string returned by the INFO command. This is decoupled * by the INFO command itself as we need to report the same information * on memory corruption problems. */ -sds genRedisInfoString(const char *section) { +sds genRedisInfoString(dict *section_dict, int all_sections, int everything) { sds info = sdsempty(); time_t uptime = server.unixtime-server.stat_starttime; int j; - int allsections = 0, defsections = 0, everything = 0, modules = 0; int sections = 0; - - if (section == NULL) section = "default"; - allsections = strcasecmp(section,"all") == 0; - defsections = strcasecmp(section,"default") == 0; - everything = strcasecmp(section,"everything") == 0; - modules = strcasecmp(section,"modules") == 0; - if (everything) allsections = 1; + if (everything) all_sections = 1; /* Server */ - if (allsections || defsections || !strcasecmp(section,"server")) { + if (all_sections || (dictFind(section_dict,"server") != NULL)) { static int call_uname = 1; static struct utsname name; char *mode; @@ -4975,7 +5166,7 @@ sds genRedisInfoString(const char *section) { } /* Clients */ - if (allsections || defsections || !strcasecmp(section,"clients")) { + if (all_sections || (dictFind(section_dict,"clients") != NULL)) { size_t maxin, maxout; getExpansiveClientsInfo(&maxin,&maxout); if (sections++) info = sdscat(info,"\r\n"); @@ -4999,7 +5190,7 @@ sds genRedisInfoString(const char *section) { } /* Memory */ - if (allsections || defsections || !strcasecmp(section,"memory")) { + if (all_sections || (dictFind(section_dict,"memory") != NULL)) { char hmem[64]; char peak_hmem[64]; char total_system_hmem[64]; @@ -5144,7 +5335,7 @@ sds genRedisInfoString(const char *section) { } /* Persistence */ - if (allsections || defsections || !strcasecmp(section,"persistence")) { + if (all_sections || (dictFind(section_dict,"persistence") != NULL)) { if (sections++) info = sdscat(info,"\r\n"); double fork_perc = 0; if (server.stat_module_progress) { @@ -5171,6 +5362,7 @@ sds genRedisInfoString(const char *section) { "rdb_last_bgsave_status:%s\r\n" "rdb_last_bgsave_time_sec:%jd\r\n" "rdb_current_bgsave_time_sec:%jd\r\n" + "rdb_saves:%lld\r\n" "rdb_last_cow_size:%zu\r\n" "rdb_last_load_keys_expired:%lld\r\n" "rdb_last_load_keys_loaded:%lld\r\n" @@ -5180,6 +5372,7 @@ sds genRedisInfoString(const char *section) { "aof_last_rewrite_time_sec:%jd\r\n" "aof_current_rewrite_time_sec:%jd\r\n" "aof_last_bgrewrite_status:%s\r\n" + "aof_rewrites:%lld\r\n" "aof_last_write_status:%s\r\n" "aof_last_cow_size:%zu\r\n" "module_fork_in_progress:%d\r\n" @@ -5199,6 +5392,7 @@ sds genRedisInfoString(const char *section) { (intmax_t)server.rdb_save_time_last, (intmax_t)((server.child_type != CHILD_TYPE_RDB) ? -1 : time(NULL)-server.rdb_save_time_start), + server.stat_rdb_saves, server.stat_rdb_cow_bytes, server.rdb_last_load_keys_expired, server.rdb_last_load_keys_loaded, @@ -5209,6 +5403,7 @@ sds genRedisInfoString(const char *section) { (intmax_t)((server.child_type != CHILD_TYPE_AOF) ? -1 : time(NULL)-server.aof_rewrite_time_start), (server.aof_lastbgrewrite_status == C_OK) ? "ok" : "err", + server.stat_aof_rewrites, (server.aof_last_write_status == C_OK && aof_bio_fsync_status == C_OK) ? "ok" : "err", server.stat_aof_cow_bytes, @@ -5273,7 +5468,7 @@ sds genRedisInfoString(const char *section) { } /* Stats */ - if (allsections || defsections || !strcasecmp(section,"stats")) { + if (all_sections || (dictFind(section_dict,"stats") != NULL)) { long long stat_total_reads_processed, stat_total_writes_processed; long long stat_net_input_bytes, stat_net_output_bytes; long long current_eviction_exceeded_time = server.stat_last_eviction_exceeded_time ? @@ -5330,7 +5525,9 @@ sds genRedisInfoString(const char *section) { "total_reads_processed:%lld\r\n" "total_writes_processed:%lld\r\n" "io_threaded_reads_processed:%lld\r\n" - "io_threaded_writes_processed:%lld\r\n", + "io_threaded_writes_processed:%lld\r\n" + "reply_buffer_shrinks:%lld\r\n" + "reply_buffer_expands:%lld\r\n", server.stat_numconnections, server.stat_numcommands, getInstantaneousMetric(STATS_METRIC_COMMAND), @@ -5373,11 +5570,13 @@ sds genRedisInfoString(const char *section) { stat_total_reads_processed, stat_total_writes_processed, server.stat_io_reads_processed, - server.stat_io_writes_processed); + server.stat_io_writes_processed, + server.stat_reply_buffer_shrinks, + server.stat_reply_buffer_expands); } /* Replication */ - if (allsections || defsections || !strcasecmp(section,"replication")) { + if (all_sections || (dictFind(section_dict,"replication") != NULL)) { if (sections++) info = sdscat(info,"\r\n"); info = sdscatprintf(info, "# Replication\r\n" @@ -5513,7 +5712,7 @@ sds genRedisInfoString(const char *section) { } /* CPU */ - if (allsections || defsections || !strcasecmp(section,"cpu")) { + if (all_sections || (dictFind(section_dict,"cpu") != NULL)) { if (sections++) info = sdscat(info,"\r\n"); struct rusage self_ru, c_ru; @@ -5541,20 +5740,21 @@ sds genRedisInfoString(const char *section) { } /* Modules */ - if (allsections || defsections || !strcasecmp(section,"modules")) { + if (all_sections || (dictFind(section_dict,"module_list") != NULL) || (dictFind(section_dict,"modules") != NULL)) { if (sections++) info = sdscat(info,"\r\n"); info = sdscatprintf(info,"# Modules\r\n"); info = genModulesInfoString(info); } /* Command statistics */ - if (allsections || !strcasecmp(section,"commandstats")) { + if (all_sections || (dictFind(section_dict,"commandstats") != NULL)) { if (sections++) info = sdscat(info,"\r\n"); info = sdscatprintf(info, "# Commandstats\r\n"); info = genRedisInfoStringCommandStats(info, server.commands); } + /* Error statistics */ - if (allsections || defsections || !strcasecmp(section,"errorstats")) { + if (all_sections || (dictFind(section_dict,"errorstats") != NULL)) { if (sections++) info = sdscat(info,"\r\n"); info = sdscat(info, "# Errorstats\r\n"); raxIterator ri; @@ -5573,7 +5773,7 @@ sds genRedisInfoString(const char *section) { } /* Latency by percentile distribution per command */ - if (allsections || !strcasecmp(section,"latencystats")) { + if (all_sections || (dictFind(section_dict,"latencystats") != NULL)) { if (sections++) info = sdscat(info,"\r\n"); info = sdscatprintf(info, "# Latencystats\r\n"); if (server.latency_tracking_enabled) { @@ -5582,7 +5782,7 @@ sds genRedisInfoString(const char *section) { } /* Cluster */ - if (allsections || defsections || !strcasecmp(section,"cluster")) { + if (all_sections || (dictFind(section_dict,"cluster") != NULL)) { if (sections++) info = sdscat(info,"\r\n"); info = sdscatprintf(info, "# Cluster\r\n" @@ -5591,7 +5791,7 @@ sds genRedisInfoString(const char *section) { } /* Key space */ - if (allsections || defsections || !strcasecmp(section,"keyspace")) { + if (all_sections || (dictFind(section_dict,"keyspace") != NULL)) { if (sections++) info = sdscat(info,"\r\n"); info = sdscatprintf(info, "# Keyspace\r\n"); for (j = 0; j < server.dbnum; j++) { @@ -5610,10 +5810,10 @@ sds genRedisInfoString(const char *section) { /* Get info from modules. * if user asked for "everything" or "modules", or a specific section * that's not found yet. */ - if (everything || modules || - (!allsections && !defsections && sections==0)) { + if (everything || dictFind(section_dict, "modules") != NULL || sections < (int)dictSize(section_dict)) { + info = modulesCollectInfo(info, - everything || modules ? NULL: section, + everything || dictFind(section_dict, "modules") != NULL ? NULL: section_dict, 0, /* not a crash report */ sections); } @@ -5625,16 +5825,14 @@ void infoCommand(client *c) { sentinelInfoCommand(c); return; } - - char *section = c->argc == 2 ? c->argv[1]->ptr : "default"; - - if (c->argc > 2) { - addReplyErrorObject(c,shared.syntaxerr); - return; - } - sds info = genRedisInfoString(section); + int all_sections = 0; + int everything = 0; + dict *sections_dict = genInfoSectionDict(c->argv+1, c->argc-1, NULL, &all_sections, &everything); + sds info = genRedisInfoString(sections_dict, all_sections, everything); addReplyVerbatim(c,info,sdslen(info),"txt"); sdsfree(info); + releaseInfoSectionDict(sections_dict); + return; } void monitorCommand(client *c) { diff --git a/src/server.h b/src/server.h index 485c051fd..4da7a010f 100644 --- a/src/server.h +++ b/src/server.h @@ -57,6 +57,10 @@ #include <systemd/sd-daemon.h> #endif +#ifndef static_assert +#define static_assert(expr, lit) extern char __static_assert_failure[(expr) ? 1:-1] +#endif + typedef long long mstime_t; /* millisecond time type. */ typedef long long ustime_t; /* microsecond time type. */ @@ -105,6 +109,7 @@ typedef long long ustime_t; /* microsecond time type. */ #define PROTO_SHARED_SELECT_CMDS 10 #define OBJ_SHARED_INTEGERS 10000 #define OBJ_SHARED_BULKHDR_LEN 32 +#define OBJ_SHARED_HDR_STRLEN(_len_) (((_len_) < 10) ? 4 : 5) /* see shared.mbulkhdr etc. */ #define LOG_MAX_LEN 1024 /* Default maximum length of syslog messages.*/ #define AOF_REWRITE_ITEMS_PER_CMD 64 #define AOF_ANNOTATION_LINE_MAX_LEN 1024 @@ -159,11 +164,14 @@ typedef long long ustime_t; /* microsecond time type. */ #define PROTO_INLINE_MAX_SIZE (1024*64) /* Max size of inline reads */ #define PROTO_MBULK_BIG_ARG (1024*32) #define PROTO_RESIZE_THRESHOLD (1024*32) /* Threshold for determining whether to resize query buffer */ +#define PROTO_REPLY_MIN_BYTES (1024) /* the lower limit on reply buffer size */ #define LONG_STR_SIZE 21 /* Bytes needed for long -> str + '\0' */ #define REDIS_AUTOSYNC_BYTES (1024*1024*4) /* Sync file every 4MB. */ #define LIMIT_PENDING_QUERYBUF (4*1024*1024) /* 4mb */ +#define REPLY_BUFFER_DEFAULT_PEAK_RESET_TIME 5000 /* 5 seconds */ + /* When configuring the server eventloop, we setup it so that the total number * of file descriptors we can handle are server.maxclients + RESERVED_FDS + * a few more to stay safe. Since RESERVED_FDS defaults to 32, we add 96 @@ -210,6 +218,7 @@ extern int configOOMScoreAdjValuesDefaults[CONFIG_OOM_COUNT]; #define CMD_NO_MULTI (1ULL<<24) #define CMD_MOVABLE_KEYS (1ULL<<25) /* populated by populateCommandMovableKeys */ #define CMD_ALLOW_BUSY ((1ULL<<26)) +#define CMD_MODULE_GETCHANNELS (1ULL<<27) /* Use the modules getchannels interface. */ /* Command flags that describe ACLs categories. */ #define ACL_CATEGORY_KEYSPACE (1ULL<<0) @@ -262,12 +271,23 @@ extern int configOOMScoreAdjValuesDefaults[CONFIG_OOM_COUNT]; #define CMD_KEY_DELETE (1ULL<<7) /* Explicitly deletes some content * from the value of the key. */ /* Other flags: */ -#define CMD_KEY_CHANNEL (1ULL<<8) /* PUBSUB shard channel */ +#define CMD_KEY_NOT_KEY (1ULL<<8) /* A 'fake' key that should be routed + * like a key in cluster mode but is + * excluded from other key checks. */ #define CMD_KEY_INCOMPLETE (1ULL<<9) /* Means that the keyspec might not point * out to all keys it should cover */ #define CMD_KEY_VARIABLE_FLAGS (1ULL<<10) /* Means that some keys might have * different flags depending on arguments */ +/* Key flags for when access type is unknown */ +#define CMD_KEY_FULL_ACCESS (CMD_KEY_RW | CMD_KEY_ACCESS | CMD_KEY_UPDATE) + +/* Channel flags share the same flag space as the key flags */ +#define CMD_CHANNEL_PATTERN (1ULL<<11) /* The argument is a channel pattern */ +#define CMD_CHANNEL_SUBSCRIBE (1ULL<<12) /* The command subscribes to channels */ +#define CMD_CHANNEL_UNSUBSCRIBE (1ULL<<13) /* The command unsubscribes to channels */ +#define CMD_CHANNEL_PUBLISH (1ULL<<14) /* The command publishes to channels. */ + /* AOF states */ #define AOF_OFF 0 /* AOF is off */ #define AOF_ON 1 /* AOF is on */ @@ -1076,6 +1096,9 @@ typedef struct client { robj **original_argv; /* Arguments of original command if arguments were rewritten. */ size_t argv_len_sum; /* Sum of lengths of objects in argv list. */ struct redisCommand *cmd, *lastcmd; /* Last command executed. */ + struct redisCommand *realcmd; /* The original command that was executed by the client, + Used to update error stats in case the c->cmd was modified + during the command invocation (like on GEOADD for example). */ user *user; /* User associated with this connection. If the user is set to NULL the connection can do anything (admin). */ @@ -1084,6 +1107,7 @@ typedef struct client { long bulklen; /* Length of bulk argument in multi bulk request. */ list *reply; /* List of reply objects to send to the client. */ unsigned long long reply_bytes; /* Tot bytes of objects in reply list. */ + list *deferred_reply_errors; /* Used for module thread safe contexts. */ size_t sentlen; /* Amount of bytes already sent in the current buffer or object being sent. */ time_t ctime; /* Client creation time. */ @@ -1159,14 +1183,11 @@ typedef struct client { * i.e. the next offset to send. */ /* Response buffer */ + size_t buf_peak; /* Peak used size of buffer in last 5 sec interval. */ + mstime_t buf_peak_last_reset_time; /* keeps the last time the buffer peak value was reset */ int bufpos; size_t buf_usable_size; /* Usable size of buffer. */ - /* Note that 'buf' must be the last field of client struct, because memory - * allocator may give us more memory than our apply for reducing fragments, - * but we want to make full use of given memory, i.e. we may access the - * memory after 'buf'. To avoid make others fields corrupt, 'buf' must be - * the last one. */ - char buf[PROTO_REPLY_CHUNK_BYTES]; + char *buf; } client; struct saveparam { @@ -1205,14 +1226,16 @@ struct sharedObjectsStruct { *rpop, *lpop, *lpush, *rpoplpush, *lmove, *blmove, *zpopmin, *zpopmax, *emptyscan, *multi, *exec, *left, *right, *hset, *srem, *xgroup, *xclaim, *script, *replconf, *eval, *persist, *set, *pexpireat, *pexpire, - *time, *pxat, *absttl, *retrycount, *force, *justid, + *time, *pxat, *absttl, *retrycount, *force, *justid, *entriesread, *lastid, *ping, *setid, *keepttl, *load, *createconsumer, *getack, *special_asterick, *special_equals, *default_username, *redacted, *ssubscribebulk,*sunsubscribebulk, *select[PROTO_SHARED_SELECT_CMDS], *integers[OBJ_SHARED_INTEGERS], *mbulkhdr[OBJ_SHARED_BULKHDR_LEN], /* "*<value>\r\n" */ - *bulkhdr[OBJ_SHARED_BULKHDR_LEN]; /* "$<value>\r\n" */ + *bulkhdr[OBJ_SHARED_BULKHDR_LEN], /* "$<value>\r\n" */ + *maphdr[OBJ_SHARED_BULKHDR_LEN], /* "%<value>\r\n" */ + *sethdr[OBJ_SHARED_BULKHDR_LEN]; /* "~<value>\r\n" */ sds minstring, maxstring; }; @@ -1536,6 +1559,8 @@ struct redisServer { long long stat_total_active_defrag_time; /* Total time memory fragmentation over the limit, unit us */ monotime stat_last_active_defrag_time; /* Timestamp of current active defrag start */ size_t stat_peak_memory; /* Max used memory record */ + long long stat_aof_rewrites; /* number of aof file rewrites performed */ + long long stat_rdb_saves; /* number of rdb saves performed */ long long stat_fork_time; /* Time needed to perform latest fork() */ double stat_fork_rate; /* Fork rate in GB/sec. */ long long stat_total_forks; /* Total count of fork. */ @@ -1576,6 +1601,9 @@ struct redisServer { long long samples[STATS_METRIC_SAMPLES]; int idx; } inst_metric[STATS_METRIC_COUNT]; + long long stat_reply_buffer_shrinks; /* Total number of output buffer shrinks */ + long long stat_reply_buffer_expands; /* Total number of output buffer expands */ + /* Configuration */ int verbosity; /* Loglevel in redis.conf */ int maxidletime; /* Client timeout in seconds */ @@ -1635,7 +1663,7 @@ struct redisServer { int aof_last_write_status; /* C_OK or C_ERR */ int aof_last_write_errno; /* Valid if aof write/fsync status is ERR */ int aof_load_truncated; /* Don't stop on unexpected AOF EOF. */ - int aof_use_rdb_preamble; /* Use RDB preamble on AOF rewrites. */ + int aof_use_rdb_preamble; /* Specify base AOF to use RDB encoding on AOF rewrites. */ redisAtomic int aof_bio_fsync_status; /* Status of AOF fsync in bio job. */ redisAtomic int aof_bio_fsync_errno; /* Errno of AOF fsync in bio job. */ aofManifest *aof_manifest; /* Used to track AOFs. */ @@ -1752,10 +1780,6 @@ struct redisServer { char master_replid[CONFIG_RUN_ID_SIZE+1]; /* Master PSYNC runid. */ long long master_initial_offset; /* Master PSYNC offset. */ int repl_slave_lazy_flush; /* Lazy FLUSHALL before loading DB? */ - /* Replication script cache. */ - dict *repl_scriptcache_dict; /* SHA1 all slaves are aware of. */ - list *repl_scriptcache_fifo; /* First in, first out LRU eviction. */ - unsigned int repl_scriptcache_size; /* Max number of elements. */ /* Synchronous replication. */ list *clients_waiting_acks; /* Clients waiting in WAIT command. */ int get_ack_from_slaves; /* If true we send REPLCONF GETACK. */ @@ -1889,6 +1913,7 @@ struct redisServer { int failover_state; /* Failover state */ int cluster_allow_pubsubshard_when_down; /* Is pubsubshard allowed when the cluster is down, doesn't affect pubsub global. */ + long reply_buffer_peak_reset_time; /* The amount of time (in milliseconds) to wait between reply buffer peak resets */ }; #define MAX_KEYS_BUFFER 256 @@ -1900,7 +1925,8 @@ typedef struct { } keyReference; /* A result structure for the various getkeys function calls. It lists the - * keys as indices to the provided argv. + * keys as indices to the provided argv. This functionality is also re-used + * for returning channel information. */ typedef struct { keyReference keysbuf[MAX_KEYS_BUFFER]; /* Pre-allocated buffer, to save heap allocations */ @@ -2294,11 +2320,12 @@ extern struct sharedObjectsStruct shared; extern dictType objectKeyPointerValueDictType; extern dictType objectKeyHeapPointerValueDictType; extern dictType setDictType; +extern dictType BenchmarkDictType; extern dictType zsetDictType; extern dictType dbDictType; extern double R_Zero, R_PosInf, R_NegInf, R_Nan; extern dictType hashDictType; -extern dictType replScriptCacheDictType; +extern dictType stringSetDictType; extern dictType dbExpiresDictType; extern dictType modulesDictType; extern dictType sdsReplyDictType; @@ -2308,8 +2335,9 @@ extern dict *modules; * Functions prototypes *----------------------------------------------------------------------------*/ -/* Key arguments specs */ +/* Command metadata */ void populateCommandLegacyRangeSpec(struct redisCommand *c); +int populateArgsStructure(struct redisCommandArg *args); /* Modules */ void moduleInitModulesSystem(void); @@ -2318,6 +2346,7 @@ void modulesCron(void); int moduleLoad(const char *path, void **argv, int argc); void moduleLoadFromQueue(void); int moduleGetCommandKeysViaAPI(struct redisCommand *cmd, robj **argv, int argc, getKeysResult *result); +int moduleGetCommandChannelsViaAPI(struct redisCommand *cmd, robj **argv, int argc, getKeysResult *result); moduleType *moduleTypeLookupModuleByID(uint64_t id); void moduleTypeNameByID(char *name, uint64_t moduleid); const char *moduleTypeModuleName(moduleType *mt); @@ -2337,7 +2366,7 @@ int TerminateModuleForkChild(int child_pid, int wait); ssize_t rdbSaveModulesAux(rio *rdb, int when); int moduleAllDatatypesHandleErrors(); int moduleAllModulesHandleReplAsyncLoad(); -sds modulesCollectInfo(sds info, const char *section, int for_crash_report, int sections); +sds modulesCollectInfo(sds info, dict *sections_dict, int for_crash_report, int sections); void moduleFireServerEvent(uint64_t eid, int subid, void *data); void processModuleLoadingProgressEvent(int is_aof); int moduleTryServeClientBlockedOnKey(client *c, robj *key); @@ -2368,6 +2397,9 @@ int validateProcTitleTemplate(const char *template); int redisCommunicateSystemd(const char *sd_notify_msg); void redisSetCpuAffinity(const char *cpulist); +/* afterErrorReply flags */ +#define ERR_REPLY_FLAG_NO_STATS_UPDATE (1ULL<<0) /* Indicating that we should not update + error stats after sending error reply */ /* networking.c -- Networking and Client related operations */ client *createClient(connection *conn); void freeClient(client *c); @@ -2377,6 +2409,7 @@ int beforeNextClient(client *c); void clearClientConnectionState(client *c); void resetClient(client *c); void freeClientOriginalArgv(client *c); +void freeClientArgv(client *c); void sendReplyToClient(connection *conn); void *addReplyDeferredLen(client *c); void setDeferredArrayLen(client *c, void *node, long length); @@ -2406,6 +2439,8 @@ void addReplyBulkSds(client *c, sds s); void setDeferredReplyBulkSds(client *c, void *node, sds s); void addReplyErrorObject(client *c, robj *err); void addReplyOrErrorObject(client *c, robj *reply); +void afterErrorReply(client *c, const char *s, size_t len, int flags); +void addReplyErrorSdsEx(client *c, sds err, int flags); void addReplyErrorSds(client *c, sds err); void addReplyError(client *c, const char *err); void addReplyErrorArity(client *c); @@ -2426,6 +2461,7 @@ void addReplySubcommandSyntaxError(client *c); void addReplyLoadedModules(client *c); void copyReplicaOutputBuffer(client *dst, client *src); void addListRangeReply(client *c, robj *o, long start, long end, int reverse); +void deferredAfterErrorReply(client *c, list *errors); size_t sdsZmallocSize(sds s); size_t getStringObjectSdsUsedMemory(robj *o); void freeClientReplyValue(void *o); @@ -2477,11 +2513,14 @@ int authRequired(client *c); void clientInstallWriteHandler(client *c); #ifdef __GNUC__ +void addReplyErrorFormatEx(client *c, int flags, const char *fmt, ...) + __attribute__((format(printf, 3, 4))); void addReplyErrorFormat(client *c, const char *fmt, ...) __attribute__((format(printf, 2, 3))); void addReplyStatusFormat(client *c, const char *fmt, ...) __attribute__((format(printf, 2, 3))); #else +void addReplyErrorFormatEx(client *c, int flags, const char *fmt, ...); void addReplyErrorFormat(client *c, const char *fmt, ...); void addReplyStatusFormat(client *c, const char *fmt, ...); #endif @@ -2760,6 +2799,10 @@ typedef struct { int minex, maxex; /* are min or max exclusive? */ } zlexrangespec; +/* flags for incrCommandFailedCalls */ +#define ERROR_COMMAND_REJECTED (1<<0) /* Indicate to update the command rejected stats */ +#define ERROR_COMMAND_FAILED (1<<1) /* Indicate to update the command failed stats */ + zskiplist *zslCreate(void); void zslFree(zskiplist *zsl); zskiplistNode *zslInsert(zskiplist *zsl, double score, sds ele); @@ -2814,6 +2857,8 @@ struct redisCommand *lookupCommandBySds(sds s); struct redisCommand *lookupCommandByCStringLogic(dict *commands, const char *s); struct redisCommand *lookupCommandByCString(const char *s); struct redisCommand *lookupCommandOrOriginal(robj **argv, int argc); +void startCommandExecution(); +int incrCommandStatsOnError(struct redisCommand *cmd, int flags); void call(client *c, int flags); void alsoPropagate(int dbid, robj **argv, int argc, int target); void propagatePendingCommands(); @@ -2932,7 +2977,7 @@ void resetServerSaveParams(void); struct rewriteConfigState; /* Forward declaration to export API. */ void rewriteConfigRewriteLine(struct rewriteConfigState *state, const char *option, sds line, int force); void rewriteConfigMarkAsProcessed(struct rewriteConfigState *state, const char *option); -int rewriteConfig(char *path, int force_all); +int rewriteConfig(char *path, int force_write); void initConfigValues(); sds getConfigDebugInfo(); int allowProtectedAction(int config, client *c); @@ -3001,13 +3046,15 @@ void freeReplicationBacklogRefMemAsync(list *blocks, rax *index); /* API to get key arguments from commands */ #define GET_KEYSPEC_DEFAULT 0 -#define GET_KEYSPEC_INCLUDE_CHANNELS (1<<0) /* Consider channels as keys */ +#define GET_KEYSPEC_INCLUDE_NOT_KEYS (1<<0) /* Consider 'fake' keys as keys */ #define GET_KEYSPEC_RETURN_PARTIAL (1<<1) /* Return all keys that can be found */ int getKeysFromCommandWithSpecs(struct redisCommand *cmd, robj **argv, int argc, int search_flags, getKeysResult *result); keyReference *getKeysPrepareResult(getKeysResult *result, int numkeys); int getKeysFromCommand(struct redisCommand *cmd, robj **argv, int argc, getKeysResult *result); int doesCommandHaveKeys(struct redisCommand *cmd); +int getChannelsFromCommand(struct redisCommand *cmd, robj **argv, int argc, getKeysResult *result); +int doesCommandHaveChannelsWithFlags(struct redisCommand *cmd, int flags); void getKeysFreeResult(getKeysResult *result); int sintercardGetKeys(struct redisCommand *cmd,robj **argv, int argc, getKeysResult *result); int zunionInterDiffGetKeys(struct redisCommand *cmd,robj **argv, int argc, getKeysResult *result); @@ -3064,6 +3111,11 @@ unsigned long evalMemory(); dict* evalScriptsDict(); unsigned long evalScriptsMemory(); +typedef struct luaScript { + uint64_t flags; + robj *body; +} luaScript; + /* Blocked clients */ void processUnblockedClients(void); void blockClient(client *c, int btype); @@ -3075,7 +3127,7 @@ void disconnectAllBlockedClients(void); void handleClientsBlockedOnKeys(void); void signalKeyAsReady(redisDb *db, robj *key, int type); void blockForKeys(client *c, int btype, robj **keys, int numkeys, long count, mstime_t timeout, robj *target, struct blockPos *blockpos, streamID *ids); -void updateStatsOnUnblock(client *c, long blocked_us, long reply_us); +void updateStatsOnUnblock(client *c, long blocked_us, long reply_us, int had_errors); /* timeout.c -- Blocked clients timeout and connections timeout. */ void addClientToTimeoutTable(client *c); @@ -3125,6 +3177,7 @@ void commandCountCommand(client *c); void commandListCommand(client *c); void commandInfoCommand(client *c); void commandGetKeysCommand(client *c); +void commandGetKeysAndFlagsCommand(client *c); void commandHelpCommand(client *c); void commandDocsCommand(client *c); void setCommand(client *c); @@ -3390,7 +3443,9 @@ void _serverPanic(const char *file, int line, const char *msg, ...); void serverLogObjectDebugInfo(const robj *o); void sigsegvHandler(int sig, siginfo_t *info, void *secret); const char *getSafeInfoString(const char *s, size_t len, char **tmp); -sds genRedisInfoString(const char *section); +dict *genInfoSectionDict(robj **argv, int argc, char **defaults, int *out_all, int *out_everything); +void releaseInfoSectionDict(dict *sec); +sds genRedisInfoString(dict *section_dict, int all_sections, int everything); sds genModulesInfoString(sds info); void applyWatchdogPeriod(); void watchdogScheduleSignal(int period); diff --git a/src/stream.h b/src/stream.h index 724f2c2ad..2d4997919 100644 --- a/src/stream.h +++ b/src/stream.h @@ -15,8 +15,11 @@ typedef struct streamID { typedef struct stream { rax *rax; /* The radix tree holding the stream. */ - uint64_t length; /* Number of elements inside this stream. */ + uint64_t length; /* Current number of elements inside this stream. */ streamID last_id; /* Zero if there are yet no items. */ + streamID first_id; /* The first non-tombstone entry, zero if empty. */ + streamID max_deleted_entry_id; /* The maximal ID that was deleted. */ + uint64_t entries_added; /* All time count of elements added. */ rax *cgroups; /* Consumer groups dictionary: name -> streamCG */ } stream; @@ -34,6 +37,7 @@ typedef struct streamIterator { unsigned char *master_fields_ptr; /* Master field to emit next. */ int entry_flags; /* Flags of entry we are emitting. */ int rev; /* True if iterating end to start (reverse). */ + int skip_tombstones; /* True if not emitting tombstone entries. */ uint64_t start_key[2]; /* Start key as 128 bit big endian. */ uint64_t end_key[2]; /* End key as 128 bit big endian. */ raxIterator ri; /* Rax iterator. */ @@ -52,6 +56,11 @@ typedef struct streamCG { streamID last_id; /* Last delivered (not acknowledged) ID for this group. Consumers that will just ask for more messages will served with IDs > than this. */ + long long entries_read; /* In a perfect world (CG starts at 0-0, no dels, no + XGROUP SETID, ...), this is the total number of + group reads. In the real world, the reasoning behind + this value is detailed at the top comment of + streamEstimateDistanceFromFirstEverEntry(). */ rax *pel; /* Pending entries list. This is a radix tree that has every message delivered to consumers (without the NOACK option) that was yet not acknowledged @@ -105,6 +114,8 @@ struct client; #define SCC_NO_NOTIFY (1<<0) /* Do not notify key space if consumer created */ #define SCC_NO_DIRTIFY (1<<1) /* Do not dirty++ if consumer created */ +#define SCG_INVALID_ENTRIES_READ -1 + stream *streamNew(void); void freeStream(stream *s); unsigned long streamLength(const robj *subject); @@ -117,7 +128,7 @@ void streamIteratorStop(streamIterator *si); streamCG *streamLookupCG(stream *s, sds groupname); streamConsumer *streamLookupConsumer(streamCG *cg, sds name, int flags); streamConsumer *streamCreateConsumer(streamCG *cg, sds name, robj *key, int dbid, int flags); -streamCG *streamCreateCG(stream *s, char *name, size_t namelen, streamID *id); +streamCG *streamCreateCG(stream *s, char *name, size_t namelen, streamID *id, long long entries_read); streamNACK *streamCreateNACK(streamConsumer *consumer); void streamDecodeID(void *buf, streamID *id); int streamCompareID(streamID *a, streamID *b); @@ -131,6 +142,8 @@ int streamParseID(const robj *o, streamID *id); robj *createObjectFromStreamID(streamID *id); int streamAppendItem(stream *s, robj **argv, int64_t numfields, streamID *added_id, streamID *use_id, int seq_given); int streamDeleteItem(stream *s, streamID *id); +void streamGetEdgeID(stream *s, int first, int skip_tombstones, streamID *edge_id); +long long streamEstimateDistanceFromFirstEverEntry(stream *s, streamID *id); int64_t streamTrimByLength(stream *s, long long maxlen, int approx); int64_t streamTrimByID(stream *s, streamID minid, int approx); diff --git a/src/t_stream.c b/src/t_stream.c index e47194926..cd7d9723e 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -68,8 +68,13 @@ stream *streamNew(void) { stream *s = zmalloc(sizeof(*s)); s->rax = raxNew(); s->length = 0; + s->first_id.ms = 0; + s->first_id.seq = 0; s->last_id.ms = 0; s->last_id.seq = 0; + s->max_deleted_entry_id.seq = 0; + s->max_deleted_entry_id.ms = 0; + s->entries_added = 0; s->cgroups = NULL; /* Created on demand to save memory when not used. */ return s; } @@ -184,7 +189,10 @@ robj *streamDup(robj *o) { new_lp, NULL); } new_s->length = s->length; + new_s->first_id = s->first_id; new_s->last_id = s->last_id; + new_s->max_deleted_entry_id = s->max_deleted_entry_id; + new_s->entries_added = s->entries_added; raxStop(&ri); if (s->cgroups == NULL) return sobj; @@ -196,7 +204,8 @@ robj *streamDup(robj *o) { while (raxNext(&ri_cgroups)) { streamCG *cg = ri_cgroups.data; streamCG *new_cg = streamCreateCG(new_s, (char *)ri_cgroups.key, - ri_cgroups.key_len, &cg->last_id); + ri_cgroups.key_len, &cg->last_id, + cg->entries_read); serverAssert(new_cg != NULL); @@ -378,37 +387,21 @@ int streamCompareID(streamID *a, streamID *b) { return 0; } -void streamGetEdgeID(stream *s, int first, streamID *edge_id) +/* Retrieves the ID of the stream edge entry. An edge is either the first or + * the last ID in the stream, and may be a tombstone. To filter out tombstones, + * set the'skip_tombstones' argument to 1. */ +void streamGetEdgeID(stream *s, int first, int skip_tombstones, streamID *edge_id) { - raxIterator ri; - raxStart(&ri, s->rax); - int empty; - if (first) { - raxSeek(&ri, "^", NULL, 0); - empty = !raxNext(&ri); - } else { - raxSeek(&ri, "$", NULL, 0); - empty = !raxPrev(&ri); - } - - if (empty) { - /* Stream is empty, mark edge ID as lowest/highest possible. */ - edge_id->ms = first ? UINT64_MAX : 0; - edge_id->seq = first ? UINT64_MAX : 0; - raxStop(&ri); - return; + streamIterator si; + int64_t numfields; + streamIteratorStart(&si,s,NULL,NULL,!first); + si.skip_tombstones = skip_tombstones; + int found = streamIteratorGetID(&si,edge_id,&numfields); + if (!found) { + streamID min_id = {0, 0}, max_id = {UINT64_MAX, UINT64_MAX}; + *edge_id = first ? max_id : min_id; } - unsigned char *lp = ri.data; - - /* Read the master ID from the radix tree key. */ - streamID master_id; - streamDecodeID(ri.key, &master_id); - - /* Construct edge ID. */ - lpGetEdgeStreamID(lp, first, &master_id, edge_id); - - raxStop(&ri); } /* Adds a new item into the stream 's' having the specified number of @@ -664,7 +657,9 @@ int streamAppendItem(stream *s, robj **argv, int64_t numfields, streamID *added_ if (ri.data != lp) raxInsert(s->rax,(unsigned char*)&rax_key,sizeof(rax_key),lp,NULL); s->length++; + s->entries_added++; s->last_id = id; + if (s->length == 1) s->first_id = id; if (added_id) *added_id = id; return C_OK; } @@ -842,7 +837,7 @@ int64_t streamTrim(stream *s, streamAddTrimArgs *args) { } deleted += deleted_from_lp; - /* Now we the entries/deleted counters. */ + /* Now we update the entries/deleted counters. */ p = lpFirst(lp); lp = lpReplaceInteger(lp,&p,entries-deleted_from_lp); p = lpNext(lp,p); /* Skip deleted field. */ @@ -864,8 +859,16 @@ int64_t streamTrim(stream *s, streamAddTrimArgs *args) { break; /* If we are here, there was enough to delete in the current node, so no need to go to the next node. */ } - raxStop(&ri); + + /* Update the stream's first ID after the trimming. */ + if (s->length == 0) { + s->first_id.ms = 0; + s->first_id.seq = 0; + } else if (deleted) { + streamGetEdgeID(s,1,1,&s->first_id); + } + return deleted; } @@ -1089,9 +1092,10 @@ void streamIteratorStart(streamIterator *si, stream *s, streamID *start, streamI } } si->stream = s; - si->lp = NULL; /* There is no current listpack right now. */ + si->lp = NULL; /* There is no current listpack right now. */ si->lp_ele = NULL; /* Current listpack cursor. */ - si->rev = rev; /* Direction, if non-zero reversed, from end to start. */ + si->rev = rev; /* Direction, if non-zero reversed, from end to start. */ + si->skip_tombstones = 1; /* By default tombstones aren't emitted. */ } /* Return 1 and store the current item ID at 'id' if there are still @@ -1189,10 +1193,10 @@ int streamIteratorGetID(streamIterator *si, streamID *id, int64_t *numfields) { serverAssert(*numfields>=0); /* If current >= start, and the entry is not marked as - * deleted, emit it. */ + * deleted or tombstones are included, emit it. */ if (!si->rev) { if (memcmp(buf,si->start_key,sizeof(streamID)) >= 0 && - !(flags & STREAM_ITEM_FLAG_DELETED)) + (!si->skip_tombstones || !(flags & STREAM_ITEM_FLAG_DELETED))) { if (memcmp(buf,si->end_key,sizeof(streamID)) > 0) return 0; /* We are already out of range. */ @@ -1203,7 +1207,7 @@ int streamIteratorGetID(streamIterator *si, streamID *id, int64_t *numfields) { } } else { if (memcmp(buf,si->end_key,sizeof(streamID)) <= 0 && - !(flags & STREAM_ITEM_FLAG_DELETED)) + (!si->skip_tombstones || !(flags & STREAM_ITEM_FLAG_DELETED))) { if (memcmp(buf,si->start_key,sizeof(streamID)) < 0) return 0; /* We are already out of range. */ @@ -1270,7 +1274,7 @@ void streamIteratorRemoveEntry(streamIterator *si, streamID *current) { int64_t aux; /* We do not really delete the entry here. Instead we mark it as - * deleted flagging it, and also incrementing the count of the + * deleted by flagging it, and also incrementing the count of the * deleted entries in the listpack header. * * We start flagging: */ @@ -1314,7 +1318,7 @@ void streamIteratorRemoveEntry(streamIterator *si, streamID *current) { streamIteratorStop(si); streamIteratorStart(si,si->stream,&start,&end,si->rev); - /* TODO: perform a garbage collection here if the ration between + /* TODO: perform a garbage collection here if the ratio between * deleted and valid goes over a certain limit. */ } @@ -1325,6 +1329,20 @@ void streamIteratorStop(streamIterator *si) { raxStop(&si->ri); } +/* Return 1 if `id` exists in `s` (and not marked as deleted) */ +int streamEntryExists(stream *s, streamID *id) { + streamIterator si; + streamIteratorStart(&si,s,id,id,0); + streamID myid; + int64_t numfields; + int found = streamIteratorGetID(&si,&myid,&numfields); + streamIteratorStop(&si); + if (!found) + return 0; + serverAssert(streamCompareID(id,&myid) == 0); + return 1; +} + /* Delete the specified item ID from the stream, returning 1 if the item * was deleted 0 otherwise (if it does not exist). */ int streamDeleteItem(stream *s, streamID *id) { @@ -1372,6 +1390,148 @@ robj *createObjectFromStreamID(streamID *id) { id->ms,id->seq)); } +/* Returns non-zero if the ID is 0-0. */ +int streamIDEqZero(streamID *id) { + return !(id->ms || id->seq); +} + +/* A helper that returns non-zero if the range from 'start' to `end` + * contains a tombstone. + * + * NOTE: this assumes that the caller had verified that 'start' is less than + * 's->last_id'. */ +int streamRangeHasTombstones(stream *s, streamID *start, streamID *end) { + streamID start_id, end_id; + + if (!s->length || streamIDEqZero(&s->max_deleted_entry_id)) { + /* The stream is empty or has no tombstones. */ + return 0; + } + + if (streamCompareID(&s->first_id,&s->max_deleted_entry_id) > 0) { + /* The latest tombstone is before the first entry. */ + return 0; + } + + if (start) { + start_id = *start; + } else { + start_id.ms = 0; + start_id.seq = 0; + } + + if (end) { + end_id = *end; + } else { + end_id.ms = UINT64_MAX; + end_id.seq = UINT64_MAX; + } + + if (streamCompareID(&start_id,&s->max_deleted_entry_id) <= 0 && + streamCompareID(&s->max_deleted_entry_id,&end_id) <= 0) + { + /* start_id <= max_deleted_entry_id <= end_id: The range does include a tombstone. */ + return 1; + } + + /* The range doesn't includes a tombstone. */ + return 0; +} + +/* Replies with a consumer group's current lag, that is the number of messages + * in the stream that are yet to be delivered. In case that the lag isn't + * available due to fragmentation, the reply to the client is a null. */ +void streamReplyWithCGLag(client *c, stream *s, streamCG *cg) { + int valid = 0; + long long lag = 0; + + if (!s->entries_added) { + /* The lag of a newly-initialized stream is 0. */ + lag = 0; + valid = 1; + } else if (cg->entries_read != SCG_INVALID_ENTRIES_READ && !streamRangeHasTombstones(s,&cg->last_id,NULL)) { + /* No fragmentation ahead means that the group's logical reads counter + * is valid for performing the lag calculation. */ + lag = (long long)s->entries_added - cg->entries_read; + valid = 1; + } else { + /* Attempt to retrieve the group's last ID logical read counter. */ + long long entries_read = streamEstimateDistanceFromFirstEverEntry(s,&cg->last_id); + if (entries_read != SCG_INVALID_ENTRIES_READ) { + /* A valid counter was obtained. */ + lag = (long long)s->entries_added - entries_read; + valid = 1; + } + } + + if (valid) { + addReplyLongLong(c,lag); + } else { + addReplyNull(c); + } +} + +/* This function returns a value that is the ID's logical read counter, or its + * distance (the number of entries) from the first entry ever to have been added + * to the stream. + * + * A counter is returned only in one of the following cases: + * 1. The ID is the same as the stream's last ID. In this case, the returned + * is the same as the stream's entries_added counter. + * 2. The ID equals that of the currently first entry in the stream, and the + * stream has no tombstones. The returned value, in this case, is the result + * of subtracting the stream's length from its added_entries, incremented by + * one. + * 3. The ID less than the stream's first current entry's ID, and there are no + * tombstones. Here the estimated counter is the result of subtracting the + * stream's length from its added_entries. + * 4. The stream's added_entries is zero, meaning that no entries were ever + * added. + * + * The special return value of ULLONG_MAX signals that the counter's value isn't + * obtainable. It is returned in these cases: + * 1. The provided ID, if it even exists, is somewhere between the stream's + * current first and last entries' IDs, or in the future. + * 2. The stream contains one or more tombstones. */ +long long streamEstimateDistanceFromFirstEverEntry(stream *s, streamID *id) { + /* The counter of any ID in an empty, never-before-used stream is 0. */ + if (!s->entries_added) { + return 0; + } + + /* In the empty stream, if the ID is smaller or equal to the last ID, + * it can set to the current added_entries value. */ + if (!s->length && streamCompareID(id,&s->last_id) < 1) { + return s->entries_added; + } + + int cmp_last = streamCompareID(id,&s->last_id); + if (cmp_last == 0) { + /* Return the exact counter of the last entry in the stream. */ + return s->entries_added; + } else if (cmp_last > 0) { + /* The counter of a future ID is unknown. */ + return SCG_INVALID_ENTRIES_READ; + } + + int cmp_id_first = streamCompareID(id,&s->first_id); + int cmp_xdel_first = streamCompareID(&s->max_deleted_entry_id,&s->first_id); + if (streamIDEqZero(&s->max_deleted_entry_id) || cmp_xdel_first < 0) { + /* There's definitely no fragmentation ahead. */ + if (cmp_id_first < 0) { + /* Return the estimated counter. */ + return s->entries_added - s->length; + } else if (cmp_id_first == 0) { + /* Return the exact counter of the first entry in the stream. */ + return s->entries_added - s->length + 1; + } + } + + /* The ID is either before an XDEL that fragments the stream or an arbitrary + * ID. Either case, so we can't make a prediction. */ + return SCG_INVALID_ENTRIES_READ; +} + /* As a result of an explicit XCLAIM or XREADGROUP command, new entries * are created in the pending list of the stream and consumers. We need * to propagate this changes in the form of XCLAIM commands. */ @@ -1411,19 +1571,22 @@ void streamPropagateXCLAIM(client *c, robj *key, streamCG *group, robj *groupnam * that was consumed by XREADGROUP with the NOACK option: in that case we can't * propagate the last ID just using the XCLAIM LASTID option, so we emit * - * XGROUP SETID <key> <groupname> <id> + * XGROUP SETID <key> <groupname> <id> ENTRIESREAD <entries_read> */ void streamPropagateGroupID(client *c, robj *key, streamCG *group, robj *groupname) { - robj *argv[5]; + robj *argv[7]; argv[0] = shared.xgroup; argv[1] = shared.setid; argv[2] = key; argv[3] = groupname; argv[4] = createObjectFromStreamID(&group->last_id); + argv[5] = shared.entriesread; + argv[6] = createStringObjectFromLongLong(group->entries_read); - alsoPropagate(c->db->id,argv,5,PROPAGATE_AOF|PROPAGATE_REPL); + alsoPropagate(c->db->id,argv,7,PROPAGATE_AOF|PROPAGATE_REPL); decrRefCount(argv[4]); + decrRefCount(argv[6]); } /* We need this when we want to propagate creation of consumer that was created @@ -1462,6 +1625,10 @@ void streamPropagateConsumerCreation(client *c, robj *key, robj *groupname, sds * function will not return it to the client. * 3. An entry in the pending list will be created for every entry delivered * for the first time to this consumer. + * 4. The group's read counter is incremented if it is already valid and there + * are no future tombstones, or is invalidated (set to 0) otherwise. If the + * counter is invalid to begin with, we try to obtain it for the last + * delivered ID. * * The behavior may be modified passing non-zero flags: * @@ -1518,6 +1685,15 @@ size_t streamReplyWithRange(client *c, stream *s, streamID *start, streamID *end while(streamIteratorGetID(&si,&id,&numfields)) { /* Update the group last_id if needed. */ if (group && streamCompareID(&id,&group->last_id) > 0) { + if (group->entries_read != SCG_INVALID_ENTRIES_READ && !streamRangeHasTombstones(s,&id,NULL)) { + /* A valid counter and no future tombstones mean we can + * increment the read counter to keep tracking the group's + * progress. */ + group->entries_read++; + } else if (s->entries_added) { + /* The group's counter may be invalid, so we try to obtain it. */ + group->entries_read = streamEstimateDistanceFromFirstEverEntry(s,&id); + } group->last_id = id; /* Group last ID should be propagated only if NOACK was * specified, otherwise the last id will be included @@ -1791,7 +1967,7 @@ void streamRewriteTrimArgument(client *c, stream *s, int trim_strategy, int idx) arg = createStringObjectFromLongLong(s->length); } else { streamID first_id; - streamGetEdgeID(s, 1, &first_id); + streamGetEdgeID(s,1,0,&first_id); arg = createObjectFromStreamID(&first_id); } @@ -2284,10 +2460,10 @@ void streamFreeConsumer(streamConsumer *sc) { } /* Create a new consumer group in the context of the stream 's', having the - * specified name and last server ID. If a consumer group with the same name - * already existed NULL is returned, otherwise the pointer to the consumer - * group is returned. */ -streamCG *streamCreateCG(stream *s, char *name, size_t namelen, streamID *id) { + * specified name, last server ID and reads counter. If a consumer group with + * the same name already exists NULL is returned, otherwise the pointer to the + * consumer group is returned. */ +streamCG *streamCreateCG(stream *s, char *name, size_t namelen, streamID *id, long long entries_read) { if (s->cgroups == NULL) s->cgroups = raxNew(); if (raxFind(s->cgroups,(unsigned char*)name,namelen) != raxNotFound) return NULL; @@ -2296,6 +2472,7 @@ streamCG *streamCreateCG(stream *s, char *name, size_t namelen, streamID *id) { cg->pel = raxNew(); cg->consumers = raxNew(); cg->last_id = *id; + cg->entries_read = entries_read; raxInsert(s->cgroups,(unsigned char*)name,namelen,cg,NULL); return cg; } @@ -2375,8 +2552,8 @@ void streamDelConsumer(streamCG *cg, streamConsumer *consumer) { * Consumer groups commands * ----------------------------------------------------------------------- */ -/* XGROUP CREATE <key> <groupname> <id or $> [MKSTREAM] - * XGROUP SETID <key> <groupname> <id or $> +/* XGROUP CREATE <key> <groupname> <id or $> [MKSTREAM] [ENTRIESADDED count] + * XGROUP SETID <key> <groupname> <id or $> [ENTRIESADDED count] * XGROUP DESTROY <key> <groupname> * XGROUP CREATECONSUMER <key> <groupname> <consumer> * XGROUP DELCONSUMER <key> <groupname> <consumername> */ @@ -2386,21 +2563,33 @@ void xgroupCommand(client *c) { streamCG *cg = NULL; char *opt = c->argv[1]->ptr; /* Subcommand name. */ int mkstream = 0; + long long entries_read = SCG_INVALID_ENTRIES_READ; robj *o; - /* CREATE has an MKSTREAM option that creates the stream if it - * does not exist. */ - if (c->argc == 6 && !strcasecmp(opt,"CREATE")) { - if (strcasecmp(c->argv[5]->ptr,"MKSTREAM")) { - addReplySubcommandSyntaxError(c); - return; - } - mkstream = 1; - grpname = c->argv[3]->ptr; - } - /* Everything but the "HELP" option requires a key and group name. */ if (c->argc >= 4) { + /* Parse optional arguments for CREATE and SETID */ + int i = 5; + int create_subcmd = !strcasecmp(opt,"CREATE"); + int setid_subcmd = !strcasecmp(opt,"SETID"); + while (i < c->argc) { + if (create_subcmd && !strcasecmp(c->argv[i]->ptr,"MKSTREAM")) { + mkstream = 1; + i++; + } else if ((create_subcmd || setid_subcmd) && !strcasecmp(c->argv[i]->ptr,"ENTRIESREAD") && i + 1 < c->argc) { + if (getLongLongFromObjectOrReply(c,c->argv[i+1],&entries_read,NULL) != C_OK) + return; + if (entries_read < 0 && entries_read != SCG_INVALID_ENTRIES_READ) { + addReplyError(c,"value for ENTRIESREAD must be positive or -1"); + return; + } + i += 2; + } else { + addReplySubcommandSyntaxError(c); + return; + } + } + o = lookupKeyWrite(c->db,c->argv[2]); if (o) { if (checkType(c,o,OBJ_STREAM)) return; @@ -2440,18 +2629,20 @@ void xgroupCommand(client *c) { " Create a new consumer group. Options are:", " * MKSTREAM", " Create the empty stream if it does not exist.", +" * ENTRIESREAD entries_read", +" Set the group's entries_read counter (internal use).", "CREATECONSUMER <key> <groupname> <consumer>", " Create a new consumer in the specified group.", "DELCONSUMER <key> <groupname> <consumer>", " Remove the specified consumer.", "DESTROY <key> <groupname>", " Remove the specified group.", -"SETID <key> <groupname> <id|$>", -" Set the current group ID.", +"SETID <key> <groupname> <id|$> [ENTRIESREAD entries_read]", +" Set the current group ID and entries_read counter.", NULL }; addReplyHelp(c, help); - } else if (!strcasecmp(opt,"CREATE") && (c->argc == 5 || c->argc == 6)) { + } else if (!strcasecmp(opt,"CREATE") && (c->argc >= 5 && c->argc <= 8)) { streamID id; if (!strcmp(c->argv[4]->ptr,"$")) { if (s) { @@ -2473,7 +2664,7 @@ NULL signalModifiedKey(c,c->db,c->argv[2]); } - streamCG *cg = streamCreateCG(s,grpname,sdslen(grpname),&id); + streamCG *cg = streamCreateCG(s,grpname,sdslen(grpname),&id,entries_read); if (cg) { addReply(c,shared.ok); server.dirty++; @@ -2482,7 +2673,7 @@ NULL } else { addReplyError(c,"-BUSYGROUP Consumer Group name already exists"); } - } else if (!strcasecmp(opt,"SETID") && c->argc == 5) { + } else if (!strcasecmp(opt,"SETID") && (c->argc == 5 || c->argc == 7)) { streamID id; if (!strcmp(c->argv[4]->ptr,"$")) { id = s->last_id; @@ -2490,6 +2681,7 @@ NULL return; } cg->last_id = id; + cg->entries_read = entries_read; addReply(c,shared.ok); server.dirty++; notifyKeyspaceEvent(NOTIFY_STREAM,"xgroup-setid",c->argv[2],c->db->id); @@ -2528,16 +2720,46 @@ NULL } } -/* XSETID <stream> <id> +/* XSETID <stream> <id> [ENTRIESADDED entries_added] [MAXDELETEDID max_deleted_entry_id] * - * Set the internal "last ID" of a stream. */ + * Set the internal "last ID", "added entries" and "maximal deleted entry ID" + * of a stream. */ void xsetidCommand(client *c) { + streamID id, max_xdel_id = {0, 0}; + long long entries_added = -1; + + if (streamParseStrictIDOrReply(c,c->argv[2],&id,0,NULL) != C_OK) + return; + + int i = 3; + while (i < c->argc) { + int moreargs = (c->argc-1) - i; /* Number of additional arguments. */ + char *opt = c->argv[i]->ptr; + if (!strcasecmp(opt,"ENTRIESADDED") && moreargs) { + if (getLongLongFromObjectOrReply(c,c->argv[i+1],&entries_added,NULL) != C_OK) { + return; + } else if (entries_added < 0) { + addReplyError(c,"entries_added must be positive"); + return; + } + i += 2; + } else if (!strcasecmp(opt,"MAXDELETEDID") && moreargs) { + if (streamParseStrictIDOrReply(c,c->argv[i+1],&max_xdel_id,0,NULL) != C_OK) { + return; + } else if (streamCompareID(&id,&max_xdel_id) < 0) { + addReplyError(c,"The ID specified in XSETID is smaller than the provided max_deleted_entry_id"); + return; + } + i += 2; + } else { + addReplyErrorObject(c,shared.syntaxerr); + return; + } + } + robj *o = lookupKeyWriteOrReply(c,c->argv[1],shared.nokeyerr); if (o == NULL || checkType(c,o,OBJ_STREAM)) return; - stream *s = o->ptr; - streamID id; - if (streamParseStrictIDOrReply(c,c->argv[2],&id,0,NULL) != C_OK) return; /* If the stream has at least one item, we want to check that the user * is setting a last ID that is equal or greater than the current top @@ -2547,12 +2769,22 @@ void xsetidCommand(client *c) { streamLastValidID(s,&maxid); if (streamCompareID(&id,&maxid) < 0) { - addReplyError(c,"The ID specified in XSETID is smaller than the " - "target stream top item"); + addReplyError(c,"The ID specified in XSETID is smaller than the target stream top item"); + return; + } + + /* If an entries_added was provided, it can't be lower than the length. */ + if (entries_added != -1 && s->length > (uint64_t)entries_added) { + addReplyError(c,"The entries_added specified in XSETID is smaller than the target stream length"); return; } } + s->last_id = id; + if (entries_added != -1) + s->entries_added = entries_added; + if (!streamIDEqZero(&max_xdel_id)) + s->max_deleted_entry_id = max_xdel_id; addReply(c,shared.ok); server.dirty++; notifyKeyspaceEvent(NOTIFY_STREAM,"xsetid",c->argv[1],c->db->id); @@ -2980,23 +3212,28 @@ void xclaimCommand(client *c) { /* Lookup the ID in the group PEL. */ streamNACK *nack = raxFind(group->pel,buf,sizeof(buf)); + /* Item must exist for us to transfer it to another consumer. */ + if (!streamEntryExists(o->ptr,&id)) { + /* Clear this entry from the PEL, it no longer exists */ + if (nack != raxNotFound) { + /* Propagate this change (we are going to delete the NACK). */ + streamPropagateXCLAIM(c,c->argv[1],group,c->argv[2],c->argv[j],nack); + propagate_last_id = 0; /* Will be propagated by XCLAIM itself. */ + server.dirty++; + /* Release the NACK */ + raxRemove(group->pel,buf,sizeof(buf),NULL); + raxRemove(nack->consumer->pel,buf,sizeof(buf),NULL); + streamFreeNACK(nack); + } + continue; + } + /* If FORCE is passed, let's check if at least the entry * exists in the Stream. In such case, we'll create a new * entry in the PEL from scratch, so that XCLAIM can also * be used to create entries in the PEL. Useful for AOF * and replication of consumer groups. */ if (force && nack == raxNotFound) { - streamIterator myiterator; - streamIteratorStart(&myiterator,o->ptr,&id,&id,0); - int64_t numfields; - int found = 0; - streamID item_id; - if (streamIteratorGetID(&myiterator,&item_id,&numfields)) found = 1; - streamIteratorStop(&myiterator); - - /* Item must exist for us to create a NACK for it. */ - if (!found) continue; - /* Create the NACK. */ nack = streamCreateNACK(NULL); raxInsert(group->pel,buf,sizeof(buf),nack,NULL); @@ -3013,6 +3250,7 @@ void xclaimCommand(client *c) { mstime_t this_idle = now - nack->delivery_time; if (this_idle < minidle) continue; } + if (consumer == NULL && (consumer = streamLookupConsumer(group,name,SLC_DEFAULT)) == NULL) { @@ -3042,9 +3280,7 @@ void xclaimCommand(client *c) { if (justid) { addReplyStreamID(c,&id); } else { - size_t emitted = streamReplyWithRange(c,o->ptr,&id,&id,1,0, - NULL,NULL,STREAM_RWR_RAWENTRIES,NULL); - if (!emitted) addReplyNull(c); + serverAssert(streamReplyWithRange(c,o->ptr,&id,&id,1,0,NULL,NULL,STREAM_RWR_RAWENTRIES,NULL) == 1); } arraylen++; @@ -3138,9 +3374,9 @@ void xautoclaimCommand(client *c) { streamConsumer *consumer = NULL; long long attempts = count*10; - addReplyArrayLen(c, 2); - void *endidptr = addReplyDeferredLen(c); - void *arraylenptr = addReplyDeferredLen(c); + addReplyArrayLen(c, 3); /* We add another reply later */ + void *endidptr = addReplyDeferredLen(c); /* reply[0] */ + void *arraylenptr = addReplyDeferredLen(c); /* reply[1] */ unsigned char startkey[sizeof(streamID)]; streamEncodeID(startkey,&startid); @@ -3150,18 +3386,37 @@ void xautoclaimCommand(client *c) { size_t arraylen = 0; mstime_t now = mstime(); sds name = c->argv[3]->ptr; + streamID *deleted_ids = zmalloc(count * sizeof(streamID)); + int deleted_id_num = 0; while (attempts-- && count && raxNext(&ri)) { streamNACK *nack = ri.data; + streamID id; + streamDecodeID(ri.key, &id); + + /* Item must exist for us to transfer it to another consumer. */ + if (!streamEntryExists(o->ptr,&id)) { + /* Propagate this change (we are going to delete the NACK). */ + robj *idstr = createObjectFromStreamID(&id); + streamPropagateXCLAIM(c,c->argv[1],group,c->argv[2],idstr,nack); + decrRefCount(idstr); + server.dirty++; + /* Clear this entry from the PEL, it no longer exists */ + raxRemove(group->pel,ri.key,ri.key_len,NULL); + raxRemove(nack->consumer->pel,ri.key,ri.key_len,NULL); + streamFreeNACK(nack); + /* Remember the ID for later */ + deleted_ids[deleted_id_num++] = id; + raxSeek(&ri,">=",ri.key,ri.key_len); + continue; + } + if (minidle) { mstime_t this_idle = now - nack->delivery_time; if (this_idle < minidle) continue; } - streamID id; - streamDecodeID(ri.key, &id); - if (consumer == NULL && (consumer = streamLookupConsumer(group,name,SLC_DEFAULT)) == NULL) { @@ -3191,11 +3446,7 @@ void xautoclaimCommand(client *c) { if (justid) { addReplyStreamID(c,&id); } else { - size_t emitted = - streamReplyWithRange(c,o->ptr,&id,&id,1,0,NULL,NULL, - STREAM_RWR_RAWENTRIES,NULL); - if (!emitted) - addReplyNull(c); + serverAssert(streamReplyWithRange(c,o->ptr,&id,&id,1,0,NULL,NULL,STREAM_RWR_RAWENTRIES,NULL) == 1); } arraylen++; count--; @@ -3221,6 +3472,12 @@ void xautoclaimCommand(client *c) { setDeferredArrayLen(c,arraylenptr,arraylen); setDeferredReplyStreamID(c,endidptr,&endid); + addReplyArrayLen(c, deleted_id_num); /* reply[2] */ + for (int i = 0; i < deleted_id_num; i++) { + addReplyStreamID(c, &deleted_ids[i]); + } + zfree(deleted_ids); + preventCommandPropagation(c); } @@ -3250,8 +3507,31 @@ void xdelCommand(client *c) { /* Actually apply the command. */ int deleted = 0; + int first_entry = 0; for (int j = 2; j < c->argc; j++) { - deleted += streamDeleteItem(s,&ids[j-2]); + streamID *id = &ids[j-2]; + if (streamDeleteItem(s,id)) { + /* We want to know if the first entry in the stream was deleted + * so we can later set the new one. */ + if (streamCompareID(id,&s->first_id) == 0) { + first_entry = 1; + } + /* Update the stream's maximal tombstone if needed. */ + if (streamCompareID(id,&s->max_deleted_entry_id) > 0) { + s->max_deleted_entry_id = *id; + } + deleted++; + }; + } + + /* Update the stream's first ID. */ + if (deleted) { + if (s->length == 0) { + s->first_id.ms = 0; + s->first_id.seq = 0; + } else if (first_entry) { + streamGetEdgeID(s,1,1,&s->first_id); + } } /* Propagate the write if needed. */ @@ -3359,7 +3639,7 @@ void xinfoReplyWithStreamInfo(client *c, stream *s) { } } - addReplyMapLen(c,full ? 6 : 7); + addReplyMapLen(c,full ? 9 : 10); addReplyBulkCString(c,"length"); addReplyLongLong(c,s->length); addReplyBulkCString(c,"radix-tree-keys"); @@ -3368,6 +3648,12 @@ void xinfoReplyWithStreamInfo(client *c, stream *s) { addReplyLongLong(c,s->rax->numnodes); addReplyBulkCString(c,"last-generated-id"); addReplyStreamID(c,&s->last_id); + addReplyBulkCString(c,"max-deleted-entry-id"); + addReplyStreamID(c,&s->max_deleted_entry_id); + addReplyBulkCString(c,"entries-added"); + addReplyLongLong(c,s->entries_added); + addReplyBulkCString(c,"recorded-first-entry-id"); + addReplyStreamID(c,&s->first_id); if (!full) { /* XINFO STREAM <key> */ @@ -3406,7 +3692,7 @@ void xinfoReplyWithStreamInfo(client *c, stream *s) { raxSeek(&ri_cgroups,"^",NULL,0); while(raxNext(&ri_cgroups)) { streamCG *cg = ri_cgroups.data; - addReplyMapLen(c,5); + addReplyMapLen(c,7); /* Name */ addReplyBulkCString(c,"name"); @@ -3416,6 +3702,18 @@ void xinfoReplyWithStreamInfo(client *c, stream *s) { addReplyBulkCString(c,"last-delivered-id"); addReplyStreamID(c,&cg->last_id); + /* Read counter of the last delivered ID */ + addReplyBulkCString(c,"entries-read"); + if (cg->entries_read != SCG_INVALID_ENTRIES_READ) { + addReplyLongLong(c,cg->entries_read); + } else { + addReplyNull(c); + } + + /* Group lag */ + addReplyBulkCString(c,"lag"); + streamReplyWithCGLag(c,s,cg); + /* Group PEL count */ addReplyBulkCString(c,"pel-count"); addReplyLongLong(c,raxSize(cg->pel)); @@ -3593,7 +3891,7 @@ NULL raxSeek(&ri,"^",NULL,0); while(raxNext(&ri)) { streamCG *cg = ri.data; - addReplyMapLen(c,4); + addReplyMapLen(c,6); addReplyBulkCString(c,"name"); addReplyBulkCBuffer(c,ri.key,ri.key_len); addReplyBulkCString(c,"consumers"); @@ -3602,6 +3900,14 @@ NULL addReplyLongLong(c,raxSize(cg->pel)); addReplyBulkCString(c,"last-delivered-id"); addReplyStreamID(c,&cg->last_id); + addReplyBulkCString(c,"entries-read"); + if (cg->entries_read != SCG_INVALID_ENTRIES_READ) { + addReplyLongLong(c,cg->entries_read); + } else { + addReplyNull(c); + } + addReplyBulkCString(c,"lag"); + streamReplyWithCGLag(c,s,cg); } raxStop(&ri); } else if (!strcasecmp(opt,"STREAM")) { diff --git a/src/t_zset.c b/src/t_zset.c index e09d4528b..bc947c965 100644 --- a/src/t_zset.c +++ b/src/t_zset.c @@ -2848,7 +2848,7 @@ typedef enum { typedef struct zrange_result_handler zrange_result_handler; -typedef void (*zrangeResultBeginFunction)(zrange_result_handler *c); +typedef void (*zrangeResultBeginFunction)(zrange_result_handler *c, long length); typedef void (*zrangeResultFinalizeFunction)( zrange_result_handler *c, size_t result_count); typedef void (*zrangeResultEmitCBufferFunction)( @@ -2876,8 +2876,22 @@ struct zrange_result_handler { zrangeResultEmitLongLongFunction emitResultFromLongLong; }; -/* Result handler methods for responding the ZRANGE to clients. */ -static void zrangeResultBeginClient(zrange_result_handler *handler) { +/* Result handler methods for responding the ZRANGE to clients. + * length can be used to provide the result length in advance (avoids deferred reply overhead). + * length can be set to -1 if the result length is not know in advance. + */ +static void zrangeResultBeginClient(zrange_result_handler *handler, long length) { + if (length > 0) { + /* In case of WITHSCORES, respond with a single array in RESP2, and + * nested arrays in RESP3. We can't use a map response type since the + * client library needs to know to respect the order. */ + if (handler->withscores && (handler->client->resp == 2)) { + length *= 2; + } + addReplyArrayLen(handler->client, length); + handler->userdata = NULL; + return; + } handler->userdata = addReplyDeferredLen(handler->client); } @@ -2912,6 +2926,9 @@ static void zrangeResultEmitLongLongToClient(zrange_result_handler *handler, static void zrangeResultFinalizeClient(zrange_result_handler *handler, size_t result_count) { + /* If the reply size was know at start there's nothing left to do */ + if (!handler->userdata) + return; /* In case of WITHSCORES, respond with a single array in RESP2, and * nested arrays in RESP3. We can't use a map response type since the * client library needs to know to respect the order. */ @@ -2923,8 +2940,9 @@ static void zrangeResultFinalizeClient(zrange_result_handler *handler, } /* Result handler methods for storing the ZRANGESTORE to a zset. */ -static void zrangeResultBeginStore(zrange_result_handler *handler) +static void zrangeResultBeginStore(zrange_result_handler *handler, long length) { + UNUSED(length); handler->dstobj = createZsetListpackObject(); } @@ -3019,11 +3037,11 @@ void genericZrangebyrankCommand(zrange_result_handler *handler, if (end < 0) end = llen+end; if (start < 0) start = 0; - handler->beginResultEmission(handler); /* Invariant: start >= 0, so this test will be true when end < 0. * The range is empty when start > end or start >= length. */ if (start > end || start >= llen) { + handler->beginResultEmission(handler, 0); handler->finalizeResultEmission(handler, 0); return; } @@ -3031,6 +3049,7 @@ void genericZrangebyrankCommand(zrange_result_handler *handler, rangelen = (end-start)+1; result_cardinality = rangelen; + handler->beginResultEmission(handler, rangelen); if (zobj->encoding == OBJ_ENCODING_LISTPACK) { unsigned char *zl = zobj->ptr; unsigned char *eptr, *sptr; @@ -3124,7 +3143,7 @@ void genericZrangebyscoreCommand(zrange_result_handler *handler, int reverse) { unsigned long rangelen = 0; - handler->beginResultEmission(handler); + handler->beginResultEmission(handler, -1); /* For invalid offset, return directly. */ if (offset > 0 && offset >= (long)zsetLength(zobj)) { @@ -3409,7 +3428,7 @@ void genericZrangebylexCommand(zrange_result_handler *handler, { unsigned long rangelen = 0; - handler->beginResultEmission(handler); + handler->beginResultEmission(handler, -1); if (zobj->encoding == OBJ_ENCODING_LISTPACK) { unsigned char *zl = zobj->ptr; @@ -3647,7 +3666,7 @@ void zrangeGenericCommand(zrange_result_handler *handler, int argc_start, int st zobj = lookupKeyRead(c->db, key); if (zobj == NULL) { if (store) { - handler->beginResultEmission(handler); + handler->beginResultEmission(handler, -1); handler->finalizeResultEmission(handler, 0); } else { addReply(c, shared.emptyarray); @@ -39,6 +39,10 @@ #include <openssl/err.h> #include <openssl/rand.h> #include <openssl/pem.h> +#if OPENSSL_VERSION_NUMBER >= 0x30000000L +#include <openssl/decoder.h> +#endif +#include <sys/uio.h> #define REDIS_TLS_PROTO_TLSv1 (1<<0) #define REDIS_TLS_PROTO_TLSv1_1 (1<<1) @@ -146,14 +150,13 @@ void tlsInit(void) { */ #if OPENSSL_VERSION_NUMBER < 0x10100000L OPENSSL_config(NULL); + SSL_load_error_strings(); + SSL_library_init(); #elif OPENSSL_VERSION_NUMBER < 0x10101000L OPENSSL_init_crypto(OPENSSL_INIT_LOAD_CONFIG, NULL); #else OPENSSL_init_crypto(OPENSSL_INIT_LOAD_CONFIG|OPENSSL_INIT_ATFORK, NULL); #endif - ERR_load_crypto_strings(); - SSL_load_error_strings(); - SSL_library_init(); #ifdef USE_CRYPTO_LOCKS initCryptoLocks(); @@ -323,20 +326,46 @@ int tlsConfigure(redisTLSContextConfig *ctx_config) { if (ctx_config->prefer_server_ciphers) SSL_CTX_set_options(ctx, SSL_OP_CIPHER_SERVER_PREFERENCE); -#if defined(SSL_CTX_set_ecdh_auto) +#if ((OPENSSL_VERSION_NUMBER < 0x30000000L) && defined(SSL_CTX_set_ecdh_auto)) SSL_CTX_set_ecdh_auto(ctx, 1); #endif SSL_CTX_set_options(ctx, SSL_OP_SINGLE_DH_USE); if (ctx_config->dh_params_file) { FILE *dhfile = fopen(ctx_config->dh_params_file, "r"); - DH *dh = NULL; if (!dhfile) { serverLog(LL_WARNING, "Failed to load %s: %s", ctx_config->dh_params_file, strerror(errno)); goto error; } - dh = PEM_read_DHparams(dhfile, NULL, NULL, NULL); +#if (OPENSSL_VERSION_NUMBER >= 0x30000000L) + EVP_PKEY *pkey = NULL; + OSSL_DECODER_CTX *dctx = OSSL_DECODER_CTX_new_for_pkey( + &pkey, "PEM", NULL, "DH", OSSL_KEYMGMT_SELECT_DOMAIN_PARAMETERS, NULL, NULL); + if (!dctx) { + serverLog(LL_WARNING, "No decoder for DH params."); + fclose(dhfile); + goto error; + } + if (!OSSL_DECODER_from_fp(dctx, dhfile)) { + serverLog(LL_WARNING, "%s: failed to read DH params.", ctx_config->dh_params_file); + OSSL_DECODER_CTX_free(dctx); + fclose(dhfile); + goto error; + } + + OSSL_DECODER_CTX_free(dctx); + fclose(dhfile); + + if (SSL_CTX_set0_tmp_dh_pkey(ctx, pkey) <= 0) { + ERR_error_string_n(ERR_get_error(), errbuf, sizeof(errbuf)); + serverLog(LL_WARNING, "Failed to load DH params file: %s: %s", ctx_config->dh_params_file, errbuf); + EVP_PKEY_free(pkey); + goto error; + } + /* Not freeing pkey, it is owned by OpenSSL now */ +#else + DH *dh = PEM_read_DHparams(dhfile, NULL, NULL, NULL); fclose(dhfile); if (!dh) { serverLog(LL_WARNING, "%s: failed to read DH params.", ctx_config->dh_params_file); @@ -351,6 +380,11 @@ int tlsConfigure(redisTLSContextConfig *ctx_config) { } DH_free(dh); +#endif + } else { +#if (OPENSSL_VERSION_NUMBER >= 0x30000000L) + SSL_CTX_set_dh_auto(ctx, 1); +#endif } /* If a client-side certificate is configured, create an explicit client context */ @@ -786,6 +820,43 @@ static int connTLSWrite(connection *conn_, const void *data, size_t data_len) { return ret; } +static int connTLSWritev(connection *conn_, const struct iovec *iov, int iovcnt) { + if (iovcnt == 1) return connTLSWrite(conn_, iov[0].iov_base, iov[0].iov_len); + + /* Accumulate the amount of bytes of each buffer and check if it exceeds NET_MAX_WRITES_PER_EVENT. */ + size_t iov_bytes_len = 0; + for (int i = 0; i < iovcnt; i++) { + iov_bytes_len += iov[i].iov_len; + if (iov_bytes_len > NET_MAX_WRITES_PER_EVENT) break; + } + + /* The amount of all buffers is greater than NET_MAX_WRITES_PER_EVENT, + * which is not worth doing so much memory copying to reduce system calls, + * therefore, invoke connTLSWrite() multiple times to avoid memory copies. */ + if (iov_bytes_len > NET_MAX_WRITES_PER_EVENT) { + size_t tot_sent = 0; + for (int i = 0; i < iovcnt; i++) { + size_t sent = connTLSWrite(conn_, iov[i].iov_base, iov[i].iov_len); + if (sent <= 0) return tot_sent > 0 ? tot_sent : sent; + tot_sent += sent; + if (sent != iov[i].iov_len) break; + } + return tot_sent; + } + + /* The amount of all buffers is less than NET_MAX_WRITES_PER_EVENT, + * which is worth doing more memory copies in exchange for fewer system calls, + * so concatenate these scattered buffers into a contiguous piece of memory + * and send it away by one call to connTLSWrite(). */ + char buf[iov_bytes_len]; + size_t offset = 0; + for (int i = 0; i < iovcnt; i++) { + memcpy(buf + offset, iov[i].iov_base, iov[i].iov_len); + offset += iov[i].iov_len; + } + return connTLSWrite(conn_, buf, iov_bytes_len); +} + static int connTLSRead(connection *conn_, void *buf, size_t buf_len) { tls_connection *conn = (tls_connection *) conn_; int ret; @@ -949,6 +1020,7 @@ ConnectionType CT_TLS = { .blocking_connect = connTLSBlockingConnect, .read = connTLSRead, .write = connTLSWrite, + .writev = connTLSWritev, .close = connTLSClose, .set_write_handler = connTLSSetWriteHandler, .set_read_handler = connTLSSetReadHandler, diff --git a/src/version.h b/src/version.h index cddaae2b1..0f750bad3 100644 --- a/src/version.h +++ b/src/version.h @@ -1,2 +1,2 @@ -#define REDIS_VERSION "6.9.240" -#define REDIS_VERSION_NUM 0x000609f0 +#define REDIS_VERSION "6.9.241" +#define REDIS_VERSION_NUM 0x000609f1 diff --git a/tests/cluster/tests/15-cluster-slots.tcl b/tests/cluster/tests/15-cluster-slots.tcl index f154b7270..93b64b408 100644 --- a/tests/cluster/tests/15-cluster-slots.tcl +++ b/tests/cluster/tests/15-cluster-slots.tcl @@ -49,6 +49,34 @@ test "client can handle keys with hash tag" { $cluster close } +test "slot migration is valid from primary to another primary" { + set cluster [redis_cluster 127.0.0.1:[get_instance_attrib redis 0 port]] + set key order1 + set slot [$cluster cluster keyslot $key] + array set nodefrom [$cluster masternode_for_slot $slot] + array set nodeto [$cluster masternode_notfor_slot $slot] + + assert_equal {OK} [$nodefrom(link) cluster setslot $slot node $nodeto(id)] + assert_equal {OK} [$nodeto(link) cluster setslot $slot node $nodeto(id)] +} + +test "slot migration is invalid from primary to replica" { + set cluster [redis_cluster 127.0.0.1:[get_instance_attrib redis 0 port]] + set key order1 + set slot [$cluster cluster keyslot $key] + array set nodefrom [$cluster masternode_for_slot $slot] + + # Get replica node serving slot. + set replicanodeinfo [$cluster cluster replicas $nodefrom(id)] + puts $replicanodeinfo + set args [split $replicanodeinfo " "] + set replicaid [lindex [split [lindex $args 0] \{] 1] + puts $replicaid + + catch {[$nodefrom(link) cluster setslot $slot node $replicaid]} err + assert_match "*Target node is not a master" $err +} + if {$::tls} { test {CLUSTER SLOTS from non-TLS client in TLS cluster} { set slots_tls [R 0 cluster slots] @@ -60,4 +88,4 @@ if {$::tls} { # Compare the ports in the first row assert_no_match [lindex $slots_tls 0 3 1] [lindex $slots_plain 0 3 1] } -} +}
\ No newline at end of file diff --git a/tests/cluster/tests/23-multiple-slot-operations.tcl b/tests/cluster/tests/23-multiple-slot-operations.tcl index 965ecd5af..060ab577a 100644 --- a/tests/cluster/tests/23-multiple-slot-operations.tcl +++ b/tests/cluster/tests/23-multiple-slot-operations.tcl @@ -33,7 +33,6 @@ set master3 [Rn 2] set master4 [Rn 3] set master5 [Rn 4] - test "Continuous slots distribution" { assert_match "* 0-3276*" [$master1 CLUSTER NODES] assert_match "* 3277-6552*" [$master2 CLUSTER NODES] @@ -66,3 +65,51 @@ test "Continuous slots distribution" { assert_match "* 13105-13500 14001-15000 16001-16383*" [$master5 CLUSTER NODES] assert_match "*13105 13500*14001 15000*16001 16383*" [$master5 CLUSTER SLOTS] } + +test "ADDSLOTSRANGE command with several boundary conditions test suite" { + # Add multiple slots with incorrect argument number + assert_error "ERR wrong number of arguments for 'cluster|addslotsrange' command" {R 0 cluster ADDSLOTSRANGE 3001 3020 3030} + + # Add multiple slots with invalid input slot + assert_error "ERR Invalid or out of range slot" {R 0 cluster ADDSLOTSRANGE 3001 3020 3030 aaa} + assert_error "ERR Invalid or out of range slot" {R 0 cluster ADDSLOTSRANGE 3001 3020 3030 70000} + assert_error "ERR Invalid or out of range slot" {R 0 cluster ADDSLOTSRANGE 3001 3020 -1000 3030} + + # Add multiple slots when start slot number is greater than the end slot + assert_error "ERR start slot number 3030 is greater than end slot number 3025" {R 0 cluster ADDSLOTSRANGE 3001 3020 3030 3025} + + # Add multiple slots with busy slot + assert_error "ERR Slot 3200 is already busy" {R 0 cluster ADDSLOTSRANGE 3001 3020 3200 3250} + + # Add multiple slots with assigned multiple times + assert_error "ERR Slot 3001 specified multiple times" {R 0 cluster ADDSLOTSRANGE 3001 3020 3001 3020} +} + +test "DELSLOTSRANGE command with several boundary conditions test suite" { + # Delete multiple slots with incorrect argument number + assert_error "ERR wrong number of arguments for 'cluster|delslotsrange' command" {R 0 cluster DELSLOTSRANGE 1000 2000 2100} + assert_match "* 0-3000 3051-3276*" [$master1 CLUSTER NODES] + assert_match "*0 3000*3051 3276*" [$master1 CLUSTER SLOTS] + + # Delete multiple slots with invalid input slot + assert_error "ERR Invalid or out of range slot" {R 0 cluster DELSLOTSRANGE 1000 2000 2100 aaa} + assert_error "ERR Invalid or out of range slot" {R 0 cluster DELSLOTSRANGE 1000 2000 2100 70000} + assert_error "ERR Invalid or out of range slot" {R 0 cluster DELSLOTSRANGE 1000 2000 -2100 2200} + assert_match "* 0-3000 3051-3276*" [$master1 CLUSTER NODES] + assert_match "*0 3000*3051 3276*" [$master1 CLUSTER SLOTS] + + # Delete multiple slots when start slot number is greater than the end slot + assert_error "ERR start slot number 5800 is greater than end slot number 5750" {R 1 cluster DELSLOTSRANGE 5600 5700 5800 5750} + assert_match "* 3277-5000 5501-6552*" [$master2 CLUSTER NODES] + assert_match "*3277 5000*5501 6552*" [$master2 CLUSTER SLOTS] + + # Delete multiple slots with already unassigned + assert_error "ERR Slot 7001 is already unassigned" {R 2 cluster DELSLOTSRANGE 7001 7100 9000 9200} + assert_match "* 6553-7000 7101-8000 8501-9828*" [$master3 CLUSTER NODES] + assert_match "*6553 7000*7101 8000*8501 9828*" [$master3 CLUSTER SLOTS] + + # Delete multiple slots with assigned multiple times + assert_error "ERR Slot 12500 specified multiple times" {R 3 cluster DELSLOTSRANGE 12500 12600 12500 12600} + assert_match "* 9829-11000 12001-12100 12201-13104*" [$master4 CLUSTER NODES] + assert_match "*9829 11000*12001 12100*12201 13104*" [$master4 CLUSTER SLOTS] +} diff --git a/tests/integration/aof-multi-part.tcl b/tests/integration/aof-multi-part.tcl index 52877404c..982b6907b 100644 --- a/tests/integration/aof-multi-part.tcl +++ b/tests/integration/aof-multi-part.tcl @@ -45,7 +45,7 @@ tags {"external:skip"} { fail "AOF loading didn't fail" } - assert_equal 1 [count_message_lines $server_path/stdout "appendonly.aof.1.incr.aof doesn't exist"] + assert_equal 1 [count_message_lines $server_path/stdout "appendonly.aof.1.incr.aof .*No such file or directory"] } clean_aof_persistence $aof_dirpath @@ -100,7 +100,7 @@ tags {"external:skip"} { fail "AOF loading didn't fail" } - assert_equal 1 [count_message_lines $server_path/stdout "The AOF manifest file is invalid format"] + assert_equal 1 [count_message_lines $server_path/stdout "Invalid AOF manifest file format"] } clean_aof_persistence $aof_dirpath @@ -186,7 +186,7 @@ tags {"external:skip"} { fail "AOF loading didn't fail" } - assert_equal 2 [count_message_lines $server_path/stdout "The AOF manifest file is invalid format"] + assert_equal 2 [count_message_lines $server_path/stdout "Invalid AOF manifest file format"] } clean_aof_persistence $aof_dirpath @@ -213,7 +213,7 @@ tags {"external:skip"} { fail "AOF loading didn't fail" } - assert_equal 3 [count_message_lines $server_path/stdout "The AOF manifest file is invalid format"] + assert_equal 3 [count_message_lines $server_path/stdout "Invalid AOF manifest file format"] } clean_aof_persistence $aof_dirpath @@ -267,7 +267,7 @@ tags {"external:skip"} { fail "AOF loading didn't fail" } - assert_equal 4 [count_message_lines $server_path/stdout "The AOF manifest file is invalid format"] + assert_equal 4 [count_message_lines $server_path/stdout "Invalid AOF manifest file format"] } clean_aof_persistence $aof_dirpath @@ -584,7 +584,7 @@ tags {"external:skip"} { fail "AOF loading didn't fail" } - assert_equal 1 [count_message_lines $server_path/stdout "appendonly.aof doesn't exist"] + assert_equal 1 [count_message_lines $server_path/stdout "appendonly.aof .*No such file or directory"] } clean_aof_persistence $aof_dirpath diff --git a/tests/integration/aof.tcl b/tests/integration/aof.tcl index cab1d3745..3d8c08c51 100644 --- a/tests/integration/aof.tcl +++ b/tests/integration/aof.tcl @@ -4,6 +4,7 @@ set server_path [tmpdir server.aof] set aof_dirname "appendonlydir" set aof_basename "appendonly.aof" set aof_dirpath "$server_path/$aof_dirname" +set aof_base_file "$server_path/$aof_dirname/${aof_basename}.1$::base_aof_sufix$::aof_format_suffix" set aof_file "$server_path/$aof_dirname/${aof_basename}.1$::incr_aof_sufix$::aof_format_suffix" set aof_manifest_file "$server_path/$aof_dirname/$aof_basename$::manifest_suffix" @@ -142,7 +143,7 @@ tags {"aof external:skip"} { ## Test that redis-check-aof indeed sees this AOF is not valid test "Short read: Utility should confirm the AOF is not valid" { catch { - exec src/redis-check-aof $aof_file + exec src/redis-check-aof $aof_manifest_file } result assert_match "*not valid*" $result } @@ -154,13 +155,13 @@ tags {"aof external:skip"} { } catch { - exec src/redis-check-aof $aof_file + exec src/redis-check-aof $aof_manifest_file } result assert_match "*ok_up_to_line=8*" $result } test "Short read: Utility should be able to fix the AOF" { - set result [exec src/redis-check-aof --fix $aof_file << "y\n"] + set result [exec src/redis-check-aof --fix $aof_manifest_file << "y\n"] assert_match "*Successfully truncated AOF*" $result } @@ -444,7 +445,7 @@ tags {"aof external:skip"} { test {Truncate AOF to specific timestamp} { # truncate to timestamp 1628217473 - exec src/redis-check-aof --truncate-to-timestamp 1628217473 $aof_file + exec src/redis-check-aof --truncate-to-timestamp 1628217473 $aof_manifest_file start_server_aof [list dir $server_path] { set c [redis [dict get $srv host] [dict get $srv port] 0 $::tls] wait_done_loading $c @@ -454,7 +455,7 @@ tags {"aof external:skip"} { } # truncate to timestamp 1628217471 - exec src/redis-check-aof --truncate-to-timestamp 1628217471 $aof_file + exec src/redis-check-aof --truncate-to-timestamp 1628217471 $aof_manifest_file start_server_aof [list dir $server_path] { set c [redis [dict get $srv host] [dict get $srv port] 0 $::tls] wait_done_loading $c @@ -464,7 +465,7 @@ tags {"aof external:skip"} { } # truncate to timestamp 1628217470 - exec src/redis-check-aof --truncate-to-timestamp 1628217470 $aof_file + exec src/redis-check-aof --truncate-to-timestamp 1628217470 $aof_manifest_file start_server_aof [list dir $server_path] { set c [redis [dict get $srv host] [dict get $srv port] 0 $::tls] wait_done_loading $c @@ -473,7 +474,7 @@ tags {"aof external:skip"} { } # truncate to timestamp 1628217469 - catch {exec src/redis-check-aof --truncate-to-timestamp 1628217469 $aof_file} e + catch {exec src/redis-check-aof --truncate-to-timestamp 1628217469 $aof_manifest_file} e assert_match {*aborting*} $e } @@ -515,4 +516,120 @@ tags {"aof external:skip"} { assert_equal [r get foo] 102 } } + + test {Test redis-check-aof for old style resp AOF} { + create_aof $aof_dirpath $aof_file { + append_to_aof [formatCommand set foo hello] + append_to_aof [formatCommand set bar world] + } + + catch { + exec src/redis-check-aof $aof_file + } result + assert_match "*Start checking Old-Style AOF*is valid*" $result + } + + test {Test redis-check-aof for old style rdb-preamble AOF} { + catch { + exec src/redis-check-aof tests/assets/rdb-preamble.aof + } result + assert_match "*Start checking Old-Style AOF*RDB preamble is OK, proceeding with AOF tail*is valid*" $result + } + + test {Test redis-check-aof for Multi Part AOF with resp AOF base} { + create_aof $aof_dirpath $aof_base_file { + append_to_aof [formatCommand set foo hello] + append_to_aof [formatCommand set bar world] + } + + create_aof $aof_dirpath $aof_file { + append_to_aof [formatCommand set foo hello] + append_to_aof [formatCommand set bar world] + } + + create_aof_manifest $aof_dirpath $aof_manifest_file { + append_to_manifest "file appendonly.aof.1.base.aof seq 1 type b\n" + append_to_manifest "file appendonly.aof.1.incr.aof seq 1 type i\n" + } + + catch { + exec src/redis-check-aof $aof_manifest_file + } result + assert_match "*Start checking Multi Part AOF*Start to check BASE AOF (RESP format)*BASE AOF*is valid*Start to check INCR files*INCR AOF*is valid*All AOF files and manifest are valid*" $result + } + + test {Test redis-check-aof for Multi Part AOF with rdb-preamble AOF base} { + exec cp tests/assets/rdb-preamble.aof $aof_base_file + + create_aof $aof_dirpath $aof_file { + append_to_aof [formatCommand set foo hello] + append_to_aof [formatCommand set bar world] + } + + create_aof_manifest $aof_dirpath $aof_manifest_file { + append_to_manifest "file appendonly.aof.1.base.aof seq 1 type b\n" + append_to_manifest "file appendonly.aof.1.incr.aof seq 1 type i\n" + } + + catch { + exec src/redis-check-aof $aof_manifest_file + } result + assert_match "*Start checking Multi Part AOF*Start to check BASE AOF (RDB format)*DB preamble is OK, proceeding with AOF tail*BASE AOF*is valid*Start to check INCR files*INCR AOF*is valid*All AOF files and manifest are valid*" $result + } + + test {Test redis-check-aof only truncates the last file for Multi Part AOF in fix mode} { + create_aof $aof_dirpath $aof_base_file { + append_to_aof [formatCommand set foo hello] + append_to_aof [formatCommand multi] + append_to_aof [formatCommand set bar world] + } + + create_aof $aof_dirpath $aof_file { + append_to_aof [formatCommand set foo hello] + append_to_aof [formatCommand set bar world] + } + + create_aof_manifest $aof_dirpath $aof_manifest_file { + append_to_manifest "file appendonly.aof.1.base.aof seq 1 type b\n" + append_to_manifest "file appendonly.aof.1.incr.aof seq 1 type i\n" + } + + catch { + exec src/redis-check-aof $aof_manifest_file + } result + assert_match "*not valid*" $result + + catch { + exec src/redis-check-aof --fix $aof_manifest_file + } result + assert_match "*Failed to truncate AOF*because it is not the last file*" $result + } + + test {Test redis-check-aof only truncates the last file for Multi Part AOF in truncate-to-timestamp mode} { + create_aof $aof_dirpath $aof_base_file { + append_to_aof "#TS:1628217470\r\n" + append_to_aof [formatCommand set foo1 bar1] + append_to_aof "#TS:1628217471\r\n" + append_to_aof [formatCommand set foo2 bar2] + append_to_aof "#TS:1628217472\r\n" + append_to_aof "#TS:1628217473\r\n" + append_to_aof [formatCommand set foo3 bar3] + append_to_aof "#TS:1628217474\r\n" + } + + create_aof $aof_dirpath $aof_file { + append_to_aof [formatCommand set foo hello] + append_to_aof [formatCommand set bar world] + } + + create_aof_manifest $aof_dirpath $aof_manifest_file { + append_to_manifest "file appendonly.aof.1.base.aof seq 1 type b\n" + append_to_manifest "file appendonly.aof.1.incr.aof seq 1 type i\n" + } + + catch { + exec src/redis-check-aof --truncate-to-timestamp 1628217473 $aof_manifest_file + } result + assert_match "*Failed to truncate AOF*to timestamp*because it is not the last file*" $result + } } diff --git a/tests/integration/corrupt-dump.tcl b/tests/integration/corrupt-dump.tcl index 86c7dd246..d2491306a 100644 --- a/tests/integration/corrupt-dump.tcl +++ b/tests/integration/corrupt-dump.tcl @@ -193,9 +193,8 @@ test {corrupt payload: listpack invalid size header} { test {corrupt payload: listpack too long entry len} { start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { r config set sanitize-dump-payload no - r restore key 0 "\x0F\x01\x10\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x02\x40\x55\x55\x00\x00\x00\x0F\x00\x01\x01\x00\x01\x02\x01\x88\x31\x00\x00\x00\x00\x00\x00\x00\x09\x88\x32\x00\x00\x00\x00\x00\x00\x00\x09\x00\x01\x00\x01\x00\x01\x00\x01\x02\x02\x89\x31\x00\x00\x00\x00\x00\x00\x00\x09\x88\x61\x00\x00\x00\x00\x00\x00\x00\x09\x88\x32\x00\x00\x00\x00\x00\x00\x00\x09\x88\x62\x00\x00\x00\x00\x00\x00\x00\x09\x08\x01\xFF\x0A\x01\x00\x00\x09\x00\x40\x63\xC9\x37\x03\xA2\xE5\x68" catch { - r xinfo stream key full + r restore key 0 "\x0F\x01\x10\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x02\x40\x55\x55\x00\x00\x00\x0F\x00\x01\x01\x00\x01\x02\x01\x88\x31\x00\x00\x00\x00\x00\x00\x00\x09\x88\x32\x00\x00\x00\x00\x00\x00\x00\x09\x00\x01\x00\x01\x00\x01\x00\x01\x02\x02\x89\x31\x00\x00\x00\x00\x00\x00\x00\x09\x88\x61\x00\x00\x00\x00\x00\x00\x00\x09\x88\x32\x00\x00\x00\x00\x00\x00\x00\x09\x88\x62\x00\x00\x00\x00\x00\x00\x00\x09\x08\x01\xFF\x0A\x01\x00\x00\x09\x00\x40\x63\xC9\x37\x03\xA2\xE5\x68" } err assert_equal [count_log_message 0 "crashed by signal"] 0 assert_equal [count_log_message 0 "ASSERTION FAILED"] 1 @@ -205,9 +204,9 @@ test {corrupt payload: listpack too long entry len} { test {corrupt payload: listpack very long entry len} { start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { r config set sanitize-dump-payload no - r restore key 0 "\x0F\x01\x10\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x02\x40\x55\x55\x00\x00\x00\x0F\x00\x01\x01\x00\x01\x02\x01\x88\x31\x00\x00\x00\x00\x00\x00\x00\x09\x88\x32\x00\x00\x00\x00\x00\x00\x00\x09\x00\x01\x00\x01\x00\x01\x00\x01\x02\x02\x88\x31\x00\x00\x00\x00\x00\x00\x00\x09\x88\x61\x00\x00\x00\x00\x00\x00\x00\x09\x88\x32\x00\x00\x00\x00\x00\x00\x00\x09\x9C\x62\x00\x00\x00\x00\x00\x00\x00\x09\x08\x01\xFF\x0A\x01\x00\x00\x09\x00\x63\x6F\x42\x8E\x7C\xB5\xA2\x9D" catch { - r xinfo stream key full + # This will catch migrated payloads from v6.2.x + r restore key 0 "\x0F\x01\x10\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x02\x40\x55\x55\x00\x00\x00\x0F\x00\x01\x01\x00\x01\x02\x01\x88\x31\x00\x00\x00\x00\x00\x00\x00\x09\x88\x32\x00\x00\x00\x00\x00\x00\x00\x09\x00\x01\x00\x01\x00\x01\x00\x01\x02\x02\x88\x31\x00\x00\x00\x00\x00\x00\x00\x09\x88\x61\x00\x00\x00\x00\x00\x00\x00\x09\x88\x32\x00\x00\x00\x00\x00\x00\x00\x09\x9C\x62\x00\x00\x00\x00\x00\x00\x00\x09\x08\x01\xFF\x0A\x01\x00\x00\x09\x00\x63\x6F\x42\x8E\x7C\xB5\xA2\x9D" } err assert_equal [count_log_message 0 "crashed by signal"] 0 assert_equal [count_log_message 0 "ASSERTION FAILED"] 1 diff --git a/tests/integration/replication-4.tcl b/tests/integration/replication-4.tcl index 8cccd9678..b8c50308a 100644 --- a/tests/integration/replication-4.tcl +++ b/tests/integration/replication-4.tcl @@ -46,6 +46,12 @@ start_server {tags {"repl external:skip"}} { set master_port [srv -1 port] set slave [srv 0 client] + # Load some functions to be used later + $master FUNCTION load lua test replace { + redis.register_function{function_name='f_default_flags', callback=function(keys, args) return redis.call('get',keys[1]) end, flags={}} + redis.register_function{function_name='f_no_writes', callback=function(keys, args) return redis.call('get',keys[1]) end, flags={'no-writes'}} + } + test {First server should have role slave after SLAVEOF} { $slave slaveof $master_host $master_port wait_replica_online $master @@ -54,28 +60,46 @@ start_server {tags {"repl external:skip"}} { test {With min-slaves-to-write (1,3): master should be writable} { $master config set min-slaves-max-lag 3 $master config set min-slaves-to-write 1 - $master set foo bar - } {OK} + assert_equal OK [$master set foo 123] + assert_equal OK [$master eval "return redis.call('set','foo',12345)" 0] + } test {With min-slaves-to-write (2,3): master should not be writable} { $master config set min-slaves-max-lag 3 $master config set min-slaves-to-write 2 - catch {$master set foo bar} e - set e - } {NOREPLICAS*} + assert_error "*NOREPLICAS*" {$master set foo bar} + assert_error "*NOREPLICAS*" {$master eval "redis.call('set','foo','bar')" 0} + } + + test {With min-slaves-to-write function without no-write flag} { + assert_error "*NOREPLICAS*" {$master fcall f_default_flags 1 foo} + assert_equal "12345" [$master fcall f_no_writes 1 foo] + } + + test {With not enough good slaves, read in Lua script is still accepted} { + $master config set min-slaves-max-lag 3 + $master config set min-slaves-to-write 1 + $master eval "redis.call('set','foo','bar')" 0 + + $master config set min-slaves-to-write 2 + $master eval "return redis.call('get','foo')" 0 + } {bar} test {With min-slaves-to-write: master not writable with lagged slave} { $master config set min-slaves-max-lag 2 $master config set min-slaves-to-write 1 - assert {[$master set foo bar] eq {OK}} + assert_equal OK [$master set foo 123] + assert_equal OK [$master eval "return redis.call('set','foo',12345)" 0] + # Killing a slave to make it become a lagged slave. exec kill -SIGSTOP [srv 0 pid] + # Waiting for slave kill. wait_for_condition 100 100 { - [catch {$master set foo bar}] != 0 + [catch {$master set foo 123}] != 0 } else { fail "Master didn't become readonly" } - catch {$master set foo bar} err - assert_match {NOREPLICAS*} $err + assert_error "*NOREPLICAS*" {$master set foo 123} + assert_error "*NOREPLICAS*" {$master eval "return redis.call('set','foo',12345)" 0} exec kill -SIGCONT [srv 0 pid] } } diff --git a/tests/integration/replication.tcl b/tests/integration/replication.tcl index 2af07e806..05f62d5e8 100644 --- a/tests/integration/replication.tcl +++ b/tests/integration/replication.tcl @@ -318,7 +318,7 @@ foreach mdl {no yes} { stop_write_load $load_handle4 # Make sure no more commands processed - wait_load_handlers_disconnected + wait_load_handlers_disconnected -3 wait_for_ofs_sync $master [lindex $slaves 0] wait_for_ofs_sync $master [lindex $slaves 1] @@ -428,7 +428,7 @@ foreach testType {Successful Aborted} { } else { fail "Replica didn't get into loading mode" } - + assert_equal [s -1 async_loading] 0 } @@ -444,7 +444,7 @@ foreach testType {Successful Aborted} { } else { fail "Replica didn't disconnect" } - + test {Diskless load swapdb (different replid): old database is exposed after replication fails} { # Ensure we see old values from replica assert_equal [$replica get mykey] "myvalue" @@ -551,10 +551,10 @@ foreach testType {Successful Aborted} { } else { fail "Replica didn't get into async_loading mode" } - + assert_equal [s -1 loading] 0 } - + test {Diskless load swapdb (async_loading): old database is exposed while async replication is in progress} { # Ensure we still see old values while async_loading is in progress and also not LOADING status assert_equal [$replica get mykey] "myvalue" @@ -598,7 +598,7 @@ foreach testType {Successful Aborted} { } else { fail "Replica didn't disconnect" } - + test {Diskless load swapdb (async_loading): old database is exposed after async replication fails} { # Ensure we see old values from replica assert_equal [$replica get mykey] "myvalue" @@ -1222,14 +1222,100 @@ test {replica can handle EINTR if use diskless load} { # Wait for the replica to start reading the rdb set res [wait_for_log_messages -1 {"*Loading DB in memory*"} 0 200 10] set loglines [lindex $res 1] - + # Wait till we see the watchgod log line AFTER the loading started wait_for_log_messages -1 {"*WATCHDOG TIMER EXPIRED*"} $loglines 200 10 - + # Make sure we're still loading, and that there was just one full sync attempt - assert ![log_file_matches [srv -1 stdout] "*Reconnecting to MASTER*"] + assert ![log_file_matches [srv -1 stdout] "*Reconnecting to MASTER*"] assert_equal 1 [s 0 sync_full] assert_equal 1 [s -1 loading] } } } {} {external:skip} + +start_server {tags {"repl" "external:skip"}} { + test "replica do not write the reply to the replication link - SYNC (_addReplyToBufferOrList)" { + set rd [redis_deferring_client] + set lines [count_log_lines 0] + + $rd sync + $rd ping + catch {$rd read} e + if {$::verbose} { puts "SYNC _addReplyToBufferOrList: $e" } + assert_equal "PONG" [r ping] + + # Check we got the warning logs about the PING command. + verify_log_message 0 "*Replica generated a reply to command 'ping', disconnecting it: *" $lines + + $rd close + catch {exec kill -9 [get_child_pid 0]} + waitForBgsave r + } + + test "replica do not write the reply to the replication link - SYNC (addReplyDeferredLen)" { + set rd [redis_deferring_client] + set lines [count_log_lines 0] + + $rd sync + $rd xinfo help + catch {$rd read} e + if {$::verbose} { puts "SYNC addReplyDeferredLen: $e" } + assert_equal "PONG" [r ping] + + # Check we got the warning logs about the XINFO HELP command. + verify_log_message 0 "*Replica generated a reply to command 'xinfo|help', disconnecting it: *" $lines + + $rd close + catch {exec kill -9 [get_child_pid 0]} + waitForBgsave r + } + + test "replica do not write the reply to the replication link - PSYNC (_addReplyToBufferOrList)" { + set rd [redis_deferring_client] + set lines [count_log_lines 0] + + $rd psync replicationid -1 + assert_match {FULLRESYNC * 0} [$rd read] + $rd get foo + catch {$rd read} e + if {$::verbose} { puts "PSYNC _addReplyToBufferOrList: $e" } + assert_equal "PONG" [r ping] + + # Check we got the warning logs about the GET command. + verify_log_message 0 "*Replica generated a reply to command 'get', disconnecting it: *" $lines + verify_log_message 0 "*== CRITICAL == This master is sending an error to its replica: *" $lines + verify_log_message 0 "*Replica can't interact with the keyspace*" $lines + + $rd close + catch {exec kill -9 [get_child_pid 0]} + waitForBgsave r + } + + test "replica do not write the reply to the replication link - PSYNC (addReplyDeferredLen)" { + set rd [redis_deferring_client] + set lines [count_log_lines 0] + + $rd psync replicationid -1 + assert_match {FULLRESYNC * 0} [$rd read] + $rd slowlog get + catch {$rd read} e + if {$::verbose} { puts "PSYNC addReplyDeferredLen: $e" } + assert_equal "PONG" [r ping] + + # Check we got the warning logs about the SLOWLOG GET command. + verify_log_message 0 "*Replica generated a reply to command 'slowlog|get', disconnecting it: *" $lines + + $rd close + catch {exec kill -9 [get_child_pid 0]} + waitForBgsave r + } + + test "PSYNC with wrong offset should throw error" { + # It used to accept the FULL SYNC, but also replied with an error. + assert_error {ERR value is not an integer or out of range} {r psync replicationid offset_str} + set logs [exec tail -n 100 < [srv 0 stdout]] + assert_match {*Replica * asks for synchronization but with a wrong offset} $logs + assert_equal "PONG" [r ping] + } +} diff --git a/tests/modules/Makefile b/tests/modules/Makefile index ec8e6de89..ce842f3af 100644 --- a/tests/modules/Makefile +++ b/tests/modules/Makefile @@ -2,11 +2,12 @@ # find the OS uname_S := $(shell sh -c 'uname -s 2>/dev/null || echo not') +warning_cflags = -W -Wall -Wno-missing-field-initializers ifeq ($(uname_S),Darwin) - SHOBJ_CFLAGS ?= -W -Wall -dynamic -fno-common -g -ggdb -std=c99 -O2 + SHOBJ_CFLAGS ?= $(warning_cflags) -dynamic -fno-common -g -ggdb -std=c99 -O2 SHOBJ_LDFLAGS ?= -bundle -undefined dynamic_lookup else # Linux, others - SHOBJ_CFLAGS ?= -W -Wall -fno-common -g -ggdb -std=c99 -O2 + SHOBJ_CFLAGS ?= $(warning_cflags) -fno-common -g -ggdb -std=c99 -O2 SHOBJ_LDFLAGS ?= -shared endif @@ -40,6 +41,7 @@ TEST_MODULES = \ keyspace_events.so \ blockedclient.so \ getkeys.so \ + getchannels.so \ test_lazyfree.so \ timer.so \ defragtest.so \ @@ -51,6 +53,7 @@ TEST_MODULES = \ list.so \ subcommands.so \ reply.so \ + cmdintrospection.so \ eventloop.so .PHONY: all diff --git a/tests/modules/aclcheck.c b/tests/modules/aclcheck.c index cc8d263fd..0e9c9af29 100644 --- a/tests/modules/aclcheck.c +++ b/tests/modules/aclcheck.c @@ -15,11 +15,13 @@ int set_aclcheck_key(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { const char *flags = RedisModule_StringPtrLen(argv[1], NULL); if (!strcasecmp(flags, "W")) { - permissions = REDISMODULE_KEY_PERMISSION_WRITE; + permissions = REDISMODULE_CMD_KEY_UPDATE; } else if (!strcasecmp(flags, "R")) { - permissions = REDISMODULE_KEY_PERMISSION_READ; + permissions = REDISMODULE_CMD_KEY_ACCESS; } else if (!strcasecmp(flags, "*")) { - permissions = REDISMODULE_KEY_PERMISSION_ALL; + permissions = REDISMODULE_CMD_KEY_UPDATE | REDISMODULE_CMD_KEY_ACCESS; + } else if (!strcasecmp(flags, "~")) { + permissions = 0; /* Requires either read or write */ } else { RedisModule_ReplyWithError(ctx, "INVALID FLAGS"); return REDISMODULE_OK; @@ -58,7 +60,7 @@ int publish_aclcheck_channel(RedisModuleCtx *ctx, RedisModuleString **argv, int /* Check that the pubsub channel can be accessed */ RedisModuleString *user_name = RedisModule_GetCurrentUserName(ctx); RedisModuleUser *user = RedisModule_GetModuleUserFromUserName(user_name); - int ret = RedisModule_ACLCheckChannelPermissions(user, argv[1], 1); + int ret = RedisModule_ACLCheckChannelPermissions(user, argv[1], REDISMODULE_CMD_CHANNEL_SUBSCRIBE); if (ret != 0) { RedisModule_ReplyWithError(ctx, "DENIED CHANNEL"); RedisModule_FreeModuleUser(user); diff --git a/tests/modules/cmdintrospection.c b/tests/modules/cmdintrospection.c new file mode 100644 index 000000000..aba817e14 --- /dev/null +++ b/tests/modules/cmdintrospection.c @@ -0,0 +1,157 @@ +#include "redismodule.h" + +#define UNUSED(V) ((void) V) + +int cmd_xadd(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + UNUSED(argv); + UNUSED(argc); + RedisModule_ReplyWithSimpleString(ctx, "OK"); + return REDISMODULE_OK; +} + +int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + REDISMODULE_NOT_USED(argv); + REDISMODULE_NOT_USED(argc); + if (RedisModule_Init(ctx, "cmdintrospection", 1, REDISMODULE_APIVER_1) == REDISMODULE_ERR) + return REDISMODULE_ERR; + + if (RedisModule_CreateCommand(ctx,"cmdintrospection.xadd",cmd_xadd,"write deny-oom random fast",0,0,0) == REDISMODULE_ERR) + return REDISMODULE_ERR; + + RedisModuleCommand *xadd = RedisModule_GetCommand(ctx,"cmdintrospection.xadd"); + + RedisModuleCommandInfo info = { + .version = REDISMODULE_COMMAND_INFO_VERSION, + .arity = -5, + .summary = "Appends a new entry to a stream", + .since = "5.0.0", + .complexity = "O(1) when adding a new entry, O(N) when trimming where N being the number of entries evicted.", + .tips = "nondeterministic_output", + .history = (RedisModuleCommandHistoryEntry[]){ + /* NOTE: All versions specified should be the module's versions, not + * Redis'! We use Redis versions in this example for the purpose of + * testing (comparing the output with the output of the vanilla + * XADD). */ + {"6.2.0", "Added the `NOMKSTREAM` option, `MINID` trimming strategy and the `LIMIT` option."}, + {"7.0.0", "Added support for the `<ms>-*` explicit ID form."}, + {0} + }, + .key_specs = (RedisModuleCommandKeySpec[]){ + { + .notes = "UPDATE instead of INSERT because of the optional trimming feature", + .flags = REDISMODULE_CMD_KEY_RW | REDISMODULE_CMD_KEY_UPDATE, + .begin_search_type = REDISMODULE_KSPEC_BS_INDEX, + .bs.index.pos = 1, + .find_keys_type = REDISMODULE_KSPEC_FK_RANGE, + .fk.range = {0,1,0} + }, + {0} + }, + .args = (RedisModuleCommandArg[]){ + { + .name = "key", + .type = REDISMODULE_ARG_TYPE_KEY, + .key_spec_index = 0 + }, + { + .name = "nomkstream", + .type = REDISMODULE_ARG_TYPE_PURE_TOKEN, + .token = "NOMKSTREAM", + .since = "6.2.0", + .flags = REDISMODULE_CMD_ARG_OPTIONAL + }, + { + .name = "trim", + .type = REDISMODULE_ARG_TYPE_BLOCK, + .flags = REDISMODULE_CMD_ARG_OPTIONAL, + .subargs = (RedisModuleCommandArg[]){ + { + .name = "strategy", + .type = REDISMODULE_ARG_TYPE_ONEOF, + .subargs = (RedisModuleCommandArg[]){ + { + .name = "maxlen", + .type = REDISMODULE_ARG_TYPE_PURE_TOKEN, + .token = "MAXLEN", + }, + { + .name = "minid", + .type = REDISMODULE_ARG_TYPE_PURE_TOKEN, + .token = "MINID", + .since = "6.2.0", + }, + {0} + } + }, + { + .name = "operator", + .type = REDISMODULE_ARG_TYPE_ONEOF, + .flags = REDISMODULE_CMD_ARG_OPTIONAL, + .subargs = (RedisModuleCommandArg[]){ + { + .name = "equal", + .type = REDISMODULE_ARG_TYPE_PURE_TOKEN, + .token = "=" + }, + { + .name = "approximately", + .type = REDISMODULE_ARG_TYPE_PURE_TOKEN, + .token = "~" + }, + {0} + } + }, + { + .name = "threshold", + .type = REDISMODULE_ARG_TYPE_STRING, + }, + { + .name = "count", + .type = REDISMODULE_ARG_TYPE_INTEGER, + .token = "LIMIT", + .since = "6.2.0", + .flags = REDISMODULE_CMD_ARG_OPTIONAL + }, + {0} + } + }, + { + .name = "id_or_auto", + .type = REDISMODULE_ARG_TYPE_ONEOF, + .subargs = (RedisModuleCommandArg[]){ + { + .name = "auto_id", + .type = REDISMODULE_ARG_TYPE_PURE_TOKEN, + .token = "*" + }, + { + .name = "id", + .type = REDISMODULE_ARG_TYPE_STRING, + }, + {0} + } + }, + { + .name = "field_value", + .type = REDISMODULE_ARG_TYPE_BLOCK, + .flags = REDISMODULE_CMD_ARG_MULTIPLE, + .subargs = (RedisModuleCommandArg[]){ + { + .name = "field", + .type = REDISMODULE_ARG_TYPE_STRING, + }, + { + .name = "value", + .type = REDISMODULE_ARG_TYPE_STRING, + }, + {0} + } + }, + {0} + } + }; + if (RedisModule_SetCommandInfo(xadd, &info) == REDISMODULE_ERR) + return REDISMODULE_ERR; + + return REDISMODULE_OK; +} diff --git a/tests/modules/getchannels.c b/tests/modules/getchannels.c new file mode 100644 index 000000000..330531d1a --- /dev/null +++ b/tests/modules/getchannels.c @@ -0,0 +1,69 @@ +#include "redismodule.h" +#include <strings.h> +#include <assert.h> +#include <unistd.h> +#include <errno.h> + +/* A sample with declarable channels, that are used to validate against ACLs */ +int getChannels_subscribe(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + if ((argc - 1) % 3 != 0) { + RedisModule_WrongArity(ctx); + return REDISMODULE_OK; + } + char *err = NULL; + + /* getchannels.command [[subscribe|unsubscribe|publish] [pattern|literal] <channel> ...] + * This command marks the given channel is accessed based on the + * provided modifiers. */ + for (int i = 1; i < argc; i += 3) { + const char *operation = RedisModule_StringPtrLen(argv[i], NULL); + const char *type = RedisModule_StringPtrLen(argv[i+1], NULL); + int flags = 0; + + if (!strcasecmp(operation, "subscribe")) { + flags |= REDISMODULE_CMD_CHANNEL_SUBSCRIBE; + } else if (!strcasecmp(operation, "unsubscribe")) { + flags |= REDISMODULE_CMD_CHANNEL_UNSUBSCRIBE; + } else if (!strcasecmp(operation, "publish")) { + flags |= REDISMODULE_CMD_CHANNEL_PUBLISH; + } else { + err = "Invalid channel operation"; + break; + } + + if (!strcasecmp(type, "literal")) { + /* No op */ + } else if (!strcasecmp(type, "pattern")) { + flags |= REDISMODULE_CMD_CHANNEL_PATTERN; + } else { + err = "Invalid channel type"; + break; + } + if (RedisModule_IsChannelsPositionRequest(ctx)) { + RedisModule_ChannelAtPosWithFlags(ctx, i+2, flags); + } + } + + if (!RedisModule_IsChannelsPositionRequest(ctx)) { + if (err) { + RedisModule_ReplyWithError(ctx, err); + } else { + /* Normal implementation would go here, but for tests just return okay */ + RedisModule_ReplyWithSimpleString(ctx, "OK"); + } + } + + return REDISMODULE_OK; +} + +int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + REDISMODULE_NOT_USED(argv); + REDISMODULE_NOT_USED(argc); + if (RedisModule_Init(ctx, "getchannels", 1, REDISMODULE_APIVER_1) == REDISMODULE_ERR) + return REDISMODULE_ERR; + + if (RedisModule_CreateCommand(ctx, "getchannels.command", getChannels_subscribe, "getchannels-api", 0, 0, 0) == REDISMODULE_ERR) + return REDISMODULE_ERR; + + return REDISMODULE_OK; +} diff --git a/tests/modules/getkeys.c b/tests/modules/getkeys.c index acb8a1295..cee3b3e13 100644 --- a/tests/modules/getkeys.c +++ b/tests/modules/getkeys.c @@ -44,6 +44,40 @@ int getkeys_command(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) return REDISMODULE_OK; } +int getkeys_command_with_flags(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) +{ + int i; + int count = 0; + + /* Handle getkeys-api introspection */ + if (RedisModule_IsKeysPositionRequest(ctx)) { + for (i = 0; i < argc; i++) { + size_t len; + const char *str = RedisModule_StringPtrLen(argv[i], &len); + + if (len == 3 && !strncasecmp(str, "key", 3) && i + 1 < argc) + RedisModule_KeyAtPosWithFlags(ctx, i + 1, REDISMODULE_CMD_KEY_RO | REDISMODULE_CMD_KEY_ACCESS); + } + + return REDISMODULE_OK; + } + + /* Handle real command invocation */ + RedisModule_ReplyWithArray(ctx, REDISMODULE_POSTPONED_LEN); + for (i = 0; i < argc; i++) { + size_t len; + const char *str = RedisModule_StringPtrLen(argv[i], &len); + + if (len == 3 && !strncasecmp(str, "key", 3) && i + 1 < argc) { + RedisModule_ReplyWithString(ctx, argv[i+1]); + count++; + } + } + RedisModule_ReplySetArrayLength(ctx, count); + + return REDISMODULE_OK; +} + int getkeys_fixed(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { int i; @@ -57,19 +91,22 @@ int getkeys_fixed(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) /* Introspect a command using RM_GetCommandKeys() and returns the list * of keys. Essentially this is COMMAND GETKEYS implemented in a module. + * INTROSPECT <with-flags> <cmd> <args> */ int getkeys_introspect(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { - UNUSED(argv); - UNUSED(argc); + long long with_flags = 0; - if (argc < 3) { + if (argc < 4) { RedisModule_WrongArity(ctx); return REDISMODULE_OK; } - int num_keys; - int *keyidx = RedisModule_GetCommandKeys(ctx, &argv[1], argc - 1, &num_keys); + if (RedisModule_StringToLongLong(argv[1],&with_flags) != REDISMODULE_OK) + return RedisModule_ReplyWithError(ctx,"ERR invalid integer"); + + int num_keys, *keyflags = NULL; + int *keyidx = RedisModule_GetCommandKeysWithFlags(ctx, &argv[2], argc - 2, &num_keys, with_flags ? &keyflags : NULL); if (!keyidx) { if (!errno) @@ -93,10 +130,27 @@ int getkeys_introspect(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) int i; RedisModule_ReplyWithArray(ctx, num_keys); - for (i = 0; i < num_keys; i++) - RedisModule_ReplyWithString(ctx, argv[1 + keyidx[i]]); + for (i = 0; i < num_keys; i++) { + if (!with_flags) { + RedisModule_ReplyWithString(ctx, argv[2 + keyidx[i]]); + continue; + } + RedisModule_ReplyWithArray(ctx, 2); + RedisModule_ReplyWithString(ctx, argv[2 + keyidx[i]]); + char* sflags = ""; + if (keyflags[i] & REDISMODULE_CMD_KEY_RO) + sflags = "RO"; + else if (keyflags[i] & REDISMODULE_CMD_KEY_RW) + sflags = "RW"; + else if (keyflags[i] & REDISMODULE_CMD_KEY_OW) + sflags = "OW"; + else if (keyflags[i] & REDISMODULE_CMD_KEY_RM) + sflags = "RM"; + RedisModule_ReplyWithCString(ctx, sflags); + } RedisModule_Free(keyidx); + RedisModule_Free(keyflags); } return REDISMODULE_OK; @@ -111,6 +165,9 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) if (RedisModule_CreateCommand(ctx,"getkeys.command", getkeys_command,"getkeys-api",0,0,0) == REDISMODULE_ERR) return REDISMODULE_ERR; + if (RedisModule_CreateCommand(ctx,"getkeys.command_with_flags", getkeys_command_with_flags,"getkeys-api",0,0,0) == REDISMODULE_ERR) + return REDISMODULE_ERR; + if (RedisModule_CreateCommand(ctx,"getkeys.fixed", getkeys_fixed,"",2,4,1) == REDISMODULE_ERR) return REDISMODULE_ERR; diff --git a/tests/modules/keyspecs.c b/tests/modules/keyspecs.c index 18291019d..32a6bebaa 100644 --- a/tests/modules/keyspecs.c +++ b/tests/modules/keyspecs.c @@ -1,116 +1,185 @@ #include "redismodule.h" -#ifdef INCLUDE_UNRELEASED_KEYSPEC_API #define UNUSED(V) ((void) V) -int kspec_legacy(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { +/* This function implements all commands in this module. All we care about is + * the COMMAND metadata anyway. */ +int kspec_impl(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { UNUSED(argv); UNUSED(argc); RedisModule_ReplyWithSimpleString(ctx, "OK"); return REDISMODULE_OK; } -int kspec_complex1(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { - UNUSED(argv); - UNUSED(argc); - RedisModule_ReplyWithSimpleString(ctx, "OK"); - return REDISMODULE_OK; -} - -int kspec_complex2(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { - UNUSED(argv); - UNUSED(argc); - RedisModule_ReplyWithSimpleString(ctx, "OK"); +int createKspecNone(RedisModuleCtx *ctx) { + /* A command without keyspecs; only the legacy (first,last,step) triple (MSET like spec). */ + if (RedisModule_CreateCommand(ctx,"kspec.none",kspec_impl,"",1,-1,2) == REDISMODULE_ERR) + return REDISMODULE_ERR; return REDISMODULE_OK; } -int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { - REDISMODULE_NOT_USED(argv); - REDISMODULE_NOT_USED(argc); - - int spec_id; - - if (RedisModule_Init(ctx, "keyspecs", 1, REDISMODULE_APIVER_1)== REDISMODULE_ERR) +int createKspecTwoRanges(RedisModuleCtx *ctx) { + /* Test that two position/range-based key specs are combined to produce the + * legacy (first,last,step) values representing both keys. */ + if (RedisModule_CreateCommand(ctx,"kspec.tworanges",kspec_impl,"",0,0,0) == REDISMODULE_ERR) + return REDISMODULE_ERR; + + RedisModuleCommand *command = RedisModule_GetCommand(ctx,"kspec.tworanges"); + RedisModuleCommandInfo info = { + .version = REDISMODULE_COMMAND_INFO_VERSION, + .arity = -2, + .key_specs = (RedisModuleCommandKeySpec[]){ + { + .flags = REDISMODULE_CMD_KEY_RO | REDISMODULE_CMD_KEY_ACCESS, + .begin_search_type = REDISMODULE_KSPEC_BS_INDEX, + .bs.index.pos = 1, + .find_keys_type = REDISMODULE_KSPEC_FK_RANGE, + .fk.range = {0,1,0} + }, + { + .flags = REDISMODULE_CMD_KEY_RW | REDISMODULE_CMD_KEY_UPDATE, + .begin_search_type = REDISMODULE_KSPEC_BS_INDEX, + .bs.index.pos = 2, + /* Omitted find_keys_type is shorthand for RANGE {0,1,0} */ + }, + {0} + } + }; + if (RedisModule_SetCommandInfo(command, &info) == REDISMODULE_ERR) return REDISMODULE_ERR; - /* Test legacy range "gluing" */ - if (RedisModule_CreateCommand(ctx,"kspec.legacy",kspec_legacy,"",0,0,0) == REDISMODULE_ERR) - return REDISMODULE_ERR; - RedisModuleCommand *legacy = RedisModule_GetCommand(ctx,"kspec.legacy"); + return REDISMODULE_OK; +} - if (RedisModule_AddCommandKeySpec(legacy,"RO ACCESS",&spec_id) == REDISMODULE_ERR) - return REDISMODULE_ERR; - if (RedisModule_SetCommandKeySpecBeginSearchIndex(legacy,spec_id,1) == REDISMODULE_ERR) - return REDISMODULE_ERR; - if (RedisModule_SetCommandKeySpecFindKeysRange(legacy,spec_id,0,1,0) == REDISMODULE_ERR) +int createKspecKeyword(RedisModuleCtx *ctx) { + /* Only keyword-based specs. The legacy triple is wiped and set to (0,0,0). */ + if (RedisModule_CreateCommand(ctx,"kspec.keyword",kspec_impl,"",3,-1,1) == REDISMODULE_ERR) + return REDISMODULE_ERR; + + RedisModuleCommand *command = RedisModule_GetCommand(ctx,"kspec.keyword"); + RedisModuleCommandInfo info = { + .version = REDISMODULE_COMMAND_INFO_VERSION, + .key_specs = (RedisModuleCommandKeySpec[]){ + { + .flags = REDISMODULE_CMD_KEY_RO | REDISMODULE_CMD_KEY_ACCESS, + .begin_search_type = REDISMODULE_KSPEC_BS_KEYWORD, + .bs.keyword.keyword = "KEYS", + .bs.keyword.startfrom = 1, + .find_keys_type = REDISMODULE_KSPEC_FK_RANGE, + .fk.range = {-1,1,0} + }, + {0} + } + }; + if (RedisModule_SetCommandInfo(command, &info) == REDISMODULE_ERR) return REDISMODULE_ERR; - if (RedisModule_AddCommandKeySpec(legacy,"RW UPDATE",&spec_id) == REDISMODULE_ERR) - return REDISMODULE_ERR; - if (RedisModule_SetCommandKeySpecBeginSearchIndex(legacy,spec_id,2) == REDISMODULE_ERR) - return REDISMODULE_ERR; - if (RedisModule_SetCommandKeySpecFindKeysRange(legacy,spec_id,0,1,0) == REDISMODULE_ERR) - return REDISMODULE_ERR; + return REDISMODULE_OK; +} - /* First is legacy, rest are new specs */ - if (RedisModule_CreateCommand(ctx,"kspec.complex1",kspec_complex1,"",1,1,1) == REDISMODULE_ERR) +int createKspecComplex1(RedisModuleCtx *ctx) { + /* First is a range a single key. The rest are keyword-based specs. */ + if (RedisModule_CreateCommand(ctx,"kspec.complex1",kspec_impl,"",1,1,1) == REDISMODULE_ERR) + return REDISMODULE_ERR; + + RedisModuleCommand *command = RedisModule_GetCommand(ctx,"kspec.complex1"); + RedisModuleCommandInfo info = { + .version = REDISMODULE_COMMAND_INFO_VERSION, + .key_specs = (RedisModuleCommandKeySpec[]){ + { + .flags = REDISMODULE_CMD_KEY_RO, + .begin_search_type = REDISMODULE_KSPEC_BS_INDEX, + .bs.index.pos = 1, + }, + { + .flags = REDISMODULE_CMD_KEY_RW | REDISMODULE_CMD_KEY_UPDATE, + .begin_search_type = REDISMODULE_KSPEC_BS_KEYWORD, + .bs.keyword.keyword = "STORE", + .bs.keyword.startfrom = 2, + }, + { + .flags = REDISMODULE_CMD_KEY_RO | REDISMODULE_CMD_KEY_ACCESS, + .begin_search_type = REDISMODULE_KSPEC_BS_KEYWORD, + .bs.keyword.keyword = "KEYS", + .bs.keyword.startfrom = 2, + .find_keys_type = REDISMODULE_KSPEC_FK_KEYNUM, + .fk.keynum = {0,1,1} + }, + {0} + } + }; + if (RedisModule_SetCommandInfo(command, &info) == REDISMODULE_ERR) return REDISMODULE_ERR; - RedisModuleCommand *complex1 = RedisModule_GetCommand(ctx,"kspec.complex1"); - if (RedisModule_AddCommandKeySpec(complex1,"RW UPDATE",&spec_id) == REDISMODULE_ERR) - return REDISMODULE_ERR; - if (RedisModule_SetCommandKeySpecBeginSearchKeyword(complex1,spec_id,"STORE",2) == REDISMODULE_ERR) - return REDISMODULE_ERR; - if (RedisModule_SetCommandKeySpecFindKeysRange(complex1,spec_id,0,1,0) == REDISMODULE_ERR) - return REDISMODULE_ERR; - - if (RedisModule_AddCommandKeySpec(complex1,"RO ACCESS",&spec_id) == REDISMODULE_ERR) - return REDISMODULE_ERR; - if (RedisModule_SetCommandKeySpecBeginSearchKeyword(complex1,spec_id,"KEYS",2) == REDISMODULE_ERR) - return REDISMODULE_ERR; - if (RedisModule_SetCommandKeySpecFindKeysKeynum(complex1,spec_id,0,1,1) == REDISMODULE_ERR) - return REDISMODULE_ERR; + return REDISMODULE_OK; +} +int createKspecComplex2(RedisModuleCtx *ctx) { /* First is not legacy, more than STATIC_KEYS_SPECS_NUM specs */ - if (RedisModule_CreateCommand(ctx,"kspec.complex2",kspec_complex2,"",0,0,0) == REDISMODULE_ERR) - return REDISMODULE_ERR; - RedisModuleCommand *complex2 = RedisModule_GetCommand(ctx,"kspec.complex2"); - - if (RedisModule_AddCommandKeySpec(complex2,"RW UPDATE",&spec_id) == REDISMODULE_ERR) - return REDISMODULE_ERR; - if (RedisModule_SetCommandKeySpecBeginSearchKeyword(complex2,spec_id,"STORE",5) == REDISMODULE_ERR) - return REDISMODULE_ERR; - if (RedisModule_SetCommandKeySpecFindKeysRange(complex2,spec_id,0,1,0) == REDISMODULE_ERR) - return REDISMODULE_ERR; - - if (RedisModule_AddCommandKeySpec(complex2,"RO ACCESS",&spec_id) == REDISMODULE_ERR) - return REDISMODULE_ERR; - if (RedisModule_SetCommandKeySpecBeginSearchIndex(complex2,spec_id,1) == REDISMODULE_ERR) - return REDISMODULE_ERR; - if (RedisModule_SetCommandKeySpecFindKeysRange(complex2,spec_id,0,1,0) == REDISMODULE_ERR) + if (RedisModule_CreateCommand(ctx,"kspec.complex2",kspec_impl,"",0,0,0) == REDISMODULE_ERR) + return REDISMODULE_ERR; + + RedisModuleCommand *command = RedisModule_GetCommand(ctx,"kspec.complex2"); + RedisModuleCommandInfo info = { + .version = REDISMODULE_COMMAND_INFO_VERSION, + .key_specs = (RedisModuleCommandKeySpec[]){ + { + .flags = REDISMODULE_CMD_KEY_RW | REDISMODULE_CMD_KEY_UPDATE, + .begin_search_type = REDISMODULE_KSPEC_BS_KEYWORD, + .bs.keyword.keyword = "STORE", + .bs.keyword.startfrom = 5, + .find_keys_type = REDISMODULE_KSPEC_FK_RANGE, + .fk.range = {0,1,0} + }, + { + .flags = REDISMODULE_CMD_KEY_RO | REDISMODULE_CMD_KEY_ACCESS, + .begin_search_type = REDISMODULE_KSPEC_BS_INDEX, + .bs.index.pos = 1, + .find_keys_type = REDISMODULE_KSPEC_FK_RANGE, + .fk.range = {0,1,0} + }, + { + .flags = REDISMODULE_CMD_KEY_RO | REDISMODULE_CMD_KEY_ACCESS, + .begin_search_type = REDISMODULE_KSPEC_BS_INDEX, + .bs.index.pos = 2, + .find_keys_type = REDISMODULE_KSPEC_FK_RANGE, + .fk.range = {0,1,0} + }, + { + .flags = REDISMODULE_CMD_KEY_RW | REDISMODULE_CMD_KEY_UPDATE, + .begin_search_type = REDISMODULE_KSPEC_BS_INDEX, + .bs.index.pos = 3, + .find_keys_type = REDISMODULE_KSPEC_FK_KEYNUM, + .fk.keynum = {0,1,1} + }, + { + .flags = REDISMODULE_CMD_KEY_RW | REDISMODULE_CMD_KEY_UPDATE, + .begin_search_type = REDISMODULE_KSPEC_BS_KEYWORD, + .bs.keyword.keyword = "MOREKEYS", + .bs.keyword.startfrom = 5, + .find_keys_type = REDISMODULE_KSPEC_FK_RANGE, + .fk.range = {-1,1,0} + }, + {0} + } + }; + if (RedisModule_SetCommandInfo(command, &info) == REDISMODULE_ERR) return REDISMODULE_ERR; - if (RedisModule_AddCommandKeySpec(complex2,"RO ACCESS",&spec_id) == REDISMODULE_ERR) - return REDISMODULE_ERR; - if (RedisModule_SetCommandKeySpecBeginSearchIndex(complex2,spec_id,2) == REDISMODULE_ERR) - return REDISMODULE_ERR; - if (RedisModule_SetCommandKeySpecFindKeysRange(complex2,spec_id,0,1,0) == REDISMODULE_ERR) - return REDISMODULE_ERR; + return REDISMODULE_OK; +} - if (RedisModule_AddCommandKeySpec(complex2,"RW UPDATE",&spec_id) == REDISMODULE_ERR) - return REDISMODULE_ERR; - if (RedisModule_SetCommandKeySpecBeginSearchIndex(complex2,spec_id,3) == REDISMODULE_ERR) - return REDISMODULE_ERR; - if (RedisModule_SetCommandKeySpecFindKeysKeynum(complex2,spec_id,0,1,1) == REDISMODULE_ERR) - return REDISMODULE_ERR; +int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + REDISMODULE_NOT_USED(argv); + REDISMODULE_NOT_USED(argc); - if (RedisModule_AddCommandKeySpec(complex2,"RW UPDATE",&spec_id) == REDISMODULE_ERR) - return REDISMODULE_ERR; - if (RedisModule_SetCommandKeySpecBeginSearchKeyword(complex2,spec_id,"MOREKEYS",5) == REDISMODULE_ERR) - return REDISMODULE_ERR; - if (RedisModule_SetCommandKeySpecFindKeysRange(complex2,spec_id,-1,1,0) == REDISMODULE_ERR) + if (RedisModule_Init(ctx, "keyspecs", 1, REDISMODULE_APIVER_1) == REDISMODULE_ERR) return REDISMODULE_ERR; + if (createKspecNone(ctx) == REDISMODULE_ERR) return REDISMODULE_ERR; + if (createKspecTwoRanges(ctx) == REDISMODULE_ERR) return REDISMODULE_ERR; + if (createKspecKeyword(ctx) == REDISMODULE_ERR) return REDISMODULE_ERR; + if (createKspecComplex1(ctx) == REDISMODULE_ERR) return REDISMODULE_ERR; + if (createKspecComplex2(ctx) == REDISMODULE_ERR) return REDISMODULE_ERR; return REDISMODULE_OK; } -#endif diff --git a/tests/modules/subcommands.c b/tests/modules/subcommands.c index 51a760c9c..7cb337331 100644 --- a/tests/modules/subcommands.c +++ b/tests/modules/subcommands.c @@ -29,11 +29,7 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) REDISMODULE_NOT_USED(argv); REDISMODULE_NOT_USED(argc); -#ifdef INCLUDE_UNRELEASED_KEYSPEC_API - int spec_id; -#endif - - if (RedisModule_Init(ctx, "subcommands", 1, REDISMODULE_APIVER_1)== REDISMODULE_ERR) + if (RedisModule_Init(ctx, "subcommands", 1, REDISMODULE_APIVER_1) == REDISMODULE_ERR) return REDISMODULE_ERR; if (RedisModule_CreateCommand(ctx,"subcommands.bitarray",NULL,"",0,0,0) == REDISMODULE_ERR) @@ -43,28 +39,40 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) if (RedisModule_CreateSubcommand(parent,"set",cmd_set,"",0,0,0) == REDISMODULE_ERR) return REDISMODULE_ERR; RedisModuleCommand *subcmd = RedisModule_GetCommand(ctx,"subcommands.bitarray|set"); - -#ifdef INCLUDE_UNRELEASED_KEYSPEC_API - if (RedisModule_AddCommandKeySpec(subcmd,"RW UPDATE",&spec_id) == REDISMODULE_ERR) - return REDISMODULE_ERR; - if (RedisModule_SetCommandKeySpecBeginSearchIndex(subcmd,spec_id,1) == REDISMODULE_ERR) - return REDISMODULE_ERR; - if (RedisModule_SetCommandKeySpecFindKeysRange(subcmd,spec_id,0,1,0) == REDISMODULE_ERR) + RedisModuleCommandInfo cmd_set_info = { + .version = REDISMODULE_COMMAND_INFO_VERSION, + .key_specs = (RedisModuleCommandKeySpec[]){ + { + .flags = REDISMODULE_CMD_KEY_RW | REDISMODULE_CMD_KEY_UPDATE, + .begin_search_type = REDISMODULE_KSPEC_BS_INDEX, + .bs.index.pos = 1, + .find_keys_type = REDISMODULE_KSPEC_FK_RANGE, + .fk.range = {0,1,0} + }, + {0} + } + }; + if (RedisModule_SetCommandInfo(subcmd, &cmd_set_info) == REDISMODULE_ERR) return REDISMODULE_ERR; -#endif if (RedisModule_CreateSubcommand(parent,"get",cmd_get,"",0,0,0) == REDISMODULE_ERR) return REDISMODULE_ERR; subcmd = RedisModule_GetCommand(ctx,"subcommands.bitarray|get"); - -#ifdef INCLUDE_UNRELEASED_KEYSPEC_API - if (RedisModule_AddCommandKeySpec(subcmd,"RO ACCESS",&spec_id) == REDISMODULE_ERR) - return REDISMODULE_ERR; - if (RedisModule_SetCommandKeySpecBeginSearchIndex(subcmd,spec_id,1) == REDISMODULE_ERR) - return REDISMODULE_ERR; - if (RedisModule_SetCommandKeySpecFindKeysRange(subcmd,spec_id,0,1,0) == REDISMODULE_ERR) + RedisModuleCommandInfo cmd_get_info = { + .version = REDISMODULE_COMMAND_INFO_VERSION, + .key_specs = (RedisModuleCommandKeySpec[]){ + { + .flags = REDISMODULE_CMD_KEY_RO | REDISMODULE_CMD_KEY_ACCESS, + .begin_search_type = REDISMODULE_KSPEC_BS_INDEX, + .bs.index.pos = 1, + .find_keys_type = REDISMODULE_KSPEC_FK_RANGE, + .fk.range = {0,1,0} + }, + {0} + } + }; + if (RedisModule_SetCommandInfo(subcmd, &cmd_get_info) == REDISMODULE_ERR) return REDISMODULE_ERR; -#endif /* Get the name of the command currently running. */ if (RedisModule_CreateCommand(ctx,"subcommands.parent_get_fullname",cmd_get_fullname,"",0,0,0) == REDISMODULE_ERR) diff --git a/tests/sentinel/tests/03-runtime-reconf.tcl b/tests/sentinel/tests/03-runtime-reconf.tcl index ad9284e41..3e930646a 100644 --- a/tests/sentinel/tests/03-runtime-reconf.tcl +++ b/tests/sentinel/tests/03-runtime-reconf.tcl @@ -44,5 +44,22 @@ test "Sentinel Set with other error situations" { # unknown parameter option assert_error "ERR Unknown option or number of arguments for SENTINEL SET 'fakeoption'" {S 0 SENTINEL SET mymaster fakeoption fakevalue} -} + # save new config to disk failed + set info [S 0 SENTINEL master mymaster] + set origin_quorum [dict get $info quorum] + set update_quorum [expr $origin_quorum+1] + set sentinel_id 0 + set configfilename [file join "sentinel_$sentinel_id" "sentinel.conf"] + set configfilename_bak [file join "sentinel_$sentinel_id" "sentinel.conf.bak"] + + file rename $configfilename $configfilename_bak + file mkdir $configfilename + + catch {[S 0 SENTINEL SET mymaster quorum $update_quorum]} err + + file delete $configfilename + file rename $configfilename_bak $configfilename + + assert_match "ERR Failed to save config file*" $err +} diff --git a/tests/sentinel/tests/13-info-command.tcl b/tests/sentinel/tests/13-info-command.tcl new file mode 100644 index 000000000..ef9dc0113 --- /dev/null +++ b/tests/sentinel/tests/13-info-command.tcl @@ -0,0 +1,47 @@ +source "../tests/includes/init-tests.tcl" + +test "info command with at most one argument" { + set subCommandList {} + foreach arg {"" "all" "default" "everything"} { + if {$arg == ""} { + set info [S 0 info] + } else { + set info [S 0 info $arg] + } + assert { [string match "*redis_version*" $info] } + assert { [string match "*maxclients*" $info] } + assert { [string match "*used_cpu_user*" $info] } + assert { [string match "*sentinel_tilt*" $info] } + assert { ![string match "*used_memory*" $info] } + assert { ![string match "*rdb_last_bgsave*" $info] } + assert { ![string match "*master_repl_offset*" $info] } + assert { ![string match "*cluster_enabled*" $info] } + } +} + +test "info command with one sub-section" { + set info [S 0 info cpu] + assert { [string match "*used_cpu_user*" $info] } + assert { ![string match "*sentinel_tilt*" $info] } + assert { ![string match "*redis_version*" $info] } + + set info [S 0 info sentinel] + assert { [string match "*sentinel_tilt*" $info] } + assert { ![string match "*used_cpu_user*" $info] } + assert { ![string match "*redis_version*" $info] } +} + +test "info command with multiple sub-sections" { + set info [S 0 info server sentinel replication] + assert { [string match "*redis_version*" $info] } + assert { [string match "*sentinel_tilt*" $info] } + assert { ![string match "*used_memory*" $info] } + assert { ![string match "*used_cpu_user*" $info] } + + set info [S 0 info cpu all] + assert { [string match "*used_cpu_user*" $info] } + assert { [string match "*sentinel_tilt*" $info] } + assert { [string match "*redis_version*" $info] } + assert { ![string match "*used_memory*" $info] } + assert { ![string match "*master_repl_offset*" $info] } +} diff --git a/tests/support/redis.tcl b/tests/support/redis.tcl index 2c89de6ce..5743be5f4 100644 --- a/tests/support/redis.tcl +++ b/tests/support/redis.tcl @@ -35,6 +35,7 @@ array set ::redis::addr {} array set ::redis::blocking {} array set ::redis::deferred {} array set ::redis::readraw {} +array set ::redis::attributes {} ;# Holds the RESP3 attributes from the last call array set ::redis::reconnect {} array set ::redis::tls {} array set ::redis::callback {} @@ -105,6 +106,7 @@ proc ::redis::__dispatch__raw__ {id method argv} { set argv [lrange $argv 0 end-1] } if {[info command ::redis::__method__$method] eq {}} { + catch {unset ::redis::attributes($id)} set cmd "*[expr {[llength $argv]+1}]\r\n" append cmd "$[string length $method]\r\n$method\r\n" foreach a $argv { @@ -165,6 +167,7 @@ proc ::redis::__method__close {id fd} { catch {unset ::redis::blocking($id)} catch {unset ::redis::deferred($id)} catch {unset ::redis::readraw($id)} + catch {unset ::redis::attributes($id)} catch {unset ::redis::reconnect($id)} catch {unset ::redis::tls($id)} catch {unset ::redis::state($id)} @@ -185,6 +188,10 @@ proc ::redis::__method__readraw {id fd val} { set ::redis::readraw($id) $val } +proc ::redis::__method__attributes {id fd} { + set _ $::redis::attributes($id) +} + proc ::redis::redis_write {fd buf} { puts -nonewline $fd $buf } @@ -286,8 +293,8 @@ proc ::redis::redis_read_reply {id fd} { * {return [redis_multi_bulk_read $id $fd]} % {return [redis_read_map $id $fd]} | { - # ignore attributes for now (nowhere to store them) - redis_read_map $id $fd + set attrib [redis_read_map $id $fd] + set ::redis::attributes($id) $attrib continue } default { diff --git a/tests/support/test.tcl b/tests/support/test.tcl index db3a81e06..f5de12256 100644 --- a/tests/support/test.tcl +++ b/tests/support/test.tcl @@ -24,10 +24,10 @@ proc assert_no_match {pattern value} { } } -proc assert_match {pattern value} { +proc assert_match {pattern value {detail ""}} { if {![string match $pattern $value]} { set context "(context: [info frame -1])" - error "assertion:Expected '$value' to match '$pattern' $context" + error "assertion:Expected '$value' to match '$pattern' $context $detail" } } @@ -84,9 +84,9 @@ proc assert_range {value min max {detail ""}} { proc assert_error {pattern code {detail ""}} { if {[catch {uplevel 1 $code} error]} { - assert_match $pattern $error + assert_match $pattern $error $detail } else { - assert_failed "assertion:Expected an error but nothing was caught" $detail + assert_failed "Expected an error matching '$pattern' but got '$error'" $detail } } diff --git a/tests/support/util.tcl b/tests/support/util.tcl index 5fc319254..46c9654c8 100644 --- a/tests/support/util.tcl +++ b/tests/support/util.tcl @@ -89,7 +89,7 @@ proc waitForBgsave r { puts -nonewline "\nWaiting for background save to finish... " flush stdout } - after 1000 + after 50 } else { break } @@ -103,7 +103,7 @@ proc waitForBgrewriteaof r { puts -nonewline "\nWaiting for background AOF rewrite to finish... " flush stdout } - after 1000 + after 50 } else { break } @@ -647,7 +647,7 @@ proc latencyrstat_percentiles {cmd r} { proc generate_fuzzy_traffic_on_key {key duration} { # Commands per type, blocking commands removed - # TODO: extract these from help.h or elsewhere, and improve to include other types + # TODO: extract these from COMMAND DOCS, and improve to include other types set string_commands {APPEND BITCOUNT BITFIELD BITOP BITPOS DECR DECRBY GET GETBIT GETRANGE GETSET INCR INCRBY INCRBYFLOAT MGET MSET MSETNX PSETEX SET SETBIT SETEX SETNX SETRANGE LCS STRLEN} set hash_commands {HDEL HEXISTS HGET HGETALL HINCRBY HINCRBYFLOAT HKEYS HLEN HMGET HMSET HSCAN HSET HSETNX HSTRLEN HVALS HRANDFIELD} set zset_commands {ZADD ZCARD ZCOUNT ZINCRBY ZINTERSTORE ZLEXCOUNT ZPOPMAX ZPOPMIN ZRANGE ZRANGEBYLEX ZRANGEBYSCORE ZRANK ZREM ZREMRANGEBYLEX ZREMRANGEBYRANK ZREMRANGEBYSCORE ZREVRANGE ZREVRANGEBYLEX ZREVRANGEBYSCORE ZREVRANK ZSCAN ZSCORE ZUNIONSTORE ZRANDMEMBER} diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl index 1a5096937..277fa3803 100644 --- a/tests/test_helper.tcl +++ b/tests/test_helper.tcl @@ -20,6 +20,7 @@ set ::all_tests { unit/keyspace unit/scan unit/info + unit/info-command unit/type/string unit/type/incr unit/type/list @@ -92,6 +93,7 @@ set ::all_tests { unit/cluster unit/client-eviction unit/violations + unit/replybufsize } # Index to the next test to run in the ::all_tests list. set ::next_test 0 diff --git a/tests/unit/acl.tcl b/tests/unit/acl.tcl index 494c3847e..0a9ffb250 100644 --- a/tests/unit/acl.tcl +++ b/tests/unit/acl.tcl @@ -7,6 +7,11 @@ start_server {tags {"acl external:skip"}} { r ACL setuser newuser } + test {Usernames can not contain spaces or null characters} { + catch {r ACL setuser "a a"} err + set err + } {*Usernames can't contain spaces or null characters*} + test {New users start disabled} { r ACL setuser newuser >passwd1 catch {r AUTH newuser passwd1} err @@ -699,6 +704,23 @@ start_server {tags {"acl external:skip"}} { catch {[r ping]} e assert_match "*I/O error*" $e } + + test {ACL GENPASS command failed test} { + catch {r ACL genpass -236} err1 + catch {r ACL genpass 5000} err2 + assert_match "*ACL GENPASS argument must be the number*" $err1 + assert_match "*ACL GENPASS argument must be the number*" $err2 + } + + test {Default user can not be removed} { + catch {r ACL deluser default} err + set err + } {ERR The 'default' user cannot be removed} + + test {ACL load non-existing configured ACL file} { + catch {r ACL load} err + set err + } {*Redis instance is not configured to use an ACL file*} } set server_path [tmpdir "server.acl"] diff --git a/tests/unit/auth.tcl b/tests/unit/auth.tcl index 6fa5e0c13..4a4d7564c 100644 --- a/tests/unit/auth.tcl +++ b/tests/unit/auth.tcl @@ -3,6 +3,11 @@ start_server {tags {"auth external:skip"}} { catch {r auth foo} err set _ $err } {ERR*any password*} + + test {Arity check for auth command} { + catch {r auth a b c} err + set _ $err + } {*syntax error*} } start_server {tags {"auth external:skip"} overrides {requirepass foobar}} { diff --git a/tests/unit/client-eviction.tcl b/tests/unit/client-eviction.tcl index 0b1b8b281..949ac8f3d 100644 --- a/tests/unit/client-eviction.tcl +++ b/tests/unit/client-eviction.tcl @@ -45,6 +45,10 @@ proc mb {v} { return [expr $v * 1024 * 1024] } +proc kb {v} { + return [expr $v * 1024] +} + start_server {} { set maxmemory_clients 3000000 r config set maxmemory-clients $maxmemory_clients @@ -213,7 +217,7 @@ start_server {} { r debug pause-cron 0 $rr close $redirected_c close - } + } {0} {needs:debug} test "client evicted due to client tracking prefixes" { r flushdb @@ -391,6 +395,7 @@ start_server {} { test "evict clients only until below limit" { set client_count 10 set client_mem [mb 1] + r debug replybuffer-peak-reset-time never r config set maxmemory-clients 0 r client setname control r client no-evict on @@ -433,19 +438,23 @@ start_server {} { set connected_clients [llength [lsearch -all [split [string trim [r client list]] "\r\n"] *name=client*]] assert {$connected_clients == [expr $client_count / 2]} + # Restore the peak reset time to default + r debug replybuffer-peak-reset-time reset + foreach rr $rrs {$rr close} - } + } {} {needs:debug} } start_server {} { test "evict clients in right order (large to small)" { # Note that each size step needs to be at least x2 larger than previous step # because of how the client-eviction size bucktting works - set sizes [list 100000 [mb 1] [mb 3]] + set sizes [list [kb 128] [mb 1] [mb 3]] set clients_per_size 3 r client setname control r client no-evict on r config set maxmemory-clients 0 + r debug replybuffer-peak-reset-time never # Run over all sizes and create some clients using up that size set total_client_mem 0 @@ -470,7 +479,6 @@ start_server {} { # Account total client memory usage incr total_mem [expr $clients_per_size * $client_mem] } - incr total_mem [client_field control tot-mem] # Make sure all clients are connected set clients [split [string trim [r client list]] "\r\n"] @@ -481,8 +489,9 @@ start_server {} { # For each size reduce maxmemory-clients so relevant clients should be evicted # do this from largest to smallest foreach size [lreverse $sizes] { + set control_mem [client_field control tot-mem] set total_mem [expr $total_mem - $clients_per_size * $size] - r config set maxmemory-clients $total_mem + r config set maxmemory-clients [expr $total_mem + $control_mem] set clients [split [string trim [r client list]] "\r\n"] # Verify only relevant clients were evicted for {set i 0} {$i < [llength $sizes]} {incr i} { @@ -495,8 +504,12 @@ start_server {} { } } } + + # Restore the peak reset time to default + r debug replybuffer-peak-reset-time reset + foreach rr $rrs {$rr close} - } + } {} {needs:debug} } } diff --git a/tests/unit/geo.tcl b/tests/unit/geo.tcl index bd93ea4ba..e6afb211b 100644 --- a/tests/unit/geo.tcl +++ b/tests/unit/geo.tcl @@ -293,6 +293,31 @@ start_server {tags {"geo"}} { test {GEORADIUSBYMEMBER simple (sorted)} { r georadiusbymember nyc "wtc one" 7 km } {{wtc one} {union square} {central park n/q/r} 4545 {lic market}} + + test {GEORADIUSBYMEMBER search areas contain satisfied points in oblique direction} { + r del k1 + + r geoadd k1 -0.15307903289794921875 85 n1 0.3515625 85.00019260486917005437 n2 + set ret1 [r GEORADIUSBYMEMBER k1 n1 4891.94 m] + assert_equal $ret1 {n1 n2} + + r zrem k1 n1 n2 + r geoadd k1 -4.95211958885192871094 85 n3 11.25 85.0511 n4 + set ret2 [r GEORADIUSBYMEMBER k1 n3 156544 m] + assert_equal $ret2 {n3 n4} + + r zrem k1 n3 n4 + r geoadd k1 -45 65.50900022111811438208 n5 90 85.0511 n6 + set ret3 [r GEORADIUSBYMEMBER k1 n5 5009431 m] + assert_equal $ret3 {n5 n6} + } + + test {GEORADIUSBYMEMBER crossing pole search} { + r del k1 + r geoadd k1 45 65 n1 -135 85.05 n2 + set ret [r GEORADIUSBYMEMBER k1 n1 5009431 m] + assert_equal $ret {n1 n2} + } test {GEOSEARCH FROMMEMBER simple (sorted)} { r geosearch nyc frommember "wtc one" bybox 14 14 km diff --git a/tests/unit/info-command.tcl b/tests/unit/info-command.tcl new file mode 100644 index 000000000..bc24ed256 --- /dev/null +++ b/tests/unit/info-command.tcl @@ -0,0 +1,62 @@ +start_server {tags {"info and its relative command"}} { + test "info command with at most one sub command" { + foreach arg {"" "all" "default" "everything"} { + if {$arg == ""} { + set info [r 0 info] + } else { + set info [r 0 info $arg] + } + + assert { [string match "*redis_version*" $info] } + assert { [string match "*used_cpu_user*" $info] } + assert { ![string match "*sentinel_tilt*" $info] } + assert { [string match "*used_memory*" $info] } + if {$arg == "" || $arg == "default"} { + assert { ![string match "*rejected_calls*" $info] } + } else { + assert { [string match "*rejected_calls*" $info] } + } + } + } + + test "info command with one sub-section" { + set info [r info cpu] + assert { [string match "*used_cpu_user*" $info] } + assert { ![string match "*sentinel_tilt*" $info] } + assert { ![string match "*used_memory*" $info] } + + set info [r info sentinel] + assert { ![string match "*sentinel_tilt*" $info] } + assert { ![string match "*used_memory*" $info] } + + set info [r info commandSTATS] ;# test case insensitive compare + assert { ![string match "*used_memory*" $info] } + assert { [string match "*rejected_calls*" $info] } + } + + test "info command with multiple sub-sections" { + set info [r info cpu sentinel] + assert { [string match "*used_cpu_user*" $info] } + assert { ![string match "*sentinel_tilt*" $info] } + assert { ![string match "*master_repl_offset*" $info] } + + set info [r info cpu all] + assert { [string match "*used_cpu_user*" $info] } + assert { ![string match "*sentinel_tilt*" $info] } + assert { [string match "*used_memory*" $info] } + assert { [string match "*master_repl_offset*" $info] } + assert { [string match "*rejected_calls*" $info] } + # check that we didn't get the same info twice + assert { ![string match "*used_cpu_user_children*used_cpu_user_children*" $info] } + + set info [r info cpu default] + assert { [string match "*used_cpu_user*" $info] } + assert { ![string match "*sentinel_tilt*" $info] } + assert { [string match "*used_memory*" $info] } + assert { [string match "*master_repl_offset*" $info] } + assert { ![string match "*rejected_calls*" $info] } + # check that we didn't get the same info twice + assert { ![string match "*used_cpu_user_children*used_cpu_user_children*" $info] } + } + +} diff --git a/tests/unit/info.tcl b/tests/unit/info.tcl index b211e6c91..759e5bc0b 100644 --- a/tests/unit/info.tcl +++ b/tests/unit/info.tcl @@ -238,6 +238,7 @@ start_server {tags {"info" "external:skip"}} { assert_equal [s total_error_replies] 1 r config resetstat assert_match {} [errorstat OOM] + r config set maxmemory 0 } test {errorstats: rejected call by authorization error} { @@ -253,6 +254,25 @@ start_server {tags {"info" "external:skip"}} { assert_equal [s total_error_replies] 1 r config resetstat assert_match {} [errorstat NOPERM] + r auth default "" } + + test {errorstats: blocking commands} { + r config resetstat + set rd [redis_deferring_client] + $rd client id + set rd_id [$rd read] + r del list1{t} + + $rd blpop list1{t} 0 + wait_for_blocked_client + r client unblock $rd_id error + assert_error {UNBLOCKED*} {$rd read} + assert_match {*count=1*} [errorstat UNBLOCKED] + assert_match {*calls=1,*,rejected_calls=0,failed_calls=1} [cmdstat blpop] + assert_equal [s total_error_replies] 1 + $rd close + } + } } diff --git a/tests/unit/introspection-2.tcl b/tests/unit/introspection-2.tcl index 40124e035..46dac50b7 100644 --- a/tests/unit/introspection-2.tcl +++ b/tests/unit/introspection-2.tcl @@ -33,6 +33,15 @@ start_server {tags {"introspection"}} { assert_match {} [cmdstat zadd] } {} {needs:config-resetstat} + test {errors stats for GEOADD} { + r config resetstat + # make sure geo command will failed + r set foo 1 + assert_error {WRONGTYPE Operation against a key holding the wrong kind of value*} {r GEOADD foo 0 0 bar} + assert_match {*calls=1*,rejected_calls=0,failed_calls=1*} [cmdstat geoadd] + assert_match {} [cmdstat zadd] + } {} {needs:config-resetstat} + test {command stats for EXPIRE} { r config resetstat r SET foo bar @@ -81,6 +90,13 @@ start_server {tags {"introspection"}} { assert_equal {key} [r command getkeys get key] } + test {COMMAND GETKEYSANDFLAGS} { + assert_equal {{k1 {OW update}}} [r command getkeysandflags set k1 v1] + assert_equal {{k1 {OW update}} {k2 {OW update}}} [r command getkeysandflags mset k1 v1 k2 v2] + assert_equal {{k1 {RW access delete}} {k2 {RW insert}}} [r command getkeysandflags LMOVE k1 k2 left right] + assert_equal {{k1 {RO access}} {k2 {OW update}}} [r command getkeysandflags sort k1 store k2] + } + test {COMMAND GETKEYS MEMORY USAGE} { assert_equal {key} [r command getkeys memory usage key] } diff --git a/tests/unit/introspection.tcl b/tests/unit/introspection.tcl index 33a41ee50..59c96b5ad 100644 --- a/tests/unit/introspection.tcl +++ b/tests/unit/introspection.tcl @@ -7,7 +7,7 @@ start_server {tags {"introspection"}} { test {CLIENT LIST} { r client list - } {id=* addr=*:* laddr=*:* fd=* name=* age=* idle=* flags=N db=* sub=0 psub=0 multi=-1 qbuf=26 qbuf-free=* argv-mem=* multi-mem=0 obl=0 oll=0 omem=0 tot-mem=* events=r cmd=client|list user=* redir=-1 resp=2*} + } {id=* addr=*:* laddr=*:* fd=* name=* age=* idle=* flags=N db=* sub=0 psub=0 multi=-1 qbuf=26 qbuf-free=* argv-mem=* multi-mem=0 rbs=* rbp=* obl=0 oll=0 omem=0 tot-mem=* events=r cmd=client|list user=* redir=-1 resp=2*} test {CLIENT LIST with IDs} { set myid [r client id] @@ -17,7 +17,7 @@ start_server {tags {"introspection"}} { test {CLIENT INFO} { r client info - } {id=* addr=*:* laddr=*:* fd=* name=* age=* idle=* flags=N db=* sub=0 psub=0 multi=-1 qbuf=26 qbuf-free=* argv-mem=* multi-mem=0 obl=0 oll=0 omem=0 tot-mem=* events=r cmd=client|info user=* redir=-1 resp=2*} + } {id=* addr=*:* laddr=*:* fd=* name=* age=* idle=* flags=N db=* sub=0 psub=0 multi=-1 qbuf=26 qbuf-free=* argv-mem=* multi-mem=0 rbs=* rbp=* obl=0 oll=0 omem=0 tot-mem=* events=r cmd=client|info user=* redir=-1 resp=2*} test {CLIENT KILL with illegal arguments} { assert_error "ERR wrong number of arguments for 'client|kill' command" {r client kill} diff --git a/tests/unit/memefficiency.tcl b/tests/unit/memefficiency.tcl index 299dd658b..e6663ce06 100644 --- a/tests/unit/memefficiency.tcl +++ b/tests/unit/memefficiency.tcl @@ -157,6 +157,88 @@ start_server {tags {"defrag external:skip"} overrides {appendonly yes auto-aof-r } r config set appendonly no r config set key-load-delay 0 + + test "Active defrag eval scripts" { + r flushdb + r script flush sync + r config resetstat + r config set hz 100 + r config set activedefrag no + r config set active-defrag-threshold-lower 5 + r config set active-defrag-cycle-min 65 + r config set active-defrag-cycle-max 75 + r config set active-defrag-ignore-bytes 1500kb + r config set maxmemory 0 + + set n 50000 + + # Populate memory with interleaving script-key pattern of same size + set dummy_script "--[string repeat x 400]\nreturn " + set rd [redis_deferring_client] + for {set j 0} {$j < $n} {incr j} { + set val "$dummy_script[format "%06d" $j]" + $rd script load $val + $rd set k$j $val + } + for {set j 0} {$j < $n} {incr j} { + $rd read ; # Discard script load replies + $rd read ; # Discard set replies + } + after 120 ;# serverCron only updates the info once in 100ms + if {$::verbose} { + puts "used [s allocator_allocated]" + puts "rss [s allocator_active]" + puts "frag [s allocator_frag_ratio]" + puts "frag_bytes [s allocator_frag_bytes]" + } + assert_lessthan [s allocator_frag_ratio] 1.05 + + # Delete all the keys to create fragmentation + for {set j 0} {$j < $n} {incr j} { $rd del k$j } + for {set j 0} {$j < $n} {incr j} { $rd read } ; # Discard del replies + $rd close + after 120 ;# serverCron only updates the info once in 100ms + if {$::verbose} { + puts "used [s allocator_allocated]" + puts "rss [s allocator_active]" + puts "frag [s allocator_frag_ratio]" + puts "frag_bytes [s allocator_frag_bytes]" + } + assert_morethan [s allocator_frag_ratio] 1.4 + + catch {r config set activedefrag yes} e + if {[r config get activedefrag] eq "activedefrag yes"} { + + # wait for the active defrag to start working (decision once a second) + wait_for_condition 50 100 { + [s active_defrag_running] ne 0 + } else { + fail "defrag not started." + } + + # wait for the active defrag to stop working + wait_for_condition 500 100 { + [s active_defrag_running] eq 0 + } else { + after 120 ;# serverCron only updates the info once in 100ms + puts [r info memory] + puts [r memory malloc-stats] + fail "defrag didn't stop." + } + + # test the the fragmentation is lower + after 120 ;# serverCron only updates the info once in 100ms + if {$::verbose} { + puts "used [s allocator_allocated]" + puts "rss [s allocator_active]" + puts "frag [s allocator_frag_ratio]" + puts "frag_bytes [s allocator_frag_bytes]" + } + assert_lessthan_equal [s allocator_frag_ratio] 1.05 + } + # Flush all script to make sure we don't crash after defragging them + r script flush sync + } {OK} test "Active defrag big keys" { r flushdb diff --git a/tests/unit/moduleapi/aclcheck.tcl b/tests/unit/moduleapi/aclcheck.tcl index a6df4f7c9..953f4bf05 100644 --- a/tests/unit/moduleapi/aclcheck.tcl +++ b/tests/unit/moduleapi/aclcheck.tcl @@ -26,6 +26,12 @@ start_server {tags {"modules acl"}} { catch {r aclcheck.set.check.key "*" v 5} e assert_match "*DENIED KEY*" $e + assert_equal [r aclcheck.set.check.key "~" x 5] OK + assert_equal [r aclcheck.set.check.key "~" y 5] OK + assert_equal [r aclcheck.set.check.key "~" z 5] OK + catch {r aclcheck.set.check.key "~" v 5} e + assert_match "*DENIED KEY*" $e + assert_equal [r aclcheck.set.check.key "W" y 5] OK catch {r aclcheck.set.check.key "W" v 5} e assert_match "*DENIED KEY*" $e diff --git a/tests/unit/moduleapi/blockedclient.tcl b/tests/unit/moduleapi/blockedclient.tcl index 523d7ba69..ea2d6f5a4 100644 --- a/tests/unit/moduleapi/blockedclient.tcl +++ b/tests/unit/moduleapi/blockedclient.tcl @@ -180,6 +180,37 @@ start_server {tags {"modules"}} { assert_no_match "*name=myclient*" $clients } + test {module client error stats} { + r config resetstat + + # simple module command that replies with string error + assert_error "NULL reply returned" {r do_rm_call hgetalllll} + assert_equal [errorrstat NULL r] {count=1} + + # module command that replies with string error from bg thread + assert_error "NULL reply returned" {r do_bg_rm_call hgetalllll} + assert_equal [errorrstat NULL r] {count=2} + + # module command that returns an arity error + r do_rm_call set x x + assert_error "ERR wrong number of arguments for 'do_rm_call' command" {r do_rm_call} + assert_equal [errorrstat ERR r] {count=1} + + # RM_Call that propagates an error + assert_error "WRONGTYPE*" {r do_rm_call hgetall x} + assert_equal [errorrstat WRONGTYPE r] {count=1} + assert_match {*calls=1,*,rejected_calls=0,failed_calls=1} [cmdrstat hgetall r] + + # RM_Call from bg thread that propagates an error + assert_error "WRONGTYPE*" {r do_bg_rm_call hgetall x} + assert_equal [errorrstat WRONGTYPE r] {count=2} + assert_match {*calls=2,*,rejected_calls=0,failed_calls=2} [cmdrstat hgetall r] + + assert_equal [s total_error_replies] 5 + assert_match {*calls=4,*,rejected_calls=0,failed_calls=3} [cmdrstat do_rm_call r] + assert_match {*calls=2,*,rejected_calls=0,failed_calls=2} [cmdrstat do_bg_rm_call r] + } + test "Unload the module - blockedclient" { assert_equal {OK} [r module unload blockedclient] } diff --git a/tests/unit/moduleapi/cmdintrospection.tcl b/tests/unit/moduleapi/cmdintrospection.tcl new file mode 100644 index 000000000..375b3e406 --- /dev/null +++ b/tests/unit/moduleapi/cmdintrospection.tcl @@ -0,0 +1,42 @@ +set testmodule [file normalize tests/modules/cmdintrospection.so] + +start_server {tags {"modules"}} { + r module load $testmodule + + # cmdintrospection.xadd mimics XADD with regards to how + # what COMMAND exposes. There are two differences: + # + # 1. cmdintrospection.xadd (and all module commands) do not have ACL categories + # 2. cmdintrospection.xadd's `group` is "module" + # + # This tests verify that, apart from the above differences, the output of + # COMMAND INFO and COMMAND DOCS are identical for the two commands. + test "Module command introspection via COMMAND INFO" { + set redis_reply [lindex [r command info xadd] 0] + set module_reply [lindex [r command info cmdintrospection.xadd] 0] + for {set i 1} {$i < [llength $redis_reply]} {incr i} { + if {$i == 2} { + # Remove the "module" flag + set mylist [lindex $module_reply $i] + set idx [lsearch $mylist "module"] + set mylist [lreplace $mylist $idx $idx] + lset module_reply $i $mylist + } + if {$i == 6} { + # Skip ACL categories + continue + } + assert_equal [lindex $redis_reply $i] [lindex $module_reply $i] + } + } + + test "Module command introspection via COMMAND DOCS" { + set redis_reply [dict create {*}[lindex [r command docs xadd] 1]] + set module_reply [dict create {*}[lindex [r command docs cmdintrospection.xadd] 1]] + # Compare the maps. We need to pop "group" first. + dict unset redis_reply group + dict unset module_reply group + + assert_equal $redis_reply $module_reply + } +} diff --git a/tests/unit/moduleapi/getchannels.tcl b/tests/unit/moduleapi/getchannels.tcl new file mode 100644 index 000000000..e8f557dcc --- /dev/null +++ b/tests/unit/moduleapi/getchannels.tcl @@ -0,0 +1,40 @@ +set testmodule [file normalize tests/modules/getchannels.so] + +start_server {tags {"modules"}} { + r module load $testmodule + + # Channels are currently used to just validate ACLs, so test them here + r ACL setuser testuser +@all resetchannels &channel &pattern* + + test "module getchannels-api with literals - ACL" { + assert_equal "OK" [r ACL DRYRUN testuser getchannels.command subscribe literal channel subscribe literal pattern1] + assert_equal "OK" [r ACL DRYRUN testuser getchannels.command publish literal channel publish literal pattern1] + assert_equal "OK" [r ACL DRYRUN testuser getchannels.command unsubscribe literal channel unsubscribe literal pattern1] + + assert_equal "This user has no permissions to access the 'nopattern1' channel" [r ACL DRYRUN testuser getchannels.command subscribe literal channel subscribe literal nopattern1] + assert_equal "This user has no permissions to access the 'nopattern1' channel" [r ACL DRYRUN testuser getchannels.command publish literal channel subscribe literal nopattern1] + assert_equal "OK" [r ACL DRYRUN testuser getchannels.command unsubscribe literal channel unsubscribe literal nopattern1] + + assert_equal "This user has no permissions to access the 'otherchannel' channel" [r ACL DRYRUN testuser getchannels.command subscribe literal otherchannel subscribe literal pattern1] + assert_equal "This user has no permissions to access the 'otherchannel' channel" [r ACL DRYRUN testuser getchannels.command publish literal otherchannel subscribe literal pattern1] + assert_equal "OK" [r ACL DRYRUN testuser getchannels.command unsubscribe literal otherchannel unsubscribe literal pattern1] + } + + test "module getchannels-api with patterns - ACL" { + assert_equal "OK" [r ACL DRYRUN testuser getchannels.command subscribe pattern pattern*] + assert_equal "OK" [r ACL DRYRUN testuser getchannels.command publish pattern pattern*] + assert_equal "OK" [r ACL DRYRUN testuser getchannels.command unsubscribe pattern pattern*] + + assert_equal "This user has no permissions to access the 'pattern1' channel" [r ACL DRYRUN testuser getchannels.command subscribe pattern pattern1 subscribe pattern pattern*] + assert_equal "This user has no permissions to access the 'pattern1' channel" [r ACL DRYRUN testuser getchannels.command publish pattern pattern1 subscribe pattern pattern*] + assert_equal "OK" [r ACL DRYRUN testuser getchannels.command unsubscribe pattern pattern1 unsubscribe pattern pattern*] + + assert_equal "This user has no permissions to access the 'otherpattern*' channel" [r ACL DRYRUN testuser getchannels.command subscribe pattern otherpattern* subscribe pattern pattern*] + assert_equal "This user has no permissions to access the 'otherpattern*' channel" [r ACL DRYRUN testuser getchannels.command publish pattern otherpattern* subscribe pattern pattern*] + assert_equal "OK" [r ACL DRYRUN testuser getchannels.command unsubscribe pattern otherpattern* unsubscribe pattern pattern*] + } + + test "Unload the module - getchannels" { + assert_equal {OK} [r module unload getchannels] + } +} diff --git a/tests/unit/moduleapi/getkeys.tcl b/tests/unit/moduleapi/getkeys.tcl index 6061fe8cf..734c55fa2 100644 --- a/tests/unit/moduleapi/getkeys.tcl +++ b/tests/unit/moduleapi/getkeys.tcl @@ -16,32 +16,64 @@ start_server {tags {"modules"}} { r command getkeys getkeys.command arg1 arg2 key key1 arg3 key key2 key key3 } {key1 key2 key3} + test {COMMAND GETKEYS correctly reports a movable keys module command using flags} { + r command getkeys getkeys.command_with_flags arg1 arg2 key key1 arg3 key key2 key key3 + } {key1 key2 key3} + + test {COMMAND GETKEYSANDFLAGS correctly reports a movable keys module command not using flags} { + r command getkeysandflags getkeys.command arg1 arg2 key key1 arg3 key key2 + } {{key1 {RW access update}} {key2 {RW access update}}} + + test {COMMAND GETKEYSANDFLAGS correctly reports a movable keys module command using flags} { + r command getkeysandflags getkeys.command_with_flags arg1 arg2 key key1 arg3 key key2 key key3 + } {{key1 {RO access}} {key2 {RO access}} {key3 {RO access}}} + test {RM_GetCommandKeys on non-existing command} { - catch {r getkeys.introspect non-command key1 key2} e + catch {r getkeys.introspect 0 non-command key1 key2} e set _ $e } {*ENOENT*} test {RM_GetCommandKeys on built-in fixed keys command} { - r getkeys.introspect set key1 value1 + r getkeys.introspect 0 set key1 value1 } {key1} + test {RM_GetCommandKeys on built-in fixed keys command with flags} { + r getkeys.introspect 1 set key1 value1 + } {{key1 OW}} + test {RM_GetCommandKeys on EVAL} { - r getkeys.introspect eval "" 4 key1 key2 key3 key4 arg1 arg2 + r getkeys.introspect 0 eval "" 4 key1 key2 key3 key4 arg1 arg2 } {key1 key2 key3 key4} test {RM_GetCommandKeys on a movable keys module command} { - r getkeys.introspect getkeys.command arg1 arg2 key key1 arg3 key key2 key key3 + r getkeys.introspect 0 getkeys.command arg1 arg2 key key1 arg3 key key2 key key3 } {key1 key2 key3} test {RM_GetCommandKeys on a non-movable module command} { - r getkeys.introspect getkeys.fixed arg1 key1 key2 key3 arg2 + r getkeys.introspect 0 getkeys.fixed arg1 key1 key2 key3 arg2 } {key1 key2 key3} test {RM_GetCommandKeys with bad arity} { - catch {r getkeys.introspect set key} e + catch {r getkeys.introspect 0 set key} e set _ $e } {*EINVAL*} + # user that can only read from "read" keys, write to "write" keys, and read+write to "RW" keys + r ACL setuser testuser +@all %R~read* %W~write* %RW~rw* + + test "module getkeys-api - ACL" { + # legacy triple didn't provide flags, so they require both read and write + assert_equal "OK" [r ACL DRYRUN testuser getkeys.command key rw] + assert_equal "This user has no permissions to access the 'read' key" [r ACL DRYRUN testuser getkeys.command key read] + assert_equal "This user has no permissions to access the 'write' key" [r ACL DRYRUN testuser getkeys.command key write] + } + + test "module getkeys-api with flags - ACL" { + assert_equal "OK" [r ACL DRYRUN testuser getkeys.command_with_flags key rw] + assert_equal "OK" [r ACL DRYRUN testuser getkeys.command_with_flags key read] + assert_equal "This user has no permissions to access the 'write' key" [r ACL DRYRUN testuser getkeys.command_with_flags key write] + } + test "Unload the module - getkeys" { assert_equal {OK} [r module unload getkeys] } diff --git a/tests/unit/moduleapi/infotest.tcl b/tests/unit/moduleapi/infotest.tcl index 0d07aaa7e..354487a19 100644 --- a/tests/unit/moduleapi/infotest.tcl +++ b/tests/unit/moduleapi/infotest.tcl @@ -64,7 +64,7 @@ start_server {tags {"modules"}} { } test {module info one module} { - set info [r info INFOTEST] + set info [r info INFOtest] ;# test case insensitive compare # info all does not contain modules assert { [string match "*Spanish*" $info] } assert { ![string match "*used_memory*" $info] } @@ -72,7 +72,7 @@ start_server {tags {"modules"}} { } {-2} test {module info one section} { - set info [r info INFOTEST_SPANISH] + set info [r info INFOtest_SpanisH] ;# test case insensitive compare assert { ![string match "*used_memory*" $info] } assert { ![string match "*Italian*" $info] } assert { ![string match "*infotest_global*" $info] } @@ -90,6 +90,31 @@ start_server {tags {"modules"}} { assert_match {*infotest_unsafe_field:value=1*} $info } + test {module info multiply sections without all, everything, default keywords} { + set info [r info replication INFOTEST] + assert { [string match "*Spanish*" $info] } + assert { ![string match "*used_memory*" $info] } + assert { [string match "*repl_offset*" $info] } + } + + test {module info multiply sections with all keyword and modules} { + set info [r info all modules] + assert { [string match "*cluster*" $info] } + assert { [string match "*cmdstat_info*" $info] } + assert { [string match "*infotest_global*" $info] } + } + + test {module info multiply sections with everything keyword} { + set info [r info replication everything cpu] + assert { [string match "*client_recent*" $info] } + assert { [string match "*cmdstat_info*" $info] } + assert { [string match "*Italian*" $info] } + # check that we didn't get the same info twice + assert { ![string match "*used_cpu_user_children*used_cpu_user_children*" $info] } + assert { ![string match "*Italian*Italian*" $info] } + field $info infotest_dos + } {2} + test "Unload the module - infotest" { assert_equal {OK} [r module unload infotest] } diff --git a/tests/unit/moduleapi/keyspecs.tcl b/tests/unit/moduleapi/keyspecs.tcl index cb9f1851a..60d3fe5d3 100644 --- a/tests/unit/moduleapi/keyspecs.tcl +++ b/tests/unit/moduleapi/keyspecs.tcl @@ -1,12 +1,26 @@ set testmodule [file normalize tests/modules/keyspecs.so] -if 0 { ; # Test suite disabled due to planned API changes start_server {tags {"modules"}} { r module load $testmodule - test "Module key specs: Legacy" { - set reply [lindex [r command info kspec.legacy] 0] - # Verify (first, last, step) + test "Module key specs: No spec, only legacy triple" { + set reply [lindex [r command info kspec.none] 0] + # Verify (first, last, step) and not movablekeys + assert_equal [lindex $reply 2] {module} + assert_equal [lindex $reply 3] 1 + assert_equal [lindex $reply 4] -1 + assert_equal [lindex $reply 5] 2 + # Verify key-spec auto-generated from the legacy triple + set keyspecs [lindex $reply 8] + assert_equal [llength $keyspecs] 1 + assert_equal [lindex $keyspecs 0] {flags {RW access update variable_flags} begin_search {type index spec {index 1}} find_keys {type range spec {lastkey -1 keystep 2 limit 0}}} + assert_equal [r command getkeys kspec.none key1 val1 key2 val2] {key1 key2} + } + + test "Module key specs: Two ranges" { + set reply [lindex [r command info kspec.tworanges] 0] + # Verify (first, last, step) and not movablekeys + assert_equal [lindex $reply 2] {module} assert_equal [lindex $reply 3] 1 assert_equal [lindex $reply 4] 2 assert_equal [lindex $reply 5] 1 @@ -14,24 +28,41 @@ start_server {tags {"modules"}} { set keyspecs [lindex $reply 8] assert_equal [lindex $keyspecs 0] {flags {RO access} begin_search {type index spec {index 1}} find_keys {type range spec {lastkey 0 keystep 1 limit 0}}} assert_equal [lindex $keyspecs 1] {flags {RW update} begin_search {type index spec {index 2}} find_keys {type range spec {lastkey 0 keystep 1 limit 0}}} + assert_equal [r command getkeys kspec.tworanges foo bar baz quux] {foo bar} + } + + test "Module key specs: Keyword-only spec clears the legacy triple" { + set reply [lindex [r command info kspec.keyword] 0] + # Verify (first, last, step) and movablekeys + assert_equal [lindex $reply 2] {module movablekeys} + assert_equal [lindex $reply 3] 0 + assert_equal [lindex $reply 4] 0 + assert_equal [lindex $reply 5] 0 + # Verify key-specs + set keyspecs [lindex $reply 8] + assert_equal [lindex $keyspecs 0] {flags {RO access} begin_search {type keyword spec {keyword KEYS startfrom 1}} find_keys {type range spec {lastkey -1 keystep 1 limit 0}}} + assert_equal [r command getkeys kspec.keyword foo KEYS bar baz] {bar baz} } test "Module key specs: Complex specs, case 1" { set reply [lindex [r command info kspec.complex1] 0] - # Verify (first, last, step) + # Verify (first, last, step) and movablekeys + assert_equal [lindex $reply 2] {module movablekeys} assert_equal [lindex $reply 3] 1 assert_equal [lindex $reply 4] 1 assert_equal [lindex $reply 5] 1 # Verify key-specs set keyspecs [lindex $reply 8] - assert_equal [lindex $keyspecs 0] {flags {} begin_search {type index spec {index 1}} find_keys {type range spec {lastkey 0 keystep 1 limit 0}}} + assert_equal [lindex $keyspecs 0] {flags RO begin_search {type index spec {index 1}} find_keys {type range spec {lastkey 0 keystep 1 limit 0}}} assert_equal [lindex $keyspecs 1] {flags {RW update} begin_search {type keyword spec {keyword STORE startfrom 2}} find_keys {type range spec {lastkey 0 keystep 1 limit 0}}} assert_equal [lindex $keyspecs 2] {flags {RO access} begin_search {type keyword spec {keyword KEYS startfrom 2}} find_keys {type keynum spec {keynumidx 0 firstkey 1 keystep 1}}} + assert_equal [r command getkeys kspec.complex1 foo dummy KEYS 1 bar baz STORE quux] {foo quux bar} } test "Module key specs: Complex specs, case 2" { set reply [lindex [r command info kspec.complex2] 0] - # Verify (first, last, step) + # Verify (first, last, step) and movablekeys + assert_equal [lindex $reply 2] {module movablekeys} assert_equal [lindex $reply 3] 1 assert_equal [lindex $reply 4] 2 assert_equal [lindex $reply 5] 1 @@ -42,17 +73,42 @@ start_server {tags {"modules"}} { assert_equal [lindex $keyspecs 2] {flags {RO access} begin_search {type index spec {index 2}} find_keys {type range spec {lastkey 0 keystep 1 limit 0}}} assert_equal [lindex $keyspecs 3] {flags {RW update} begin_search {type index spec {index 3}} find_keys {type keynum spec {keynumidx 0 firstkey 1 keystep 1}}} assert_equal [lindex $keyspecs 4] {flags {RW update} begin_search {type keyword spec {keyword MOREKEYS startfrom 5}} find_keys {type range spec {lastkey -1 keystep 1 limit 0}}} + assert_equal [r command getkeys kspec.complex2 foo bar 2 baz quux banana STORE dst dummy MOREKEYS hey ho] {dst foo bar baz quux hey ho} } test "Module command list filtering" { ;# Note: we piggyback this tcl file to test the general functionality of command list filtering set reply [r command list filterby module keyspecs] - assert_equal [lsort $reply] {kspec.complex1 kspec.complex2 kspec.legacy} + assert_equal [lsort $reply] {kspec.complex1 kspec.complex2 kspec.keyword kspec.none kspec.tworanges} + assert_equal [r command getkeys kspec.complex2 foo bar 2 baz quux banana STORE dst dummy MOREKEYS hey ho] {dst foo bar baz quux hey ho} + } + + test {COMMAND GETKEYSANDFLAGS correctly reports module key-spec without flags} { + r command getkeysandflags kspec.none key1 val1 key2 val2 + } {{key1 {RW access update variable_flags}} {key2 {RW access update variable_flags}}} + + test {COMMAND GETKEYSANDFLAGS correctly reports module key-spec flags} { + r command getkeysandflags kspec.keyword keys key1 key2 key3 + } {{key1 {RO access}} {key2 {RO access}} {key3 {RO access}}} + + # user that can only read from "read" keys, write to "write" keys, and read+write to "RW" keys + r ACL setuser testuser +@all %R~read* %W~write* %RW~rw* + + test "Module key specs: No spec, only legacy triple - ACL" { + # legacy triple didn't provide flags, so they require both read and write + assert_equal "OK" [r ACL DRYRUN testuser kspec.none rw val1] + assert_equal "This user has no permissions to access the 'read' key" [r ACL DRYRUN testuser kspec.none read val1] + assert_equal "This user has no permissions to access the 'write' key" [r ACL DRYRUN testuser kspec.none write val1] + } + + test "Module key specs: tworanges - ACL" { + assert_equal "OK" [r ACL DRYRUN testuser kspec.tworanges read write] + assert_equal "OK" [r ACL DRYRUN testuser kspec.tworanges rw rw] + assert_equal "This user has no permissions to access the 'read' key" [r ACL DRYRUN testuser kspec.tworanges rw read] + assert_equal "This user has no permissions to access the 'write' key" [r ACL DRYRUN testuser kspec.tworanges write rw] } test "Unload the module - keyspecs" { assert_equal {OK} [r module unload keyspecs] } } - -} ; # Test suite disabled diff --git a/tests/unit/moduleapi/subcommands.tcl b/tests/unit/moduleapi/subcommands.tcl index d696f9ad2..11d243243 100644 --- a/tests/unit/moduleapi/subcommands.tcl +++ b/tests/unit/moduleapi/subcommands.tcl @@ -8,20 +8,15 @@ start_server {tags {"modules"}} { set command_reply [r command info subcommands.bitarray] set first_cmd [lindex $command_reply 0] set subcmds_in_command [lsort [lindex $first_cmd 9]] - if 0 { ; # Keyspecs disabled due to planned changes in keyspec API - assert_equal [lindex $subcmds_in_command 0] {subcommands.bitarray|get -2 module 1 1 1 {} {} {{flags {RO access} begin_search {type index spec {index 1}} find_keys {type range spec {lastkey 0 keystep 1 limit 0}}}} {}} - assert_equal [lindex $subcmds_in_command 1] {subcommands.bitarray|set -2 module 1 1 1 {} {} {{flags {RW update} begin_search {type index spec {index 1}} find_keys {type range spec {lastkey 0 keystep 1 limit 0}}}} {}} - } else { ; # The same asserts without the key specs - assert_equal [lindex $subcmds_in_command 0] {subcommands.bitarray|get -2 module 0 0 0 {} {} {} {}} - assert_equal [lindex $subcmds_in_command 1] {subcommands.bitarray|set -2 module 0 0 0 {} {} {} {}} - } + assert_equal [lindex $subcmds_in_command 0] {subcommands.bitarray|get -2 module 1 1 1 {} {} {{flags {RO access} begin_search {type index spec {index 1}} find_keys {type range spec {lastkey 0 keystep 1 limit 0}}}} {}} + assert_equal [lindex $subcmds_in_command 1] {subcommands.bitarray|set -2 module 1 1 1 {} {} {{flags {RW update} begin_search {type index spec {index 1}} find_keys {type range spec {lastkey 0 keystep 1 limit 0}}}} {}} # Verify that module subcommands are displayed correctly in COMMAND DOCS set docs_reply [r command docs subcommands.bitarray] set docs [dict create {*}[lindex $docs_reply 1]] set subcmds_in_cmd_docs [dict create {*}[dict get $docs subcommands]] - assert_equal [dict get $subcmds_in_cmd_docs "subcommands.bitarray|get"] {summary {} since {} group module} - assert_equal [dict get $subcmds_in_cmd_docs "subcommands.bitarray|set"] {summary {} since {} group module} + assert_equal [dict get $subcmds_in_cmd_docs "subcommands.bitarray|get"] {group module} + assert_equal [dict get $subcmds_in_cmd_docs "subcommands.bitarray|set"] {group module} } test "Module pure-container command fails on arity error" { diff --git a/tests/unit/moduleapi/testrdb.tcl b/tests/unit/moduleapi/testrdb.tcl index 8d76a11bc..a01bcb30b 100644 --- a/tests/unit/moduleapi/testrdb.tcl +++ b/tests/unit/moduleapi/testrdb.tcl @@ -47,6 +47,12 @@ tags "modules" { } } + test {Verify module options info} { + start_server [list overrides [list loadmodule "$testmodule"]] { + assert_match "*\[handle-io-errors|handle-repl-async-load\]*" [r info modules] + } + } + tags {repl} { test {diskless loading short read with module} { start_server [list overrides [list loadmodule "$testmodule"]] { diff --git a/tests/unit/moduleapi/timer.tcl b/tests/unit/moduleapi/timer.tcl index c04f80b23..4e9dd0f09 100644 --- a/tests/unit/moduleapi/timer.tcl +++ b/tests/unit/moduleapi/timer.tcl @@ -26,6 +26,8 @@ start_server {tags {"modules"}} { assert_equal "timer-incr-key" [lindex $info 0] set remaining [lindex $info 1] assert {$remaining < 10000 && $remaining > 1} + # Stop the timer after get timer test + assert_equal 1 [r test.stoptimer $id] } test {RM_StopTimer: basic sanity} { @@ -54,7 +56,43 @@ start_server {tags {"modules"}} { assert_equal {} [r test.gettimer $id] } - test "Unload the module - timer" { + test "Module can be unloaded when timer was finished" { + r set "timer-incr-key" 0 + r test.createtimer 500 timer-incr-key + + # Make sure the Timer has not been fired + assert_equal 0 [r get timer-incr-key] + # Module can not be unloaded since the timer was ongoing + catch {r module unload timer} err + assert_match {*the module holds timer that is not fired*} $err + + # Wait to be sure timer has been finished + wait_for_condition 10 500 { + [r get timer-incr-key] == 1 + } else { + fail "Timer not fired" + } + + # Timer fired, can be unloaded now. + assert_equal {OK} [r module unload timer] + } + + test "Module can be unloaded when timer was stopped" { + r module load $testmodule + r set "timer-incr-key" 0 + set id [r test.createtimer 5000 timer-incr-key] + + # Module can not be unloaded since the timer was ongoing + catch {r module unload timer} err + assert_match {*the module holds timer that is not fired*} $err + + # Stop the timer + assert_equal 1 [r test.stoptimer $id] + + # Make sure the Timer has not been fired + assert_equal 0 [r get timer-incr-key] + + # Timer has stopped, can be unloaded now. assert_equal {OK} [r module unload timer] } } diff --git a/tests/unit/multi.tcl b/tests/unit/multi.tcl index 8a0b731d5..63d85d26b 100644 --- a/tests/unit/multi.tcl +++ b/tests/unit/multi.tcl @@ -132,18 +132,61 @@ start_server {tags {"multi"}} { } {} {cluster:skip} test {EXEC fail on lazy expired WATCHed key} { - r flushall + r del key r debug set-active-expire 0 - r del key - r set key 1 px 2 - r watch key + for {set j 0} {$j < 10} {incr j} { + r set key 1 px 100 + r watch key + after 101 + r multi + r incr key + + set res [r exec] + if {$res eq {}} break + } + if {$::verbose} { puts "EXEC fail on lazy expired WATCHed key attempts: $j" } + + r debug set-active-expire 1 + set _ $res + } {} {needs:debug} + + test {WATCH stale keys should not fail EXEC} { + r del x + r debug set-active-expire 0 + r set x foo px 1 + after 2 + r watch x + r multi + r ping + assert_equal {PONG} [r exec] + r debug set-active-expire 1 + } {OK} {needs:debug} - after 100 + test {Delete WATCHed stale keys should not fail EXEC} { + r del x + r debug set-active-expire 0 + r set x foo px 1 + after 2 + r watch x + # EXISTS triggers lazy expiry/deletion + assert_equal 0 [r exists x] + r multi + r ping + assert_equal {PONG} [r exec] + r debug set-active-expire 1 + } {OK} {needs:debug} + test {FLUSHDB while watching stale keys should not fail EXEC} { + r del x + r debug set-active-expire 0 + r set x foo px 1 + after 2 + r watch x + r flushdb r multi - r incr key - assert_equal [r exec] {} + r ping + assert_equal {PONG} [r exec] r debug set-active-expire 1 } {OK} {needs:debug} @@ -245,6 +288,52 @@ start_server {tags {"multi"}} { r exec } {} {singledb:skip} + test {SWAPDB does not touch watched stale keys} { + r flushall + r select 1 + r debug set-active-expire 0 + r set x foo px 1 + after 2 + r watch x + r swapdb 0 1 ; # expired key replaced with no key => no change + r multi + r ping + assert_equal {PONG} [r exec] + r debug set-active-expire 1 + } {OK} {singledb:skip needs:debug} + + test {SWAPDB does not touch non-existing key replaced with stale key} { + r flushall + r select 0 + r debug set-active-expire 0 + r set x foo px 1 + after 2 + r select 1 + r watch x + r swapdb 0 1 ; # no key replaced with expired key => no change + r multi + r ping + assert_equal {PONG} [r exec] + r debug set-active-expire 1 + } {OK} {singledb:skip needs:debug} + + test {SWAPDB does not touch stale key replaced with another stale key} { + r flushall + r debug set-active-expire 0 + r select 1 + r set x foo px 1 + r select 0 + r set x bar px 1 + after 2 + r select 1 + r watch x + r swapdb 0 1 ; # no key replaced with expired key => no change + r multi + r ping + assert_equal {PONG} [r exec] + r debug set-active-expire 1 + } {OK} {singledb:skip needs:debug} + test {WATCH is able to remember the DB a key belongs to} { r select 5 r set x 30 diff --git a/tests/unit/protocol.tcl b/tests/unit/protocol.tcl index ec4a1a4aa..50305bd27 100644 --- a/tests/unit/protocol.tcl +++ b/tests/unit/protocol.tcl @@ -139,13 +139,17 @@ start_server {tags {"protocol network"}} { test {RESP3 attributes} { r hello 3 - set res [r debug protocol attrib] - # currently the parser in redis.tcl ignores the attributes + assert_equal {Some real reply following the attribute} [r debug protocol attrib] + assert_equal {key-popularity {key:123 90}} [r attributes] + + # make sure attributes are not kept from previous command + r ping + assert_error {*attributes* no such element in array} {r attributes} # restore state r hello 2 - set _ $res - } {Some real reply following the attribute} {needs:debug resp3} + set _ "" + } {} {needs:debug resp3} test {RESP3 attributes readraw} { r hello 3 diff --git a/tests/unit/replybufsize.tcl b/tests/unit/replybufsize.tcl new file mode 100644 index 000000000..9377a8fd3 --- /dev/null +++ b/tests/unit/replybufsize.tcl @@ -0,0 +1,47 @@ +proc get_reply_buffer_size {cname} { + + set clients [split [string trim [r client list]] "\r\n"] + set c [lsearch -inline $clients *name=$cname*] + if {![regexp rbs=(\[a-zA-Z0-9-\]+) $c - rbufsize]} { + error "field rbus not found in $c" + } + return $rbufsize +} + +start_server {tags {"replybufsize"}} { + + test {verify reply buffer limits} { + # In order to reduce test time we can set the peak reset time very low + r debug replybuffer-peak-reset-time 100 + + # Create a simple idle test client + variable tc [redis_client] + $tc client setname test_client + + # make sure the client is idle for 1 seconds to make it shrink the reply buffer + wait_for_condition 10 100 { + [get_reply_buffer_size test_client] >= 1024 && [get_reply_buffer_size test_client] < 2046 + } else { + set rbs [get_reply_buffer_size test_client] + fail "reply buffer of idle client is $rbs after 1 seconds" + } + + r set bigval [string repeat x 32768] + + # In order to reduce test time we can set the peak reset time very low + r debug replybuffer-peak-reset-time never + + wait_for_condition 10 100 { + [$tc get bigval ; get_reply_buffer_size test_client] >= 16384 && [get_reply_buffer_size test_client] < 32768 + } else { + set rbs [get_reply_buffer_size test_client] + fail "reply buffer of busy client is $rbs after 1 seconds" + } + + # Restore the peak reset time to default + r debug replybuffer-peak-reset-time reset + + $tc close + } {0} {needs:debug} +} +
\ No newline at end of file diff --git a/tests/unit/scripting.tcl b/tests/unit/scripting.tcl index f16555350..6c40844c3 100644 --- a/tests/unit/scripting.tcl +++ b/tests/unit/scripting.tcl @@ -73,10 +73,10 @@ start_server {tags {"scripting"}} { test {EVAL - Lua error reply -> Redis protocol type conversion} { catch { - run_script {return {err='this is an error'}} 0 + run_script {return {err='ERR this is an error'}} 0 } e set _ $e - } {this is an error} + } {ERR this is an error} test {EVAL - Lua table -> Redis protocol type conversion} { run_script {return {1,2,3,'ciao',{1,2}}} 0 @@ -378,7 +378,7 @@ start_server {tags {"scripting"}} { r set foo bar catch {run_script_ro {redis.call('del', KEYS[1]);} 1 foo} e set e - } {*Write commands are not allowed from read-only scripts*} + } {ERR Write commands are not allowed from read-only scripts*} if {$is_eval eq 1} { # script command is only relevant for is_eval Lua @@ -439,12 +439,12 @@ start_server {tags {"scripting"}} { test {Globals protection reading an undeclared global variable} { catch {run_script {return a} 0} e set e - } {*ERR*attempted to access * global*} + } {ERR*attempted to access * global*} test {Globals protection setting an undeclared global*} { catch {run_script {a=10} 0} e set e - } {*ERR*attempted to create global*} + } {ERR*attempted to create global*} test {Test an example script DECR_IF_GT} { set decr_if_gt { @@ -599,8 +599,8 @@ start_server {tags {"scripting"}} { } {ERR Number of keys can't be negative} test {Scripts can handle commands with incorrect arity} { - assert_error "*Wrong number of args calling Redis command from script" {run_script "redis.call('set','invalid')" 0} - assert_error "*Wrong number of args calling Redis command from script" {run_script "redis.call('incr')" 0} + assert_error "ERR Wrong number of args calling Redis command from script*" {run_script "redis.call('set','invalid')" 0} + assert_error "ERR Wrong number of args calling Redis command from script*" {run_script "redis.call('incr')" 0} } test {Correct handling of reused argv (issue #1939)} { @@ -701,6 +701,32 @@ start_server {tags {"scripting"}} { return redis.call("EXISTS", "key") } 0] 0 } + + test "Script ACL check" { + r acl setuser bob on {>123} {+@scripting} {+set} {~x*} + assert_equal [r auth bob 123] {OK} + + # Check permission granted + assert_equal [run_script { + return redis.acl_check_cmd('set','xx',1) + } 1 xx] 1 + + # Check permission denied unauthorised command + assert_equal [run_script { + return redis.acl_check_cmd('hset','xx','f',1) + } 1 xx] {} + + # Check permission denied unauthorised key + # Note: we don't pass the "yy" key as an argument to the script so key acl checks won't block the script + assert_equal [run_script { + return redis.acl_check_cmd('set','yy',1) + } 0] {} + + # Check error due to invalid command + assert_error {ERR *Invalid command passed to redis.acl_check_cmd()*} {run_script { + return redis.acl_check_cmd('invalid-cmd','arg') + } 0} + } } # Start a new server since the last test in this stanza will kill the @@ -1262,7 +1288,7 @@ start_server {tags {"scripting"}} { r config set maxmemory 1 # Fail to execute deny-oom command in OOM condition (backwards compatibility mode without flags) - assert_error {ERR Error running script *OOM command not allowed when used memory > 'maxmemory'.} { + assert_error {OOM command not allowed when used memory > 'maxmemory'*} { r eval { redis.call('set','x',1) return 1 @@ -1293,7 +1319,7 @@ start_server {tags {"scripting"}} { } test "no-writes shebang flag" { - assert_error {ERR Error running script *Write commands are not allowed from read-only scripts.} { + assert_error {ERR Write commands are not allowed from read-only scripts*} { r eval {#!lua flags=no-writes redis.call('set','x',1) return 1 @@ -1374,3 +1400,154 @@ start_server {tags {"scripting"}} { set _ {} } {} {external:skip} } + +# Additional eval only tests +start_server {tags {"scripting"}} { + test "Consistent eval error reporting" { + r config resetstat + r config set maxmemory 1 + # Script aborted due to Redis state (OOM) should report script execution error with detailed internal error + assert_error {OOM command not allowed when used memory > 'maxmemory'*} { + r eval {return redis.call('set','x','y')} 1 x + } + assert_equal [errorrstat OOM r] {count=1} + assert_equal [s total_error_replies] {1} + assert_match {calls=0*rejected_calls=1,failed_calls=0*} [cmdrstat set r] + assert_match {calls=1*rejected_calls=0,failed_calls=1*} [cmdrstat eval r] + + # redis.pcall() failure due to Redis state (OOM) returns lua error table with Redis error message without '-' prefix + r config resetstat + assert_equal [ + r eval { + local t = redis.pcall('set','x','y') + if t['err'] == "OOM command not allowed when used memory > 'maxmemory'." then + return 1 + else + return 0 + end + } 1 x + ] 1 + # error stats were not incremented + assert_equal [errorrstat ERR r] {} + assert_equal [errorrstat OOM r] {count=1} + assert_equal [s total_error_replies] {1} + assert_match {calls=0*rejected_calls=1,failed_calls=0*} [cmdrstat set r] + assert_match {calls=1*rejected_calls=0,failed_calls=0*} [cmdrstat eval r] + + # Returning an error object from lua is handled as a valid RESP error result. + r config resetstat + assert_error {OOM command not allowed when used memory > 'maxmemory'.} { + r eval { return redis.pcall('set','x','y') } 1 x + } + assert_equal [errorrstat ERR r] {} + assert_equal [errorrstat OOM r] {count=1} + assert_equal [s total_error_replies] {1} + assert_match {calls=0*rejected_calls=1,failed_calls=0*} [cmdrstat set r] + assert_match {calls=1*rejected_calls=0,failed_calls=1*} [cmdrstat eval r] + + r config set maxmemory 0 + r config resetstat + # Script aborted due to error result of Redis command + assert_error {ERR DB index is out of range*} { + r eval {return redis.call('select',99)} 0 + } + assert_equal [errorrstat ERR r] {count=1} + assert_equal [s total_error_replies] {1} + assert_match {calls=1*rejected_calls=0,failed_calls=1*} [cmdrstat select r] + assert_match {calls=1*rejected_calls=0,failed_calls=1*} [cmdrstat eval r] + + # redis.pcall() failure due to error in Redis command returns lua error table with redis error message without '-' prefix + r config resetstat + assert_equal [ + r eval { + local t = redis.pcall('select',99) + if t['err'] == "ERR DB index is out of range" then + return 1 + else + return 0 + end + } 0 + ] 1 + assert_equal [errorrstat ERR r] {count=1} ; + assert_equal [s total_error_replies] {1} + assert_match {calls=1*rejected_calls=0,failed_calls=1*} [cmdrstat select r] + assert_match {calls=1*rejected_calls=0,failed_calls=0*} [cmdrstat eval r] + + # Script aborted due to scripting specific error state (write cmd with eval_ro) should report script execution error with detailed internal error + r config resetstat + assert_error {ERR Write commands are not allowed from read-only scripts*} { + r eval_ro {return redis.call('set','x','y')} 1 x + } + assert_equal [errorrstat ERR r] {count=1} + assert_equal [s total_error_replies] {1} + assert_match {calls=0*rejected_calls=1,failed_calls=0*} [cmdrstat set r] + assert_match {calls=1*rejected_calls=0,failed_calls=1*} [cmdrstat eval_ro r] + + # redis.pcall() failure due to scripting specific error state (write cmd with eval_ro) returns lua error table with Redis error message without '-' prefix + r config resetstat + assert_equal [ + r eval_ro { + local t = redis.pcall('set','x','y') + if t['err'] == "ERR Write commands are not allowed from read-only scripts." then + return 1 + else + return 0 + end + } 1 x + ] 1 + assert_equal [errorrstat ERR r] {count=1} + assert_equal [s total_error_replies] {1} + assert_match {calls=0*rejected_calls=1,failed_calls=0*} [cmdrstat set r] + assert_match {calls=1*rejected_calls=0,failed_calls=0*} [cmdrstat eval_ro r] + + r config resetstat + # make sure geoadd will failed + r set Sicily 1 + assert_error {WRONGTYPE Operation against a key holding the wrong kind of value*} { + r eval {return redis.call('GEOADD', 'Sicily', '13.361389', '38.115556', 'Palermo', '15.087269', '37.502669', 'Catania')} 1 x + } + assert_equal [errorrstat WRONGTYPE r] {count=1} + assert_equal [s total_error_replies] {1} + assert_match {calls=1*rejected_calls=0,failed_calls=1*} [cmdrstat geoadd r] + assert_match {calls=1*rejected_calls=0,failed_calls=1*} [cmdrstat eval r] + } {} {cluster:skip} + + test "LUA redis.error_reply API" { + r config resetstat + assert_error {MY_ERR_CODE custom msg} { + r eval {return redis.error_reply("MY_ERR_CODE custom msg")} 0 + } + assert_equal [errorrstat MY_ERR_CODE r] {count=1} + } + + test "LUA redis.error_reply API with empty string" { + r config resetstat + assert_error {ERR} { + r eval {return redis.error_reply("")} 0 + } + assert_equal [errorrstat ERR r] {count=1} + } + + test "LUA redis.status_reply API" { + r config resetstat + r readraw 1 + assert_equal [ + r eval {return redis.status_reply("MY_OK_CODE custom msg")} 0 + ] {+MY_OK_CODE custom msg} + r readraw 0 + assert_equal [errorrstat MY_ERR_CODE r] {} ;# error stats were not incremented + } + + test "LUA test pcall" { + assert_equal [ + r eval {local status, res = pcall(function() return 1 end); return 'status: ' .. tostring(status) .. ' result: ' .. res} 0 + ] {status: true result: 1} + } + + test "LUA test pcall with error" { + assert_match {status: false result:*Script attempted to access nonexistent global variable 'foo'} [ + r eval {local status, res = pcall(function() return foo end); return 'status: ' .. tostring(status) .. ' result: ' .. res} 0 + ] + } +} + diff --git a/tests/unit/type/stream-cgroups.tcl b/tests/unit/type/stream-cgroups.tcl index ae8da27b8..27cbc686e 100644 --- a/tests/unit/type/stream-cgroups.tcl +++ b/tests/unit/type/stream-cgroups.tcl @@ -281,14 +281,20 @@ start_server { } test {XGROUP DESTROY should unblock XREADGROUP with -NOGROUP} { + r config resetstat r del mystream r XGROUP CREATE mystream mygroup $ MKSTREAM set rd [redis_deferring_client] $rd XREADGROUP GROUP mygroup Alice BLOCK 0 STREAMS mystream ">" wait_for_blocked_clients_count 1 r XGROUP DESTROY mystream mygroup - assert_error "*NOGROUP*" {$rd read} + assert_error "NOGROUP*" {$rd read} $rd close + + # verify command stats, error stats and error counter work on failed blocked command + assert_match {*count=1*} [errorrstat NOGROUP r] + assert_match {*calls=1,*,rejected_calls=0,failed_calls=1} [cmdrstat xreadgroup r] + assert_equal [s total_error_replies] 1 } test {RENAME can unblock XREADGROUP with data} { @@ -355,24 +361,22 @@ start_server { # Delete item 2 from the stream. Now consumer 1 has PEL that contains # only item 3. Try to use consumer 2 to claim the deleted item 2 - # from the PEL of consumer 1, this should return nil + # from the PEL of consumer 1, this should be NOP r XDEL mystream $id2 set reply [ r XCLAIM mystream mygroup consumer2 10 $id2 ] - assert {[llength $reply] == 1} - assert_equal "" [lindex $reply 0] + assert {[llength $reply] == 0} # Delete item 3 from the stream. Now consumer 1 has PEL that is empty. # Try to use consumer 2 to claim the deleted item 3 from the PEL - # of consumer 1, this should return nil + # of consumer 1, this should be NOP after 200 r XDEL mystream $id3 set reply [ r XCLAIM mystream mygroup consumer2 10 $id3 ] - assert {[llength $reply] == 1} - assert_equal "" [lindex $reply 0] + assert {[llength $reply] == 0} } test {XCLAIM without JUSTID increments delivery count} { @@ -445,6 +449,7 @@ start_server { set id1 [r XADD mystream * a 1] set id2 [r XADD mystream * b 2] set id3 [r XADD mystream * c 3] + set id4 [r XADD mystream * d 4] r XGROUP CREATE mystream mygroup 0 # Consumer 1 reads item 1 from the stream without acknowledgements. @@ -454,7 +459,7 @@ start_server { assert_equal [lindex $reply 0 1 0 1] {a 1} after 200 set reply [r XAUTOCLAIM mystream mygroup consumer2 10 - COUNT 1] - assert_equal [llength $reply] 2 + assert_equal [llength $reply] 3 assert_equal [lindex $reply 0] "0-0" assert_equal [llength [lindex $reply 1]] 1 assert_equal [llength [lindex $reply 1 0]] 2 @@ -462,7 +467,7 @@ start_server { assert_equal [lindex $reply 1 0 1] {a 1} # Consumer 1 reads another 2 items from stream - r XREADGROUP GROUP mygroup consumer1 count 2 STREAMS mystream > + r XREADGROUP GROUP mygroup consumer1 count 3 STREAMS mystream > # For min-idle-time after 200 @@ -471,33 +476,37 @@ start_server { # only item 3. Try to use consumer 2 to claim the deleted item 2 # from the PEL of consumer 1, this should return nil r XDEL mystream $id2 + + # id1 and id3 are self-claimed here but not id2 ('count' was set to 2) + # we make sure id2 is indeed skipped (the cursor points to id4) set reply [r XAUTOCLAIM mystream mygroup consumer2 10 - COUNT 2] - # id1 is self-claimed here but not id2 ('count' was set to 2) - assert_equal [llength $reply] 2 - assert_equal [lindex $reply 0] $id3 + + assert_equal [llength $reply] 3 + assert_equal [lindex $reply 0] $id4 assert_equal [llength [lindex $reply 1]] 2 assert_equal [llength [lindex $reply 1 0]] 2 assert_equal [llength [lindex $reply 1 0 1]] 2 assert_equal [lindex $reply 1 0 1] {a 1} - assert_equal [lindex $reply 1 1] "" + assert_equal [lindex $reply 1 1 1] {c 3} # Delete item 3 from the stream. Now consumer 1 has PEL that is empty. # Try to use consumer 2 to claim the deleted item 3 from the PEL # of consumer 1, this should return nil after 200 - r XDEL mystream $id3 + + r XDEL mystream $id4 + + # id1 and id3 are self-claimed here but not id2 and id4 ('count' is default 100) set reply [r XAUTOCLAIM mystream mygroup consumer2 10 - JUSTID] - # id1 is self-claimed here but not id2 and id3 ('count' is default 100) # we also test the JUSTID modifier here. note that, when using JUSTID, # deleted entries are returned in reply (consistent with XCLAIM). - assert_equal [llength $reply] 2 - assert_equal [lindex $reply 0] "0-0" - assert_equal [llength [lindex $reply 1]] 3 + assert_equal [llength $reply] 3 + assert_equal [lindex $reply 0] {0-0} + assert_equal [llength [lindex $reply 1]] 2 assert_equal [lindex $reply 1 0] $id1 - assert_equal [lindex $reply 1 1] $id2 - assert_equal [lindex $reply 1 2] $id3 + assert_equal [lindex $reply 1 1] $id3 } test {XAUTOCLAIM as an iterator} { @@ -518,7 +527,7 @@ start_server { # Claim 2 entries set reply [r XAUTOCLAIM mystream mygroup consumer2 10 - COUNT 2] - assert_equal [llength $reply] 2 + assert_equal [llength $reply] 3 set cursor [lindex $reply 0] assert_equal $cursor $id3 assert_equal [llength [lindex $reply 1]] 2 @@ -527,7 +536,7 @@ start_server { # Claim 2 more entries set reply [r XAUTOCLAIM mystream mygroup consumer2 10 $cursor COUNT 2] - assert_equal [llength $reply] 2 + assert_equal [llength $reply] 3 set cursor [lindex $reply 0] assert_equal $cursor $id5 assert_equal [llength [lindex $reply 1]] 2 @@ -536,7 +545,7 @@ start_server { # Claim last entry set reply [r XAUTOCLAIM mystream mygroup consumer2 10 $cursor COUNT 1] - assert_equal [llength $reply] 2 + assert_equal [llength $reply] 3 set cursor [lindex $reply 0] assert_equal $cursor {0-0} assert_equal [llength [lindex $reply 1]] 1 @@ -548,6 +557,56 @@ start_server { assert_error "ERR COUNT must be > 0" {r XAUTOCLAIM key group consumer 1 1 COUNT 0} } + test {XCLAIM with XDEL} { + r DEL x + r XADD x 1-0 f v + r XADD x 2-0 f v + r XADD x 3-0 f v + r XGROUP CREATE x grp 0 + assert_equal [r XREADGROUP GROUP grp Alice STREAMS x >] {{x {{1-0 {f v}} {2-0 {f v}} {3-0 {f v}}}}} + r XDEL x 2-0 + assert_equal [r XCLAIM x grp Bob 0 1-0 2-0 3-0] {{1-0 {f v}} {3-0 {f v}}} + assert_equal [r XPENDING x grp - + 10 Alice] {} + } + + test {XCLAIM with trimming} { + r DEL x + r config set stream-node-max-entries 2 + r XADD x 1-0 f v + r XADD x 2-0 f v + r XADD x 3-0 f v + r XGROUP CREATE x grp 0 + assert_equal [r XREADGROUP GROUP grp Alice STREAMS x >] {{x {{1-0 {f v}} {2-0 {f v}} {3-0 {f v}}}}} + r XTRIM x MAXLEN 1 + assert_equal [r XCLAIM x grp Bob 0 1-0 2-0 3-0] {{3-0 {f v}}} + assert_equal [r XPENDING x grp - + 10 Alice] {} + } + + test {XAUTOCLAIM with XDEL} { + r DEL x + r XADD x 1-0 f v + r XADD x 2-0 f v + r XADD x 3-0 f v + r XGROUP CREATE x grp 0 + assert_equal [r XREADGROUP GROUP grp Alice STREAMS x >] {{x {{1-0 {f v}} {2-0 {f v}} {3-0 {f v}}}}} + r XDEL x 2-0 + assert_equal [r XAUTOCLAIM x grp Bob 0 0-0] {0-0 {{1-0 {f v}} {3-0 {f v}}} 2-0} + assert_equal [r XPENDING x grp - + 10 Alice] {} + } + + test {XCLAIM with trimming} { + r DEL x + r config set stream-node-max-entries 2 + r XADD x 1-0 f v + r XADD x 2-0 f v + r XADD x 3-0 f v + r XGROUP CREATE x grp 0 + assert_equal [r XREADGROUP GROUP grp Alice STREAMS x >] {{x {{1-0 {f v}} {2-0 {f v}} {3-0 {f v}}}}} + r XTRIM x MAXLEN 1 + assert_equal [r XAUTOCLAIM x grp Bob 0 0-0] {0-0 {{3-0 {f v}}} {1-0 2-0}} + assert_equal [r XPENDING x grp - + 10 Alice] {} + } + test {XINFO FULL output} { r del x r XADD x 100 a 1 @@ -564,22 +623,30 @@ start_server { r XDEL x 103 set reply [r XINFO STREAM x FULL] - assert_equal [llength $reply] 12 - assert_equal [lindex $reply 1] 4 ;# stream length - assert_equal [lindex $reply 9] "{100-0 {a 1}} {101-0 {b 1}} {102-0 {c 1}} {104-0 {f 1}}" ;# entries - assert_equal [lindex $reply 11 0 1] "g1" ;# first group name - assert_equal [lindex $reply 11 0 7 0 0] "100-0" ;# first entry in group's PEL - assert_equal [lindex $reply 11 0 9 0 1] "Alice" ;# first consumer - assert_equal [lindex $reply 11 0 9 0 7 0 0] "100-0" ;# first entry in first consumer's PEL - assert_equal [lindex $reply 11 1 1] "g2" ;# second group name - assert_equal [lindex $reply 11 1 9 0 1] "Charlie" ;# first consumer - assert_equal [lindex $reply 11 1 9 0 7 0 0] "100-0" ;# first entry in first consumer's PEL - assert_equal [lindex $reply 11 1 9 0 7 1 0] "101-0" ;# second entry in first consumer's PEL + assert_equal [llength $reply] 18 + assert_equal [dict get $reply length] 4 + assert_equal [dict get $reply entries] "{100-0 {a 1}} {101-0 {b 1}} {102-0 {c 1}} {104-0 {f 1}}" + + # First consumer group + set group [lindex [dict get $reply groups] 0] + assert_equal [dict get $group name] "g1" + assert_equal [lindex [dict get $group pending] 0 0] "100-0" + set consumer [lindex [dict get $group consumers] 0] + assert_equal [dict get $consumer name] "Alice" + assert_equal [lindex [dict get $consumer pending] 0 0] "100-0" ;# first entry in first consumer's PEL + + # Second consumer group + set group [lindex [dict get $reply groups] 1] + assert_equal [dict get $group name] "g2" + set consumer [lindex [dict get $group consumers] 0] + assert_equal [dict get $consumer name] "Charlie" + assert_equal [lindex [dict get $consumer pending] 0 0] "100-0" ;# first entry in first consumer's PEL + assert_equal [lindex [dict get $consumer pending] 1 0] "101-0" ;# second entry in first consumer's PEL set reply [r XINFO STREAM x FULL COUNT 1] - assert_equal [llength $reply] 12 - assert_equal [lindex $reply 1] 4 - assert_equal [lindex $reply 9] "{100-0 {a 1}}" + assert_equal [llength $reply] 18 + assert_equal [dict get $reply length] 4 + assert_equal [dict get $reply entries] "{100-0 {a 1}}" } test {XGROUP CREATECONSUMER: create consumer if does not exist} { @@ -643,7 +710,7 @@ start_server { set grpinfo [r xinfo groups mystream] r debug loadaof - assert {[r xinfo groups mystream] == $grpinfo} + assert_equal [r xinfo groups mystream] $grpinfo set reply [r xinfo consumers mystream mygroup] set consumer_info [lindex $reply 0] assert_equal [lindex $consumer_info 1] "Alice" ;# consumer name @@ -682,6 +749,154 @@ start_server { } } + test {Consumer group read counter and lag in empty streams} { + r DEL x + r XGROUP CREATE x g1 0 MKSTREAM + + set reply [r XINFO STREAM x FULL] + set group [lindex [dict get $reply groups] 0] + assert_equal [dict get $reply max-deleted-entry-id] "0-0" + assert_equal [dict get $reply entries-added] 0 + assert_equal [dict get $group entries-read] {} + assert_equal [dict get $group lag] 0 + + r XADD x 1-0 data a + r XDEL x 1-0 + + set reply [r XINFO STREAM x FULL] + set group [lindex [dict get $reply groups] 0] + assert_equal [dict get $reply max-deleted-entry-id] "1-0" + assert_equal [dict get $reply entries-added] 1 + assert_equal [dict get $group entries-read] {} + assert_equal [dict get $group lag] 0 + } + + test {Consumer group read counter and lag sanity} { + r DEL x + r XADD x 1-0 data a + r XADD x 2-0 data b + r XADD x 3-0 data c + r XADD x 4-0 data d + r XADD x 5-0 data e + r XGROUP CREATE x g1 0 + + set reply [r XINFO STREAM x FULL] + set group [lindex [dict get $reply groups] 0] + assert_equal [dict get $group entries-read] {} + assert_equal [dict get $group lag] 5 + + r XREADGROUP GROUP g1 c11 COUNT 1 STREAMS x > + set reply [r XINFO STREAM x FULL] + set group [lindex [dict get $reply groups] 0] + assert_equal [dict get $group entries-read] 1 + assert_equal [dict get $group lag] 4 + + r XREADGROUP GROUP g1 c12 COUNT 10 STREAMS x > + set reply [r XINFO STREAM x FULL] + set group [lindex [dict get $reply groups] 0] + assert_equal [dict get $group entries-read] 5 + assert_equal [dict get $group lag] 0 + + r XADD x 6-0 data f + set reply [r XINFO STREAM x FULL] + set group [lindex [dict get $reply groups] 0] + assert_equal [dict get $group entries-read] 5 + assert_equal [dict get $group lag] 1 + } + + test {Consumer group lag with XDELs} { + r DEL x + r XADD x 1-0 data a + r XADD x 2-0 data b + r XADD x 3-0 data c + r XADD x 4-0 data d + r XADD x 5-0 data e + r XDEL x 3-0 + r XGROUP CREATE x g1 0 + r XGROUP CREATE x g2 0 + + set reply [r XINFO STREAM x FULL] + set group [lindex [dict get $reply groups] 0] + assert_equal [dict get $group entries-read] {} + assert_equal [dict get $group lag] {} + + r XREADGROUP GROUP g1 c11 COUNT 1 STREAMS x > + set reply [r XINFO STREAM x FULL] + set group [lindex [dict get $reply groups] 0] + assert_equal [dict get $group entries-read] {} + assert_equal [dict get $group lag] {} + + r XREADGROUP GROUP g1 c11 COUNT 1 STREAMS x > + set reply [r XINFO STREAM x FULL] + set group [lindex [dict get $reply groups] 0] + assert_equal [dict get $group entries-read] {} + assert_equal [dict get $group lag] {} + + r XREADGROUP GROUP g1 c11 COUNT 1 STREAMS x > + set reply [r XINFO STREAM x FULL] + set group [lindex [dict get $reply groups] 0] + assert_equal [dict get $group entries-read] {} + assert_equal [dict get $group lag] {} + + r XREADGROUP GROUP g1 c11 COUNT 1 STREAMS x > + set reply [r XINFO STREAM x FULL] + set group [lindex [dict get $reply groups] 0] + assert_equal [dict get $group entries-read] 5 + assert_equal [dict get $group lag] 0 + + r XADD x 6-0 data f + set reply [r XINFO STREAM x FULL] + set group [lindex [dict get $reply groups] 0] + assert_equal [dict get $group entries-read] 5 + assert_equal [dict get $group lag] 1 + + r XTRIM x MINID = 3-0 + set reply [r XINFO STREAM x FULL] + set group [lindex [dict get $reply groups] 0] + assert_equal [dict get $group entries-read] 5 + assert_equal [dict get $group lag] 1 + set group [lindex [dict get $reply groups] 1] + assert_equal [dict get $group entries-read] {} + assert_equal [dict get $group lag] 3 + + r XTRIM x MINID = 5-0 + set reply [r XINFO STREAM x FULL] + set group [lindex [dict get $reply groups] 0] + assert_equal [dict get $group entries-read] 5 + assert_equal [dict get $group lag] 1 + set group [lindex [dict get $reply groups] 1] + assert_equal [dict get $group entries-read] {} + assert_equal [dict get $group lag] 2 + } + + test {Loading from legacy (Redis <= v6.2.x, rdb_ver < 10) persistence} { + # The payload was DUMPed from a v5 instance after: + # XADD x 1-0 data a + # XADD x 2-0 data b + # XADD x 3-0 data c + # XADD x 4-0 data d + # XADD x 5-0 data e + # XADD x 6-0 data f + # XDEL x 3-0 + # XGROUP CREATE x g1 0 + # XGROUP CREATE x g2 0 + # XREADGROUP GROUP g1 c11 COUNT 4 STREAMS x > + # XTRIM x MAXLEN = 2 + + r DEL x + r RESTORE x 0 "\x0F\x01\x10\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\xC3\x40\x4A\x40\x57\x16\x57\x00\x00\x00\x23\x00\x02\x01\x04\x01\x01\x01\x84\x64\x61\x74\x61\x05\x00\x01\x03\x01\x00\x20\x01\x03\x81\x61\x02\x04\x20\x0A\x00\x01\x40\x0A\x00\x62\x60\x0A\x00\x02\x40\x0A\x00\x63\x60\x0A\x40\x22\x01\x81\x64\x20\x0A\x40\x39\x20\x0A\x00\x65\x60\x0A\x00\x05\x40\x0A\x00\x66\x20\x0A\x00\xFF\x02\x06\x00\x02\x02\x67\x31\x05\x00\x04\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x3E\xF7\x83\x43\x7A\x01\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00\x3E\xF7\x83\x43\x7A\x01\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x3E\xF7\x83\x43\x7A\x01\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x3E\xF7\x83\x43\x7A\x01\x00\x00\x01\x01\x03\x63\x31\x31\x3E\xF7\x83\x43\x7A\x01\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x02\x67\x32\x00\x00\x00\x00\x09\x00\x3D\x52\xEF\x68\x67\x52\x1D\xFA" + + set reply [r XINFO STREAM x FULL] + assert_equal [dict get $reply max-deleted-entry-id] "0-0" + assert_equal [dict get $reply entries-added] 2 + set group [lindex [dict get $reply groups] 0] + assert_equal [dict get $group entries-read] 1 + assert_equal [dict get $group lag] 1 + set group [lindex [dict get $reply groups] 1] + assert_equal [dict get $group entries-read] 0 + assert_equal [dict get $group lag] 2 + } + start_server {tags {"external:skip"}} { set master [srv -1 client] set master_host [srv -1 host] @@ -733,6 +948,46 @@ start_server { } } + start_server {tags {"external:skip"}} { + set master [srv -1 client] + set master_host [srv -1 host] + set master_port [srv -1 port] + set replica [srv 0 client] + + foreach autoclaim {0 1} { + test "Replication tests of XCLAIM with deleted entries (autclaim=$autoclaim)" { + $replica replicaof $master_host $master_port + wait_for_condition 50 100 { + [s 0 master_link_status] eq {up} + } else { + fail "Replication not started." + } + + $master DEL x + $master XADD x 1-0 f v + $master XADD x 2-0 f v + $master XADD x 3-0 f v + $master XADD x 4-0 f v + $master XADD x 5-0 f v + $master XGROUP CREATE x grp 0 + assert_equal [$master XREADGROUP GROUP grp Alice STREAMS x >] {{x {{1-0 {f v}} {2-0 {f v}} {3-0 {f v}} {4-0 {f v}} {5-0 {f v}}}}} + wait_for_ofs_sync $master $replica + assert_equal [llength [$replica XPENDING x grp - + 10 Alice]] 5 + $master XDEL x 2-0 + $master XDEL x 4-0 + if {$autoclaim} { + assert_equal [$master XAUTOCLAIM x grp Bob 0 0-0] {0-0 {{1-0 {f v}} {3-0 {f v}} {5-0 {f v}}} {2-0 4-0}} + wait_for_ofs_sync $master $replica + assert_equal [llength [$replica XPENDING x grp - + 10 Alice]] 0 + } else { + assert_equal [$master XCLAIM x grp Bob 0 1-0 2-0 3-0 4-0] {{1-0 {f v}} {3-0 {f v}}} + wait_for_ofs_sync $master $replica + assert_equal [llength [$replica XPENDING x grp - + 10 Alice]] 1 + } + } + } + } + start_server {tags {"stream needs:debug"} overrides {appendonly yes aof-use-rdb-preamble no}} { test {Empty stream with no lastid can be rewrite into AOF correctly} { r XGROUP CREATE mystream group-name $ MKSTREAM @@ -742,7 +997,7 @@ start_server { waitForBgrewriteaof r r debug loadaof assert {[dict get [r xinfo stream mystream] length] == 0} - assert {[r xinfo groups mystream] == $grpinfo} + assert_equal [r xinfo groups mystream] $grpinfo } } } diff --git a/tests/unit/type/stream.tcl b/tests/unit/type/stream.tcl index 7ba3ed116..bd689cd29 100644 --- a/tests/unit/type/stream.tcl +++ b/tests/unit/type/stream.tcl @@ -760,7 +760,9 @@ start_server {tags {"stream xsetid"}} { test {XSETID can set a specific ID} { r XSETID mystream "200-0" - assert {[dict get [r xinfo stream mystream] last-generated-id] == "200-0"} + set reply [r XINFO stream mystream] + assert_equal [dict get $reply last-generated-id] "200-0" + assert_equal [dict get $reply entries-added] 1 } test {XSETID cannot SETID with smaller ID} { @@ -774,6 +776,98 @@ start_server {tags {"stream xsetid"}} { catch {r XSETID stream 1-1} err set _ $err } {ERR no such key} + + test {XSETID cannot run with an offset but without a maximal tombstone} { + catch {r XSETID stream 1-1 0} err + set _ $err + } {ERR syntax error} + + test {XSETID cannot run with a maximal tombstone but without an offset} { + catch {r XSETID stream 1-1 0-0} err + set _ $err + } {ERR syntax error} + + test {XSETID errors on negstive offset} { + catch {r XSETID stream 1-1 ENTRIESADDED -1 MAXDELETEDID 0-0} err + set _ $err + } {ERR*must be positive} + + test {XSETID cannot set the maximal tombstone with larger ID} { + r DEL x + r XADD x 1-0 a b + + catch {r XSETID x "1-0" ENTRIESADDED 1 MAXDELETEDID "2-0" } err + r XADD mystream MAXLEN 0 * a b + set err + } {ERR*smaller*} + + test {XSETID cannot set the offset to less than the length} { + r DEL x + r XADD x 1-0 a b + + catch {r XSETID x "1-0" ENTRIESADDED 0 MAXDELETEDID "0-0" } err + r XADD mystream MAXLEN 0 * a b + set err + } {ERR*smaller*} +} + +start_server {tags {"stream offset"}} { + test {XADD advances the entries-added counter and sets the recorded-first-entry-id} { + r DEL x + r XADD x 1-0 data a + + set reply [r XINFO STREAM x FULL] + assert_equal [dict get $reply entries-added] 1 + assert_equal [dict get $reply recorded-first-entry-id] "1-0" + + r XADD x 2-0 data a + set reply [r XINFO STREAM x FULL] + assert_equal [dict get $reply entries-added] 2 + assert_equal [dict get $reply recorded-first-entry-id] "1-0" + } + + test {XDEL/TRIM are reflected by recorded first entry} { + r DEL x + r XADD x 1-0 data a + r XADD x 2-0 data a + r XADD x 3-0 data a + r XADD x 4-0 data a + r XADD x 5-0 data a + + set reply [r XINFO STREAM x FULL] + assert_equal [dict get $reply entries-added] 5 + assert_equal [dict get $reply recorded-first-entry-id] "1-0" + + r XDEL x 2-0 + set reply [r XINFO STREAM x FULL] + assert_equal [dict get $reply recorded-first-entry-id] "1-0" + + r XDEL x 1-0 + set reply [r XINFO STREAM x FULL] + assert_equal [dict get $reply recorded-first-entry-id] "3-0" + + r XTRIM x MAXLEN = 2 + set reply [r XINFO STREAM x FULL] + assert_equal [dict get $reply recorded-first-entry-id] "4-0" + } + + test {Maxmimum XDEL ID behaves correctly} { + r DEL x + r XADD x 1-0 data a + r XADD x 2-0 data b + r XADD x 3-0 data c + + set reply [r XINFO STREAM x FULL] + assert_equal [dict get $reply max-deleted-entry-id] "0-0" + + r XDEL x 2-0 + set reply [r XINFO STREAM x FULL] + assert_equal [dict get $reply max-deleted-entry-id] "2-0" + + r XDEL x 1-0 + set reply [r XINFO STREAM x FULL] + assert_equal [dict get $reply max-deleted-entry-id] "2-0" + } } start_server {tags {"stream needs:debug"} overrides {appendonly yes aof-use-rdb-preamble no}} { @@ -796,7 +890,7 @@ start_server {tags {"stream needs:debug"} overrides {appendonly yes aof-use-rdb- waitForBgrewriteaof r r debug loadaof assert {[dict get [r xinfo stream mystream] length] == 1} - assert {[dict get [r xinfo stream mystream] last-generated-id] == "2-2"} + assert_equal [dict get [r xinfo stream mystream] last-generated-id] "2-2" } } diff --git a/utils/create-cluster/create-cluster b/utils/create-cluster/create-cluster index 729666830..d97ee2b9c 100755 --- a/utils/create-cluster/create-cluster +++ b/utils/create-cluster/create-cluster @@ -28,7 +28,7 @@ then while [ $((PORT < ENDPORT)) != "0" ]; do PORT=$((PORT+1)) echo "Starting $PORT" - $BIN_PATH/redis-server --port $PORT --protected-mode $PROTECTED_MODE --cluster-enabled yes --cluster-config-file nodes-${PORT}.conf --cluster-node-timeout $TIMEOUT --appendonly yes --appendfilename appendonly-${PORT}.aof --dbfilename dump-${PORT}.rdb --logfile ${PORT}.log --daemonize yes ${ADDITIONAL_OPTIONS} + $BIN_PATH/redis-server --port $PORT --protected-mode $PROTECTED_MODE --cluster-enabled yes --cluster-config-file nodes-${PORT}.conf --cluster-node-timeout $TIMEOUT --appendonly yes --appendfilename appendonly-${PORT}.aof --appenddirname appendonlydir-${PORT} --dbfilename dump-${PORT}.rdb --logfile ${PORT}.log --daemonize yes ${ADDITIONAL_OPTIONS} done exit 0 fi @@ -95,20 +95,25 @@ fi if [ "$1" == "clean" ] then + echo "Cleaning *.log" rm -rf *.log - rm -rf appendonly*.aof - rm -rf dump*.rdb - rm -rf nodes*.conf + echo "Cleaning appendonlydir-*" + rm -rf appendonlydir-* + echo "Cleaning dump-*.rdb" + rm -rf dump-*.rdb + echo "Cleaning nodes-*.conf" + rm -rf nodes-*.conf exit 0 fi if [ "$1" == "clean-logs" ] then + echo "Cleaning *.log" rm -rf *.log exit 0 fi -echo "Usage: $0 [start|create|stop|watch|tail|clean|call]" +echo "Usage: $0 [start|create|stop|watch|tail|tailall|clean|clean-logs|call]" echo "start -- Launch Redis Cluster instances." echo "create [-f] -- Create a cluster using redis-cli --cluster create." echo "stop -- Stop Redis Cluster instances." diff --git a/src/modules/gendoc.rb b/utils/generate-module-api-doc.rb index f83b1ad9d..c6872e903 100644..100755 --- a/src/modules/gendoc.rb +++ b/utils/generate-module-api-doc.rb @@ -1,3 +1,5 @@ +#!/usr/bin/env ruby + # coding: utf-8 # gendoc.rb -- Converts the top-comments inside module.c to modules API # reference documentation in markdown format. @@ -78,6 +80,7 @@ def docufy(src,i) puts "<span id=\"#{name}\"></span>\n\n" puts "### `#{name}`\n\n" puts " #{proto}\n" + puts "**Available since:** #{$since[name]}\n\n" if $since[name] comment = "" while true i = i-1 @@ -135,8 +138,9 @@ def is_func_line(src, i) end puts "# Modules API reference\n\n" -puts "<!-- This file is generated from module.c using gendoc.rb -->\n\n" -src = File.open(File.dirname(__FILE__) ++ "/../module.c").to_a +puts "<!-- This file is generated from module.c using\n" +puts " utils/generate-module-api-doc.rb -->\n\n" +src = File.open(File.dirname(__FILE__) ++ "/../src/module.c").to_a # Build function index $index = {} @@ -148,6 +152,24 @@ src.each_with_index do |line,i| end end +# Populate the 'since' map (name => version) if we're in a git repo. +$since = {} +git_dir = File.dirname(__FILE__) ++ "/../.git" +if File.directory?(git_dir) && `which git` != "" + `git --git-dir="#{git_dir}" tag --sort=v:refname`.each_line do |version| + next if version !~ /^(\d+)\.\d+\.\d+?$/ || $1.to_i < 4 + version.chomp! + `git --git-dir="#{git_dir}" cat-file blob "#{version}:src/module.c"`.each_line do |line| + if line =~ /^\w.*[ \*]RM_([A-z0-9]+)/ + name = "RedisModule_#{$1}" + if ! $since[name] + $since[name] = version + end + end + end + end +end + # Print TOC puts "## Sections\n\n" src.each_with_index do |_line,i| |