Sanitize dump payload: fuzz tester and fixes for segfaults and leaks it exposed

The test creates keys with various encodings, DUMP them, corrupt the payload
and RESTORES it.
It utilizes the recently added use-exit-on-panic config to distinguish between
 asserts and segfaults.
If the restore succeeds, it runs random commands on the key to attempt to
trigger a crash.

It runs in two modes, one with deep sanitation enabled and one without.
In the first one we don't expect any assertions or segfaults, in the second one
we expect assertions, but no segfaults.
We also check for leaks and invalid reads using valgrind, and if we find them
we print the commands that lead to that issue.

Changes in the code (other than the test):
- Replace a few NPD (null pointer deference) flows and division by zero with an
  assertion, so that it doesn't fail the test. (since we set the server to use
  `exit` rather than `abort` on assertion).
- Fix quite a lot of flows in rdb.c that could have lead to memory leaks in
  RESTORE command (since it now responds with an error rather than panic)
- Add a DEBUG flag for SET-SKIP-CHECKSUM-VALIDATION so that the test don't need
  to bother with faking a valid checksum
- Remove a pile of code in serverLogObjectDebugInfo which is actually unsafe to
  run in the crash report (see comments in the code)
- fix a missing boundary check in lzf_decompress

test suite infra improvements:
- be able to run valgrind checks before the process terminates
- rotate log files when restarting servers

2 files changed, 138 insertions, 10 deletions

diff --git a/tests/support/server.tcl b/tests/support/server.tcl
index ae93ad007..e5b167a35 100644
--- a/tests/support/server.tcl
+++ b/tests/support/server.tcl
@@ -13,7 +13,7 @@ proc start_server_error {config_file error} {
 }
 
 proc check_valgrind_errors stderr {
-    set res [find_valgrind_errors $stderr]
+    set res [find_valgrind_errors $stderr true]
     if {$res != ""} {
         send_data_packet $::test_server_fd err "Valgrind error: $res\n"
     }
@@ -437,7 +437,7 @@ proc start_server {options {code undefined}} {
 
         while 1 {
             # check that the server actually started and is ready for connections
-            if {[exec grep -i "Ready to accept" | wc -l < $stdout] > 0} {
+            if {[count_message_lines $stdout "Ready to accept"] > 0} {
                 break
             }
             after 10
@@ -511,13 +511,19 @@ proc start_server {options {code undefined}} {
     }
 }
 
-proc restart_server {level wait_ready} {
+proc restart_server {level wait_ready rotate_logs} {
     set srv [lindex $::servers end+$level]
     kill_server $srv
 
+    set pid [dict get $srv "pid"]
     set stdout [dict get $srv "stdout"]
     set stderr [dict get $srv "stderr"]
-    set config_file [dict get $srv "config_file"]
+    if {$rotate_logs} {
+        set ts [clock format [clock seconds] -format %y%m%d%H%M%S]
+        file rename $stdout $stdout.$ts.$pid
+        file rename $stderr $stderr.$ts.$pid
+    }
+    set prev_ready_count [count_message_lines $stdout "Ready to accept"]
 
     # if we're inside a test, write the test name to the server log file
     if {[info exists ::cur_test]} {
@@ -526,7 +532,7 @@ proc restart_server {level wait_ready} {
         close $fd
     }
 
-    set prev_ready_count [exec grep -i "Ready to accept" | wc -l < $stdout]
+    set config_file [dict get $srv "config_file"]
 
     set pid [spawn_server $config_file $stdout $stderr]
 
@@ -541,7 +547,7 @@ proc restart_server {level wait_ready} {
     if {$wait_ready} {
         while 1 {
             # check that the server actually started and is ready for connections
-            if {[exec grep -i "Ready to accept" | wc -l < $stdout] > $prev_ready_count + 1} {
+            if {[count_message_lines $stdout "Ready to accept"] > $prev_ready_count} {
                 break
             }
             after 10
diff --git a/tests/support/util.tcl b/tests/support/util.tcl
index aadec7cd1..040a7f7d9 100644
--- a/tests/support/util.tcl
+++ b/tests/support/util.tcl
@@ -454,22 +454,31 @@ proc colorstr {color str} {
     }
 }
 
-proc find_valgrind_errors {stderr} {
+proc find_valgrind_errors {stderr on_termination} {
     set fd [open $stderr]
     set buf [read $fd]
     close $fd
 
     # Look for stack trace (" at 0x") and other errors (Invalid, Mismatched, etc).
     # Look for "Warnings", but not the "set address range perms". These don't indicate any real concern.
-    # Look for the absense of a leak free summary (happens when redis isn't terminated properly).
+    # corrupt-dump unit, not sure why but it seems they don't indicate any real concern.
     if {[regexp -- { at 0x} $buf] ||
         [regexp -- {^(?=.*Warning)(?:(?!set address range perms).)*$} $buf] ||
         [regexp -- {Invalid} $buf] ||
         [regexp -- {Mismatched} $buf] ||
         [regexp -- {uninitialized} $buf] ||
         [regexp -- {has a fishy} $buf] ||
-        [regexp -- {overlap} $buf] ||
-        (![regexp -- {definitely lost: 0 bytes} $buf] &&
+        [regexp -- {overlap} $buf]} {
+        return $buf
+    }
+
+    # If the process didn't terminate yet, we can't look for the summary report
+    if {!$on_termination} {
+        return ""
+    }
+
+    # Look for the absense of a leak free summary (happens when redis isn't terminated properly).
+    if {(![regexp -- {definitely lost: 0 bytes} $buf] &&
          ![regexp -- {no leaks are possible} $buf])} {
         return $buf
     }
@@ -547,3 +556,116 @@ proc cmdrstat {cmd r} {
         set _ $value
     }
 }
+
+proc generate_fuzzy_traffic_on_key {key duration} {
+    # Commands per type, blocking commands removed
+    # TODO: extract these from help.h or elsewhere, and improve to include other types
+    set string_commands {APPEND BITCOUNT BITFIELD BITOP BITPOS DECR DECRBY GET GETBIT GETRANGE GETSET INCR INCRBY INCRBYFLOAT MGET MSET MSETNX PSETEX SET SETBIT SETEX SETNX SETRANGE STRALGO STRLEN}
+    set hash_commands {HDEL HEXISTS HGET HGETALL HINCRBY HINCRBYFLOAT HKEYS HLEN HMGET HMSET HSCAN HSET HSETNX HSTRLEN HVALS}
+    set zset_commands {ZADD ZCARD ZCOUNT ZINCRBY ZINTERSTORE ZLEXCOUNT ZPOPMAX ZPOPMIN ZRANGE ZRANGEBYLEX ZRANGEBYSCORE ZRANK ZREM ZREMRANGEBYLEX ZREMRANGEBYRANK ZREMRANGEBYSCORE ZREVRANGE ZREVRANGEBYLEX ZREVRANGEBYSCORE ZREVRANK ZSCAN ZSCORE ZUNIONSTORE}
+    set list_commands {LINDEX LINSERT LLEN LPOP LPOS LPUSH LPUSHX LRANGE LREM LSET LTRIM RPOP RPOPLPUSH RPUSH RPUSHX}
+    set set_commands {SADD SCARD SDIFF SDIFFSTORE SINTER SINTERSTORE SISMEMBER SMEMBERS SMOVE SPOP SRANDMEMBER SREM SSCAN SUNION SUNIONSTORE}
+    set stream_commands {XACK XADD XCLAIM XDEL XGROUP XINFO XLEN XPENDING XRANGE XREAD XREADGROUP XREVRANGE XTRIM}
+    set commands [dict create string $string_commands hash $hash_commands zset $zset_commands list $list_commands set $set_commands stream $stream_commands]
+
+    set type [r type $key]
+    set cmds [dict get $commands $type]
+    set start_time [clock seconds]
+    set sent {}
+    set succeeded 0
+    while {([clock seconds]-$start_time) < $duration} {
+        # find a random command for our key type
+        set cmd_idx [expr {int(rand()*[llength $cmds])}]
+        set cmd [lindex $cmds $cmd_idx]
+        # get the command details from redis
+        if { [ catch {
+            set cmd_info [lindex [r command info $cmd] 0]
+        } err ] } {
+            # if we failed, it means redis crashed after the previous command
+            return $sent
+        }
+        # try to build a valid command argument
+        set arity [lindex $cmd_info 1]
+        set arity [expr $arity < 0 ? - $arity: $arity]
+        set firstkey [lindex $cmd_info 3]
+        set i 1
+        if {$cmd == "XINFO"} {
+            lappend cmd "STREAM"
+            lappend cmd $key
+            lappend cmd "FULL"
+            incr i 3
+        }
+        if {$cmd == "XREAD"} {
+            lappend cmd "STREAMS"
+            lappend cmd $key
+            randpath {
+                lappend cmd \$
+            } {
+                lappend cmd [randomValue]
+            }
+            incr i 3
+        }
+        if {$cmd == "XADD"} {
+            lappend cmd $key
+            randpath {
+                lappend cmd "*"
+            } {
+                lappend cmd [randomValue]
+            }
+            lappend cmd [randomValue]
+            lappend cmd [randomValue]
+            incr i 4
+        }
+        for {} {$i < $arity} {incr i} {
+            if {$i == $firstkey} {
+                lappend cmd $key
+            } else {
+                lappend cmd [randomValue]
+            }
+        }
+        # execute the command, we expect commands to fail on syntax errors
+        lappend sent $cmd
+        if { ! [ catch {
+            r {*}$cmd
+        } err ] } {
+            incr succeeded
+        }
+    }
+
+    # print stats so that we know if we managed to generate commands that actually made senes
+    #if {$::verbose} {
+    #    set count [llength $sent]
+    #    puts "Fuzzy traffic sent: $count, succeeded: $succeeded"
+    #}
+
+    # return the list of commands we sent
+    return $sent
+}
+
+# write line to server log file
+proc write_log_line {srv_idx msg} {
+    set logfile [srv $srv_idx stdout]
+    set fd [open $logfile "a+"]
+    puts $fd "### $msg"
+    close $fd
+}
+
+proc string2printable s {
+    set res {}
+    set has_special_chars false
+    foreach i [split $s {}] {
+        scan $i %c int
+        # non printable characters, including space and excluding: " \ $ { }
+        if {$int < 32 || $int > 122 || $int == 34 || $int == 36 || $int == 92} {
+            set has_special_chars true
+        }
+        # TCL8.5 has issues mixing \x notation and normal chars in the same
+        # source code string, so we'll convert the entire string.
+        append res \\x[format %02X $int]
+    }
+    if {!$has_special_chars} {
+        return $s
+    }
+    set res "\"$res\""
+    return $res
+}