From a0050e5030ef525cd4640c6be8ed5df1cf55e1ea Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 24 Apr 2014 11:08:22 +0200 Subject: Sentinel test files / directories layout improved. The test now runs in a self-contained directory. The general abstractions to run the tests in an environment where mutliple instances are executed at the same time was extrapolated into instances.tcl, that will be reused to test Redis Cluster. --- runtest-sentinel | 2 +- tests/instances.tcl | 395 ++++++++++++++++++++++++++ tests/sentinel-tests/00-base.tcl | 126 --------- tests/sentinel-tests/01-conf-update.tcl | 39 --- tests/sentinel-tests/02-slaves-reconf.tcl | 83 ------ tests/sentinel-tests/03-runtime-reconf.tcl | 1 - tests/sentinel-tests/04-slave-selection.tcl | 5 - tests/sentinel-tests/includes/init-tests.tcl | 67 ----- tests/sentinel-tmp/.gitignore | 2 - tests/sentinel.tcl | 402 --------------------------- tests/sentinel/run.tcl | 19 ++ tests/sentinel/tests/00-base.tcl | 126 +++++++++ tests/sentinel/tests/01-conf-update.tcl | 39 +++ tests/sentinel/tests/02-slaves-reconf.tcl | 83 ++++++ tests/sentinel/tests/03-runtime-reconf.tcl | 1 + tests/sentinel/tests/04-slave-selection.tcl | 5 + tests/sentinel/tests/includes/init-tests.tcl | 67 +++++ tests/sentinel/tmp/.gitignore | 2 + 18 files changed, 738 insertions(+), 726 deletions(-) create mode 100644 tests/instances.tcl delete mode 100644 tests/sentinel-tests/00-base.tcl delete mode 100644 tests/sentinel-tests/01-conf-update.tcl delete mode 100644 tests/sentinel-tests/02-slaves-reconf.tcl delete mode 100644 tests/sentinel-tests/03-runtime-reconf.tcl delete mode 100644 tests/sentinel-tests/04-slave-selection.tcl delete mode 100644 tests/sentinel-tests/includes/init-tests.tcl delete mode 100644 tests/sentinel-tmp/.gitignore delete mode 100644 tests/sentinel.tcl create mode 100644 tests/sentinel/run.tcl create mode 100644 tests/sentinel/tests/00-base.tcl create mode 100644 tests/sentinel/tests/01-conf-update.tcl create mode 100644 tests/sentinel/tests/02-slaves-reconf.tcl create mode 100644 tests/sentinel/tests/03-runtime-reconf.tcl create mode 100644 tests/sentinel/tests/04-slave-selection.tcl create mode 100644 tests/sentinel/tests/includes/init-tests.tcl create mode 100644 tests/sentinel/tmp/.gitignore diff --git a/runtest-sentinel b/runtest-sentinel index 1650eea73..3fb1ef615 100755 --- a/runtest-sentinel +++ b/runtest-sentinel @@ -11,4 +11,4 @@ then echo "You need tcl 8.5 or newer in order to run the Redis Sentinel test" exit 1 fi -$TCLSH tests/sentinel.tcl $* +$TCLSH tests/sentinel/run.tcl $* diff --git a/tests/instances.tcl b/tests/instances.tcl new file mode 100644 index 000000000..7cc798dcd --- /dev/null +++ b/tests/instances.tcl @@ -0,0 +1,395 @@ +# Multi-instance test framework. +# This is used in order to test Sentinel and Redis Cluster, and provides +# basic capabilities for spawning and handling N parallel Redis / Sentinel +# instances. +# +# Copyright (C) 2014 Salvatore Sanfilippo antirez@gmail.com +# This softare is released under the BSD License. See the COPYING file for +# more information. + +package require Tcl 8.5 + +set tcl_precision 17 +source ../support/redis.tcl +source ../support/util.tcl +source ../support/server.tcl +source ../support/test.tcl + +set ::verbose 0 +set ::pause_on_error 0 +set ::simulate_error 0 +set ::sentinel_instances {} +set ::redis_instances {} +set ::sentinel_base_port 20000 +set ::redis_base_port 30000 +set ::instances_count 5 ; # How many Sentinels / Instances we use at max +set ::pids {} ; # We kill everything at exit +set ::dirs {} ; # We remove all the temp dirs at exit +set ::run_matching {} ; # If non empty, only tests matching pattern are run. + +if {[catch {cd tmp}]} { + puts "tmp directory not found." + puts "Please run this test from the Redis source root." + exit 1 +} + +# Spawn a redis or sentinel instance, depending on 'type'. +proc spawn_instance {type base_port count {conf {}}} { + for {set j 0} {$j < $count} {incr j} { + set port [find_available_port $base_port] + incr base_port + puts "Starting $type #$j at port $port" + + # Create a directory for this instance. + set dirname "${type}_${j}" + lappend ::dirs $dirname + catch {exec rm -rf $dirname} + file mkdir $dirname + + # Write the instance config file. + set cfgfile [file join $dirname $type.conf] + set cfg [open $cfgfile w] + puts $cfg "port $port" + puts $cfg "dir ./$dirname" + puts $cfg "logfile log.txt" + # Add additional config files + foreach directive $conf { + puts $cfg $directive + } + close $cfg + + # Finally exec it and remember the pid for later cleanup. + if {$type eq "redis"} { + set prgname redis-server + } elseif {$type eq "sentinel"} { + set prgname redis-sentinel + } else { + error "Unknown instance type." + } + set pid [exec ../../../src/${prgname} $cfgfile &] + lappend ::pids $pid + + # Check availability + if {[server_is_up 127.0.0.1 $port 100] == 0} { + abort_sentinel_test "Problems starting $type #$j: ping timeout" + } + + # Push the instance into the right list + lappend ::${type}_instances [list \ + pid $pid \ + host 127.0.0.1 \ + port $port \ + link [redis 127.0.0.1 $port] \ + ] + } +} + +proc cleanup {} { + puts "Cleaning up..." + foreach pid $::pids { + catch {exec kill -9 $pid} + } + foreach dir $::dirs { + catch {exec rm -rf $dir} + } +} + +proc abort_sentinel_test msg { + puts "WARNING: Aborting the test." + puts ">>>>>>>> $msg" + cleanup + exit 1 +} + +proc parse_options {} { + for {set j 0} {$j < [llength $::argv]} {incr j} { + set opt [lindex $::argv $j] + set val [lindex $::argv [expr $j+1]] + if {$opt eq "--single"} { + incr j + set ::run_matching "*${val}*" + } elseif {$opt eq "--pause-on-error"} { + set ::pause_on_error 1 + } elseif {$opt eq "--fail"} { + set ::simulate_error 1 + } elseif {$opt eq "--help"} { + puts "Hello, I'm sentinel.tcl and I run Sentinel unit tests." + puts "\nOptions:" + puts "--single Only runs tests specified by pattern." + puts "--pause-on-error Pause for manual inspection on error." + puts "--fail Simulate a test failure." + puts "--help Shows this help." + exit 0 + } else { + puts "Unknown option $opt" + exit 1 + } + } +} + +# If --pause-on-error option was passed at startup this function is called +# on error in order to give the developer a chance to understand more about +# the error condition while the instances are still running. +proc pause_on_error {} { + puts "" + puts [colorstr yellow "*** Please inspect the error now ***"] + puts "\nType \"continue\" to resume the test, \"help\" for help screen.\n" + while 1 { + puts -nonewline "> " + flush stdout + set line [gets stdin] + set argv [split $line " "] + set cmd [lindex $argv 0] + if {$cmd eq {continue}} { + break + } elseif {$cmd eq {show-sentinel-logs}} { + set count 10 + if {[lindex $argv 1] ne {}} {set count [lindex $argv 1]} + foreach_sentinel_id id { + puts "=== SENTINEL $id ====" + puts [exec tail -$count sentinel_$id/log.txt] + puts "---------------------\n" + } + } elseif {$cmd eq {ls}} { + foreach_redis_id id { + puts -nonewline "Redis $id" + set errcode [catch { + set str {} + append str "@[RI $id tcp_port]: " + append str "[RI $id role] " + if {[RI $id role] eq {slave}} { + append str "[RI $id master_host]:[RI $id master_port]" + } + set str + } retval] + if {$errcode} { + puts " -- $retval" + } else { + puts $retval + } + } + foreach_sentinel_id id { + puts -nonewline "Sentinel $id" + set errcode [catch { + set str {} + append str "@[SI $id tcp_port]: " + append str "[join [S $id sentinel get-master-addr-by-name mymaster]]" + set str + } retval] + if {$errcode} { + puts " -- $retval" + } else { + puts $retval + } + } + } elseif {$cmd eq {help}} { + puts "ls List Sentinel and Redis instances." + puts "show-sentinel-logs \[N\] Show latest N lines of logs." + puts "S cmd ... arg Call command in Sentinel ." + puts "R cmd ... arg Call command in Redis ." + puts "SI Show Sentinel INFO ." + puts "RI Show Sentinel INFO ." + puts "continue Resume test." + } else { + set errcode [catch {eval $line} retval] + if {$retval ne {}} {puts "$retval"} + } + } +} + +# We redefine 'test' as for Sentinel we don't use the server-client +# architecture for the test, everything is sequential. +proc test {descr code} { + set ts [clock format [clock seconds] -format %H:%M:%S] + puts -nonewline "$ts> $descr: " + flush stdout + + if {[catch {set retval [uplevel 1 $code]} error]} { + if {[string match "assertion:*" $error]} { + set msg [string range $error 10 end] + puts [colorstr red $msg] + if {$::pause_on_error} pause_on_error + puts "(Jumping to next unit after error)" + return -code continue + } else { + # Re-raise, let handler up the stack take care of this. + error $error $::errorInfo + } + } else { + puts [colorstr green OK] + } +} + +proc run_tests {} { + set tests [lsort [glob ../tests/*]] + foreach test $tests { + if {$::run_matching ne {} && [string match $::run_matching $test] == 0} { + continue + } + if {[file isdirectory $test]} continue + puts [colorstr yellow "Testing unit: [lindex [file split $test] end]"] + source $test + } +} + +# The "S" command is used to interact with the N-th Sentinel. +# The general form is: +# +# S command arg arg arg ... +# +# Example to ping the Sentinel 0 (first instance): S 0 PING +proc S {n args} { + set s [lindex $::sentinel_instances $n] + [dict get $s link] {*}$args +} + +# Like R but to chat with Redis instances. +proc R {n args} { + set r [lindex $::redis_instances $n] + [dict get $r link] {*}$args +} + +proc get_info_field {info field} { + set fl [string length $field] + append field : + foreach line [split $info "\n"] { + set line [string trim $line "\r\n "] + if {[string range $line 0 $fl] eq $field} { + return [string range $line [expr {$fl+1}] end] + } + } + return {} +} + +proc SI {n field} { + get_info_field [S $n info] $field +} + +proc RI {n field} { + get_info_field [R $n info] $field +} + +# Iterate over IDs of sentinel or redis instances. +proc foreach_instance_id {instances idvar code} { + upvar 1 $idvar id + for {set id 0} {$id < [llength $instances]} {incr id} { + set errcode [catch {uplevel 1 $code} result] + if {$errcode == 1} { + error $result $::errorInfo $::errorCode + } elseif {$errcode == 4} { + continue + } elseif {$errcode == 3} { + break + } elseif {$errcode != 0} { + return -code $errcode $result + } + } +} + +proc foreach_sentinel_id {idvar code} { + set errcode [catch {uplevel 1 [list foreach_instance_id $::sentinel_instances $idvar $code]} result] + return -code $errcode $result +} + +proc foreach_redis_id {idvar code} { + set errcode [catch {uplevel 1 [list foreach_instance_id $::redis_instances $idvar $code]} result] + return -code $errcode $result +} + +# Get the specific attribute of the specified instance type, id. +proc get_instance_attrib {type id attrib} { + dict get [lindex [set ::${type}_instances] $id] $attrib +} + +# Set the specific attribute of the specified instance type, id. +proc set_instance_attrib {type id attrib newval} { + set d [lindex [set ::${type}_instances] $id] + dict set d $attrib $newval + lset ::${type}_instances $id $d +} + +# Create a master-slave cluster of the given number of total instances. +# The first instance "0" is the master, all others are configured as +# slaves. +proc create_redis_master_slave_cluster n { + foreach_redis_id id { + if {$id == 0} { + # Our master. + R $id slaveof no one + R $id flushall + } elseif {$id < $n} { + R $id slaveof [get_instance_attrib redis 0 host] \ + [get_instance_attrib redis 0 port] + } else { + # Instances not part of the cluster. + R $id slaveof no one + } + } + # Wait for all the slaves to sync. + wait_for_condition 1000 50 { + [RI 0 connected_slaves] == ($n-1) + } else { + fail "Unable to create a master-slaves cluster." + } +} + +proc get_instance_id_by_port {type port} { + foreach_${type}_id id { + if {[get_instance_attrib $type $id port] == $port} { + return $id + } + } + fail "Instance $type port $port not found." +} + +# Kill an instance of the specified type/id with SIGKILL. +# This function will mark the instance PID as -1 to remember that this instance +# is no longer running and will remove its PID from the list of pids that +# we kill at cleanup. +# +# The instance can be restarted with restart-instance. +proc kill_instance {type id} { + set pid [get_instance_attrib $type $id pid] + if {$pid == -1} { + error "You tried to kill $type $id twice." + } + exec kill -9 $pid + set_instance_attrib $type $id pid -1 + set_instance_attrib $type $id link you_tried_to_talk_with_killed_instance + + # Remove the PID from the list of pids to kill at exit. + set ::pids [lsearch -all -inline -not -exact $::pids $pid] +} + +# Return true of the instance of the specified type/id is killed. +proc instance_is_killed {type id} { + set pid [get_instance_attrib $type $id pid] + return $pid == -1 +} + +# Restart an instance previously killed by kill_instance +proc restart_instance {type id} { + set dirname "${type}_${id}" + set cfgfile [file join $dirname $type.conf] + set port [get_instance_attrib $type $id port] + + # Execute the instance with its old setup and append the new pid + # file for cleanup. + if {$type eq "redis"} { + set prgname redis-server + } else { + set prgname redis-sentinel + } + set pid [exec ../../../src/${prgname} $cfgfile &] + set_instance_attrib $type $id pid $pid + lappend ::pids $pid + + # Check that the instance is running + if {[server_is_up 127.0.0.1 $port 100] == 0} { + abort_sentinel_test "Problems starting $type #$j: ping timeout" + } + + # Connect with it with a fresh link + set_instance_attrib $type $id link [redis 127.0.0.1 $port] +} + diff --git a/tests/sentinel-tests/00-base.tcl b/tests/sentinel-tests/00-base.tcl deleted file mode 100644 index 26758de09..000000000 --- a/tests/sentinel-tests/00-base.tcl +++ /dev/null @@ -1,126 +0,0 @@ -# Check the basic monitoring and failover capabilities. - -source "../sentinel-tests/includes/init-tests.tcl" - -if {$::simulate_error} { - test "This test will fail" { - fail "Simulated error" - } -} - -test "Basic failover works if the master is down" { - set old_port [RI $master_id tcp_port] - set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] - assert {[lindex $addr 1] == $old_port} - kill_instance redis $master_id - foreach_sentinel_id id { - wait_for_condition 1000 50 { - [lindex [S $id SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 1] != $old_port - } else { - fail "At least one Sentinel did not received failover info" - } - } - restart_instance redis $master_id - set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] - set master_id [get_instance_id_by_port redis [lindex $addr 1]] -} - -test "New master [join $addr {:}] role matches" { - assert {[RI $master_id role] eq {master}} -} - -test "All the other slaves now point to the new master" { - foreach_redis_id id { - if {$id != $master_id && $id != 0} { - wait_for_condition 1000 50 { - [RI $id master_port] == [lindex $addr 1] - } else { - fail "Redis ID $id not configured to replicate with new master" - } - } - } -} - -test "The old master eventually gets reconfigured as a slave" { - wait_for_condition 1000 50 { - [RI 0 master_port] == [lindex $addr 1] - } else { - fail "Old master not reconfigured as slave of new master" - } -} - -test "ODOWN is not possible without N (quorum) Sentinels reports" { - foreach_sentinel_id id { - S $id SENTINEL SET mymaster quorum [expr $sentinels+1] - } - set old_port [RI $master_id tcp_port] - set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] - assert {[lindex $addr 1] == $old_port} - kill_instance redis $master_id - - # Make sure failover did not happened. - set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] - assert {[lindex $addr 1] == $old_port} - restart_instance redis $master_id -} - -test "Failover is not possible without majority agreement" { - foreach_sentinel_id id { - S $id SENTINEL SET mymaster quorum $quorum - } - - # Crash majority of sentinels - for {set id 0} {$id < $quorum} {incr id} { - kill_instance sentinel $id - } - - # Kill the current master - kill_instance redis $master_id - - # Make sure failover did not happened. - set addr [S $quorum SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] - assert {[lindex $addr 1] == $old_port} - restart_instance redis $master_id - - # Cleanup: restart Sentinels to monitor the master. - for {set id 0} {$id < $quorum} {incr id} { - restart_instance sentinel $id - } -} - -test "Failover works if we configure for absolute agreement" { - foreach_sentinel_id id { - S $id SENTINEL SET mymaster quorum $sentinels - } - - # Wait for Sentinels to monitor the master again - foreach_sentinel_id id { - wait_for_condition 1000 50 { - [dict get [S $id SENTINEL MASTER mymaster] info-refresh] < 100000 - } else { - fail "At least one Sentinel is not monitoring the master" - } - } - - kill_instance redis $master_id - - foreach_sentinel_id id { - wait_for_condition 1000 50 { - [lindex [S $id SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 1] != $old_port - } else { - fail "At least one Sentinel did not received failover info" - } - } - restart_instance redis $master_id - set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] - set master_id [get_instance_id_by_port redis [lindex $addr 1]] - - # Set the min ODOWN agreement back to strict majority. - foreach_sentinel_id id { - S $id SENTINEL SET mymaster quorum $quorum - } -} - -test "New master [join $addr {:}] role matches" { - assert {[RI $master_id role] eq {master}} -} diff --git a/tests/sentinel-tests/01-conf-update.tcl b/tests/sentinel-tests/01-conf-update.tcl deleted file mode 100644 index 4625ebd4c..000000000 --- a/tests/sentinel-tests/01-conf-update.tcl +++ /dev/null @@ -1,39 +0,0 @@ -# Test Sentinel configuration consistency after partitions heal. - -source "../sentinel-tests/includes/init-tests.tcl" - -test "We can failover with Sentinel 1 crashed" { - set old_port [RI $master_id tcp_port] - set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] - assert {[lindex $addr 1] == $old_port} - - # Crash Sentinel 1 - kill_instance sentinel 1 - - kill_instance redis $master_id - foreach_sentinel_id id { - if {$id != 1} { - wait_for_condition 1000 50 { - [lindex [S $id SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 1] != $old_port - } else { - fail "Sentinel $id did not received failover info" - } - } - } - restart_instance redis $master_id - set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] - set master_id [get_instance_id_by_port redis [lindex $addr 1]] -} - -test "After Sentinel 1 is restarted, its config gets updated" { - restart_instance sentinel 1 - wait_for_condition 1000 50 { - [lindex [S 1 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 1] != $old_port - } else { - fail "Restarted Sentinel did not received failover info" - } -} - -test "New master [join $addr {:}] role matches" { - assert {[RI $master_id role] eq {master}} -} diff --git a/tests/sentinel-tests/02-slaves-reconf.tcl b/tests/sentinel-tests/02-slaves-reconf.tcl deleted file mode 100644 index 843c62dcc..000000000 --- a/tests/sentinel-tests/02-slaves-reconf.tcl +++ /dev/null @@ -1,83 +0,0 @@ -# Check that slaves are reconfigured at a latter time if they are partitioned. -# -# Here we should test: -# 1) That slaves point to the new master after failover. -# 2) That partitioned slaves point to new master when they are partitioned -# away during failover and return at a latter time. - -source "../sentinel-tests/includes/init-tests.tcl" - -proc 03_test_slaves_replication {} { - uplevel 1 { - test "Check that slaves replicate from current master" { - set master_port [RI $master_id tcp_port] - foreach_redis_id id { - if {$id == $master_id} continue - if {[instance_is_killed redis $id]} continue - wait_for_condition 1000 50 { - [RI $id master_port] == $master_port - } else { - fail "Redis slave $id is replicating from wrong master" - } - } - } - } -} - -proc 03_crash_and_failover {} { - uplevel 1 { - test "Crash the master and force a failover" { - set old_port [RI $master_id tcp_port] - set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] - assert {[lindex $addr 1] == $old_port} - kill_instance redis $master_id - foreach_sentinel_id id { - wait_for_condition 1000 50 { - [lindex [S $id SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 1] != $old_port - } else { - fail "At least one Sentinel did not received failover info" - } - } - restart_instance redis $master_id - set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] - set master_id [get_instance_id_by_port redis [lindex $addr 1]] - } - } -} - -03_test_slaves_replication -03_crash_and_failover -03_test_slaves_replication - -test "Kill a slave instance" { - foreach_redis_id id { - if {$id == $master_id} continue - set killed_slave_id $id - kill_instance redis $id - break - } -} - -03_crash_and_failover -03_test_slaves_replication - -test "Wait for failover to end" { - set inprogress 1 - while {$inprogress} { - set inprogress 0 - foreach_sentinel_id id { - if {[dict exists [S $id SENTINEL MASTER mymaster] failover-state]} { - incr inprogress - } - } - if {$inprogress} {after 100} - } -} - -test "Restart killed slave and test replication of slaves again..." { - restart_instance redis $killed_slave_id -} - -# Now we check if the slave rejoining the partition is reconfigured even -# if the failover finished. -03_test_slaves_replication diff --git a/tests/sentinel-tests/03-runtime-reconf.tcl b/tests/sentinel-tests/03-runtime-reconf.tcl deleted file mode 100644 index 426596c37..000000000 --- a/tests/sentinel-tests/03-runtime-reconf.tcl +++ /dev/null @@ -1 +0,0 @@ -# Test runtime reconfiguration command SENTINEL SET. diff --git a/tests/sentinel-tests/04-slave-selection.tcl b/tests/sentinel-tests/04-slave-selection.tcl deleted file mode 100644 index 3d2ca6484..000000000 --- a/tests/sentinel-tests/04-slave-selection.tcl +++ /dev/null @@ -1,5 +0,0 @@ -# Test slave selection algorithm. -# -# This unit should test: -# 1) That when there are no suitable slaves no failover is performed. -# 2) That among the available slaves, the one with better offset is picked. diff --git a/tests/sentinel-tests/includes/init-tests.tcl b/tests/sentinel-tests/includes/init-tests.tcl deleted file mode 100644 index cb359ea1b..000000000 --- a/tests/sentinel-tests/includes/init-tests.tcl +++ /dev/null @@ -1,67 +0,0 @@ -# Initialization tests -- most units will start including this. - -test "(init) Restart killed instances" { - foreach type {redis sentinel} { - foreach_${type}_id id { - if {[get_instance_attrib $type $id pid] == -1} { - puts -nonewline "$type/$id " - flush stdout - restart_instance $type $id - } - } - } -} - -set redis_slaves 4 -test "(init) Create a master-slaves cluster of [expr $redis_slaves+1] instances" { - create_redis_master_slave_cluster [expr {$redis_slaves+1}] -} -set master_id 0 - -test "(init) Sentinels can start monitoring a master" { - set sentinels [llength $::sentinel_instances] - set quorum [expr {$sentinels/2+1}] - foreach_sentinel_id id { - catch {S $id SENTINEL REMOVE mymaster} - S $id SENTINEL MONITOR mymaster \ - [get_instance_attrib redis $master_id host] \ - [get_instance_attrib redis $master_id port] $quorum - } - foreach_sentinel_id id { - assert {[S $id sentinel master mymaster] ne {}} - S $id SENTINEL SET mymaster down-after-milliseconds 2000 - S $id SENTINEL SET mymaster failover-timeout 20000 - S $id SENTINEL SET mymaster parallel-syncs 10 - } -} - -test "(init) Sentinels can talk with the master" { - foreach_sentinel_id id { - wait_for_condition 100 50 { - [catch {S $id SENTINEL GET-MASTER-ADDR-BY-NAME mymaster}] == 0 - } else { - fail "Sentinel $id can't talk with the master." - } - } -} - -test "(init) Sentinels are able to auto-discover other sentinels" { - set sentinels [llength $::sentinel_instances] - foreach_sentinel_id id { - wait_for_condition 100 50 { - [dict get [S $id SENTINEL MASTER mymaster] num-other-sentinels] == ($sentinels-1) - } else { - fail "At least some sentinel can't detect some other sentinel" - } - } -} - -test "(init) Sentinels are able to auto-discover slaves" { - foreach_sentinel_id id { - wait_for_condition 100 50 { - [dict get [S $id SENTINEL MASTER mymaster] num-slaves] == $redis_slaves - } else { - fail "At least some sentinel can't detect some slave" - } - } -} diff --git a/tests/sentinel-tmp/.gitignore b/tests/sentinel-tmp/.gitignore deleted file mode 100644 index f581f73e2..000000000 --- a/tests/sentinel-tmp/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -redis_* -sentinel_* diff --git a/tests/sentinel.tcl b/tests/sentinel.tcl deleted file mode 100644 index 6b1476d96..000000000 --- a/tests/sentinel.tcl +++ /dev/null @@ -1,402 +0,0 @@ -# Sentinel test suite. Copyright (C) 2014 Salvatore Sanfilippo antirez@gmail.com -# This softare is released under the BSD License. See the COPYING file for -# more information. - -package require Tcl 8.5 - -set tcl_precision 17 -source tests/support/redis.tcl -source tests/support/util.tcl -source tests/support/server.tcl -source tests/support/test.tcl - -set ::verbose 0 -set ::pause_on_error 0 -set ::simulate_error 0 -set ::sentinel_instances {} -set ::redis_instances {} -set ::sentinel_base_port 20000 -set ::redis_base_port 30000 -set ::instances_count 5 ; # How many Sentinels / Instances we use at max -set ::pids {} ; # We kill everything at exit -set ::dirs {} ; # We remove all the temp dirs at exit -set ::run_matching {} ; # If non empty, only tests matching pattern are run. - -if {[catch {cd tests/sentinel-tmp}]} { - puts "tests/sentinel-tmp directory not found." - puts "Please run this test from the Redis source root." - exit 1 -} - -# Spawn a redis or sentinel instance, depending on 'type'. -proc spawn_instance {type base_port count {conf {}}} { - for {set j 0} {$j < $count} {incr j} { - set port [find_available_port $base_port] - incr base_port - puts "Starting $type #$j at port $port" - - # Create a directory for this instance. - set dirname "${type}_${j}" - lappend ::dirs $dirname - catch {exec rm -rf $dirname} - file mkdir $dirname - - # Write the instance config file. - set cfgfile [file join $dirname $type.conf] - set cfg [open $cfgfile w] - puts $cfg "port $port" - puts $cfg "dir ./$dirname" - puts $cfg "logfile log.txt" - # Add additional config files - foreach directive $conf { - puts $cfg $directive - } - close $cfg - - # Finally exec it and remember the pid for later cleanup. - if {$type eq "redis"} { - set prgname redis-server - } elseif {$type eq "sentinel"} { - set prgname redis-sentinel - } else { - error "Unknown instance type." - } - set pid [exec ../../src/${prgname} $cfgfile &] - lappend ::pids $pid - - # Check availability - if {[server_is_up 127.0.0.1 $port 100] == 0} { - abort_sentinel_test "Problems starting $type #$j: ping timeout" - } - - # Push the instance into the right list - lappend ::${type}_instances [list \ - pid $pid \ - host 127.0.0.1 \ - port $port \ - link [redis 127.0.0.1 $port] \ - ] - } -} - -proc cleanup {} { - puts "Cleaning up..." - foreach pid $::pids { - catch {exec kill -9 $pid} - } - foreach dir $::dirs { - catch {exec rm -rf $dir} - } -} - -proc abort_sentinel_test msg { - puts "WARNING: Aborting the test." - puts ">>>>>>>> $msg" - cleanup - exit 1 -} - -proc parse_options {} { - for {set j 0} {$j < [llength $::argv]} {incr j} { - set opt [lindex $::argv $j] - set val [lindex $::argv [expr $j+1]] - if {$opt eq "--single"} { - incr j - set ::run_matching "*${val}*" - } elseif {$opt eq "--pause-on-error"} { - set ::pause_on_error 1 - } elseif {$opt eq "--fail"} { - set ::simulate_error 1 - } elseif {$opt eq "--help"} { - puts "Hello, I'm sentinel.tcl and I run Sentinel unit tests." - puts "\nOptions:" - puts "--single Only runs tests specified by pattern." - puts "--pause-on-error Pause for manual inspection on error." - puts "--fail Simulate a test failure." - puts "--help Shows this help." - exit 0 - } else { - puts "Unknown option $opt" - exit 1 - } - } -} - -proc main {} { - parse_options - spawn_instance sentinel $::sentinel_base_port $::instances_count - spawn_instance redis $::redis_base_port $::instances_count - run_tests - cleanup -} - -# If --pause-on-error option was passed at startup this function is called -# on error in order to give the developer a chance to understand more about -# the error condition while the instances are still running. -proc pause_on_error {} { - puts "" - puts [colorstr yellow "*** Please inspect the error now ***"] - puts "\nType \"continue\" to resume the test, \"help\" for help screen.\n" - while 1 { - puts -nonewline "> " - flush stdout - set line [gets stdin] - set argv [split $line " "] - set cmd [lindex $argv 0] - if {$cmd eq {continue}} { - break - } elseif {$cmd eq {show-sentinel-logs}} { - set count 10 - if {[lindex $argv 1] ne {}} {set count [lindex $argv 1]} - foreach_sentinel_id id { - puts "=== SENTINEL $id ====" - puts [exec tail -$count sentinel_$id/log.txt] - puts "---------------------\n" - } - } elseif {$cmd eq {ls}} { - foreach_redis_id id { - puts -nonewline "Redis $id" - set errcode [catch { - set str {} - append str "@[RI $id tcp_port]: " - append str "[RI $id role] " - if {[RI $id role] eq {slave}} { - append str "[RI $id master_host]:[RI $id master_port]" - } - set str - } retval] - if {$errcode} { - puts " -- $retval" - } else { - puts $retval - } - } - foreach_sentinel_id id { - puts -nonewline "Sentinel $id" - set errcode [catch { - set str {} - append str "@[SI $id tcp_port]: " - append str "[join [S $id sentinel get-master-addr-by-name mymaster]]" - set str - } retval] - if {$errcode} { - puts " -- $retval" - } else { - puts $retval - } - } - } elseif {$cmd eq {help}} { - puts "ls List Sentinel and Redis instances." - puts "show-sentinel-logs \[N\] Show latest N lines of logs." - puts "S cmd ... arg Call command in Sentinel ." - puts "R cmd ... arg Call command in Redis ." - puts "SI Show Sentinel INFO ." - puts "RI Show Sentinel INFO ." - puts "continue Resume test." - } else { - set errcode [catch {eval $line} retval] - if {$retval ne {}} {puts "$retval"} - } - } -} - -# We redefine 'test' as for Sentinel we don't use the server-client -# architecture for the test, everything is sequential. -proc test {descr code} { - set ts [clock format [clock seconds] -format %H:%M:%S] - puts -nonewline "$ts> $descr: " - flush stdout - - if {[catch {set retval [uplevel 1 $code]} error]} { - if {[string match "assertion:*" $error]} { - set msg [string range $error 10 end] - puts [colorstr red $msg] - if {$::pause_on_error} pause_on_error - puts "(Jumping to next unit after error)" - return -code continue - } else { - # Re-raise, let handler up the stack take care of this. - error $error $::errorInfo - } - } else { - puts [colorstr green OK] - } -} - -proc run_tests {} { - set tests [lsort [glob ../sentinel-tests/*]] - foreach test $tests { - if {$::run_matching ne {} && [string match $::run_matching $test] == 0} { - continue - } - if {[file isdirectory $test]} continue - puts [colorstr yellow "Testing unit: [lindex [file split $test] end]"] - source $test - } -} - -# The "S" command is used to interact with the N-th Sentinel. -# The general form is: -# -# S command arg arg arg ... -# -# Example to ping the Sentinel 0 (first instance): S 0 PING -proc S {n args} { - set s [lindex $::sentinel_instances $n] - [dict get $s link] {*}$args -} - -# Like R but to chat with Redis instances. -proc R {n args} { - set r [lindex $::redis_instances $n] - [dict get $r link] {*}$args -} - -proc get_info_field {info field} { - set fl [string length $field] - append field : - foreach line [split $info "\n"] { - set line [string trim $line "\r\n "] - if {[string range $line 0 $fl] eq $field} { - return [string range $line [expr {$fl+1}] end] - } - } - return {} -} - -proc SI {n field} { - get_info_field [S $n info] $field -} - -proc RI {n field} { - get_info_field [R $n info] $field -} - -# Iterate over IDs of sentinel or redis instances. -proc foreach_instance_id {instances idvar code} { - upvar 1 $idvar id - for {set id 0} {$id < [llength $instances]} {incr id} { - set errcode [catch {uplevel 1 $code} result] - if {$errcode == 1} { - error $result $::errorInfo $::errorCode - } elseif {$errcode == 4} { - continue - } elseif {$errcode == 3} { - break - } elseif {$errcode != 0} { - return -code $errcode $result - } - } -} - -proc foreach_sentinel_id {idvar code} { - set errcode [catch {uplevel 1 [list foreach_instance_id $::sentinel_instances $idvar $code]} result] - return -code $errcode $result -} - -proc foreach_redis_id {idvar code} { - set errcode [catch {uplevel 1 [list foreach_instance_id $::redis_instances $idvar $code]} result] - return -code $errcode $result -} - -# Get the specific attribute of the specified instance type, id. -proc get_instance_attrib {type id attrib} { - dict get [lindex [set ::${type}_instances] $id] $attrib -} - -# Set the specific attribute of the specified instance type, id. -proc set_instance_attrib {type id attrib newval} { - set d [lindex [set ::${type}_instances] $id] - dict set d $attrib $newval - lset ::${type}_instances $id $d -} - -# Create a master-slave cluster of the given number of total instances. -# The first instance "0" is the master, all others are configured as -# slaves. -proc create_redis_master_slave_cluster n { - foreach_redis_id id { - if {$id == 0} { - # Our master. - R $id slaveof no one - R $id flushall - } elseif {$id < $n} { - R $id slaveof [get_instance_attrib redis 0 host] \ - [get_instance_attrib redis 0 port] - } else { - # Instances not part of the cluster. - R $id slaveof no one - } - } - # Wait for all the slaves to sync. - wait_for_condition 1000 50 { - [RI 0 connected_slaves] == ($n-1) - } else { - fail "Unable to create a master-slaves cluster." - } -} - -proc get_instance_id_by_port {type port} { - foreach_${type}_id id { - if {[get_instance_attrib $type $id port] == $port} { - return $id - } - } - fail "Instance $type port $port not found." -} - -# Kill an instance of the specified type/id with SIGKILL. -# This function will mark the instance PID as -1 to remember that this instance -# is no longer running and will remove its PID from the list of pids that -# we kill at cleanup. -# -# The instance can be restarted with restart-instance. -proc kill_instance {type id} { - set pid [get_instance_attrib $type $id pid] - if {$pid == -1} { - error "You tried to kill $type $id twice." - } - exec kill -9 $pid - set_instance_attrib $type $id pid -1 - set_instance_attrib $type $id link you_tried_to_talk_with_killed_instance - - # Remove the PID from the list of pids to kill at exit. - set ::pids [lsearch -all -inline -not -exact $::pids $pid] -} - -# Return true of the instance of the specified type/id is killed. -proc instance_is_killed {type id} { - set pid [get_instance_attrib $type $id pid] - return $pid == -1 -} - -# Restart an instance previously killed by kill_instance -proc restart_instance {type id} { - set dirname "${type}_${id}" - set cfgfile [file join $dirname $type.conf] - set port [get_instance_attrib $type $id port] - - # Execute the instance with its old setup and append the new pid - # file for cleanup. - if {$type eq "redis"} { - set prgname redis-server - } else { - set prgname redis-sentinel - } - set pid [exec ../../src/${prgname} $cfgfile &] - set_instance_attrib $type $id pid $pid - lappend ::pids $pid - - # Check that the instance is running - if {[server_is_up 127.0.0.1 $port 100] == 0} { - abort_sentinel_test "Problems starting $type #$j: ping timeout" - } - - # Connect with it with a fresh link - set_instance_attrib $type $id link [redis 127.0.0.1 $port] -} - -if {[catch main e]} { - puts $::errorInfo - cleanup -} diff --git a/tests/sentinel/run.tcl b/tests/sentinel/run.tcl new file mode 100644 index 000000000..78d2f61bc --- /dev/null +++ b/tests/sentinel/run.tcl @@ -0,0 +1,19 @@ +# Sentinel test suite. Copyright (C) 2014 Salvatore Sanfilippo antirez@gmail.com +# This softare is released under the BSD License. See the COPYING file for +# more information. + +cd tests/sentinel +source ../instances.tcl + +proc main {} { + parse_options + spawn_instance sentinel $::sentinel_base_port $::instances_count + spawn_instance redis $::redis_base_port $::instances_count + run_tests + cleanup +} + +if {[catch main e]} { + puts $::errorInfo + cleanup +} diff --git a/tests/sentinel/tests/00-base.tcl b/tests/sentinel/tests/00-base.tcl new file mode 100644 index 000000000..a79d0c371 --- /dev/null +++ b/tests/sentinel/tests/00-base.tcl @@ -0,0 +1,126 @@ +# Check the basic monitoring and failover capabilities. + +source "../tests/includes/init-tests.tcl" + +if {$::simulate_error} { + test "This test will fail" { + fail "Simulated error" + } +} + +test "Basic failover works if the master is down" { + set old_port [RI $master_id tcp_port] + set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] + assert {[lindex $addr 1] == $old_port} + kill_instance redis $master_id + foreach_sentinel_id id { + wait_for_condition 1000 50 { + [lindex [S $id SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 1] != $old_port + } else { + fail "At least one Sentinel did not received failover info" + } + } + restart_instance redis $master_id + set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] + set master_id [get_instance_id_by_port redis [lindex $addr 1]] +} + +test "New master [join $addr {:}] role matches" { + assert {[RI $master_id role] eq {master}} +} + +test "All the other slaves now point to the new master" { + foreach_redis_id id { + if {$id != $master_id && $id != 0} { + wait_for_condition 1000 50 { + [RI $id master_port] == [lindex $addr 1] + } else { + fail "Redis ID $id not configured to replicate with new master" + } + } + } +} + +test "The old master eventually gets reconfigured as a slave" { + wait_for_condition 1000 50 { + [RI 0 master_port] == [lindex $addr 1] + } else { + fail "Old master not reconfigured as slave of new master" + } +} + +test "ODOWN is not possible without N (quorum) Sentinels reports" { + foreach_sentinel_id id { + S $id SENTINEL SET mymaster quorum [expr $sentinels+1] + } + set old_port [RI $master_id tcp_port] + set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] + assert {[lindex $addr 1] == $old_port} + kill_instance redis $master_id + + # Make sure failover did not happened. + set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] + assert {[lindex $addr 1] == $old_port} + restart_instance redis $master_id +} + +test "Failover is not possible without majority agreement" { + foreach_sentinel_id id { + S $id SENTINEL SET mymaster quorum $quorum + } + + # Crash majority of sentinels + for {set id 0} {$id < $quorum} {incr id} { + kill_instance sentinel $id + } + + # Kill the current master + kill_instance redis $master_id + + # Make sure failover did not happened. + set addr [S $quorum SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] + assert {[lindex $addr 1] == $old_port} + restart_instance redis $master_id + + # Cleanup: restart Sentinels to monitor the master. + for {set id 0} {$id < $quorum} {incr id} { + restart_instance sentinel $id + } +} + +test "Failover works if we configure for absolute agreement" { + foreach_sentinel_id id { + S $id SENTINEL SET mymaster quorum $sentinels + } + + # Wait for Sentinels to monitor the master again + foreach_sentinel_id id { + wait_for_condition 1000 50 { + [dict get [S $id SENTINEL MASTER mymaster] info-refresh] < 100000 + } else { + fail "At least one Sentinel is not monitoring the master" + } + } + + kill_instance redis $master_id + + foreach_sentinel_id id { + wait_for_condition 1000 50 { + [lindex [S $id SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 1] != $old_port + } else { + fail "At least one Sentinel did not received failover info" + } + } + restart_instance redis $master_id + set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] + set master_id [get_instance_id_by_port redis [lindex $addr 1]] + + # Set the min ODOWN agreement back to strict majority. + foreach_sentinel_id id { + S $id SENTINEL SET mymaster quorum $quorum + } +} + +test "New master [join $addr {:}] role matches" { + assert {[RI $master_id role] eq {master}} +} diff --git a/tests/sentinel/tests/01-conf-update.tcl b/tests/sentinel/tests/01-conf-update.tcl new file mode 100644 index 000000000..4998104d2 --- /dev/null +++ b/tests/sentinel/tests/01-conf-update.tcl @@ -0,0 +1,39 @@ +# Test Sentinel configuration consistency after partitions heal. + +source "../tests/includes/init-tests.tcl" + +test "We can failover with Sentinel 1 crashed" { + set old_port [RI $master_id tcp_port] + set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] + assert {[lindex $addr 1] == $old_port} + + # Crash Sentinel 1 + kill_instance sentinel 1 + + kill_instance redis $master_id + foreach_sentinel_id id { + if {$id != 1} { + wait_for_condition 1000 50 { + [lindex [S $id SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 1] != $old_port + } else { + fail "Sentinel $id did not received failover info" + } + } + } + restart_instance redis $master_id + set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] + set master_id [get_instance_id_by_port redis [lindex $addr 1]] +} + +test "After Sentinel 1 is restarted, its config gets updated" { + restart_instance sentinel 1 + wait_for_condition 1000 50 { + [lindex [S 1 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 1] != $old_port + } else { + fail "Restarted Sentinel did not received failover info" + } +} + +test "New master [join $addr {:}] role matches" { + assert {[RI $master_id role] eq {master}} +} diff --git a/tests/sentinel/tests/02-slaves-reconf.tcl b/tests/sentinel/tests/02-slaves-reconf.tcl new file mode 100644 index 000000000..868bae5ec --- /dev/null +++ b/tests/sentinel/tests/02-slaves-reconf.tcl @@ -0,0 +1,83 @@ +# Check that slaves are reconfigured at a latter time if they are partitioned. +# +# Here we should test: +# 1) That slaves point to the new master after failover. +# 2) That partitioned slaves point to new master when they are partitioned +# away during failover and return at a latter time. + +source "../tests/includes/init-tests.tcl" + +proc 03_test_slaves_replication {} { + uplevel 1 { + test "Check that slaves replicate from current master" { + set master_port [RI $master_id tcp_port] + foreach_redis_id id { + if {$id == $master_id} continue + if {[instance_is_killed redis $id]} continue + wait_for_condition 1000 50 { + [RI $id master_port] == $master_port + } else { + fail "Redis slave $id is replicating from wrong master" + } + } + } + } +} + +proc 03_crash_and_failover {} { + uplevel 1 { + test "Crash the master and force a failover" { + set old_port [RI $master_id tcp_port] + set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] + assert {[lindex $addr 1] == $old_port} + kill_instance redis $master_id + foreach_sentinel_id id { + wait_for_condition 1000 50 { + [lindex [S $id SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 1] != $old_port + } else { + fail "At least one Sentinel did not received failover info" + } + } + restart_instance redis $master_id + set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] + set master_id [get_instance_id_by_port redis [lindex $addr 1]] + } + } +} + +03_test_slaves_replication +03_crash_and_failover +03_test_slaves_replication + +test "Kill a slave instance" { + foreach_redis_id id { + if {$id == $master_id} continue + set killed_slave_id $id + kill_instance redis $id + break + } +} + +03_crash_and_failover +03_test_slaves_replication + +test "Wait for failover to end" { + set inprogress 1 + while {$inprogress} { + set inprogress 0 + foreach_sentinel_id id { + if {[dict exists [S $id SENTINEL MASTER mymaster] failover-state]} { + incr inprogress + } + } + if {$inprogress} {after 100} + } +} + +test "Restart killed slave and test replication of slaves again..." { + restart_instance redis $killed_slave_id +} + +# Now we check if the slave rejoining the partition is reconfigured even +# if the failover finished. +03_test_slaves_replication diff --git a/tests/sentinel/tests/03-runtime-reconf.tcl b/tests/sentinel/tests/03-runtime-reconf.tcl new file mode 100644 index 000000000..426596c37 --- /dev/null +++ b/tests/sentinel/tests/03-runtime-reconf.tcl @@ -0,0 +1 @@ +# Test runtime reconfiguration command SENTINEL SET. diff --git a/tests/sentinel/tests/04-slave-selection.tcl b/tests/sentinel/tests/04-slave-selection.tcl new file mode 100644 index 000000000..3d2ca6484 --- /dev/null +++ b/tests/sentinel/tests/04-slave-selection.tcl @@ -0,0 +1,5 @@ +# Test slave selection algorithm. +# +# This unit should test: +# 1) That when there are no suitable slaves no failover is performed. +# 2) That among the available slaves, the one with better offset is picked. diff --git a/tests/sentinel/tests/includes/init-tests.tcl b/tests/sentinel/tests/includes/init-tests.tcl new file mode 100644 index 000000000..cb359ea1b --- /dev/null +++ b/tests/sentinel/tests/includes/init-tests.tcl @@ -0,0 +1,67 @@ +# Initialization tests -- most units will start including this. + +test "(init) Restart killed instances" { + foreach type {redis sentinel} { + foreach_${type}_id id { + if {[get_instance_attrib $type $id pid] == -1} { + puts -nonewline "$type/$id " + flush stdout + restart_instance $type $id + } + } + } +} + +set redis_slaves 4 +test "(init) Create a master-slaves cluster of [expr $redis_slaves+1] instances" { + create_redis_master_slave_cluster [expr {$redis_slaves+1}] +} +set master_id 0 + +test "(init) Sentinels can start monitoring a master" { + set sentinels [llength $::sentinel_instances] + set quorum [expr {$sentinels/2+1}] + foreach_sentinel_id id { + catch {S $id SENTINEL REMOVE mymaster} + S $id SENTINEL MONITOR mymaster \ + [get_instance_attrib redis $master_id host] \ + [get_instance_attrib redis $master_id port] $quorum + } + foreach_sentinel_id id { + assert {[S $id sentinel master mymaster] ne {}} + S $id SENTINEL SET mymaster down-after-milliseconds 2000 + S $id SENTINEL SET mymaster failover-timeout 20000 + S $id SENTINEL SET mymaster parallel-syncs 10 + } +} + +test "(init) Sentinels can talk with the master" { + foreach_sentinel_id id { + wait_for_condition 100 50 { + [catch {S $id SENTINEL GET-MASTER-ADDR-BY-NAME mymaster}] == 0 + } else { + fail "Sentinel $id can't talk with the master." + } + } +} + +test "(init) Sentinels are able to auto-discover other sentinels" { + set sentinels [llength $::sentinel_instances] + foreach_sentinel_id id { + wait_for_condition 100 50 { + [dict get [S $id SENTINEL MASTER mymaster] num-other-sentinels] == ($sentinels-1) + } else { + fail "At least some sentinel can't detect some other sentinel" + } + } +} + +test "(init) Sentinels are able to auto-discover slaves" { + foreach_sentinel_id id { + wait_for_condition 100 50 { + [dict get [S $id SENTINEL MASTER mymaster] num-slaves] == $redis_slaves + } else { + fail "At least some sentinel can't detect some slave" + } + } +} diff --git a/tests/sentinel/tmp/.gitignore b/tests/sentinel/tmp/.gitignore new file mode 100644 index 000000000..f581f73e2 --- /dev/null +++ b/tests/sentinel/tmp/.gitignore @@ -0,0 +1,2 @@ +redis_* +sentinel_* -- cgit v1.2.1