diff options
author | Madelyn Olson <34459052+madolson@users.noreply.github.com> | 2022-12-22 17:37:00 -0800 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-12-22 17:37:00 -0800 |
commit | 7379d2219620d809943f32a5d26137a077bbae37 (patch) | |
tree | e0c37d65021af5ac4c6532c34b298b202b914959 /tests/cluster/tests | |
parent | 9e1a00d663c9212e6e992feb690089a5eac686f3 (diff) | |
download | redis-7379d2219620d809943f32a5d26137a077bbae37.tar.gz |
Harden init-tests for cluster tests (#11635)
Attempt to harden cluster init-tests by doing two things:
* Retry up to 3 times to join the cluster. Cluster meet is entirely idempotent, so it should stabilize if we missed a node.
* Validate the connection is actually established, not just exists in the cluster list. Nodes can exist in handshake, but might later get dropped.
Diffstat (limited to 'tests/cluster/tests')
-rw-r--r-- | tests/cluster/tests/includes/init-tests.tcl | 22 |
1 files changed, 19 insertions, 3 deletions
diff --git a/tests/cluster/tests/includes/init-tests.tcl b/tests/cluster/tests/includes/init-tests.tcl index fc5897a1a..4875a0106 100644 --- a/tests/cluster/tests/includes/init-tests.tcl +++ b/tests/cluster/tests/includes/init-tests.tcl @@ -48,7 +48,9 @@ test "Cluster nodes hard reset" { } } -test "Cluster Join and auto-discovery test" { +# Helper function to attempt to have each node in a cluster +# meet each other. +proc join_nodes_in_cluster {} { # Join node 0 with 1, 1 with 2, ... and so forth. # If auto-discovery works all nodes will know every other node # eventually. @@ -63,11 +65,25 @@ test "Cluster Join and auto-discovery test" { foreach_redis_id id { wait_for_condition 1000 50 { - [llength [get_cluster_nodes $id]] == [llength $ids] + [llength [get_cluster_nodes $id connected]] == [llength $ids] } else { - fail "Cluster failed to join into a full mesh." + return 0 } } + return 1 +} + +test "Cluster Join and auto-discovery test" { + # Use multiple attempts since sometimes nodes timeout + # while attempting to connect. + for {set attempts 3} {$attempts > 0} {incr attempts -1} { + if {[join_nodes_in_cluster] == 1} { + break + } + } + if {$attempts == 0} { + fail "Cluster failed to form full mesh" + } } test "Before slots allocation, all nodes report cluster failure" { |