From e342bedc837e146c912546e92ded000a33a7d2ba Mon Sep 17 00:00:00 2001 From: Moti Cohen Date: Tue, 5 Apr 2022 17:07:59 +0300 Subject: Stabilize Sentinel tests - refine failover-timeout & tilt-period (#10518) Sentinel once in a while experience Sentinel TILT period or leader election failure cycle. The problem is that those default timeout are too big and once it happens, it breaks our tests. Suggesting: - Reducing failover-timeout from 20 to 10sec (actually it is multiplied by 2 and reach 40sec of timeout) - Modify tilt-period from default of 30sec to 5sec. When TILT period happens it might lead to failover in our tests, and might cause also to failover cycle cycle failure. Sentinel tests should `wait_for_condition` up to 50seconds, where needed, to be stable in case having single TILT period or failover failure cycle. In addition relax timing configuration for "manual failover" Sentinel test (was modified several months ago as part of an effort to reduce tests runtime) --- tests/sentinel/tests/05-manual.tcl | 6 +++--- tests/sentinel/tests/includes/init-tests.tcl | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) (limited to 'tests/sentinel') diff --git a/tests/sentinel/tests/05-manual.tcl b/tests/sentinel/tests/05-manual.tcl index 28c5560a6..a0004eb75 100644 --- a/tests/sentinel/tests/05-manual.tcl +++ b/tests/sentinel/tests/05-manual.tcl @@ -3,9 +3,9 @@ source "../tests/includes/init-tests.tcl" foreach_sentinel_id id { - S $id sentinel debug info-period 1000 - S $id sentinel debug default-down-after 3000 - S $id sentinel debug publish-period 500 + S $id sentinel debug info-period 2000 + S $id sentinel debug default-down-after 6000 + S $id sentinel debug publish-period 1000 } test "Manual failover works" { diff --git a/tests/sentinel/tests/includes/init-tests.tcl b/tests/sentinel/tests/includes/init-tests.tcl index fe9a61815..ddb131903 100644 --- a/tests/sentinel/tests/includes/init-tests.tcl +++ b/tests/sentinel/tests/includes/init-tests.tcl @@ -28,7 +28,8 @@ test "(init) Sentinels can start monitoring a master" { foreach_sentinel_id id { assert {[S $id sentinel master mymaster] ne {}} S $id SENTINEL SET mymaster down-after-milliseconds 2000 - S $id SENTINEL SET mymaster failover-timeout 20000 + S $id SENTINEL SET mymaster failover-timeout 10000 + S $id SENTINEL debug tilt-period 5000 S $id SENTINEL SET mymaster parallel-syncs 10 if {$::leaked_fds_file != "" && [exec uname] == "Linux"} { S $id SENTINEL SET mymaster notification-script ../../tests/helpers/check_leaked_fds.tcl -- cgit v1.2.1