Improved the reliability of cluster replica sync tests (#9628)

Improved the reliability of cluster replica sync tests

2 files changed, 40 insertions, 23 deletions

diff --git a/tests/cluster/tests/22-replica-in-sync.tcl b/tests/cluster/tests/22-replica-in-sync.tcl
index c147cb068..db1590560 100644
--- a/tests/cluster/tests/22-replica-in-sync.tcl
+++ b/tests/cluster/tests/22-replica-in-sync.tcl
@@ -27,26 +27,25 @@ proc is_replica_online {info_repl} {
 
 set master_id 0
 
-test "Fill up" {
-    R $master_id debug populate 10000000 key 100
+test "Fill up primary with data" {
+    # Set 1 MB of data
+    R $master_id debug populate 1000 key 1000
 }
 
 test "Add new node as replica" {
-    set replica_id [cluster_find_available_slave 1]
-    set master_myself [get_myself $master_id]
-    set replica_myself [get_myself $replica_id]
-    set replica [dict get $replica_myself id]
-    R $replica_id cluster replicate [dict get $master_myself id]
+    set replica_id 1
+    set replica [R $replica_id CLUSTER MYID]
+    R $replica_id cluster replicate [R $master_id CLUSTER MYID]
 }
 
 test "Check digest and replica state" {
-    R 1 readonly
     wait_for_condition 1000 50 {
         [is_in_slots $master_id $replica]
     } else {
         fail "New replica didn't appear in the slots"
     }
-    wait_for_condition 1000 50 {
+
+    wait_for_condition 100 50 {
         [is_replica_online [R $master_id info replication]]
     } else {
         fail "Replica is down for too long"
@@ -57,10 +56,20 @@ test "Check digest and replica state" {
 
 test "Replica in loading state is hidden" {
     # Kill replica client for master and load new data to the primary
-    R $master_id multi
     R $master_id config set repl-backlog-size 100
+
+    # Set the key load delay so that it will take at least
+    # 2 seconds to fully load the data.
+    R $replica_id config set key-load-delay 4000
+
+    # Trigger event loop processing every 1024 bytes, this trigger
+    # allows us to send and receive cluster messages, so we are setting
+    # it low so that the cluster messages are sent more frequently.
+    R $replica_id config set loading-process-events-interval-bytes 1024
+
+    R $master_id multi
     R $master_id client kill type replica
-    set num 10000
+    set num 100
     set value [string repeat A 1024]
     for {set j 0} {$j < $num} {incr j} {
         set key "{0}"
@@ -69,28 +78,34 @@ test "Replica in loading state is hidden" {
     }
     R $master_id exec
 
-    # Check that replica started loading
-    wait_for_condition 1000 50 {
-        [s $replica_id loading] eq 1
+    # The master will be the last to know the replica
+    # is loading, so we will wait on that and assert
+    # the replica is loading afterwards. 
+    wait_for_condition 100 50 {
+        ![is_in_slots $master_id $replica]
     } else {
-        fail "Replica didn't enter loading state"
+        fail "Replica was always present in cluster slots"
     }
-    # Check that replica is not in cluster slots
-    assert {![is_in_slots $master_id $replica]}
+    assert_equal 1 [s $replica_id loading]
 
-    # Wait for sync to finish
-    wait_for_condition 1000 50 {
-        [s $replica_id loading] eq 0
+    # Wait for the replica to finish full-sync and become online
+    wait_for_condition 200 50 {
+        [s $replica_id master_link_status] eq "up"
     } else {
-        fail "Replica is in loading state for too long"
+        fail "Replica didn't finish loading"
     }
 
-    # Check replica is back to cluster slots
-    wait_for_condition 1000 50 {
+    # Return configs to default values
+    R $replica_id config set loading-process-events-interval-bytes 2097152
+    R $replica_id config set key-load-delay 0
+
+    # Check replica is back in cluster slots
+    wait_for_condition 100 50 {
         [is_in_slots $master_id $replica] 
     } else {
         fail "Replica is not back to slots"
     }
+    assert_equal 1 [is_in_slots $replica_id $replica] 
 }
 
 test "Check disconnected replica not hidden from slots" {
diff --git a/tests/cluster/tests/includes/init-tests.tcl b/tests/cluster/tests/includes/init-tests.tcl
index 466ab8f25..caedc13e2 100644
--- a/tests/cluster/tests/includes/init-tests.tcl
+++ b/tests/cluster/tests/includes/init-tests.tcl
@@ -39,6 +39,8 @@ test "Cluster nodes hard reset" {
         R $id EXEC
         R $id config set cluster-node-timeout $node_timeout
         R $id config set cluster-slave-validity-factor 10
+        R $id config set loading-process-events-interval-bytes 2097152
+        R $id config set key-load-delay 0
         R $id config rewrite
     }
 }