fix cluster rebalance test race (#10207)

Try to fix the rebalance cluster test that's failing with ASAN daily:

Looks like `redis-cli --cluster rebalance` gets `ERR Please use SETSLOT only with masters` in `clusterManagerMoveSlot()`.
it happens when `12-replica-migration-2.tcl` is run with ASAN in GH Actions.
in `Resharding all the master #0 slots away from it`

So the fix (assuming i got it right) is to call `redis-cli --cluster check` before `--cluster rebalance`.
p.s. it looks like a few other checks in these tests needed that wait, added them too.

Other changes:
* in instances.tcl, make sure to catch tcl test crashes and let the rest of the code proceed, so that if there was
  a redis crash, we'll find it and print it too.
* redis-cli, try to make sure it prints an error instead of silently exiting.

specifically about redis-cli:
1. clusterManagerMoveSlot used to print an error, only if the caller also asked for it (should be the other way around).
2. clusterManagerCommandReshard asked for an error, but didn't use it (probably tried to avoid the double print).
3. clusterManagerCommandRebalance didn't ask for the error, now it does.
4. making sure that other places in clusterManagerCommandRebalance print something before exiting with an error.

3 files changed, 32 insertions, 11 deletions

diff --git a/tests/cluster/tests/12-replica-migration-2.tcl b/tests/cluster/tests/12-replica-migration-2.tcl
index aecae0429..679eaa3a4 100644
--- a/tests/cluster/tests/12-replica-migration-2.tcl
+++ b/tests/cluster/tests/12-replica-migration-2.tcl
@@ -6,6 +6,7 @@
 
 source "../tests/includes/init-tests.tcl"
 source "../../../tests/support/cli.tcl"
+source "../tests/includes/utils.tcl"
 
 # Create a cluster with 5 master and 15 slaves, to make sure there are no
 # empty masters and make rebalancing simpler to handle during the test.
@@ -29,6 +30,10 @@ test "Each master should have at least two replicas attached" {
     }
 }
 
+test "Wait cluster to be stable" {
+    wait_cluster_stable
+}
+
 test "Set allow-replica-migration yes" {
     foreach_redis_id id {
         R $id CONFIG SET cluster-allow-replica-migration yes
@@ -53,10 +58,13 @@ test "Master #0 should lose its replicas" {
     }
 }
 
+# Wait for the cluster config to propagate before attempting a
+# new resharding.
+test "Wait cluster to be stable" {
+    wait_cluster_stable
+}
+
 test "Resharding back some slot to master #0" {
-    # Wait for the cluster config to propagate before attempting a
-    # new resharding.
-    after 10000
     set output [exec \
         ../../../src/redis-cli --cluster rebalance \
         127.0.0.1:[get_instance_attrib redis 0 port] \
@@ -65,6 +73,10 @@ test "Resharding back some slot to master #0" {
         --cluster-use-empty-masters  >@ stdout]
 }
 
+test "Wait cluster to be stable" {
+    wait_cluster_stable
+}
+
 test "Master #0 should re-acquire one or more replicas" {
     wait_for_condition 1000 50 {
         [llength [lindex [R 0 role] 2]] >= 1
diff --git a/tests/cluster/tests/12.1-replica-migration-3.tcl b/tests/cluster/tests/12.1-replica-migration-3.tcl
index 25d14a56e..1227fd71a 100644
--- a/tests/cluster/tests/12.1-replica-migration-3.tcl
+++ b/tests/cluster/tests/12.1-replica-migration-3.tcl
@@ -4,6 +4,7 @@
 # migrate when master becomes empty.
 
 source "../tests/includes/init-tests.tcl"
+source "../tests/includes/utils.tcl"
 
 # Create a cluster with 5 master and 15 slaves, to make sure there are no
 # empty masters and make rebalancing simpler to handle during the test.
@@ -33,6 +34,10 @@ test "Set allow-replica-migration no" {
     }
 }
 
+test "Wait cluster to be stable" {
+    wait_cluster_stable
+}
+
 set master0_id [dict get [get_myself 0] id]
 test "Resharding all the master #0 slots away from it" {
     set output [exec \
@@ -43,14 +48,7 @@ test "Resharding all the master #0 slots away from it" {
 }
 
 test "Wait cluster to be stable" {
-    wait_for_condition 1000 50 {
-        [catch {exec ../../../src/redis-cli --cluster \
-            check 127.0.0.1:[get_instance_attrib redis 0 port] \
-            {*}[rediscli_tls_config "../../../tests"] \
-            }] == 0
-    } else {
-        fail "Cluster doesn't stabilize"
-    }
+    wait_cluster_stable
 }
 
 test "Master #0 still should have its replicas" {
diff --git a/tests/cluster/tests/includes/utils.tcl b/tests/cluster/tests/includes/utils.tcl
index 48c40a050..c1b0fe6b7 100644
--- a/tests/cluster/tests/includes/utils.tcl
+++ b/tests/cluster/tests/includes/utils.tcl
@@ -23,3 +23,14 @@ proc fix_cluster {addr} {
         fail "Cluster could not settle with configuration"
     }
 }
+
+proc wait_cluster_stable {} {
+    wait_for_condition 1000 50 {
+        [catch {exec ../../../src/redis-cli --cluster \
+            check 127.0.0.1:[get_instance_attrib redis 0 port] \
+            {*}[rediscli_tls_config "../../../tests"] \
+            }] == 0
+    } else {
+        fail "Cluster doesn't stabilize"
+    }
+}
\ No newline at end of file