Add cluster slot migration tests (#8649)

Add tests for fixing migrating slot at all stages:

1. when migration is half inited on "migrating" node
2. when migration is half inited on "importing" node
3. migration inited, but not finished
4. migration is half finished on "migrating" node
5. migration is half finished on "importing" node

Also add tests for many simultaneous slot migrations.

Co-authored-by: Yossi Gottlieb <yossigo@gmail.com>

3 files changed, 154 insertions, 0 deletions

diff --git a/tests/cluster/tests/20-half-migrated-slot.tcl b/tests/cluster/tests/20-half-migrated-slot.tcl
new file mode 100644
index 000000000..6b5a8c516
--- /dev/null
+++ b/tests/cluster/tests/20-half-migrated-slot.tcl
@@ -0,0 +1,86 @@
+# Tests for fixing migrating slot at all stages:
+# 1. when migration is half inited on "migrating" node
+# 2. when migration is half inited on "importing" node
+# 3. migration inited, but not finished
+# 4. migration is half finished on "migrating" node
+# 5. migration is half finished on "importing" node
+source "../tests/includes/init-tests.tcl"
+source "../tests/includes/utils.tcl"
+
+test "Create a 2 nodes cluster" {
+    create_cluster 2 0
+}
+
+test "Cluster is up" {
+    assert_cluster_state ok
+}
+
+set cluster [redis_cluster 127.0.0.1:[get_instance_attrib redis 0 port]]
+catch {unset nodefrom}
+catch {unset nodeto}
+
+proc reset_cluster {} {
+    uplevel 1 {
+        $cluster refresh_nodes_map
+        array set nodefrom [$cluster masternode_for_slot 609]
+        array set nodeto [$cluster masternode_notfor_slot 609]
+    }
+}
+
+reset_cluster
+
+$cluster set aga xyz
+
+test "Half init migration in 'migrating' is fixable" {
+    $nodefrom(link) cluster setslot 609 migrating $nodeto(id)
+    fix_cluster $nodefrom(addr)
+    assert {[$cluster get aga] eq "xyz"}
+}
+
+test "Half init migration in 'importing' is fixable" {
+    $nodeto(link) cluster setslot 609 importing $nodefrom(id)
+    fix_cluster $nodefrom(addr)
+    assert {[$cluster get aga] eq "xyz"}
+}
+
+test "Init migration and move key" {
+    $nodefrom(link) cluster setslot 609 migrating $nodeto(id)
+    $nodeto(link) cluster setslot 609 importing $nodefrom(id)
+    $nodefrom(link) migrate $nodeto(host) $nodeto(port) aga 0 10000
+    assert {[$cluster get aga] eq "xyz"}
+    fix_cluster $nodefrom(addr)
+    assert {[$cluster get aga] eq "xyz"}
+}
+
+reset_cluster
+
+test "Move key again" {
+    $nodefrom(link) cluster setslot 609 migrating $nodeto(id)
+    $nodeto(link) cluster setslot 609 importing $nodefrom(id)
+    $nodefrom(link) migrate $nodeto(host) $nodeto(port) aga 0 10000
+    assert {[$cluster get aga] eq "xyz"}
+}
+
+test "Half-finish migration" {
+    # half finish migration on 'migrating' node
+    $nodefrom(link) cluster setslot 609 node $nodeto(id)
+    fix_cluster $nodefrom(addr)
+    assert {[$cluster get aga] eq "xyz"}
+}
+
+reset_cluster
+
+test "Move key back" {
+    # 'aga' key is in 609 slot
+    $nodefrom(link) cluster setslot 609 migrating $nodeto(id)
+    $nodeto(link) cluster setslot 609 importing $nodefrom(id)
+    $nodefrom(link) migrate $nodeto(host) $nodeto(port) aga 0 10000
+    assert {[$cluster get aga] eq "xyz"}
+}
+
+test "Half-finish importing" {
+    # Now we half finish 'importing' node
+    $nodeto(link) cluster setslot 609 node $nodeto(id)
+    fix_cluster $nodefrom(addr)
+    assert {[$cluster get aga] eq "xyz"}
+}
diff --git a/tests/cluster/tests/21-many-slot-migration.tcl b/tests/cluster/tests/21-many-slot-migration.tcl
new file mode 100644
index 000000000..62df0ce79
--- /dev/null
+++ b/tests/cluster/tests/21-many-slot-migration.tcl
@@ -0,0 +1,51 @@
+# Tests for many simlutaneous migrations.
+
+source "../tests/includes/init-tests.tcl"
+source "../tests/includes/utils.tcl"
+
+test "Create a 10 nodes cluster" {
+    create_cluster 10 10
+}
+
+test "Cluster is up" {
+    assert_cluster_state ok
+}
+
+set cluster [redis_cluster 127.0.0.1:[get_instance_attrib redis 0 port]]
+catch {unset nodefrom}
+catch {unset nodeto}
+
+$cluster refresh_nodes_map
+
+test "Set many keys" {
+    for {set i 0} {$i < 40000} {incr i} {
+        $cluster set key:$i val:$i
+    }
+}
+
+test "Keys are accessible" {
+    for {set i 0} {$i < 40000} {incr i} {
+        assert { [$cluster get key:$i] eq "val:$i" }
+    }
+}
+
+test "Init migration of many slots" {
+    for {set slot 0} {$slot < 1000} {incr slot} {
+        array set nodefrom [$cluster masternode_for_slot $slot]
+        array set nodeto [$cluster masternode_notfor_slot $slot]
+
+        $nodefrom(link) cluster setslot $slot migrating $nodeto(id)
+        $nodeto(link) cluster setslot $slot importing $nodefrom(id)
+    }
+}
+
+test "Fix cluster" {
+    fix_cluster $nodefrom(addr)
+}
+
+test "Keys are accessible" {
+    for {set i 0} {$i < 40000} {incr i} {
+        assert { [$cluster get key:$i] eq "val:$i" }
+    }
+}
+
diff --git a/tests/cluster/tests/includes/utils.tcl b/tests/cluster/tests/includes/utils.tcl
new file mode 100644
index 000000000..202db7350
--- /dev/null
+++ b/tests/cluster/tests/includes/utils.tcl
@@ -0,0 +1,17 @@
+source "../../../tests/support/cli.tcl"
+
+proc fix_cluster {addr} {
+    set code [catch {
+        exec ../../../src/redis-cli {*}[rediscli_tls_config "../../../tests"] --cluster fix $addr << yes
+    } result]
+    if {$code != 0 && $::verbose} {
+        puts $result
+    }
+    assert {$code == 0}
+    assert_cluster_state ok
+    wait_for_condition 1000 10 {
+        [catch {exec ../../../src/redis-cli {*}[rediscli_tls_config "../../../tests"] --cluster check $addr} _] == 0
+    } else {
+        fail "Cluster could not settle with configuration"
+    }
+}