blob: 7fb1a8befc4808e76cc45cd322cb7bda9ca5a3c6 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
|
# Check the basic monitoring and failover capabilities.
source "../tests/includes/init-tests.tcl"
if {$::simulate_error} {
test "This test will fail" {
fail "Simulated error"
}
}
test "Basic failover works if the master is down" {
set old_port [RI $master_id tcp_port]
set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster]
assert {[lindex $addr 1] == $old_port}
kill_instance redis $master_id
foreach_sentinel_id id {
wait_for_condition 1000 50 {
[lindex [S $id SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 1] != $old_port
} else {
fail "At least one Sentinel did not receive failover info"
}
}
restart_instance redis $master_id
set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster]
set master_id [get_instance_id_by_port redis [lindex $addr 1]]
}
test "New master [join $addr {:}] role matches" {
assert {[RI $master_id role] eq {master}}
}
test "All the other slaves now point to the new master" {
foreach_redis_id id {
if {$id != $master_id && $id != 0} {
wait_for_condition 1000 50 {
[RI $id master_port] == [lindex $addr 1]
} else {
fail "Redis ID $id not configured to replicate with new master"
}
}
}
}
test "The old master eventually gets reconfigured as a slave" {
wait_for_condition 1000 50 {
[RI 0 master_port] == [lindex $addr 1]
} else {
fail "Old master not reconfigured as slave of new master"
}
}
test "ODOWN is not possible without N (quorum) Sentinels reports" {
foreach_sentinel_id id {
S $id SENTINEL SET mymaster quorum [expr $sentinels+1]
}
set old_port [RI $master_id tcp_port]
set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster]
assert {[lindex $addr 1] == $old_port}
kill_instance redis $master_id
# Make sure failover did not happened.
set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster]
assert {[lindex $addr 1] == $old_port}
restart_instance redis $master_id
}
test "Failover is not possible without majority agreement" {
foreach_sentinel_id id {
S $id SENTINEL SET mymaster quorum $quorum
}
# Crash majority of sentinels
for {set id 0} {$id < $quorum} {incr id} {
kill_instance sentinel $id
}
# Kill the current master
kill_instance redis $master_id
# Make sure failover did not happened.
set addr [S $quorum SENTINEL GET-MASTER-ADDR-BY-NAME mymaster]
assert {[lindex $addr 1] == $old_port}
restart_instance redis $master_id
# Cleanup: restart Sentinels to monitor the master.
for {set id 0} {$id < $quorum} {incr id} {
restart_instance sentinel $id
}
}
test "Failover works if we configure for absolute agreement" {
foreach_sentinel_id id {
S $id SENTINEL SET mymaster quorum $sentinels
}
# Wait for Sentinels to monitor the master again
foreach_sentinel_id id {
wait_for_condition 1000 50 {
[dict get [S $id SENTINEL MASTER mymaster] info-refresh] < 100000
} else {
fail "At least one Sentinel is not monitoring the master"
}
}
kill_instance redis $master_id
foreach_sentinel_id id {
wait_for_condition 1000 50 {
[lindex [S $id SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 1] != $old_port
} else {
fail "At least one Sentinel did not receive failover info"
}
}
restart_instance redis $master_id
set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster]
set master_id [get_instance_id_by_port redis [lindex $addr 1]]
# Set the min ODOWN agreement back to strict majority.
foreach_sentinel_id id {
S $id SENTINEL SET mymaster quorum $quorum
}
}
test "New master [join $addr {:}] role matches" {
assert {[RI $master_id role] eq {master}}
}
|
