blob: 761ee82d10582a228ce3c09ae587dd2da05d2a18 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
|
# Check the basic monitoring and failover capabilities.
source "../tests/includes/start-init-tests.tcl"
source "../tests/includes/init-tests.tcl"
foreach_sentinel_id id {
S $id sentinel debug default-down-after 1000
}
if {$::simulate_error} {
test "This test will fail" {
fail "Simulated error"
}
}
test "Sentinel commands sanity check" {
foreach_sentinel_id id {
assert_equal {72} [llength [S $id command list]]
assert_equal {15} [S $id command count]
}
}
test "Basic failover works if the master is down" {
set old_port [RPort $master_id]
set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster]
assert {[lindex $addr 1] == $old_port}
kill_instance redis $master_id
foreach_sentinel_id id {
S $id sentinel debug ping-period 500
S $id sentinel debug ask-period 500
wait_for_condition 1000 100 {
[lindex [S $id SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 1] != $old_port
} else {
fail "At least one Sentinel did not receive failover info"
}
}
restart_instance redis $master_id
set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster]
set master_id [get_instance_id_by_port redis [lindex $addr 1]]
}
test "New master [join $addr {:}] role matches" {
assert {[RI $master_id role] eq {master}}
}
test "All the other slaves now point to the new master" {
foreach_redis_id id {
if {$id != $master_id && $id != 0} {
wait_for_condition 1000 50 {
[RI $id master_port] == [lindex $addr 1]
} else {
fail "Redis ID $id not configured to replicate with new master"
}
}
}
}
test "The old master eventually gets reconfigured as a slave" {
wait_for_condition 1000 50 {
[RI 0 master_port] == [lindex $addr 1]
} else {
fail "Old master not reconfigured as slave of new master"
}
}
test "ODOWN is not possible without N (quorum) Sentinels reports" {
foreach_sentinel_id id {
S $id SENTINEL SET mymaster quorum [expr $sentinels+1]
}
set old_port [RPort $master_id]
set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster]
assert {[lindex $addr 1] == $old_port}
kill_instance redis $master_id
# Make sure failover did not happened.
set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster]
assert {[lindex $addr 1] == $old_port}
restart_instance redis $master_id
}
test "Failover is not possible without majority agreement" {
foreach_sentinel_id id {
S $id SENTINEL SET mymaster quorum $quorum
}
# Crash majority of sentinels
for {set id 0} {$id < $quorum} {incr id} {
kill_instance sentinel $id
}
# Kill the current master
kill_instance redis $master_id
# Make sure failover did not happened.
set addr [S $quorum SENTINEL GET-MASTER-ADDR-BY-NAME mymaster]
assert {[lindex $addr 1] == $old_port}
restart_instance redis $master_id
# Cleanup: restart Sentinels to monitor the master.
for {set id 0} {$id < $quorum} {incr id} {
restart_instance sentinel $id
}
}
test "Failover works if we configure for absolute agreement" {
foreach_sentinel_id id {
S $id SENTINEL SET mymaster quorum $sentinels
}
# Wait for Sentinels to monitor the master again
foreach_sentinel_id id {
wait_for_condition 1000 100 {
[dict get [S $id SENTINEL MASTER mymaster] info-refresh] < 100000
} else {
fail "At least one Sentinel is not monitoring the master"
}
}
kill_instance redis $master_id
foreach_sentinel_id id {
wait_for_condition 1000 100 {
[lindex [S $id SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 1] != $old_port
} else {
fail "At least one Sentinel did not receive failover info"
}
}
restart_instance redis $master_id
set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster]
set master_id [get_instance_id_by_port redis [lindex $addr 1]]
# Set the min ODOWN agreement back to strict majority.
foreach_sentinel_id id {
S $id SENTINEL SET mymaster quorum $quorum
}
}
test "New master [join $addr {:}] role matches" {
assert {[RI $master_id role] eq {master}}
}
|
