blob: a719a0bf37fa859e440020ad0dd3516afecd7b47 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
|
# Failover stress test.
# In this test a different node is killed in a loop for N
# iterations. The test checks that certain properties
# are preseved across iterations.
source "../tests/includes/init-tests.tcl"
test "Create a 5 nodes cluster" {
create_cluster 5 5
}
test "Cluster is up" {
assert_cluster_state ok
}
test "Enable AOF in all the instances" {
foreach_redis_id id {
R $id config set appendonly yes
# We use "appendfsync no" because it's fast but also guarantees that
# write(2) is performed before replying to client.
R $id config set appendfsync no
}
foreach_redis_id id {
wait_for_condition 1000 500 {
[RI $id aof_rewrite_in_progress] == 0 &&
[RI $id aof_enabled] == 1
} else {
fail "Failed to enable AOF on instance #$id"
}
}
}
# Return nno-zero if the specified PID is about a process still in execution,
# otherwise 0 is returned.
proc process_is_running {pid} {
# PS should return with an error if PID is non existing,
# and catch will return non-zero. We want to return non-zero if
# the PID exists, so we invert the return value with expr not operator.
expr {![catch {exec ps -p $pid}]}
}
# Our resharding test performs the following actions:
#
# - N commands are sent to the cluster in the course of the test.
# - Every command selects a random key from key:0 to key:MAX-1.
# - The operation RPUSH key <randomvalue> is perforemd.
# - Tcl remembers into an array all the values pushed to each list.
# - After N/2 commands, the resharding process is started in background.
# - The test continues while the resharding is in progress.
# - At the end of the test, we wait for the resharding process to stop.
# - Finally the keys are checked to see if they contain the value they should.
set numkeys 50000
set numops 200000
set cluster [redis_cluster 127.0.0.1:[get_instance_attrib redis 0 port]]
catch {unset content}
array set content {}
set tribpid {}
test "Cluster consistency during live resharding" {
for {set j 0} {$j < $numops} {incr j} {
# Trigger the resharding once we execute half the ops.
if {$tribpid ne {} &&
($j % 10000) == 0 &&
![process_is_running $tribpid]} {
set tribpid {}
}
if {$j >= $numops/2 && $tribpid eq {}} {
puts -nonewline "...Starting resharding..."
flush stdout
set target [dict get [get_myself [randomInt 5]] id]
set tribpid [lindex [exec \
../../../src/redis-trib.rb reshard \
--from all \
--to $target \
--slots 100 \
--yes \
127.0.0.1:[get_instance_attrib redis 0 port] \
| [info nameofexecutable] \
../tests/helpers/onlydots.tcl \
&] 0]
}
# Write random data to random list.
set listid [randomInt $numkeys]
set key "key:$listid"
set ele [randomValue]
# We write both with Lua scripts and with plain commands.
# This way we are able to stress Lua -> Redis command invocation
# as well, that has tests to prevent Lua to write into wrong
# hash slots.
if {$listid % 2} {
$cluster rpush $key $ele
} else {
$cluster eval {redis.call("rpush",KEYS[1],ARGV[1])} 1 $key $ele
}
lappend content($key) $ele
if {($j % 1000) == 0} {
puts -nonewline W; flush stdout
}
}
# Wait for the resharding process to end
wait_for_condition 1000 500 {
[process_is_running $tribpid] == 0
} else {
fail "Resharding is not terminating after some time."
}
}
test "Verify $numkeys keys for consistency with logical content" {
# Check that the Redis Cluster content matches our logical content.
foreach {key value} [array get content] {
assert {[$cluster lrange $key 0 -1] eq $value}
}
}
test "Crash and restart all the instances" {
foreach_redis_id id {
kill_instance redis $id
restart_instance redis $id
}
}
test "Cluster should eventually be up again" {
assert_cluster_state ok
}
test "Verify $numkeys keys after the crash & restart" {
# Check that the Redis Cluster content matches our logical content.
foreach {key value} [array get content] {
assert {[$cluster lrange $key 0 -1] eq $value}
}
}
test "Disable AOF in all the instances" {
foreach_redis_id id {
R $id config set appendonly no
}
}
|
