1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
|
//
// Jumbo tests for refineCollectionShardKey.
//
(function() {
'use strict';
const st = new ShardingTest({mongos: 1, shards: 2, other: {chunkSize: 1, enableAutoSplit: true}});
const primaryShard = st.shard0.shardName;
const secondaryShard = st.shard1.shardName;
const kDbName = 'test';
const kCollName = 'foo';
const kNestedCollName = 'nested_foo';
const kNsName = kDbName + '.' + kCollName;
const kNestedNsName = kDbName + '.' + kNestedCollName;
const kConfigChunks = 'config.chunks';
const kZoneName = 'testZone';
function generateJumboChunk(ns, isForNestedCase) {
const big = 'X'.repeat(10000);
let x = 0;
let bulk = st.s.getCollection(ns).initializeUnorderedBulkOp();
// Create sufficient documents to generate a jumbo chunk, and use the same shard key in all
// of them so that the chunk cannot be split and moved.
for (let i = 0; i < 500; i++) {
if (isForNestedCase) {
bulk.insert({x: x, y: {z: i}, big: big});
} else {
bulk.insert({x: x, y: i, big: big});
}
}
assert.commandWorked(bulk.execute());
}
function runBalancer() {
st.startBalancer();
let numRounds = 0;
// Let the balancer run for 3 rounds.
assert.soon(() => {
st.awaitBalancerRound();
st.printShardingStatus(true);
numRounds++;
return (numRounds === 3);
}, 'Balancer failed to run for 3 rounds', 1000 * 60 * 10);
st.stopBalancer();
}
function validateBalancerBeforeRefine(ns) {
runBalancer();
// Confirm that the jumbo chunk has not been split or moved from the primary shard.
const jumboChunk = st.s.getCollection(kConfigChunks).find({ns: ns}).toArray();
assert.eq(1, jumboChunk.length);
assert.eq(true, jumboChunk[0].jumbo);
assert.eq(primaryShard, jumboChunk[0].shard);
}
function validateBalancerAfterRefine(ns, newField) {
runBalancer();
// Confirm that the jumbo chunk has been split and some chunks moved to the secondary shard.
const chunks = st.s.getCollection(kConfigChunks)
.find({
ns: ns,
min: {$lte: {x: 0, [newField]: MaxKey}},
max: {$gt: {x: 0, [newField]: MinKey}}
})
.toArray();
assert.lt(1, chunks.length);
assert.eq(true, chunks.some((chunk) => {
return (chunk.shard === secondaryShard);
}));
}
function validateMoveChunkBeforeRefine(ns) {
assert.commandFailedWithCode(
st.s.adminCommand({moveChunk: ns, find: {x: 0}, to: secondaryShard}),
ErrorCodes.ChunkTooBig);
// Confirm that the jumbo chunk has not been split or moved from the primary shard.
const jumboChunk = st.s.getCollection(kConfigChunks).find({ns: ns}).toArray();
assert.eq(1, jumboChunk.length);
assert.eq(primaryShard, jumboChunk[0].shard);
}
function validateMoveChunkAfterRefine(ns, newField) {
// Manually split the jumbo chunk before moving the smaller chunks to the secondary shard.
// We split a total of 4 times to ensure that each chunk is below the max chunk size and so
// moveChunk succeeds.
for (let i = 1; i <= 4; i++) {
assert.commandWorked(st.s.adminCommand({split: ns, middle: {x: 0, [newField]: i * 125}}));
}
const chunksToMove = st.s.getCollection(kConfigChunks)
.find({
ns: ns,
min: {$lte: {x: 0, [newField]: MaxKey}},
max: {$gt: {x: 0, [newField]: MinKey}}
})
.toArray();
chunksToMove.forEach((chunk) => {
assert.commandWorked(st.s.adminCommand(
{moveChunk: ns, find: {x: 0, [newField]: chunk.min[newField]}, to: secondaryShard}));
});
// Confirm that the jumbo chunk has been split and all chunks moved to the secondary shard.
const chunks = st.s.getCollection(kConfigChunks)
.find({
ns: ns,
min: {$lte: {x: 0, [newField]: MaxKey}},
max: {$gt: {x: 0, [newField]: MinKey}}
})
.toArray();
assert.lt(1, chunks.length);
chunks.forEach((chunk) => {
assert.eq(secondaryShard, chunk.shard);
});
}
// This test generates a jumbo chunk that cannot be split due to low shard key cardinality. It
// verifies that the balancer cannot split the chunk. After refining the shard key with
// 'refineCollectionShardKey', it verifies that the balancer can now split and move the chunk.
jsTestLog('********** BALANCER JUMBO TEST **********');
//
// With a non nested shard key.
//
// NOTE: The current shard key is {x: 1}.
assert.commandWorked(st.s.adminCommand({enableSharding: kDbName}));
st.ensurePrimaryShard(kDbName, primaryShard);
assert.commandWorked(st.s.adminCommand({shardCollection: kNsName, key: {x: 1}}));
generateJumboChunk(kNsName, false /* isForNestedCase */);
// Create a zone covering the entire range of shard keys to force the balancer to try and fail
// to move the jumbo chunk.
assert.commandWorked(st.s.adminCommand({addShardToZone: secondaryShard, zone: kZoneName}));
assert.commandWorked(st.s.adminCommand(
{updateZoneKeyRange: kNsName, min: {x: MinKey}, max: {x: MaxKey}, zone: kZoneName}));
validateBalancerBeforeRefine(kNsName);
// NOTE: The shard key is now {x: 1, y: 1}.
assert.commandWorked(st.s.getCollection(kNsName).createIndex({x: 1, y: 1}));
assert.commandWorked(st.s.adminCommand({refineCollectionShardKey: kNsName, key: {x: 1, y: 1}}));
validateBalancerAfterRefine(kNsName, "y");
assert(st.s.getCollection(kNsName).drop());
//
// With a nested shard key.
//
assert.commandWorked(st.s.adminCommand({enableSharding: kDbName}));
st.ensurePrimaryShard(kDbName, primaryShard);
assert.commandWorked(st.s.adminCommand({shardCollection: kNestedNsName, key: {x: 1}}));
generateJumboChunk(kNestedNsName, true /* isForNestedCase */);
// Create a zone covering the entire range of shard keys to force the balancer to try and fail
// to move the jumbo chunk.
assert.commandWorked(st.s.adminCommand({addShardToZone: secondaryShard, zone: kZoneName}));
assert.commandWorked(st.s.adminCommand(
{updateZoneKeyRange: kNestedNsName, min: {x: MinKey}, max: {x: MaxKey}, zone: kZoneName}));
validateBalancerBeforeRefine(kNestedNsName);
assert.commandWorked(st.s.getCollection(kNestedNsName).createIndex({x: 1, "y.z": 1}));
assert.commandWorked(
st.s.adminCommand({refineCollectionShardKey: kNestedNsName, key: {x: 1, "y.z": 1}}));
validateBalancerAfterRefine(kNestedNsName, "y.z");
assert(st.s.getCollection(kNestedNsName).drop());
// This test generates a jumbo chunk that cannot be split due to low shard key cardinality. It
// verifies that one cannot manually move the chunk. After refining the shard key with
// 'refineCollectionShardKey', it verifies that one can now manually split and move the chunk.
jsTestLog('********** MANUAL (i.e. MOVE CHUNK) JUMBO TEST **********');
//
// With a non nested shard key.
//
// NOTE: The current shard key is {x: 1}.
assert.commandWorked(st.s.adminCommand({enableSharding: kDbName}));
st.ensurePrimaryShard(kDbName, primaryShard);
assert.commandWorked(st.s.adminCommand({shardCollection: kNsName, key: {x: 1}}));
generateJumboChunk(kNsName, false /* isForNestedCase */);
validateMoveChunkBeforeRefine(kNsName);
// NOTE: The shard key is now {x: 1, y: 1}.
assert.commandWorked(st.s.getCollection(kNsName).createIndex({x: 1, y: 1}));
assert.commandWorked(st.s.adminCommand({refineCollectionShardKey: kNsName, key: {x: 1, y: 1}}));
validateMoveChunkAfterRefine(kNsName, "y");
assert(st.s.getCollection(kNsName).drop());
//
// With a nested shard key.
//
assert.commandWorked(st.s.adminCommand({enableSharding: kDbName}));
st.ensurePrimaryShard(kDbName, primaryShard);
assert.commandWorked(st.s.adminCommand({shardCollection: kNestedNsName, key: {x: 1}}));
generateJumboChunk(kNestedNsName, true /* isForNestedCase */);
validateMoveChunkBeforeRefine(kNestedNsName);
assert.commandWorked(st.s.getCollection(kNestedNsName).createIndex({x: 1, "y.z": 1}));
assert.commandWorked(
st.s.adminCommand({refineCollectionShardKey: kNestedNsName, key: {x: 1, "y.z": 1}}));
validateMoveChunkAfterRefine(kNestedNsName, "y.z");
st.stop();
})();
|
