1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
|
(function() {
'use strict';
// 3.2.1 is the final version to use the old style replSetUpdatePosition command.
var oldVersion = "3.2.1";
var newVersion = "latest";
var nodes = { n1 : { binVersion : oldVersion },
n2 : { binVersion : newVersion },
n3 : { binVersion : oldVersion },
n4 : { binVersion : newVersion },
n5 : { binVersion : oldVersion } };
var host = getHostName();
var name = 'tags';
var replTest = new ReplSetTest({name: name, nodes: nodes, useBridge: true});
var nodes = replTest.nodeList();
var conns = replTest.startSet();
var port = replTest.ports;
replTest.initiate({
_id: name,
members : [
{
_id: 0,
host: nodes[0],
tags: {
server: '0',
dc: 'ny',
ny: '1',
rack: 'ny.rk1',
},
},
{
_id: 1,
host: nodes[1],
priority: 2,
tags: {
server: '1',
dc: 'ny',
ny: '2',
rack: 'ny.rk1',
},
},
{
_id: 2,
host: nodes[2],
priority: 3,
tags: {
server: '2',
dc: 'ny',
ny: '3',
rack: 'ny.rk2',
2: 'this',
},
},
{
_id: 3,
host: nodes[3],
tags: {
server: '3',
dc: 'sf',
sf: '1',
rack: 'sf.rk1',
},
},
{
_id: 4,
host: nodes[4],
tags: {
server: '4',
dc: 'sf',
sf: '2',
rack: 'sf.rk2',
},
},
],
settings : {
getLastErrorModes : {
'2 dc and 3 server': {
dc: 2,
server: 3,
},
'1 and 2': {
2: 1,
server: 1,
},
'2': {
2: 1,
},
'3 and 4': {
sf: 2,
},
'3 or 4': {
sf: 1,
},
},
},
});
replTest.awaitReplication();
// Create collection to guard against timeouts due to file allocation.
assert.commandWorked(replTest.getPrimary().getDB('foo').createCollection('bar'));
replTest.awaitReplication();
var ensurePrimary = function(nodeId, expectedWritableNodes) {
jsTestLog('Node ' + nodeId + ' (' + replTest.nodes[nodeId].host + ') should be primary.');
replTest.waitForState(replTest.nodes[nodeId], ReplSetTest.State.PRIMARY, 60 * 1000);
primary = replTest.getPrimary();
primary.forceWriteMode('commands');
var writeConcern = {writeConcern: {w: expectedWritableNodes, wtimeout: 30 * 1000}};
assert.writeOK(primary.getDB('foo').bar.insert({x: 100}, writeConcern));
return primary;
};
// 2 should eventually stage a priority takeover from the primary.
var primary = ensurePrimary(2, 3);
jsTestLog('primary is now 2');
var config = assert.commandWorked(primary.adminCommand({replSetGetConfig: 1})).config;
jsTestLog('test configuration = ' + tojson(config));
jsTestLog('Setting up partitions: [0-1-2] [3] [4]');
conns[0].disconnect(conns[3]);
conns[0].disconnect(conns[4]);
conns[1].disconnect(conns[3]);
conns[1].disconnect(conns[4]);
conns[2].disconnect(conns[3]);
conns[2].disconnect(conns[4]);
conns[3].disconnect(conns[4]);
jsTestLog('Done setting up partitions');
jsTestLog('partitions: nodes with each set of brackets [N1, N2, N3] form a complete network.');
jsTestLog('partitions: [0-1-2] [3] [4] (only nodes 0 and 1 can replicate from primary node 2');
var doc = {x: 1};
// This timeout should be shorter in duration than the server parameter maxSyncSourceLagSecs.
// Some writes are expected to block for this 'timeout' duration before failing.
// Depending on the order of heartbeats (containing last committed op time) received
// by a node, it might hang up on its sync source. This may cause some of the write concern
// tests to fail.
var timeout = 20 * 1000;
jsTestLog('test1');
primary = ensurePrimary(2, 3);
jsTestLog('Non-existent write concern should be rejected.');
options = {writeConcern: {w: 'blahblah', wtimeout: timeout}};
assert.writeOK(primary.getDB('foo').bar.insert(doc));
var result = assert.writeError(primary.getDB('foo').bar.insert(doc, options));
assert.neq(null, result.getWriteConcernError());
assert.eq(ErrorCodes.UnknownReplWriteConcern, result.getWriteConcernError().code,
tojson(result.getWriteConcernError()));
jsTestLog('Write concern "3 or 4" should fail - 3 and 4 are not connected to the primary.');
var options = {writeConcern: {w: '3 or 4', wtimeout: timeout}};
assert.writeOK(primary.getDB('foo').bar.insert(doc));
result = primary.getDB('foo').bar.insert(doc, options);
assert.neq(null, result.getWriteConcernError());
assert(result.getWriteConcernError().errInfo.wtimeout);
conns[1].reconnect(conns[4]);
jsTestLog('partitions: [0-1-2] [1-4] [3] ' +
'(all nodes besides node 3 can replicate from primary node 2)');
primary = ensurePrimary(2, 4);
jsTestLog('Write concern "3 or 4" should work - 4 is now connected to the primary ' +
primary.host + ' via node 1 ' + replTest.nodes[1].host);
options = {writeConcern: {w: '3 or 4', wtimeout: timeout}};
assert.writeOK(primary.getDB('foo').bar.insert(doc));
assert.writeOK(primary.getDB('foo').bar.insert(doc, options));
jsTestLog('Write concern "3 and 4" should fail - 3 is not connected to the primary.');
options = {writeConcern: {w: '3 and 4', wtimeout: timeout}};
assert.writeOK(primary.getDB('foo').bar.insert(doc));
result = assert.writeError(primary.getDB('foo').bar.insert(doc, options));
assert.neq(null, result.getWriteConcernError());
assert(result.getWriteConcernError().errInfo.wtimeout, tojson(result.getWriteConcernError()));
conns[3].reconnect(conns[4]);
jsTestLog('partitions: [0-1-2] [1-4] [3-4] ' +
'(all secondaries can replicate from primary node 2)');
primary = ensurePrimary(2, 5);
jsTestLog('31004 should sync from 31001 (31026)');
jsTestLog('31003 should sync from 31004 (31024)');
jsTestLog('Write concern "3 and 4" should work - ' +
'nodes 3 and 4 are connected to primary via node 1.');
options = {writeConcern: {w: '3 and 4', wtimeout: timeout}};
assert.writeOK(primary.getDB('foo').bar.insert(doc));
assert.writeOK(primary.getDB('foo').bar.insert(doc, options));
jsTestLog('Write concern "2" - writes to primary only.');
options = {writeConcern: {w: '2', wtimeout: 0}};
assert.writeOK(primary.getDB('foo').bar.insert(doc));
assert.writeOK(primary.getDB('foo').bar.insert(doc, options));
jsTestLog('Write concern "1 and 2"');
options = {writeConcern: {w: '1 and 2', wtimeout: 0}};
assert.writeOK(primary.getDB('foo').bar.insert(doc));
assert.writeOK(primary.getDB('foo').bar.insert(doc, options));
jsTestLog('Write concern "2 dc and 3 server"');
primary = ensurePrimary(2, 5);
options = {writeConcern: {w: '2 dc and 3 server', wtimeout: timeout}};
assert.writeOK(primary.getDB('foo').bar.insert(doc));
assert.writeOK(primary.getDB('foo').bar.insert(doc, options));
jsTestLog('Bringing down current primary node 2 ' + primary.host +
' to allow next higher priority node 1 ' + replTest.nodes[1].host +
' to become primary.');
// Is this necessary since 3 will be connected to the new primary via node 4?
conns[1].reconnect(conns[3]);
conns[2].disconnect(conns[0]);
conns[2].disconnect(conns[1]);
// Is this necessary when we partition node 2 off from the rest of the nodes?
replTest.stop(2);
jsTestLog('partitions: [0-1] [2] [1-3-4] ' +
'(all secondaries except down node 2 can replicate from new primary node 1)');
// Node 1 with slightly higher priority will take over.
jsTestLog('1 must become primary here because otherwise the other members will take too ' +
'long timing out their old sync threads');
primary = ensurePrimary(1, 4);
jsTestLog('Write concern "3 and 4" should still work with new primary node 1 ' + primary.host);
options = {writeConcern: {w: '3 and 4', wtimeout: timeout}};
assert.writeOK(primary.getDB('foo').bar.insert(doc));
assert.writeOK(primary.getDB('foo').bar.insert(doc, options));
jsTestLog('Write concern "2" should fail because node 2 ' + replTest.nodes[2].host +
' is down.');
options = {writeConcern: {w: '2', wtimeout: timeout}};
assert.writeOK(primary.getDB('foo').bar.insert(doc));
result = assert.writeError(primary.getDB('foo').bar.insert(doc, options));
assert.neq(null, result.getWriteConcernError());
assert(result.getWriteConcernError().errInfo.wtimeout);
replTest.stopSet();
jsTestLog('tags.js SUCCESS');
}());
|
