1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
|
/**
* This test tests that replSetGetStatus returns initial sync stats while initial sync is in
* progress.
*
* @tags: [requires_fcv_49]
*/
(function() {
"use strict";
load("jstests/libs/fail_point_util.js");
var name = 'initial_sync_replSetGetStatus';
var replSet = new ReplSetTest({
name: name,
nodes: 1,
});
replSet.startSet();
replSet.initiate();
var primary = replSet.getPrimary();
const barColl = primary.getDB('pretest').bar;
assert.commandWorked(barColl.insert({a: 1}));
assert.commandWorked(barColl.insert({a: 2}));
assert.commandWorked(barColl.insert({a: 3}));
var coll = primary.getDB('test').foo;
assert.commandWorked(coll.insert({a: 1}));
assert.commandWorked(coll.insert({a: 2}));
// Add a secondary node but make it hang before copying databases.
let secondary = replSet.add(
{rsConfig: {votes: 0, priority: 0}, setParameter: {'collectionClonerBatchSize': 2}});
secondary.setSecondaryOk();
var failPointBeforeCopying = configureFailPoint(secondary, 'initialSyncHangBeforeCopyingDatabases');
var failPointBeforeFinish = configureFailPoint(secondary, 'initialSyncHangBeforeFinish');
let failPointAfterNumDocsCopied =
configureFailPoint(secondary,
'initialSyncHangDuringCollectionClone',
{namespace: barColl.getFullName(), numDocsToClone: 2});
replSet.reInitiate();
// Wait for initial sync to pause before it copies the databases.
failPointBeforeCopying.wait();
// Test that replSetGetStatus returns the correct results while initial sync is in progress.
var res = assert.commandWorked(secondary.adminCommand({replSetGetStatus: 1}));
assert(res.initialSyncStatus,
() => "Response should have an 'initialSyncStatus' field: " + tojson(res));
res = assert.commandWorked(secondary.adminCommand({replSetGetStatus: 1, initialSync: 0}));
assert(!res.initialSyncStatus,
() => "Response should not have an 'initialSyncStatus' field: " + tojson(res));
assert.commandFailedWithCode(secondary.adminCommand({replSetGetStatus: 1, initialSync: "t"}),
ErrorCodes.TypeMismatch);
assert.commandWorked(coll.insert({a: 3}));
assert.commandWorked(coll.insert({a: 4}));
// Let initial sync continue working.
failPointBeforeCopying.off();
// Wait for initial sync to pause halfway through cloning the 'pretest.bar' collection.
failPointAfterNumDocsCopied.wait();
const pretestDbRes = assert.commandWorked(secondary.adminCommand({replSetGetStatus: 1}));
assert.gt(pretestDbRes.initialSyncStatus.totalInitialSyncElapsedMillis, 0);
assert.gt(pretestDbRes.initialSyncStatus.remainingInitialSyncEstimatedMillis, 0);
assert.gt(pretestDbRes.initialSyncStatus.approxTotalDataSize, 0);
assert.eq(pretestDbRes.initialSyncStatus.databases.pretest.collections, 1);
assert.eq(pretestDbRes.initialSyncStatus.databases.pretest.clonedCollections, 0);
let barCollRes = pretestDbRes.initialSyncStatus.databases.pretest["pretest.bar"];
assert.eq(barCollRes.documentsToCopy, 3);
assert.gte(barCollRes.documentsCopied, 2);
assert.gt(barCollRes.bytesToCopy, 0);
assert.gt(barCollRes.approxBytesCopied, 0);
assert.lt(barCollRes.approxBytesCopied, barCollRes.bytesToCopy);
assert.lt(barCollRes.approxBytesCopied, pretestDbRes.initialSyncStatus.approxTotalDataSize);
const bytesCopiedAdminDb =
pretestDbRes.initialSyncStatus.databases.admin["admin.system.version"].approxBytesCopied +
pretestDbRes.initialSyncStatus.databases.admin["admin.system.keys"].approxBytesCopied;
assert.eq(pretestDbRes.initialSyncStatus.approxTotalBytesCopied,
bytesCopiedAdminDb + barCollRes.approxBytesCopied);
assert.gt(pretestDbRes.initialSyncStatus.approxTotalBytesCopied, 0);
// The server still has the 'pretest' and 'test' dbs to finish cloning.
assert.eq(pretestDbRes.initialSyncStatus.databases.databasesCloned, 2);
assert.eq(pretestDbRes.initialSyncStatus.databases.databasesToClone, 2);
failPointAfterNumDocsCopied.off();
// Wait for initial sync to pause right before it finishes.
failPointBeforeFinish.wait();
// Test that replSetGetStatus returns the correct results when initial sync is at the very end.
const endOfCloningRes = assert.commandWorked(secondary.adminCommand({replSetGetStatus: 1}));
assert(endOfCloningRes.initialSyncStatus,
() => "Response should have an 'initialSyncStatus' field: " + tojson(res));
// It is possible that we update the config document after going through a reconfig. So make sure
// we account for this.
assert.gte(endOfCloningRes.initialSyncStatus.appliedOps, 3);
// Assert metrics have progressed in the right direction since the last time we checked the metrics.
assert.gt(endOfCloningRes.initialSyncStatus.totalInitialSyncElapsedMillis,
pretestDbRes.initialSyncStatus.totalInitialSyncElapsedMillis);
assert.lt(endOfCloningRes.initialSyncStatus.remainingInitialSyncEstimatedMillis,
pretestDbRes.initialSyncStatus.remainingInitialSyncEstimatedMillis);
assert.gt(endOfCloningRes.initialSyncStatus.approxTotalBytesCopied,
pretestDbRes.initialSyncStatus.approxTotalBytesCopied);
assert.eq(endOfCloningRes.initialSyncStatus.approxTotalDataSize,
pretestDbRes.initialSyncStatus.approxTotalDataSize);
assert.eq(endOfCloningRes.initialSyncStatus.failedInitialSyncAttempts, 0);
assert.eq(endOfCloningRes.initialSyncStatus.maxFailedInitialSyncAttempts, 10);
assert.eq(endOfCloningRes.initialSyncStatus.databases.databasesCloned, 4);
assert.eq(endOfCloningRes.initialSyncStatus.databases.databasesToClone, 0);
assert.eq(endOfCloningRes.initialSyncStatus.databases.pretest.collections, 1);
assert.eq(endOfCloningRes.initialSyncStatus.databases.pretest.clonedCollections, 1);
barCollRes = endOfCloningRes.initialSyncStatus.databases.pretest["pretest.bar"];
assert.eq(barCollRes.documentsToCopy, 3);
assert.eq(barCollRes.documentsCopied, 3);
assert.eq(barCollRes.indexes, 1);
assert.eq(barCollRes.fetchedBatches, 2);
assert.gt(barCollRes.bytesToCopy, 0);
assert.eq(barCollRes.approxBytesCopied, barCollRes.bytesToCopy);
let fooCollRes = endOfCloningRes.initialSyncStatus.databases.test["test.foo"];
assert.eq(endOfCloningRes.initialSyncStatus.databases.test.collections, 1);
assert.eq(endOfCloningRes.initialSyncStatus.databases.test.clonedCollections, 1);
assert.eq(fooCollRes.documentsToCopy, 4);
assert.eq(fooCollRes.documentsCopied, 4);
assert.eq(fooCollRes.indexes, 1);
assert.eq(fooCollRes.fetchedBatches, 2);
assert.gt(fooCollRes.bytesToCopy, 0);
assert.eq(fooCollRes.approxBytesCopied, fooCollRes.bytesToCopy);
assert.eq(endOfCloningRes.initialSyncStatus.approxTotalDataSize,
endOfCloningRes.initialSyncStatus.approxTotalBytesCopied);
assert.eq(endOfCloningRes.initialSyncStatus.approxTotalBytesCopied,
fooCollRes.approxBytesCopied + barCollRes.approxBytesCopied + bytesCopiedAdminDb);
// Let initial sync finish and get into secondary state.
failPointBeforeFinish.off();
replSet.awaitSecondaryNodes(60 * 1000);
// Test that replSetGetStatus returns the correct results after initial sync is finished.
res = assert.commandWorked(secondary.adminCommand({replSetGetStatus: 1}));
assert(!res.initialSyncStatus,
() => "Response should not have an 'initialSyncStatus' field: " + tojson(res));
assert.commandFailedWithCode(secondary.adminCommand({replSetGetStatus: 1, initialSync: "m"}),
ErrorCodes.TypeMismatch);
assert.eq(0,
secondary.getDB('local')['temp_oplog_buffer'].find().itcount(),
"Oplog buffer was not dropped after initial sync");
replSet.stopSet();
})();
|
