1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
|
/** TODO: fix all these comments
* Runs the validate command with {background:true} against all nodes (replica set members and
* standalone nodes, not sharded clusters) concurrently with running tests.
*/
'use strict';
(function() {
print("yipppeee ki ya were in a background thread!!!");
load('jstests/libs/discover_topology.js'); // For Topology and DiscoverTopology.
load('jstests/libs/parallelTester.js'); // For Thread.
if (typeof db === 'undefined') {
throw new Error(
"Expected mongo shell to be connected a server, but global 'db' object isn't defined");
}
// Disable implicit sessions so FSM workloads that kill random sessions won't interrupt the
// operations in this test that aren't resilient to interruptions.
TestData.disableImplicitSessions = true;
const conn = db.getMongo();
const topology = DiscoverTopology.findConnectedNodes(conn);
/**
* Returns true if the error code is transient and does not indicate data corruption.
*/
const isIgnorableError = function ignorableError(codeName) {
if (codeName == "NamespaceNotFound" || codeName == "Interrupted" ||
codeName == "CommandNotSupportedOnView" || codeName == "InterruptedAtShutdown" ||
codeName == "InvalidViewDefinition") {
return true;
}
return false;
};
/**
* Runs validate commands with {background:true} against 'host' for all collections it possesses.
*
* Returns the cumulative command failure results, if there are any, in an object
* { ok: 0, error: [{cmd-res}, {cmd-res}, ... ]}
* Or simply OK if all cmds were successful.
* {ok: 1}
*
* This function should not throw if everything is working properly.
*/
const validateCollectionsBackgroundThread = function validateCollectionsBackground(
host, isIgnorableErrorFunc) {
// Calls 'func' with the print() function overridden to be a no-op.
const quietly = (func) => {
const printOriginal = print;
try {
print = Function.prototype;
func();
} finally {
print = printOriginal;
}
};
// Suppress the log messages generated establishing new mongo connections. The
// run_validate_collections_background.js hook is executed frequently by resmoke.py and
// could lead to generating an overwhelming amount of log messages.
let conn;
quietly(() => {
conn = new Mongo(host);
});
assert.neq(null,
conn,
"Failed to connect to host '" + host + "' for background collection validation");
// Filter out arbiters.
if (conn.adminCommand({isMaster: 1}).arbiterOnly) {
print("Skipping background validation against test node: " + host +
" because it is an arbiter and has no data.");
return {ok: 1};
}
print("Running background validation on all collections on test node: " + host);
// Save a map of namespace to validate cmd results for any cmds that fail so that we can return
// the results afterwards.
let failedValidateResults = [];
// Validate all collections in every database.
const dbNames =
assert
.commandWorked(conn.adminCommand(
{"listDatabases": 1, "nameOnly": true, "$readPreference": {"mode": "nearest"}}))
.databases.map(function(z) {
return z.name;
});
conn.adminCommand({configureFailPoint: "crashOnMultikeyValidateFailure", mode: "alwaysOn"});
for (let dbName of dbNames) {
let db = conn.getDB(dbName);
const listCollRes = assert.commandWorked(db.runCommand({
"listCollections": 1,
"nameOnly": true,
"filter": {$or: [{type: 'collection'}, {type: {$exists: false}}]},
"$readPreference": {"mode": "nearest"},
}));
const collectionNames = new DBCommandCursor(db, listCollRes).map(function(z) {
return z.name;
});
for (let collectionName of collectionNames) {
let res = conn.getDB(dbName).getCollection(collectionName).runCommand({
"validate": collectionName,
background: true,
"$readPreference": {"mode": "nearest"}
});
if ((!res.ok && !isIgnorableErrorFunc(res.codeName)) || (res.valid === false)) {
failedValidateResults.push({"ns": dbName + "." + collectionName, "res": res});
}
}
}
conn.adminCommand({configureFailPoint: "crashOnMultikeyValidateFailure", mode: "off"});
// If any commands failed, format and return an error.
if (failedValidateResults.length) {
let errorsArray = [];
for (let nsAndRes of failedValidateResults) {
errorsArray.push({"namespace": nsAndRes.ns, "res": nsAndRes.res});
}
const heading = "Validate command(s) with {background:true} failed against mongod";
print(heading + " '" + conn.host + "': \n" + tojson(errorsArray));
return {ok: 0, error: "Validate failure (search for the following heading): " + heading};
}
return {ok: 1};
};
if (topology.type === Topology.kStandalone) {
let res = validateCollectionsBackgroundThread(topology.mongod);
assert.commandWorked(
res,
() => 'background collection validation against the standalone failed: ' + tojson(res));
} else if (topology.type === Topology.kReplicaSet) {
const threads = [];
try {
for (let replicaMember of topology.nodes) {
const thread =
new Thread(validateCollectionsBackgroundThread, replicaMember, isIgnorableError);
threads.push(thread);
thread.start();
}
} finally {
// Wait for each thread to finish and gather any errors.
let gatheredErrors = [];
const returnData = threads.map(thread => {
try {
thread.join();
// Calling returnData can cause an error thrown in the thread to be thrown again, so
// we do this in a try-catch block.
let res = thread.returnData();
if (!res.ok) {
gatheredErrors.push(res);
}
} catch (e) {
gatheredErrors.push(e);
}
});
if (gatheredErrors.length) {
throw new Error(
"Background collection validation was not successful against all replica set " +
"members: \n" + tojson(gatheredErrors));
}
}
} else {
throw new Error('Unsupported topology configuration: ' + tojson(topology));
}
})();
|
