1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
|
// Wrapper around the validate command that can be used to validate index key counts.
'use strict';
function CollectionValidator() {
load('jstests/libs/parallelTester.js');
if (!(this instanceof CollectionValidator)) {
throw new Error('Please use "new CollectionValidator()"');
}
this.validateCollections = function(db, obj) {
function dumpCollection(coll, limit) {
print('Printing indexes in: ' + coll.getFullName());
printjson(coll.getIndexes());
print('Printing the first ' + limit + ' documents in: ' + coll.getFullName());
const res = coll.find().limit(limit);
while (res.hasNext()) {
printjson(res.next());
}
}
assert.eq(typeof db, 'object', 'Invalid `db` object, is the shell connected to a mongod?');
assert.eq(typeof obj, 'object', 'The `obj` argument must be an object');
assert(obj.hasOwnProperty('full'), 'Please specify whether to use full validation');
// Failed collection validation results are saved in failed_res.
let full_res = {ok: 1, failed_res: []};
// Don't run validate on view namespaces.
let filter = {type: 'collection'};
if (jsTest.options().skipValidationOnInvalidViewDefinitions) {
// If skipValidationOnInvalidViewDefinitions=true, then we avoid resolving the view
// catalog on the admin database.
//
// TODO SERVER-25493: Remove the $exists clause once performing an initial sync from
// versions of MongoDB <= 3.2 is no longer supported.
filter = {$or: [filter, {type: {$exists: false}}]};
}
// Optionally skip collections.
if (Array.isArray(jsTest.options().skipValidationNamespaces) &&
jsTest.options().skipValidationNamespaces.length > 0) {
let skippedCollections = [];
for (let ns of jsTest.options().skipValidationNamespaces) {
// Attempt to strip the name of the database we are about to validate off of the
// namespace we wish to skip. If the replace() function does find a match with the
// database, then we know that the collection we want to skip is in the database we
// are about to validate. We will then put it in the 'filter' for later use.
const collName = ns.replace(new RegExp('^' + db.getName() + '\.'), '');
if (collName !== ns) {
skippedCollections.push({name: {$ne: collName}});
}
}
filter = {$and: [filter, ...skippedCollections]};
}
let collInfo = db.getCollectionInfos(filter);
for (let collDocument of collInfo) {
const coll = db.getCollection(collDocument['name']);
const res = coll.validate(obj);
if (!res.ok || !res.valid) {
if (jsTest.options().skipValidationOnNamespaceNotFound &&
res.codeName === "NamespaceNotFound") {
// During a 'stopStart' backup/restore on the secondary node, the actual list of
// collections can be out of date if ops are still being applied from the oplog.
// In this case we skip the collection if the ns was not found at time of
// validation and continue to next.
print('Skipping collection validation for ' + coll.getFullName() +
' since collection was not found');
continue;
}
const host = db.getMongo().host;
print('Collection validation failed on host ' + host +
' with response: ' + tojson(res));
dumpCollection(coll, 100);
full_res.failed_res.push(res);
full_res.ok = 0;
}
}
return full_res;
};
// Run a separate thread to validate collections on each server in parallel.
const validateCollectionsThread = function(validatorFunc, host, skipFCV) {
try {
print('Running validate() on ' + host);
const conn = new Mongo(host);
conn.setSlaveOk();
jsTest.authenticate(conn);
// Skip validating collections for arbiters.
if (conn.getDB('admin').isMaster('admin').arbiterOnly === true) {
print('Skipping collection validation on arbiter ' + host);
return {ok: 1};
}
const requiredFCV = jsTest.options().forceValidationWithFeatureCompatibilityVersion;
if (requiredFCV && !skipFCV) {
// Make sure this node has the desired FCV as it may take time for the updates to
// replicate to the nodes that weren't part of the w=majority.
assert.soonNoExcept(() => {
checkFCV(conn.getDB('admin'), requiredFCV);
return true;
});
}
const dbNames = conn.getDBNames();
for (let dbName of dbNames) {
const validateRes = validatorFunc(conn.getDB(dbName), {full: true});
if (validateRes.ok !== 1) {
return {ok: 0, host: host, validateRes: validateRes};
}
}
return {ok: 1};
} catch (e) {
print('Exception caught in scoped thread running validationCollections on server: ' +
host);
return {ok: 0, error: e.toString(), stack: e.stack, host: host};
}
};
this.validateNodes = function(hostList, skipFCV) {
// We run the scoped threads in a try/finally block in case any thread throws an exception,
// in which case we want to still join all the threads.
let threads = [];
try {
hostList.forEach(host => {
const thread =
new Thread(validateCollectionsThread, this.validateCollections, host, skipFCV);
threads.push(thread);
thread.start();
});
} finally {
// Wait for each thread to finish. Throw an error if any thread fails.
const returnData = threads.map(thread => {
thread.join();
return thread.returnData();
});
returnData.forEach(res => {
assert.commandWorked(res, 'Collection validation failed');
});
}
};
}
// Ensure compatibility with existing callers. Cannot use `const` or `let` here since this file may
// be loaded more than once.
var validateCollections = new CollectionValidator().validateCollections;
|
