diff options
Diffstat (limited to 'jstests/hooks')
-rw-r--r-- | jstests/hooks/run_validate_collections.js | 73 | ||||
-rw-r--r-- | jstests/hooks/validate_collections.js | 175 |
2 files changed, 128 insertions, 120 deletions
diff --git a/jstests/hooks/run_validate_collections.js b/jstests/hooks/run_validate_collections.js index d95c9eb5c9f..772c2366113 100644 --- a/jstests/hooks/run_validate_collections.js +++ b/jstests/hooks/run_validate_collections.js @@ -4,22 +4,15 @@ (function() { assert.eq(typeof db, 'object', 'Invalid `db` object, is the shell connected to a mongod?'); - load('jstests/libs/parallelTester.js'); - function getDirectConnections(conn) { + function getConnectionStrings(conn) { // If conn does not point to a repl set, then this function returns [conn]. const res = conn.adminCommand({isMaster: 1}); - const connections = []; - if (res.hasOwnProperty('hosts')) { - for (let hostString of res.hosts) { - connections.push(new Mongo(hostString)); - } + return res.hosts; } else { - connections.push(conn); + return [conn.host]; } - - return connections; } function getConfigConnStr() { @@ -42,8 +35,8 @@ return db.isMaster().msg === 'isdbgrid'; } - function getServerList() { - const serverList = []; + function getHostList() { + let hostList = []; if (isMongos()) { // We're connected to a sharded cluster through a mongos. @@ -51,7 +44,7 @@ // 1) Add all the config servers to the server list. const configConnStr = getConfigConnStr(); const configServerReplSetConn = new Mongo(configConnStr); - serverList.push(...getDirectConnections(configServerReplSetConn)); + hostList = getConnectionStrings(configServerReplSetConn); // 2) Add shard members to the server list. const configDB = db.getSiblingDB('config'); @@ -60,61 +53,17 @@ while (cursor.hasNext()) { const shard = cursor.next(); const shardReplSetConn = new Mongo(shard.host); - serverList.push(...getDirectConnections(shardReplSetConn)); + hostList.push(...getConnectionStrings(shardReplSetConn)); } } else { // We're connected to a mongod. - serverList.push(...getDirectConnections(db.getMongo())); + hostList = getConnectionStrings(db.getMongo()); } - return serverList; + return hostList; } - // Run a separate thread to validate collections on each server in parallel. - var validateCollectionsThread = function(host, testData) { - load('jstests/hooks/validate_collections.js'); // For validateCollections. - TestData = testData; // Pass the TestData object from main thread. - - try { - print('Running validate() on ' + host); - const conn = new Mongo(host); - conn.setSlaveOk(); - jsTest.authenticate(conn); - - const dbNames = conn.getDBNames(); - for (let dbName of dbNames) { - if (!validateCollections(conn.getDB(dbName), {full: true})) { - return {ok: 0}; - } - } - return {ok: 1}; - } catch (e) { - print('Exception caught in scoped thread running validationCollections on server: ' + - host); - return {ok: 0, error: e.toString(), stack: e.stack}; - } - }; - - // We run the scoped threads in a try/finally block in case any thread throws an exception, in - // which case we want to still join all the threads. - let threads = []; - const serverList = getServerList(); + load('jstests/hooks/validate_collections.js'); // For CollectionValidator. + new CollectionValidator().validateNodes(getHostList()); - try { - serverList.forEach(server => { - const thread = new ScopedThread(validateCollectionsThread, server.host, TestData); - threads.push(thread); - thread.start(); - }); - } finally { - // Wait for each thread to finish. Throw an error if any thread fails. - const returnData = threads.map(thread => { - thread.join(); - return thread.returnData(); - }); - - returnData.forEach(res => { - assert.commandWorked(res, 'Collection validation failed'); - }); - } })(); diff --git a/jstests/hooks/validate_collections.js b/jstests/hooks/validate_collections.js index 31e5d26affe..71136a3e4cc 100644 --- a/jstests/hooks/validate_collections.js +++ b/jstests/hooks/validate_collections.js @@ -1,75 +1,134 @@ // Wrapper around the validate command that can be used to validate index key counts. 'use strict'; -function validateCollections(db, obj) { - function dumpCollection(coll, limit) { - print('Printing indexes in: ' + coll.getFullName()); - printjson(coll.getIndexes()); - - print('Printing the first ' + limit + ' documents in: ' + coll.getFullName()); - var res = coll.find().limit(limit); - while (res.hasNext()) { - printjson(res.next()); - } +function CollectionValidator() { + load('jstests/libs/parallelTester.js'); + + if (!(this instanceof CollectionValidator)) { + throw new Error('Please use "new CollectionValidator()"'); } - assert.eq(typeof db, 'object', 'Invalid `db` object, is the shell connected to a mongod?'); - assert.eq(typeof obj, 'object', 'The `obj` argument must be an object'); - assert(obj.hasOwnProperty('full'), 'Please specify whether to use full validation'); + this.validateCollections = function(db, obj) { + function dumpCollection(coll, limit) { + print('Printing indexes in: ' + coll.getFullName()); + printjson(coll.getIndexes()); - var full = obj.full; + print('Printing the first ' + limit + ' documents in: ' + coll.getFullName()); + const res = coll.find().limit(limit); + while (res.hasNext()) { + printjson(res.next()); + } + } - var success = true; + assert.eq(typeof db, 'object', 'Invalid `db` object, is the shell connected to a mongod?'); + assert.eq(typeof obj, 'object', 'The `obj` argument must be an object'); + assert(obj.hasOwnProperty('full'), 'Please specify whether to use full validation'); - var adminDB = db.getSiblingDB("admin"); + const full = obj.full; - // Don't run validate on view namespaces. - let filter = {type: "collection"}; - if (jsTest.options().skipValidationOnInvalidViewDefinitions) { - // If skipValidationOnInvalidViewDefinitions=true, then we avoid resolving the view catalog - // on the admin database. - // - // TODO SERVER-25493: Remove the $exists clause once performing an initial sync from - // versions of MongoDB <= 3.2 is no longer supported. - filter = {$or: [filter, {type: {$exists: false}}]}; - } + let success = true; - // Optionally skip collections. - if (Array.isArray(jsTest.options().skipValidationNamespaces) && - jsTest.options().skipValidationNamespaces.length > 0) { - let skippedCollections = []; - for (let ns of jsTest.options().skipValidationNamespaces) { - // Strip off the database name from 'ns' to extract the collName. - const collName = ns.replace(new RegExp('^' + db.getName() + '\.'), ''); - // Skip the collection 'collName' if the db name was removed from 'ns'. - if (collName !== ns) { - skippedCollections.push({name: {$ne: collName}}); + // Don't run validate on view namespaces. + let filter = {type: "collection"}; + if (jsTest.options().skipValidationOnInvalidViewDefinitions) { + // If skipValidationOnInvalidViewDefinitions=true, then we avoid resolving the view + // catalog on the admin database. + // + // TODO SERVER-25493: Remove the $exists clause once performing an initial sync from + // versions of MongoDB <= 3.2 is no longer supported. + filter = {$or: [filter, {type: {$exists: false}}]}; + } + + // Optionally skip collections. + if (Array.isArray(jsTest.options().skipValidationNamespaces) && + jsTest.options().skipValidationNamespaces.length > 0) { + let skippedCollections = []; + for (let ns of jsTest.options().skipValidationNamespaces) { + // Strip off the database name from 'ns' to extract the collName. + const collName = ns.replace(new RegExp('^' + db.getName() + '\.'), ''); + // Skip the collection 'collName' if the db name was removed from 'ns'. + if (collName !== ns) { + skippedCollections.push({name: {$ne: collName}}); + } } + filter = {$and: [filter, ...skippedCollections]}; } - filter = {$and: [filter, ...skippedCollections]}; - } - let collInfo = db.getCollectionInfos(filter); - for (var collDocument of collInfo) { - var coll = db.getCollection(collDocument["name"]); - var res = coll.validate(full); - - if (!res.ok || !res.valid) { - if (jsTest.options().skipValidationOnNamespaceNotFound && - res.errmsg === 'ns not found') { - // During a 'stopStart' backup/restore on the secondary node, the actual list of - // collections can be out of date if ops are still being applied from the oplog. In - // this case we skip the collection if the ns was not found at time of validation - // and continue to next. - print('Skipping collection validation for ' + coll.getFullName() + - ' since collection was not found'); - continue; + let collInfo = db.getCollectionInfos(filter); + for (let collDocument of collInfo) { + const coll = db.getCollection(collDocument["name"]); + const res = coll.validate(full); + + if (!res.ok || !res.valid) { + if (jsTest.options().skipValidationOnNamespaceNotFound && + res.errmsg === 'ns not found') { + // During a 'stopStart' backup/restore on the secondary node, the actual list of + // collections can be out of date if ops are still being applied from the oplog. + // In this case we skip the collection if the ns was not found at time of + // validation and continue to next. + print('Skipping collection validation for ' + coll.getFullName() + + ' since collection was not found'); + continue; + } + print('Collection validation failed with response: ' + tojson(res)); + dumpCollection(coll, 100); + success = false; } - print('Collection validation failed with response: ' + tojson(res)); - dumpCollection(coll, 100); - success = false; } - } - return success; + return success; + }; + + // Run a separate thread to validate collections on each server in parallel. + const validateCollectionsThread = function(validatorFunc, host, testData) { + TestData = testData; // Pass the TestData object from main thread. + + try { + print('Running validate() on ' + host); + const conn = new Mongo(host); + conn.setSlaveOk(); + jsTest.authenticate(conn); + + const dbNames = conn.getDBNames(); + for (let dbName of dbNames) { + if (!validatorFunc(conn.getDB(dbName), {full: true})) { + return {ok: 0}; + } + } + return {ok: 1}; + } catch (e) { + print('Exception caught in scoped thread running validationCollections on server: ' + + host); + return {ok: 0, error: e.toString(), stack: e.stack}; + } + }; + + this.validateNodes = function(hostList) { + // We run the scoped threads in a try/finally block in case any thread throws an exception, + // in which case we want to still join all the threads. + let threads = []; + + try { + hostList.forEach(host => { + const thread = new ScopedThread( + validateCollectionsThread, this.validateCollections, host, TestData); + threads.push(thread); + thread.start(); + }); + } finally { + // Wait for each thread to finish. Throw an error if any thread fails. + const returnData = threads.map(thread => { + thread.join(); + return thread.returnData(); + }); + + returnData.forEach(res => { + assert.commandWorked(res, 'Collection validation failed'); + }); + } + }; } + +// Ensure compatability with existing callers. Cannot use `const` or `let` here since this file may +// be loaded more than once. +var validateCollections = new CollectionValidator().validateCollections; |