diff options
author | Dianna Hohensee <dianna.hohensee@mongodb.com> | 2019-10-21 18:23:49 +0000 |
---|---|---|
committer | evergreen <evergreen@mongodb.com> | 2019-10-21 18:23:49 +0000 |
commit | a5887f3a5bd5bcc7398bf2bddc8c537b679e6a58 (patch) | |
tree | 02e4cdb7aa3b103841bfc7687b3d2962b286227f /jstests | |
parent | babc552483a0f8a2fff4c20354089b78da14baeb (diff) | |
download | mongo-a5887f3a5bd5bcc7398bf2bddc8c537b679e6a58.tar.gz |
SERVER-42358 Add new background collection validation test hook. Runs concurrently with tests.
Diffstat (limited to 'jstests')
-rw-r--r-- | jstests/hooks/run_validate_collections_background.js | 179 | ||||
-rw-r--r-- | jstests/hooks/validate_collections.js | 6 |
2 files changed, 183 insertions, 2 deletions
diff --git a/jstests/hooks/run_validate_collections_background.js b/jstests/hooks/run_validate_collections_background.js new file mode 100644 index 00000000000..5363b2860c0 --- /dev/null +++ b/jstests/hooks/run_validate_collections_background.js @@ -0,0 +1,179 @@ +/** + * Runs the validate command with {background:true} against all nodes (replica set members and + * standalone nodes, not sharded clusters) concurrently with running tests. + */ + +'use strict'; + +(function() { +load('jstests/libs/discover_topology.js'); // For Topology and DiscoverTopology. +load('jstests/libs/parallelTester.js'); // For Thread. + +if (typeof db === 'undefined') { + throw new Error( + "Expected mongo shell to be connected a server, but global 'db' object isn't defined"); +} + +const conn = db.getMongo(); +const topology = DiscoverTopology.findConnectedNodes(conn); + +/** + * Returns true if the error code is transient and does not indicate data corruption. + */ +const isIgnorableError = function ignorableError(codeName) { + if (codeName == "NamespaceNotFound" || codeName == "Interrupted" || + codeName == "CommandNotSupportedOnView" || codeName == "InterruptedAtShutdown" || + codeName == "InvalidViewDefinition") { + return true; + } + return false; +}; + +/** + * Runs validate commands with {background:true} against 'host' for all collections it possesses. + * + * Returns the cumulative command failure results, if there are any, in an object + * { ok: 0, error: [{cmd-res}, {cmd-res}, ... ]} + * Or simply OK if all cmds were successful. + * {ok: 1} + * + * This function should not throw if everything is working properly. + */ +const validateCollectionsBackgroundThread = function validateCollectionsBackground( + host, isIgnorableErrorFunc) { + // Calls 'func' with the print() function overridden to be a no-op. + const quietly = (func) => { + const printOriginal = print; + try { + print = Function.prototype; + func(); + } finally { + print = printOriginal; + } + }; + + // Suppress the log messages generated establishing new mongo connections. The + // run_validate_collections_background.js hook is executed frequently by resmoke.py and + // could lead to generating an overwhelming amount of log messages. + let conn; + quietly(() => { + conn = new Mongo(host); + }); + assert.neq(null, + conn, + "Failed to connect to host '" + host + "' for background collection validation"); + + if (!conn.adminCommand("serverStatus").storageEngine.supportsCheckpoints) { + print("Skipping background validation against test node: " + host + + " because its storage engine does not support background validation (checkpoints)."); + return {ok: 1}; + } + + // Filter out arbiters. + if (conn.adminCommand({isMaster: 1}).arbiterOnly) { + print("Skipping background validation against test node: " + host + + " because it is an arbiter and has no data."); + return {ok: 1}; + } + + print("Running background validation on all collections on test node: " + host); + + // Save a map of namespace to validate cmd results for any cmds that fail so that we can return + // the results afterwards. + let failedValidateResults = []; + + // Validate all collections in every database. + + const dbNames = + assert + .commandWorked(conn.adminCommand( + {"listDatabases": 1, "nameOnly": true, "$readPreference": {"mode": "nearest"}})) + .databases.map(function(z) { + return z.name; + }); + + for (let dbName of dbNames) { + let db = conn.getDB(dbName); + + const listCollRes = assert.commandWorked(db.runCommand({ + "listCollections": 1, + "nameOnly": true, + "filter": {$or: [{type: 'collection'}, {type: {$exists: false}}]}, + "$readPreference": {"mode": "nearest"}, + })); + const collectionNames = new DBCommandCursor(db, listCollRes).map(function(z) { + return z.name; + }); + + for (let collectionName of collectionNames) { + let res = conn.getDB(dbName).getCollection(collectionName).runCommand({ + "validate": collectionName, + background: true, + "$readPreference": {"mode": "nearest"} + }); + + if ((!res.ok && !isIgnorableErrorFunc(res.codeName)) || (res.valid === false)) { + failedValidateResults.push({"ns": dbName + "." + collectionName, "res": res}); + } + } + } + + // If any commands failed, format and return an error. + if (failedValidateResults.length) { + let errorsArray = []; + for (let nsAndRes of failedValidateResults) { + errorsArray.push({"namespace": nsAndRes.ns, "res": nsAndRes.res}); + } + + const heading = "Validate command(s) with {background:true} failed against mongod"; + print(heading + " '" + conn.host + "': \n" + tojson(errorsArray)); + + return {ok: 0, error: "Validate failure (search for the following heading): " + heading}; + } + + return {ok: 1}; +}; + +if (topology.type === Topology.kStandalone) { + let res = validateCollectionsBackgroundThread(topology.mongod); + assert.commandWorked( + res, + () => 'background collection validation against the standalone failed: ' + tojson(res)); +} else if (topology.type === Topology.kReplicaSet) { + const threads = []; + try { + for (let replicaMember of topology.nodes) { + const thread = + new Thread(validateCollectionsBackgroundThread, replicaMember, isIgnorableError); + threads.push(thread); + thread.start(); + } + } finally { + // Wait for each thread to finish and gather any errors. + let gatheredErrors = []; + const returnData = threads.map(thread => { + try { + thread.join(); + + // Calling returnData can cause an error thrown in the thread to be thrown again, so + // we do this in a try-catch block. + let res = thread.returnData(); + + if (!res.ok) { + gatheredErrors.push(res); + } + } catch (e) { + gatheredErrors.push(e); + } + }); + + if (gatheredErrors.length) { + throw new Error( + "Background collection validation was not successful against all replica set " + + "members: \n" + tojson(gatheredErrors)); + } + } +} else { + throw new Error('Unsupported topology configuration: ' + tojson(topology)); +} +})(); diff --git a/jstests/hooks/validate_collections.js b/jstests/hooks/validate_collections.js index 69085b8a1d6..9f4a9e8fc5f 100644 --- a/jstests/hooks/validate_collections.js +++ b/jstests/hooks/validate_collections.js @@ -43,9 +43,11 @@ function CollectionValidator() { jsTest.options().skipValidationNamespaces.length > 0) { let skippedCollections = []; for (let ns of jsTest.options().skipValidationNamespaces) { - // Strip off the database name from 'ns' to extract the collName. + // Attempt to strip the name of the database we are about to validate off of the + // namespace we wish to skip. If the replace() function does find a match with the + // database, then we know that the collection we want to skip is in the database we + // are about to validate. We will then put it in the 'filter' for later use. const collName = ns.replace(new RegExp('^' + db.getName() + '\.'), ''); - // Skip the collection 'collName' if the db name was removed from 'ns'. if (collName !== ns) { skippedCollections.push({name: {$ne: collName}}); } |