From 86884b98e44f4a77f5b66970a6138874f2b98cf3 Mon Sep 17 00:00:00 2001 From: Max Hirschhorn Date: Sat, 20 Feb 2016 16:04:44 -0500 Subject: SERVER-21881 Use fsync+lock when checking dbhash in concurrency suite. The fsync+lock operation prevents the TTL monitor from running on the primary after awaiting replication and before getting the dbhashes from all the nodes in the replica set. Otherwise, a primary's dbhash can reflect a delete operation that hasn't yet replicated to all of the secondaries, leading to a spurious dbhash mismatch. (cherry picked from commit 41f296cdc37f92f8de0ea3d17109693c4f0befb6) --- jstests/concurrency/fsm_libs/cluster.js | 197 +++++++++++++-------- jstests/concurrency/fsm_libs/runner.js | 46 +++-- jstests/concurrency/fsm_workloads/collmod.js | 3 +- .../fsm_workloads/indexed_insert_ttl.js | 3 +- 4 files changed, 160 insertions(+), 89 deletions(-) diff --git a/jstests/concurrency/fsm_libs/cluster.js b/jstests/concurrency/fsm_libs/cluster.js index f6d5fc9a793..889cf316d85 100644 --- a/jstests/concurrency/fsm_libs/cluster.js +++ b/jstests/concurrency/fsm_libs/cluster.js @@ -109,7 +109,6 @@ var Cluster = function(options) { mongod: [] }; var nextConn = 0; - var primaries = []; var replSets = []; // TODO: Define size of replica set from options @@ -179,7 +178,6 @@ var Cluster = function(options) { i = 0; while (rsTest) { this._addReplicaSetConns(rsTest); - primaries.push(rsTest.getPrimary()); replSets.push(rsTest); ++i; rsTest = st['rs' + i]; @@ -210,7 +208,6 @@ var Cluster = function(options) { rst.awaitSecondaryNodes(); conn = rst.getPrimary(); - primaries = [conn]; replSets = [rst]; this.teardown = function teardown() { @@ -417,88 +414,142 @@ var Cluster = function(options) { return this.isSharded() && options.enableBalancer; }; - this.awaitReplication = function awaitReplication(message) { + this.checkDBHashes = function checkDBHashes(rst, dbBlacklist, phase) { assert(initialized, 'cluster must be initialized first'); - if (this.isReplication()) { - var wc = { - writeConcern: { - w: replSetNodes, // all nodes in replica set - wtimeout: 300000 // wait up to 5 minutes + assert(this.isReplication(), 'cluster is not a replica set'); + + // Use liveNodes.master instead of getPrimary() to avoid the detection of a new primary. + var primary = rst.liveNodes.master; + + var res = primary.adminCommand({ listDatabases: 1 }); + assert.commandWorked(res); + + res.databases.forEach(dbInfo => { + var dbName = dbInfo.name; + if (Array.contains(dbBlacklist, dbName)) { + return; + } + + var dbHashes = rst.getHashes(dbName); + var primaryDBHash = dbHashes.master; + assert.commandWorked(primaryDBHash); + + dbHashes.slaves.forEach(secondaryDBHash => { + assert.commandWorked(secondaryDBHash); + + var primaryNumCollections = Object.keys(primaryDBHash.collections).length; + var secondaryNumCollections = Object.keys(secondaryDBHash.collections).length; + + assert.eq(primaryNumCollections, secondaryNumCollections, + phase + ', the primary and secondary have a different number of' + + ' collections: ' + tojson(dbHashes)); + + // Only compare the dbhashes of non-capped collections because capped collections + // are not necessarily truncated at the same points across replica set members. + var collNames = Object.keys(primaryDBHash.collections).filter(collName => + !primary.getDB(dbName)[collName].isCapped()); + + collNames.forEach(collName => { + assert.eq(primaryDBHash.collections[collName], + secondaryDBHash.collections[collName], + phase + ', the primary and secondary have a different hash for the' + + ' collection ' + dbName + '.' + collName + ': ' + tojson(dbHashes)); + }); + + if (collNames.length === primaryNumCollections) { + // If the primary and secondary have the same hashes for all the collections on + // the database and there aren't any capped collections, then the hashes for the + // whole database should match. + assert.eq(primaryDBHash.md5, + secondaryDBHash.md5, + phase + ', the primary and secondary have a different hash for the ' + + dbName + ' database: ' + tojson(dbHashes)); } - }; - primaries.forEach(function(primary) { - var startTime = Date.now(); - jsTest.log(primary.host + ': awaitReplication started ' + message); - - // Insert a document with a writeConcern for all nodes in the replica set to - // ensure that all previous workload operations have completed on secondaries - var result = primary.getDB('test').fsm_teardown.insert({ a: 1 }, wc); - assert.writeOK(result, 'teardown insert failed: ' + tojson(result)); - - var totalTime = Date.now() - startTime; - jsTest.log(primary.host + ': awaitReplication ' + message + ' completed in ' + - totalTime + ' ms'); }); - } + }); }; - // Returns true if the specified DB contains a capped collection. - var containsCappedCollection = function containsCappedCollection(db) { - return db.getCollectionNames().some(coll => db[coll].isCapped()); - }; + this.checkReplicationConsistency = function checkReplicationConsistency(dbBlacklist, + phase, + ttlIndexExists) { + assert(initialized, 'cluster must be initialized first'); - // Checks dbHashes for databases that are not on the blacklist. - // All replica set nodes are checked. - this.checkDbHashes = function checkDbHashes(dbBlacklist, message) { - if (!this.isReplication() || this.isBalancerEnabled()) { + if (!this.isReplication()) { return; } - var res = this.getDB('admin').runCommand('listDatabases'); - assert.commandWorked(res); + var shouldCheckDBHashes = !this.isBalancerEnabled(); - res.databases.forEach(function(dbInfo) { - if (Array.contains(dbBlacklist, dbInfo.name)) { - return; + replSets.forEach(rst => { + var startTime = Date.now(); + var res; + + // Use liveNodes.master instead of getPrimary() to avoid the detection of a new primary. + var primary = rst.liveNodes.master; + jsTest.log('Starting consistency checks for replica set with ' + primary.host + + ' assumed to still be primary, ' + phase); + + if (shouldCheckDBHashes && ttlIndexExists) { + // Lock the primary to prevent the TTL monitor from deleting expired documents in + // the background while we are getting the dbhashes of the replica set members. + assert.commandWorked(primary.adminCommand({ fsync: 1, lock: 1 }), + phase + ', failed to lock the primary'); } - var hasCappedColl = containsCappedCollection(this.getDB(dbInfo.name)); - - replSets.forEach(function(replSet) { - var hashes = replSet.getHashes(dbInfo.name); - var masterHashes = hashes.master; - assert.commandWorked(masterHashes); - var dbHash = masterHashes.md5; - - hashes.slaves.forEach(function(slaveHashes) { - assert.commandWorked(slaveHashes); - assert.eq(Object.keys(masterHashes.collections).length, - Object.keys(slaveHashes.collections).length, - message + ' dbHash number of collections in db ' + - dbInfo.name + ' ' + tojson(hashes)); - - if (!hasCappedColl) { - // dbHash on a DB not containing a capped collection should match. - assert.eq(dbHash, - slaveHashes.md5, - message + ' dbHash inconsistency for db ' + - dbInfo.name + ' ' + tojson(hashes)); - } else { - // dbHash on a DB containing a capped collection will not return - // consistent results between the replica set, so we only - // check non-capped collections in the DB. - var collNames = Object.keys(masterHashes.collections).filter( - coll => - !this.getDB(dbInfo.name)[coll].isCapped()); - collNames.forEach(function(coll) { - assert.eq(masterHashes.collections[coll], - slaveHashes.collections[coll], - message + ' dbHash inconsistency for collection ' + coll + - ' in db ' + dbInfo.name + ' ' + tojson(hashes)); - }, this); + + var activeException = false; + var msg; + + try { + // Get the latest optime from the primary. + var replSetStatus = primary.adminCommand({ replSetGetStatus: 1 }); + assert.commandWorked(replSetStatus, + phase + ', error getting replication status'); + + var primaryInfo = replSetStatus.members.find(memberInfo => memberInfo.self); + assert(primaryInfo !== undefined, + phase + ', failed to find self in replication status: ' + + tojson(replSetStatus)); + + // Wait for all previous workload operations to complete. We use the "getLastError" + // command rather than a replicated write because the primary is currently + // fsyncLock()ed to prevent the TTL monitor from running. + res = primary.getDB('test').runCommand({ + getLastError: 1, + w: replSetNodes, + wtimeout: 5 * 60 * 1000, + wOpTime: primaryInfo.optime + }); + assert.commandWorked(res, phase + ', error awaiting replication'); + + if (shouldCheckDBHashes) { + // Compare the dbhashes of the primary and secondaries. + this.checkDBHashes(rst, dbBlacklist); + } + } catch (e) { + activeException = true; + throw e; + } finally { + if (shouldCheckDBHashes && ttlIndexExists) { + // Allow writes on the primary. + res = primary.adminCommand({ fsyncUnlock: 1 }); + + // Returning early would suppress the exception rethrown in the catch block. + if (!res.ok) { + msg = phase + ', failed to unlock the primary, which may cause this' + + ' test to hang: ' + tojson(res); + if (activeException) { + jsTest.log(msg); + } else { + throw new Error(msg); + } } - }, this); - }, this); - }, this); + } + } + + var totalTime = Date.now() - startTime; + jsTest.log('Finished consistency checks of replica set with ' + primary.host + + ' as primary in ' + totalTime + ' ms, ' + phase); + }); }; this.recordConfigServerData = function recordConfigServerData(configServer) { diff --git a/jstests/concurrency/fsm_libs/runner.js b/jstests/concurrency/fsm_libs/runner.js index 09f788a8a59..ff27744e596 100644 --- a/jstests/concurrency/fsm_libs/runner.js +++ b/jstests/concurrency/fsm_libs/runner.js @@ -427,17 +427,16 @@ var runner = (function() { jsTest.log('End of schedule'); } - function cleanupWorkload(workload, context, cluster, errors, header, dbHashBlacklist) { + function cleanupWorkload(workload, context, cluster, errors, header, dbHashBlacklist, + ttlIndexExists) { // Returns true if the workload's teardown succeeds and false if the workload's // teardown fails. try { - // Ensure that secondaries have caught up before workload teardown. - cluster.awaitReplication('before workload teardown'); - - // Check dbHash, for all DBs not in dbHashBlacklist, on all nodes - // before the workload's teardown method is called. - cluster.checkDbHashes(dbHashBlacklist, 'before workload teardown'); + // Ensure that all data has replicated correctly to the secondaries before calling the + // workload's teardown method. + var phase = 'before workload ' + workload + ' teardown'; + cluster.checkReplicationConsistency(dbHashBlacklist, phase, ttlIndexExists); } catch (e) { errors.push(new WorkloadFailure(e.toString(), e.stack, 'main', header + ' checking consistency on secondaries')); @@ -485,6 +484,7 @@ var runner = (function() { var teardownFailed = false; var startTime = Date.now(); // Initialize in case setupWorkload fails below. var totalTime; + var ttlIndexExists; jsTest.log('Workload(s) started: ' + workloads.join(' ')); @@ -522,11 +522,19 @@ var runner = (function() { 'Foreground ' + e.workloads.join(' ')))); } } finally { + // Checking that the data is consistent across the primary and secondaries requires + // additional complexity to prevent writes from occurring in the background on the + // primary due to the TTL monitor. If none of the workloads actually created any TTL + // indexes (and we dropped the data of any previous workloads), then don't expend any + // additional effort in trying to handle that case. + ttlIndexExists = workloads.some(workload => + context[workload].config.data.ttlIndexExists); + // Call each foreground workload's teardown function. After all teardowns have completed // check if any of them failed. var cleanupResults = cleanup.map(workload => cleanupWorkload(workload, context, cluster, errors, - 'Foreground', dbHashBlacklist)); + 'Foreground', dbHashBlacklist, ttlIndexExists)); teardownFailed = cleanupResults.some(success => (success === false)); totalTime = Date.now() - startTime; @@ -544,12 +552,14 @@ var runner = (function() { // Throw any existing errors so that the schedule aborts. throwError(errors); - // Ensure that secondaries have caught up after workload teardown. - cluster.awaitReplication('after workload-group teardown and data clean-up'); + // All workload data should have been dropped at this point, so there shouldn't be any TTL + // indexes. + ttlIndexExists = false; - // Check dbHash, for all DBs not in dbHashBlacklist, on all nodes - // after the workload's teardown method is called. - cluster.checkDbHashes(dbHashBlacklist, 'after workload-group teardown and data clean-up'); + // Ensure that all operations replicated correctly to the secondaries. + cluster.checkReplicationConsistency(dbHashBlacklist, + 'after workload-group teardown and data clean-up', + ttlIndexExists); } function runWorkloads(workloads, @@ -685,10 +695,18 @@ var runner = (function() { } } finally { try { + // Checking that the data is consistent across the primary and secondaries requires + // additional complexity to prevent writes from occurring in the background on the + // primary due to the TTL monitor. If none of the workloads actually created any TTL + // indexes (and we dropped the data of any previous workloads), then don't expend + // any additional effort in trying to handle that case. + var ttlIndexExists = bgWorkloads.some(bgWorkload => + bgContext[bgWorkload].config.data.ttlIndexExists); + // Call each background workload's teardown function. bgCleanup.forEach(bgWorkload => cleanupWorkload(bgWorkload, bgContext, cluster, errors, 'Background', - dbHashBlacklist)); + dbHashBlacklist, ttlIndexExists)); // TODO: Call cleanupWorkloadData() on background workloads here if no background // workload teardown functions fail. diff --git a/jstests/concurrency/fsm_workloads/collmod.js b/jstests/concurrency/fsm_workloads/collmod.js index 758a02634d9..efed90ef9d1 100644 --- a/jstests/concurrency/fsm_workloads/collmod.js +++ b/jstests/concurrency/fsm_workloads/collmod.js @@ -14,7 +14,8 @@ var $config = (function() { var data = { numDocs: 1000, - maxTTL: 5000 // max time to live + maxTTL: 5000, // max time to live + ttlIndexExists: true }; var states = (function() { diff --git a/jstests/concurrency/fsm_workloads/indexed_insert_ttl.js b/jstests/concurrency/fsm_workloads/indexed_insert_ttl.js index c803057864d..fbdbb9c523e 100644 --- a/jstests/concurrency/fsm_workloads/indexed_insert_ttl.js +++ b/jstests/concurrency/fsm_workloads/indexed_insert_ttl.js @@ -60,7 +60,8 @@ var $config = (function() { transitions: transitions, setup: setup, data: { - ttlSeconds: 5 + ttlSeconds: 5, + ttlIndexExists: true }, teardown: teardown }; -- cgit v1.2.1