diff options
author | Yu Jin Kang Park <yujin.kang@mongodb.com> | 2022-08-10 17:04:35 +0000 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2022-08-10 21:04:29 +0000 |
commit | 943c960cbd08b341d6855cb86fbf8c45977dba03 (patch) | |
tree | 1265cd005e51114e91cddc33611582854ca96e44 | |
parent | 28fe300dd7aa19a2a830a460ac982554d3087ec8 (diff) | |
download | mongo-943c960cbd08b341d6855cb86fbf8c45977dba03.tar.gz |
SERVER-67939: run_dbcheck_background.js retry on CappedPositionLost
-rw-r--r-- | jstests/hooks/run_dbcheck_background.js | 34 |
1 files changed, 25 insertions, 9 deletions
diff --git a/jstests/hooks/run_dbcheck_background.js b/jstests/hooks/run_dbcheck_background.js index 383644bbf95..f718db48b9b 100644 --- a/jstests/hooks/run_dbcheck_background.js +++ b/jstests/hooks/run_dbcheck_background.js @@ -93,16 +93,32 @@ const exceptionFilteredBackgroundDbCheck = function(hosts) { const healthlog = node.getDB('local').system.healthlog; // Regex matching strings that start without "SnapshotTooOld" const regexStringWithoutSnapTooOld = /^((?!^SnapshotTooOld).)*$/; - let errs = - healthlog.find({"severity": "error", "data.error": regexStringWithoutSnapTooOld}); - if (errs.hasNext()) { - const err = "dbCheck found inconsistency on " + node.host; - jsTestLog(err + ". Errors: "); - for (let count = 0; errs.hasNext() && count < 20; count++) { - jsTestLog(tojson(errs.next())); + + // healthlog is a capped collection, truncation during scan might cause cursor + // invalidation. Truncated data is most likely from previous tests in the fixture, so we + // should still be able to catch errors by retrying. + assert.soon(() => { + try { + let errs = healthlog.find( + {"severity": "error", "data.error": regexStringWithoutSnapTooOld}); + if (errs.hasNext()) { + const err = "dbCheck found inconsistency on " + node.host; + jsTestLog(err + ". Errors: "); + for (let count = 0; errs.hasNext() && count < 20; count++) { + jsTestLog(tojson(errs.next())); + } + assert(false, err); + } + return true; + } catch (e) { + if (e.code !== ErrorCodes.CappedPositionLost) { + throw e; + } + jsTestLog(`Retrying on CappedPositionLost error: ${tojson(e)}`); + return false; } - assert(false, err); - } + }, "healthlog scan could not complete.", 60000); + jsTestLog("Checked health log on " + node.host); }); |