diff options
author | Mickey. J Winters <mickey.winters@mongodb.com> | 2022-02-14 18:11:06 +0000 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2022-02-14 18:33:46 +0000 |
commit | caa718990a002f9260deaf521d74fdc122380c33 (patch) | |
tree | b872bb8c09870f52dd510af3949863106eb0b195 | |
parent | dd680775aa8b27f442dd7b72a7b18ad569054646 (diff) | |
download | mongo-caa718990a002f9260deaf521d74fdc122380c33.tar.gz |
SERVER-62825 add retry logic to causally_consistent_index_builds.js
-rw-r--r-- | jstests/libs/override_methods/causally_consistent_index_builds.js | 51 |
1 files changed, 33 insertions, 18 deletions
diff --git a/jstests/libs/override_methods/causally_consistent_index_builds.js b/jstests/libs/override_methods/causally_consistent_index_builds.js index cacd1312f80..430a610d323 100644 --- a/jstests/libs/override_methods/causally_consistent_index_builds.js +++ b/jstests/libs/override_methods/causally_consistent_index_builds.js @@ -7,34 +7,49 @@ load("jstests/libs/override_methods/override_helpers.js"); -// This override runs a collMod after a createIndexes command. After collMod completes +// This override runs a collMod after a createIndexes command. After collMod completes successfully // we can guarantee the background index build started earlier has also completed. We update the // command response operationTime and $clusterTime so causally consistent reads only read from // that point onwards. function runCommandWithCollMod(conn, dbName, commandName, commandObj, func, makeFuncArgs) { - if (typeof commandObj !== "object" || commandObj === null) { + if (typeof commandObj !== "object" || commandObj === null || commandName !== "createIndexes") { return func.apply(conn, makeFuncArgs(commandObj)); } - let res = func.apply(conn, makeFuncArgs(commandObj)); - if (commandName !== "createIndexes") { - return res; - } - if (!res.ok) { - return res; - } + // If checking for index build completion fails due to failover, stepdown, or concurrent drop + // before we can check to see if it's done, then this retry loop should be able to restart and + // hopefully finish the index build. This also makes it more likely that an error returned is + // one resulting from a createIndex command instead of a collMod command. The user was trying + // to do a createIndex command and is not anticipating errors from collMod. + let res; + let collModRes; + let needRetry = true; + const retryFailedMsg = () => + "can't verify index build completed. collMod failed after sending createIndex command: " + + tojson(collModRes); + for (let retry = 0; retry < 10 && needRetry; retry++) { + res = func.apply(conn, makeFuncArgs(commandObj)); + if (!res.ok) { + return res; + } + needRetry = false; - let collModCmd = {collMod: commandObj[commandName]}; - let collModRes = func.apply(conn, makeFuncArgs(collModCmd)); - - // If a follow-up collMod fails, another command was likely able to execute after the - // createIndexes command. That means it is safe to use the latest operationTime for - // causal consistency purposes. - if (!collModRes.ok) { - print('note: ignoring collMod failure after sending createIndex command: ' + - tojson(collModRes)); + let collModCmd = {collMod: commandObj[commandName]}; + assert.soon(() => { + collModRes = func.apply(conn, makeFuncArgs(collModCmd)); + if (!collModRes.ok && + collModRes.code !== ErrorCodes.BackgroundOperationInProgressForNamespace) { + needRetry = true; + return true; + } + return collModRes.ok; + }, retryFailedMsg); } + // We ran out of retries and the collMod still failed. We can't guarantee the index build + // completed so we unfortunately raise an error which the caller may have not been expecting. + assert.commandWorked(collModRes, retryFailedMsg()); + // Overwrite the createIndex command's operation and cluster times, so that the owning // session can perform causal reads. if (collModRes.hasOwnProperty("operationTime")) { |