From 59a22d074e7d690b8db0eda76747ff3985907c98 Mon Sep 17 00:00:00 2001 From: Esha Maharishi Date: Wed, 24 Apr 2019 13:36:29 -0400 Subject: SERVER-40806 Router should choose first *write* participant shard as transaction recovery shard --- .../txn_two_phase_commit_recover_decision.js | 558 ++++++++++++--------- 1 file changed, 317 insertions(+), 241 deletions(-) (limited to 'jstests') diff --git a/jstests/sharding/txn_two_phase_commit_recover_decision.js b/jstests/sharding/txn_two_phase_commit_recover_decision.js index 253a1a59ab9..eb91c23e696 100644 --- a/jstests/sharding/txn_two_phase_commit_recover_decision.js +++ b/jstests/sharding/txn_two_phase_commit_recover_decision.js @@ -20,80 +20,157 @@ // transaction coordinator being canceled after a timeout happen in a reasonable amount of time. TestData.transactionLifetimeLimitSeconds = 15; - const startNewReadOnlyTransactionThroughMongos = function() { - let res = assert.commandWorked(testDB.runCommand({ + const readFromShard0 = function({lsid, txnNumber, startTransaction}) { + let findDocumentOnShard0Command = { find: 'user', + filter: {x: -1}, lsid: lsid, txnNumber: NumberLong(txnNumber), autocommit: false, - startTransaction: true - })); + }; + + if (startTransaction) { + findDocumentOnShard0Command.startTransaction = true; + } + + let res = assert.commandWorked(testDB.runCommand(findDocumentOnShard0Command)); + assert.neq(null, res.recoveryToken); + return res.recoveryToken; + }; + + const readFromShard1 = function({lsid, txnNumber, startTransaction}) { + let findDocumentOnShard1Command = { + find: 'user', + filter: {x: 1}, + lsid: lsid, + txnNumber: NumberLong(txnNumber), + autocommit: false, + }; + + if (startTransaction) { + findDocumentOnShard1Command.startTransaction = true; + } + let res = assert.commandWorked(testDB.runCommand(findDocumentOnShard1Command)); assert.neq(null, res.recoveryToken); - assert.eq(null, res.recoveryToken.shardId); return res.recoveryToken; }; - const startNewSingleShardTransactionThroughMongos = function() { + const writeToShard0 = function({lsid, txnNumber, startTransaction}) { const updateDocumentOnShard0 = { q: {x: -1}, u: {"$set": {lastTxnNumber: txnNumber}}, upsert: true }; - let res = assert.commandWorked(testDB.runCommand({ + let updateDocumentOnShard0Command = { update: 'user', updates: [updateDocumentOnShard0], lsid: lsid, txnNumber: NumberLong(txnNumber), autocommit: false, - startTransaction: true - })); + }; + if (startTransaction) { + updateDocumentOnShard0Command.startTransaction = true; + } + + let res = assert.commandWorked(testDB.runCommand(updateDocumentOnShard0Command)); assert.neq(null, res.recoveryToken); - assert.neq(null, res.recoveryToken.shardId); return res.recoveryToken; }; - const startNewCrossShardTransactionThroughMongos = function() { - const updateDocumentOnShard0 = { - q: {x: -1}, - u: {"$set": {lastTxnNumber: txnNumber}}, - upsert: true - }; + const writeToShard1 = function({lsid, txnNumber, startTransaction}) { const updateDocumentOnShard1 = { q: {x: 1}, u: {"$set": {lastTxnNumber: txnNumber}}, upsert: true }; - let res = assert.commandWorked(testDB.runCommand({ + let updateDocumentOnShard1Command = { update: 'user', - updates: [updateDocumentOnShard0, updateDocumentOnShard1], + updates: [updateDocumentOnShard1], lsid: lsid, txnNumber: NumberLong(txnNumber), autocommit: false, - startTransaction: true - })); + }; + if (startTransaction) { + updateDocumentOnShard1Command.startTransaction = true; + } + + let res = assert.commandWorked(testDB.runCommand(updateDocumentOnShard1Command)); assert.neq(null, res.recoveryToken); - assert.neq(null, res.recoveryToken.shardId); return res.recoveryToken; }; - const runCoordinateCommit = function(txnNumber, participantList, writeConcern) { - writeConcern = writeConcern || {}; - return coordinatorPrimaryConn.adminCommand(Object.merge({ - coordinateCommitTransaction: 1, - participants: participantList, + const startNewSingleShardReadOnlyTransaction = function() { + const recoveryToken = readFromShard0({lsid, txnNumber, startTransaction: true}); + assert.eq(null, recoveryToken.recoveryShardId); + return recoveryToken; + }; + + const startNewSingleShardWriteTransaction = function() { + const recoveryToken = writeToShard0({lsid, txnNumber, startTransaction: true}); + assert.neq(null, recoveryToken.recoveryShardId); + return recoveryToken; + }; + + const startNewMultiShardReadOnlyTransaction = function() { + let recoveryToken = readFromShard0({lsid, txnNumber, startTransaction: true}); + assert.eq(null, recoveryToken.recoveryShardId); + + recoveryToken = readFromShard1({lsid, txnNumber}); + assert.eq(null, recoveryToken.recoveryShardId); + + return recoveryToken; + }; + + const startNewSingleWriteShardTransaction = function() { + let recoveryToken = readFromShard0({lsid, txnNumber, startTransaction: true}); + assert.eq(null, recoveryToken.recoveryShardId); + + recoveryToken = writeToShard1({lsid, txnNumber}); + assert.neq(null, recoveryToken.recoveryShardId); + + return recoveryToken; + }; + + const startNewMultiShardWriteTransaction = function() { + let recoveryToken = readFromShard0({lsid, txnNumber, startTransaction: true}); + assert.eq(null, recoveryToken.recoveryShardId); + + // Write to shard 1, not shard 0, otherwise the recovery shard will still be the same as the + // coordinator shard. + recoveryToken = writeToShard1({lsid, txnNumber}); + assert.neq(null, recoveryToken.recoveryShardId); + + recoveryToken = writeToShard0({lsid, txnNumber}); + assert.neq(null, recoveryToken.recoveryShardId); + + return recoveryToken; + }; + + const abortTransactionOnShardDirectly = function(shardPrimaryConn, lsid, txnNumber) { + assert.commandWorked(shardPrimaryConn.adminCommand({ + abortTransaction: 1, lsid: lsid, txnNumber: NumberLong(txnNumber), autocommit: false - }, - writeConcern)); + })); }; - const sendCommitViaOtherMongos = function(lsid, txnNumber, recoveryToken, writeConcern) { + const sendCommitViaOriginalMongos = function(lsid, txnNumber, recoveryToken) { + return st.s0.getDB('admin').runCommand({ + commitTransaction: 1, + lsid: lsid, + txnNumber: NumberLong(txnNumber), + autocommit: false, + recoveryToken: recoveryToken + }); + }; + + const sendCommitViaRecoveryMongos = function(lsid, txnNumber, recoveryToken, writeConcern) { writeConcern = writeConcern || {}; return st.s1.getDB('admin').runCommand(Object.merge({ commitTransaction: 1, @@ -105,8 +182,8 @@ writeConcern)); }; - let st = - new ShardingTest({shards: 2, rs: {nodes: 2}, mongos: 2, other: {rsOptions: {verbose: 2}}}); + let st = new ShardingTest( + {shards: 2, rs: {nodes: 2}, mongos: 2, other: {mongosOptions: {verbose: 3}}}); assert.commandWorked(st.s0.adminCommand({enableSharding: 'test'})); st.ensurePrimaryShard('test', st.shard0.name); @@ -119,311 +196,310 @@ let testDB = st.s0.getDB('test'); assert.commandWorked(testDB.runCommand({insert: 'user', documents: [{x: -10}, {x: 10}]})); - let coordinatorReplSetTest = st.rs0; - let coordinatorPrimaryConn = coordinatorReplSetTest.getPrimary(); - const lsid = {id: UUID()}; let txnNumber = 0; - const participantList = [{shardId: st.shard0.shardName}, {shardId: st.shard1.shardName}]; - (function() { - jsTest.log("coordinator's participant has never heard of the session."); - assert.commandFailedWithCode(runCoordinateCommit(txnNumber, participantList), - ErrorCodes.NoSuchTransaction); - })(); + // + // Generic test cases that are agnostic as to the transaction type + // (function() { - jsTest.log("coordinator finished coordinating an abort decision."); + jsTest.log("Testing recovering transaction with lower number than latest"); + ++txnNumber; + + const oldTxnNumber = txnNumber; + const oldRecoveryToken = startNewMultiShardWriteTransaction(); txnNumber++; + const newRecoveryToken = startNewMultiShardWriteTransaction(); + assert.commandWorked(sendCommitViaOriginalMongos(lsid, txnNumber, newRecoveryToken)); - const recoveryToken = startNewCrossShardTransactionThroughMongos(); - assert.commandWorked(st.rs0.getPrimary().adminCommand({ - abortTransaction: 1, - lsid: lsid, - txnNumber: NumberLong(txnNumber), - autocommit: false - })); - assert.commandFailedWithCode(testDB.adminCommand({ - commitTransaction: 1, - lsid: lsid, - txnNumber: NumberLong(txnNumber), - autocommit: false - }), - ErrorCodes.NoSuchTransaction); + assert.commandFailedWithCode( + sendCommitViaRecoveryMongos(lsid, oldTxnNumber, oldRecoveryToken), + ErrorCodes.TransactionTooOld); - assert.commandFailedWithCode(runCoordinateCommit(txnNumber, participantList), - ErrorCodes.NoSuchTransaction); - assert.commandFailedWithCode(runCoordinateCommit(txnNumber, []), - ErrorCodes.NoSuchTransaction); + // The client can still the recover decision for current transaction number. + assert.commandWorked(sendCommitViaRecoveryMongos(lsid, txnNumber, newRecoveryToken)); + })(); - assert.commandFailedWithCode(sendCommitViaOtherMongos(lsid, txnNumber, recoveryToken), - ErrorCodes.NoSuchTransaction); + (function() { + jsTest.log("Testing recovering transaction with higher number than latest"); + txnNumber++; + + const oldTxnNumber = txnNumber; + const oldRecoveryToken = startNewMultiShardWriteTransaction(); + txnNumber++; + const fakeRecoveryToken = {recoveryShardId: st.shard0.shardName}; + assert.commandFailedWithCode( + sendCommitViaRecoveryMongos(lsid, txnNumber, fakeRecoveryToken), + ErrorCodes.NoSuchTransaction); + + // The active transaction can still be committed. + assert.commandWorked(sendCommitViaOriginalMongos(lsid, oldTxnNumber, oldRecoveryToken)); })(); (function() { - jsTest.log("coordinator finished coordinating a commit decision."); - + jsTest.log("Testing recovering transaction whose recovery shard forgot the transaction"); txnNumber++; - const recoveryToken = startNewCrossShardTransactionThroughMongos(); - assert.commandWorked(testDB.adminCommand({ - commitTransaction: 1, - lsid: lsid, - txnNumber: NumberLong(txnNumber), - autocommit: false - })); + const recoveryToken = startNewMultiShardWriteTransaction(); + assert.commandWorked(sendCommitViaOriginalMongos(lsid, txnNumber, recoveryToken)); - assert.commandWorked(runCoordinateCommit(txnNumber, participantList)); - assert.commandWorked(runCoordinateCommit(txnNumber, [])); + assert.writeOK( + st.rs1.getPrimary().getDB("config").transactions.remove({}, false /* justOne */)); - assert.commandWorked(sendCommitViaOtherMongos(lsid, txnNumber, recoveryToken)); + assert.commandFailedWithCode(sendCommitViaRecoveryMongos(lsid, txnNumber, recoveryToken), + ErrorCodes.NoSuchTransaction); })(); (function() { - jsTest.log( - "coordinator finished coordinating a commit decision but coordinator node can't majority commit writes"); + jsTest.log("Testing that a recovery node does a noop write before returning 'aborted'"); txnNumber++; - const recoveryToken = startNewCrossShardTransactionThroughMongos(); - assert.commandWorked(testDB.adminCommand({ - commitTransaction: 1, - lsid: lsid, - txnNumber: NumberLong(txnNumber), - autocommit: false, - writeConcern: {w: "majority"} - })); - - stopReplicationOnSecondaries(coordinatorReplSetTest); - - // Do a write on the coordinator to bump the coordinator node's system last OpTime. - coordinatorPrimaryConn.getDB("dummy").getCollection("dummy").insert({dummy: 1}); + const recoveryToken = startNewMultiShardWriteTransaction(); + abortTransactionOnShardDirectly(st.rs0.getPrimary(), lsid, txnNumber); + assert.commandFailedWithCode(sendCommitViaOriginalMongos(lsid, txnNumber, recoveryToken), + ErrorCodes.NoSuchTransaction); - // While the coordinator primary cannot majority commit writes, coordinateCommitTransaction - // returns ok:1 with a writeConcern error. + const recoveryShardReplSetTest = st.rs1; - let res = runCoordinateCommit( - txnNumber, participantList, {writeConcern: {w: "majority", wtimeout: 500}}); - assert.eq(1, res.ok, tojson(res)); - checkWriteConcernTimedOut(res); + stopReplicationOnSecondaries(recoveryShardReplSetTest); - res = runCoordinateCommit(txnNumber, [], {writeConcern: {w: "majority", wtimeout: 500}}); - assert.eq(1, res.ok, tojson(res)); - checkWriteConcernTimedOut(res); + // Do a write on the recovery node to bump the recovery node's system last OpTime. + recoveryShardReplSetTest.getPrimary().getDB("dummy").getCollection("dummy").insert( + {dummy: 1}); - res = sendCommitViaOtherMongos( + // While the recovery shard primary cannot majority commit writes, commitTransaction returns + // NoSuchTransaction with a writeConcern error. + let res = sendCommitViaRecoveryMongos( lsid, txnNumber, recoveryToken, {writeConcern: {w: "majority", wtimeout: 500}}); - assert.eq(1, res.ok, tojson(res)); + assert.commandFailedWithCode(res, ErrorCodes.NoSuchTransaction); checkWriteConcernTimedOut(res); - // Once the coordinator primary can majority commit writes again, - // coordinateCommitTransaction returns ok:1 without a writeConcern error. - - restartReplicationOnSecondaries(coordinatorReplSetTest); - - assert.commandWorked( - runCoordinateCommit(txnNumber, participantList, {writeConcern: {w: "majority"}})); - assert.commandWorked(runCoordinateCommit(txnNumber, [], {writeConcern: {w: "majority"}})); - assert.commandWorked(sendCommitViaOtherMongos( - lsid, txnNumber, recoveryToken, {writeConcern: {w: "majority"}})); + // Once the recovery shard primary can majority commit writes again, commitTransaction + // returns NoSuchTransaction without a writeConcern error. + restartReplicationOnSecondaries(recoveryShardReplSetTest); + res = sendCommitViaRecoveryMongos( + lsid, txnNumber, recoveryToken, {writeConcern: {w: "majority"}}); + assert.commandFailedWithCode(res, ErrorCodes.NoSuchTransaction); + assert.eq(null, res.writeConcernError); })(); (function() { - jsTest.log( - "try to recover decision for lower transaction number than last number participant saw."); + jsTest.log("Testing that a recovery node does a noop write before returning 'committed'"); - ++txnNumber; + txnNumber++; - const oldTxnNumber = txnNumber; - const recoveryToken = startNewCrossShardTransactionThroughMongos(); - assert.commandWorked(testDB.adminCommand({ - commitTransaction: 1, - lsid: lsid, - txnNumber: NumberLong(txnNumber), - autocommit: false, - })); + const recoveryToken = startNewMultiShardWriteTransaction(); + assert.commandWorked(sendCommitViaOriginalMongos(lsid, txnNumber, recoveryToken)); - txnNumber++; + const recoveryShardReplSetTest = st.rs1; - startNewCrossShardTransactionThroughMongos(); - assert.commandWorked(testDB.adminCommand({ - commitTransaction: 1, - lsid: lsid, - txnNumber: NumberLong(txnNumber), - autocommit: false, - })); + stopReplicationOnSecondaries(recoveryShardReplSetTest); - assert.commandFailedWithCode(runCoordinateCommit(oldTxnNumber, participantList), - ErrorCodes.TransactionTooOld); - assert.commandFailedWithCode(runCoordinateCommit(oldTxnNumber, []), - ErrorCodes.TransactionTooOld); - assert.commandFailedWithCode(sendCommitViaOtherMongos(lsid, oldTxnNumber, recoveryToken), - ErrorCodes.TransactionTooOld); - - // Can still recover decision for current transaction number. - assert.commandWorked(runCoordinateCommit(txnNumber, participantList)); - assert.commandWorked(runCoordinateCommit(txnNumber, [])); - assert.commandWorked(sendCommitViaOtherMongos(lsid, txnNumber, recoveryToken)); + // Do a write on the recovery node to bump the recovery node's system last OpTime. + recoveryShardReplSetTest.getPrimary().getDB("dummy").getCollection("dummy").insert( + {dummy: 1}); + + // While the recovery shard primary cannot majority commit writes, commitTransaction returns + // ok with a writeConcern error. + let res = sendCommitViaRecoveryMongos( + lsid, txnNumber, recoveryToken, {writeConcern: {w: "majority", wtimeout: 500}}); + assert.commandWorkedIgnoringWriteConcernErrors(res); + checkWriteConcernTimedOut(res); + + // Once the recovery shard primary can majority commit writes again, commitTransaction + // returns ok without a writeConcern error. + restartReplicationOnSecondaries(recoveryShardReplSetTest); + assert.commandWorked(sendCommitViaRecoveryMongos( + lsid, txnNumber, recoveryToken, {writeConcern: {w: "majority"}})); })(); - (function() { - jsTest.log("session has been reaped."); + // + // Single-shard read-only transactions + // + (function() { + jsTest.log("Testing recovering single-shard read-only transaction that is in progress"); txnNumber++; + const recoveryToken = startNewSingleShardReadOnlyTransaction(); + assert.commandFailedWithCode(sendCommitViaRecoveryMongos(lsid, txnNumber, recoveryToken), + ErrorCodes.NoSuchTransaction); - const recoveryToken = startNewCrossShardTransactionThroughMongos(); - assert.commandWorked(testDB.adminCommand({ - commitTransaction: 1, - lsid: lsid, - txnNumber: NumberLong(txnNumber), - autocommit: false, - writeConcern: {w: "majority"} - })); + // A read-only transaction can still commit after reporting an abort decision. + assert.commandWorked(sendCommitViaOriginalMongos(lsid, txnNumber, recoveryToken)); + }()); - assert.writeOK( - coordinatorPrimaryConn.getDB("config").transactions.remove({}, false /* justOne */)); + (function() { + jsTest.log("Testing recovering single-shard read-only transaction that aborted"); + txnNumber++; + const recoveryToken = startNewSingleShardReadOnlyTransaction(); + abortTransactionOnShardDirectly(st.rs0.getPrimary(), lsid, txnNumber); + assert.commandFailedWithCode(sendCommitViaRecoveryMongos(lsid, txnNumber, recoveryToken), + ErrorCodes.NoSuchTransaction); + }()); - assert.commandFailedWithCode(runCoordinateCommit(txnNumber, participantList), + (function() { + jsTest.log("Testing recovering single-shard read-only transaction that committed"); + txnNumber++; + const recoveryToken = startNewSingleShardReadOnlyTransaction(); + assert.commandWorked(sendCommitViaOriginalMongos(lsid, txnNumber, recoveryToken)); + assert.commandFailedWithCode(sendCommitViaRecoveryMongos(lsid, txnNumber, recoveryToken), ErrorCodes.NoSuchTransaction); - assert.commandFailedWithCode(runCoordinateCommit(txnNumber, []), + }()); + + // + // Single-shard write transactions + // + + (function() { + jsTest.log("Testing recovering single-shard write transaction that in progress"); + txnNumber++; + const recoveryToken = startNewSingleShardWriteTransaction(); + assert.commandFailedWithCode(sendCommitViaRecoveryMongos(lsid, txnNumber, recoveryToken), ErrorCodes.NoSuchTransaction); - assert.commandFailedWithCode(sendCommitViaOtherMongos(lsid, txnNumber, recoveryToken), + + // A write transaction fails to commit after having reported an abort decision. + assert.commandFailedWithCode(sendCommitViaOriginalMongos(lsid, txnNumber, recoveryToken), ErrorCodes.NoSuchTransaction); - })(); + }()); (function() { - jsTest.log( - "try to recover decision for higher transaction number than participant has seen."); + jsTest.log("Testing recovering single-shard write transaction that aborted"); + txnNumber++; + const recoveryToken = startNewSingleShardWriteTransaction(); + abortTransactionOnShardDirectly(st.rs0.getPrimary(), lsid, txnNumber); + assert.commandFailedWithCode(sendCommitViaRecoveryMongos(lsid, txnNumber, recoveryToken), + ErrorCodes.NoSuchTransaction); + }()); + (function() { + jsTest.log("Testing recovering single-shard write transaction that committed"); txnNumber++; - const recoveryToken = startNewCrossShardTransactionThroughMongos(); + const recoveryToken = startNewSingleShardWriteTransaction(); + assert.commandWorked(sendCommitViaOriginalMongos(lsid, txnNumber, recoveryToken)); + assert.commandWorked(sendCommitViaRecoveryMongos(lsid, txnNumber, recoveryToken)); + }()); + + // + // Multi-shard read-only transactions + // + (function() { + jsTest.log("Testing recovering multi-shard read-only transaction that is in progress"); txnNumber++; - assert.commandFailedWithCode(runCoordinateCommit(txnNumber, participantList), - ErrorCodes.NoSuchTransaction); - assert.commandFailedWithCode(runCoordinateCommit(txnNumber, []), - ErrorCodes.NoSuchTransaction); - assert.commandFailedWithCode(sendCommitViaOtherMongos(lsid, txnNumber, recoveryToken), + const recoveryToken = startNewMultiShardReadOnlyTransaction(); + + assert.commandFailedWithCode(sendCommitViaRecoveryMongos(lsid, txnNumber, recoveryToken), ErrorCodes.NoSuchTransaction); - // We can still commit the active transaction with the lower transaction number. - assert.commandWorked(testDB.adminCommand({ - commitTransaction: 1, - lsid: lsid, - txnNumber: NumberLong(txnNumber - 1), - autocommit: false - })); + // A read-only transaction can still commit after reporting an abort decision. + assert.commandWorked(sendCommitViaOriginalMongos(lsid, txnNumber, recoveryToken)); })(); (function() { - jsTest.log("commit was never sent"); - + jsTest.log("Testing recovering multi-shard read-only transaction that aborted"); txnNumber++; - - // Start transaction and run CRUD ops on several shards. - const recoveryToken = startNewCrossShardTransactionThroughMongos(); - - // Try to recover decision from other mongos. This should preemptively cancel the - // coordinator, after which it should abort the local participant and return - // NoSuchTransaction. - assert.commandFailedWithCode(sendCommitViaOtherMongos(lsid, txnNumber, recoveryToken), + const recoveryToken = startNewMultiShardReadOnlyTransaction(); + abortTransactionOnShardDirectly(st.rs0.getPrimary(), lsid, txnNumber); + assert.commandFailedWithCode(sendCommitViaRecoveryMongos(lsid, txnNumber, recoveryToken), ErrorCodes.NoSuchTransaction); })(); (function() { - jsTest.log( - "try to recover decision for single-shard transaction that has already committed"); - + jsTest.log("Testing recovering multi-shard read-only transaction that committed"); txnNumber++; + const recoveryToken = startNewMultiShardReadOnlyTransaction(); + assert.commandWorked(sendCommitViaOriginalMongos(lsid, txnNumber, recoveryToken)); + assert.commandFailedWithCode(sendCommitViaRecoveryMongos(lsid, txnNumber, recoveryToken), + ErrorCodes.NoSuchTransaction); + })(); - const recoveryToken = startNewSingleShardTransactionThroughMongos(); + // + // Single-write-shard transactions (there are multiple participants but only one did a write) + // - // Commit the transaction from the first mongos. - assert.commandWorked(testDB.adminCommand({ - commitTransaction: 1, - lsid: lsid, - txnNumber: NumberLong(txnNumber), - autocommit: false, - recoveryToken: recoveryToken - })); + (function() { + jsTest.log("Testing recovering single-write-shard transaction that is in progress"); + txnNumber++; + const recoveryToken = startNewSingleWriteShardTransaction(); + assert.commandFailedWithCode(sendCommitViaRecoveryMongos(lsid, txnNumber, recoveryToken), + ErrorCodes.NoSuchTransaction); - // Try to recover decision from other mongos. - assert.commandWorked(sendCommitViaOtherMongos(lsid, txnNumber, recoveryToken)); + // A write transaction fails to commit after having reported an abort decision. + assert.commandFailedWithCode(sendCommitViaOriginalMongos(lsid, txnNumber, recoveryToken), + ErrorCodes.NoSuchTransaction); }()); (function() { - jsTest.log("try to recover decision for single-shard transaction that has already aborted"); - + jsTest.log( + "Testing recovering single-write-shard transaction that aborted on read-only shard" + + " but is in progress on write shard"); txnNumber++; - - const recoveryToken = startNewSingleShardTransactionThroughMongos(); - assert.commandWorked(st.rs0.getPrimary().adminCommand({ - abortTransaction: 1, - lsid: lsid, - txnNumber: NumberLong(txnNumber), - autocommit: false - })); - - // Commit the transaction from the first mongos. - assert.commandFailedWithCode(testDB.adminCommand({ - commitTransaction: 1, - lsid: lsid, - txnNumber: NumberLong(txnNumber), - autocommit: false, - recoveryToken: recoveryToken - }), + const recoveryToken = startNewSingleWriteShardTransaction(); + abortTransactionOnShardDirectly(st.rs1.getPrimary(), lsid, txnNumber); + assert.commandFailedWithCode(sendCommitViaRecoveryMongos(lsid, txnNumber, recoveryToken), ErrorCodes.NoSuchTransaction); + }()); - // Try to recover the decision from other mongos. - assert.commandFailedWithCode(sendCommitViaOtherMongos(lsid, txnNumber, recoveryToken), + (function() { + jsTest.log("Testing recovering single-write-shard transaction that aborted on write" + + " shard but is in progress on read-only shard"); + txnNumber++; + const recoveryToken = startNewSingleWriteShardTransaction(); + abortTransactionOnShardDirectly(st.rs1.getPrimary(), lsid, txnNumber); + assert.commandFailedWithCode(sendCommitViaRecoveryMongos(lsid, txnNumber, recoveryToken), ErrorCodes.NoSuchTransaction); }()); (function() { - jsTest.log("try to recover decision for read-only transaction that's in progress"); + jsTest.log("Testing recovering single-write-shard transaction that committed"); + txnNumber++; + const recoveryToken = startNewSingleWriteShardTransaction(); + assert.commandWorked(sendCommitViaOriginalMongos(lsid, txnNumber, recoveryToken)); + assert.commandWorked(sendCommitViaRecoveryMongos(lsid, txnNumber, recoveryToken)); + }()); + + // + // Multi-write-shard transactions (there are multiple participants and more than one did writes) + // + (function() { + jsTest.log("Testing recovering multi-write-shard transaction that is in progress"); txnNumber++; - const recoveryToken = startNewReadOnlyTransactionThroughMongos(); + const recoveryToken = startNewMultiShardWriteTransaction(); + assert.commandFailedWithCode(sendCommitViaRecoveryMongos(lsid, txnNumber, recoveryToken), + ErrorCodes.NoSuchTransaction); - assert.commandFailedWithCode(sendCommitViaOtherMongos(lsid, txnNumber, recoveryToken), + // A write transaction fails to commit after having reported an abort decision. + assert.commandFailedWithCode(sendCommitViaOriginalMongos(lsid, txnNumber, recoveryToken), ErrorCodes.NoSuchTransaction); })(); (function() { - jsTest.log("try to recover decision for read-only transaction that aborted"); - + jsTest.log("Testing recovering multi-write-shard transaction after coordinator finished" + + " coordinating an abort decision."); txnNumber++; - const recoveryToken = startNewReadOnlyTransactionThroughMongos(); - assert.commandWorked(st.rs0.getPrimary().adminCommand({ - abortTransaction: 1, - lsid: lsid, - txnNumber: NumberLong(txnNumber), - autocommit: false - })); - - assert.commandFailedWithCode(sendCommitViaOtherMongos(lsid, txnNumber, recoveryToken), + const recoveryToken = startNewMultiShardWriteTransaction(); + abortTransactionOnShardDirectly(st.rs0.getPrimary(), lsid, txnNumber); + assert.commandFailedWithCode(sendCommitViaOriginalMongos(lsid, txnNumber, recoveryToken), + ErrorCodes.NoSuchTransaction); + assert.commandFailedWithCode(sendCommitViaRecoveryMongos(lsid, txnNumber, recoveryToken), ErrorCodes.NoSuchTransaction); })(); (function() { - jsTest.log("try to recover decision for read-only transaction that committed"); - + jsTest.log("Testing recovering multi-write-shard transaction after coordinator finished" + + " coordinating a commit decision."); txnNumber++; - const recoveryToken = startNewReadOnlyTransactionThroughMongos(); - assert.commandWorked(testDB.adminCommand({ - commitTransaction: 1, - lsid: lsid, - txnNumber: NumberLong(txnNumber), - autocommit: false - })); - - assert.commandFailedWithCode(sendCommitViaOtherMongos(lsid, txnNumber, recoveryToken), - ErrorCodes.NoSuchTransaction); + const recoveryToken = startNewMultiShardWriteTransaction(); + assert.commandWorked(sendCommitViaOriginalMongos(lsid, txnNumber, recoveryToken)); + assert.commandWorked(sendCommitViaRecoveryMongos(lsid, txnNumber, recoveryToken)); })(); st.stop(); -- cgit v1.2.1