From 4ff797fa438d9d1eb49ae9335174fb1140fc86eb Mon Sep 17 00:00:00 2001 From: Gregory Noma Date: Thu, 10 Sep 2020 16:55:05 -0400 Subject: SERVER-50775 Use failpoint for yielding index build locks in RollbackResumableIndexBuildTest --- .../libs/rollback_resumable_index_build.js | 40 ++++++++++++---------- ...llback_resumable_index_build_bulk_load_phase.js | 12 +++---- ..._resumable_index_build_collection_scan_phase.js | 16 ++++----- .../rollback_resumable_index_build_complete.js | 8 +---- ...ack_resumable_index_build_drain_writes_phase.js | 39 +++++++++------------ 5 files changed, 49 insertions(+), 66 deletions(-) diff --git a/jstests/replsets/libs/rollback_resumable_index_build.js b/jstests/replsets/libs/rollback_resumable_index_build.js index 0ab2148e783..e5c63d91ada 100644 --- a/jstests/replsets/libs/rollback_resumable_index_build.js +++ b/jstests/replsets/libs/rollback_resumable_index_build.js @@ -7,8 +7,10 @@ const RollbackResumableIndexBuildTest = class { * rollback starts is specified by rollbackStartFailPointName. The phase that the index build * will resume from after rollback completes is specified by rollbackEndFailPointName. If * either of these points is in the drain writes phase, documents to insert into the side - * writes table must be specified by sideWrites. Documents specified by insertsToBeRolledBack - * are inserted after transitioning to rollback operations and will be rolled back. + * writes table must be specified by sideWrites. locksYieldedFailPointName specifies a point + * during the index build between rollbackEndFailPointName and rollbackStartFailPointName at + * which its locks are yielded. Documents specified by insertsToBeRolledBack are inserted after + * transitioning to rollback operations and will be rolled back. */ static run(rollbackTest, dbName, @@ -18,6 +20,7 @@ const RollbackResumableIndexBuildTest = class { rollbackStartFailPointData, rollbackEndFailPointName, rollbackEndFailPointData, + locksYieldedFailPointName, insertsToBeRolledBack, sideWrites = []) { const originalPrimary = rollbackTest.getPrimary(); @@ -29,6 +32,14 @@ const RollbackResumableIndexBuildTest = class { rollbackTest.awaitLastOpCommitted(); + // Set internalQueryExecYieldIterations to 0 and maxIndexBuildDrainBatchSize to 1 so that + // the index build is guaranteed to yield its locks between the rollback end and start + // failpoints. + assert.commandWorked( + originalPrimary.adminCommand({setParameter: 1, internalQueryExecYieldIterations: 0})); + assert.commandWorked( + originalPrimary.adminCommand({setParameter: 1, maxIndexBuildDrainBatchSize: 1})); + const coll = originalPrimary.getDB(dbName).getCollection(collName); const indexName = "rollback_resumable_index_build"; @@ -57,32 +68,23 @@ const RollbackResumableIndexBuildTest = class { assert.commandWorked(coll.insert(insertsToBeRolledBack)); - // Disable the failpoint in a parallel shell so that the primary can step down when the - // rollback test is transitioning to sync source operations before rollback. - const awaitDisableFailPointAfterContinuingInBackground = startParallelShell( - funWithArgs(function(failPointName, buildUUID) { - // Wait for the index build to be continue in the background. - checkLog.containsJson(db.getMongo(), 4760400, { - buildUUID: function(uuid) { - return uuid["uuid"]["$uuid"] === buildUUID; - } - }); - - // Disable the failpoint so that stepdown can proceed. - assert.commandWorked( - db.adminCommand({configureFailPoint: failPointName, mode: "off"})); - }, rollbackEndFp.failPointName, buildUUID), originalPrimary.port); + // Move the index build forward to a point at which its locks are yielded. This allows the + // primary to step down during the call to transitionToSyncSourceOperationsBeforeRollback() + // below. + const locksYieldedFp = configureFailPoint( + originalPrimary, locksYieldedFailPointName, {namespace: coll.getFullName()}); + rollbackEndFp.off(); + locksYieldedFp.wait(); rollbackTest.transitionToSyncSourceOperationsBeforeRollback(); - awaitDisableFailPointAfterContinuingInBackground(); - // The index creation will report as having failed due to InterruptedDueToReplStateChange, // but it is still building in the background. awaitCreateIndex(); // Wait until the index build reaches the desired starting point so that we can start the // rollback. + locksYieldedFp.off(); rollbackStartFp.wait(); // We ignore the return value here because the node will go into rollback immediately upon diff --git a/jstests/replsets/rollback_resumable_index_build_bulk_load_phase.js b/jstests/replsets/rollback_resumable_index_build_bulk_load_phase.js index 81631163f06..5b75f1b7b2c 100644 --- a/jstests/replsets/rollback_resumable_index_build_bulk_load_phase.js +++ b/jstests/replsets/rollback_resumable_index_build_bulk_load_phase.js @@ -13,12 +13,6 @@ load('jstests/replsets/libs/rollback_resumable_index_build.js'); -// TODO(SERVER-50775): Re-enable when stepdown issues are fixed in resumable index rollback tests. -if (true) { - jsTestLog('Skipping test.'); - return; -} - const dbName = "test"; const rollbackStartFailPointName = "hangIndexBuildDuringBulkLoadPhase"; const insertsToBeRolledBack = [{a: 4}, {a: 5}]; @@ -35,8 +29,9 @@ RollbackResumableIndexBuildTest.run(rollbackTest, {a: 1}, rollbackStartFailPointName, {iteration: 1}, - "hangAfterSettingUpIndexBuildUnlocked", + "hangAfterSettingUpIndexBuild", {}, + "setYieldAllLocksHang", insertsToBeRolledBack); // Rollback to the collection scan phase. @@ -47,7 +42,8 @@ RollbackResumableIndexBuildTest.run(rollbackTest, rollbackStartFailPointName, {iteration: 1}, "hangIndexBuildDuringCollectionScanPhaseBeforeInsertion", - {fieldsToMatch: {a: 2}}, + {iteration: 1}, + "setYieldAllLocksHang", insertsToBeRolledBack); rollbackTest.stop(); diff --git a/jstests/replsets/rollback_resumable_index_build_collection_scan_phase.js b/jstests/replsets/rollback_resumable_index_build_collection_scan_phase.js index 23807d85383..33abffdc2e3 100644 --- a/jstests/replsets/rollback_resumable_index_build_collection_scan_phase.js +++ b/jstests/replsets/rollback_resumable_index_build_collection_scan_phase.js @@ -13,12 +13,6 @@ load('jstests/replsets/libs/rollback_resumable_index_build.js'); -// TODO(SERVER-50775): Re-enable when stepdown issues are fixed in resumable index rollback tests. -if (true) { - jsTestLog('Skipping test.'); - return; -} - const dbName = "test"; const rollbackStartFailPointName = "hangIndexBuildDuringCollectionScanPhaseBeforeInsertion"; const insertsToBeRolledBack = [{a: 6}, {a: 7}]; @@ -34,9 +28,10 @@ RollbackResumableIndexBuildTest.run(rollbackTest, coll.getName(), {a: 1}, rollbackStartFailPointName, - {fieldsToMatch: {a: 2}}, - "hangAfterSettingUpIndexBuildUnlocked", + {iteration: 3}, + "hangAfterSettingUpIndexBuild", {}, + "setYieldAllLocksHang", insertsToBeRolledBack); // Rollback to earlier in the collection scan phase. @@ -45,9 +40,10 @@ RollbackResumableIndexBuildTest.run(rollbackTest, coll.getName(), {a: 1}, rollbackStartFailPointName, - {iteration: 4}, + {iteration: 3}, "hangIndexBuildDuringCollectionScanPhaseAfterInsertion", - {iteration: 2}, + {iteration: 1}, + "setYieldAllLocksHang", insertsToBeRolledBack); rollbackTest.stop(); diff --git a/jstests/replsets/rollback_resumable_index_build_complete.js b/jstests/replsets/rollback_resumable_index_build_complete.js index 51f5988f4c9..cb74fc96345 100644 --- a/jstests/replsets/rollback_resumable_index_build_complete.js +++ b/jstests/replsets/rollback_resumable_index_build_complete.js @@ -13,12 +13,6 @@ load('jstests/replsets/libs/rollback_resumable_index_build.js'); -// TODO(SERVER-50775): Re-enable when stepdown issues are fixed in resumable index rollback tests. -if (true) { - jsTestLog('Skipping test.'); - return; -} - const dbName = "test"; const insertsToBeRolledBack = [{a: 7}, {a: 8}]; @@ -32,7 +26,7 @@ RollbackResumableIndexBuildTest.runIndexBuildComplete(rollbackTest, dbName, coll.getName(), {a: 1}, - "hangAfterSettingUpIndexBuildUnlocked", + "hangAfterSettingUpIndexBuild", {}, insertsToBeRolledBack); diff --git a/jstests/replsets/rollback_resumable_index_build_drain_writes_phase.js b/jstests/replsets/rollback_resumable_index_build_drain_writes_phase.js index 4e025596884..922ee451e01 100644 --- a/jstests/replsets/rollback_resumable_index_build_drain_writes_phase.js +++ b/jstests/replsets/rollback_resumable_index_build_drain_writes_phase.js @@ -13,15 +13,9 @@ load('jstests/replsets/libs/rollback_resumable_index_build.js'); -// TODO(SERVER-50775): Re-enable when stepdown issues are fixed in resumable index rollback tests. -if (true) { - jsTestLog('Skipping test.'); - return; -} - const dbName = "test"; const rollbackStartFailPointName = "hangIndexBuildDuringDrainWritesPhase"; -const insertsToBeRolledBack = [{a: 13}, {a: 14}]; +const insertsToBeRolledBack = [{a: 18}, {a: 19}]; const rollbackTest = new RollbackTest(jsTestName()); const coll = rollbackTest.getPrimary().getDB(dbName).getCollection(jsTestName()); @@ -34,11 +28,12 @@ RollbackResumableIndexBuildTest.run(rollbackTest, coll.getName(), {a: 1}, rollbackStartFailPointName, - {iteration: 0}, - "hangAfterSettingUpIndexBuildUnlocked", + {iteration: 1}, + "hangAfterSettingUpIndexBuild", {}, + "hangDuringIndexBuildDrainYield", insertsToBeRolledBack, - [{a: 4}, {a: 5}]); + [{a: 4}, {a: 5}, {a: 6}]); // Rollback to the collection scan phase. RollbackResumableIndexBuildTest.run(rollbackTest, @@ -46,11 +41,12 @@ RollbackResumableIndexBuildTest.run(rollbackTest, coll.getName(), {a: 1}, rollbackStartFailPointName, - {iteration: 0}, + {iteration: 1}, "hangIndexBuildDuringCollectionScanPhaseBeforeInsertion", - {fieldsToMatch: {a: 2}}, + {iteration: 1}, + "hangDuringIndexBuildDrainYield", insertsToBeRolledBack, - [{a: 6}, {a: 7}]); + [{a: 7}, {a: 8}, {a: 9}]); // Rollback to the bulk load phase. RollbackResumableIndexBuildTest.run(rollbackTest, @@ -58,26 +54,25 @@ RollbackResumableIndexBuildTest.run(rollbackTest, coll.getName(), {a: 1}, rollbackStartFailPointName, - {iteration: 0}, + {iteration: 1}, "hangIndexBuildDuringBulkLoadPhase", {iteration: 1}, + "hangDuringIndexBuildDrainYield", insertsToBeRolledBack, - [{a: 8}, {a: 9}]); + [{a: 10}, {a: 11}, {a: 12}]); -// Rollback to earlier in the drain writes phase. We set maxIndexBuildDrainBatchSize to 1 so that -// the primary can step down between iterations. -assert.commandWorked( - rollbackTest.getPrimary().adminCommand({setParameter: 1, maxIndexBuildDrainBatchSize: 1})); +// Rollback to earlier in the drain writes phase. RollbackResumableIndexBuildTest.run(rollbackTest, dbName, coll.getName(), {a: 1}, rollbackStartFailPointName, - {iteration: 2}, + {iteration: 3}, "hangIndexBuildDuringDrainWritesPhaseSecond", - {iteration: 0}, + {iteration: 1}, + "hangDuringIndexBuildDrainYield", insertsToBeRolledBack, - [{a: 10}, {a: 11}, {a: 12}]); + [{a: 13}, {a: 14}, {a: 15}, {a: 16}, {a: 17}]); rollbackTest.stop(); })(); -- cgit v1.2.1