SERVER-53926 replace recipientSyncData (with returnAfterTimestamp) errors with interrupt status when appropriate

1 files changed, 109 insertions, 0 deletions

diff --git a/jstests/replsets/tenant_migration_recipient_sync_donor_timestamp.js b/jstests/replsets/tenant_migration_recipient_sync_donor_timestamp.js
new file mode 100644
index 00000000000..3a13f606055
--- /dev/null
+++ b/jstests/replsets/tenant_migration_recipient_sync_donor_timestamp.js
@@ -0,0 +1,109 @@
+/**
+ * Exercises the code path for the recipientSyncData command that waits until a timestamp provided
+ * by the donor is majority committed: make sure that in this code path, when the recipient is
+ * interrupted by a primary step down, the recipient properly swaps the error code to the true code
+ * (like primary step down) that the donor can retry on.
+ *
+ * @tags: [requires_fcv_49, requires_replication, incompatible_with_windows_tls]
+ */
+
+(function() {
+
+"use strict";
+load("jstests/libs/fail_point_util.js");
+load("jstests/libs/uuid_util.js");  // For extractUUIDFromObject()
+load("jstests/replsets/libs/tenant_migration_test.js");
+load("jstests/replsets/libs/tenant_migration_util.js");
+
+// Use a single node replSet to simplify the process.
+const donorRst = new ReplSetTest({
+    nodes: 1,
+    name: jsTestName() + "_donor",
+    nodeOptions: TenantMigrationUtil.makeX509OptionsForTest().donor
+});
+
+donorRst.startSet();
+donorRst.initiate();
+
+// Make the batch size small so that we can pause before all the batches are applied.
+const tenantMigrationTest = new TenantMigrationTest(
+    {name: jsTestName(), donorRst, sharedOptions: {setParameter: {tenantApplierBatchSizeOps: 2}}});
+
+if (!tenantMigrationTest.isFeatureFlagEnabled()) {
+    donorRst.stopSet();
+    jsTestLog("Skipping test because the tenant migrations feature flag is disabled");
+    return;
+}
+
+const kMigrationId = UUID();
+const kTenantId = 'testTenantId';
+const kReadPreference = {
+    mode: "primary"
+};
+const migrationOpts = {
+    migrationIdString: extractUUIDFromObject(kMigrationId),
+    tenantId: kTenantId,
+    readPreference: kReadPreference
+};
+
+const dbName = tenantMigrationTest.tenantDB(kTenantId, "testDB");
+const collName = jsTestName() + "_collection";
+
+const recipientRst = tenantMigrationTest.getRecipientRst();
+const recipientPrimary = recipientRst.getPrimary();
+
+// FailPoint to pause right before the data consistent promise is fulfilled.
+const fpBeforeDataConsistent = configureFailPoint(
+    recipientPrimary, "fpBeforeFulfillingDataConsistentPromise", {action: "hang"});
+const fpBeforeApplierFutureCalled =
+    configureFailPoint(recipientPrimary, "fpWaitUntilTimestampMajorityCommitted");
+
+tenantMigrationTest.insertDonorDB(dbName, collName);
+
+jsTestLog("Starting migration.");
+// Start the migration, and allow it to progress to the point where the _dataConsistentPromise has
+// been fulfilled.
+tenantMigrationTest.startMigration(migrationOpts);
+
+jsTestLog("Waiting for data consistent promise.");
+// Pause right before the _dataConsistentPromise is fulfilled. Therefore, the applier has
+// finished applying entries at least until dataConsistentStopDonorOpTime.
+fpBeforeDataConsistent.wait();
+
+jsTestLog("Pausing the tenant oplog applier.");
+// Pause the applier now. All the entries that the applier cannot process now are past the
+// dataConsistentStopDonorOpTime.
+const fpPauseOplogApplier =
+    configureFailPoint(recipientPrimary, "fpBeforeTenantOplogApplyingBatch");
+
+jsTestLog("Writing to donor db.");
+// Send writes to the donor. The applier will not be able to process these as it is paused.
+const docsToApply = [...Array(10).keys()].map((i) => ({a: i}));
+tenantMigrationTest.insertDonorDB(dbName, collName, docsToApply);
+
+jsTestLog("Waiting to hit failpoint in tenant oplog applier.");
+fpPauseOplogApplier.wait();
+
+jsTestLog("Allowing recipient to respond.");
+// Allow the recipient to respond to the donor for the recipientSyncData command that waits on the
+// fulfillment of the _dataConsistentPromise. The donor will then send another recipientSyncData
+// command that waits on the provided donor timestamp to be majority committed.
+fpBeforeDataConsistent.off();
+
+jsTestLog("Reach the point where we are waiting for the tenant oplog applier to catch up.");
+fpBeforeApplierFutureCalled.wait();
+fpBeforeApplierFutureCalled.off();
+
+jsTestLog("Stepping another node up.");
+// Make a new recipient primary step up. This will ask the applier to shutdown.
+assert.commandWorked(recipientRst.getSecondaries()[0].adminCommand({replSetStepUp: 1}));
+
+jsTestLog("Release the tenant oplog applier failpoint.");
+fpPauseOplogApplier.off();
+
+jsTestLog("Waiting for migration to complete.");
+assert.commandWorked(tenantMigrationTest.waitForMigrationToComplete(migrationOpts));
+
+donorRst.stopSet();
+tenantMigrationTest.stop();
+})();
\ No newline at end of file