SERVER-46194 Applying transfer mods in a migration does not handle write conflicts

(cherry picked from commit 7dbcb6b464eaedb4061d5e470313d9bae21ff03d)

3 files changed, 77 insertions, 8 deletions

diff --git a/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml b/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml
index d99a2a3e644..eeedcb03b2f 100644
--- a/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml
+++ b/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml
@@ -161,6 +161,8 @@ selector:
   - jstests/sharding/mr_merge_to_existing.js
   # Enable after SERVER-44598 is backported and available on 4.0 binaries
   - jstests/sharding/ssv_after_restart_of_shards_and_mongos_workarround.js
+  # Requires behavior that does not and will never exist in 4.0.
+  - jstests/sharding/migration_retries_on_write_conflict_exceptions.js
 executor:
   config:
     shell_options:
diff --git a/jstests/sharding/migration_retries_on_write_conflict_exceptions.js b/jstests/sharding/migration_retries_on_write_conflict_exceptions.js
new file mode 100644
index 00000000000..1ef9a4a26a2
--- /dev/null
+++ b/jstests/sharding/migration_retries_on_write_conflict_exceptions.js
@@ -0,0 +1,62 @@
+/**
+ * Tests that WriteConflictException is handled when applying transfer mods during migrations.
+ */
+(function() {
+'use strict';
+
+load("jstests/libs/fail_point_util.js");
+load('jstests/libs/parallel_shell_helpers.js');
+
+const dbName = "test";
+const collName = "foo";
+const ns = dbName + "." + collName;
+
+let st = new ShardingTest({shards: 2});
+
+// Create a sharded collection with two chunks: [-inf, 50), [50, inf)
+assert.commandWorked(st.s.adminCommand({enableSharding: dbName}));
+assert.commandWorked(st.s.adminCommand({movePrimary: dbName, to: st.shard0.shardName}));
+assert.commandWorked(st.s.adminCommand({shardCollection: ns, key: {x: 1}}));
+assert.commandWorked(st.s.adminCommand({split: ns, middle: {x: 50}}));
+
+let testDB = st.s.getDB(dbName);
+let testColl = testDB.foo;
+
+for (let i = 0; i < 100; i++) {
+    assert.commandWorked(testColl.insert({x: i}));
+}
+
+let preTransferModsFailpoint = configureFailPoint(st.shard1, "migrateThreadHangAtStep3");
+
+const awaitResult = startParallelShell(
+    funWithArgs(function(ns, toShardName) {
+        assert.commandWorked(
+            db.adminCommand({moveChunk: ns, find: {x: 50}, to: toShardName, _waitForDelete: true}));
+    }, ns, st.shard1.shardName), st.s.port);
+
+preTransferModsFailpoint.wait();
+
+// Perform each operation that will generate a transfer mod operation in the migration thread. The
+// migration thread processes inserts, updates and deletions which can all throw
+// WriteConflictException.
+for (let i = 100; i < 200; i++) {
+    assert.commandWorked(testColl.insert({x: i}));
+}
+
+for (let i = 50; i < 75; ++i) {
+    assert.commandWorked(testColl.remove({x: i}));
+}
+
+for (let i = 75; i < 100; ++i) {
+    assert.commandWorked(testColl.update({x: i}, {x: i, updated: true}));
+}
+
+// Trigger WriteConflictExceptions during writes.
+assert.commandWorked(st.shard1.adminCommand(
+    {configureFailPoint: 'WTWriteConflictException', mode: {activationProbability: 0.5}}));
+preTransferModsFailpoint.off();
+
+awaitResult();
+
+st.stop();
+})();
diff --git a/src/mongo/db/s/migration_destination_manager.cpp b/src/mongo/db/s/migration_destination_manager.cpp
index fe7ec302a99..66d95751f78 100644
--- a/src/mongo/db/s/migration_destination_manager.cpp
+++ b/src/mongo/db/s/migration_destination_manager.cpp
@@ -38,6 +38,7 @@
 
 #include "mongo/db/auth/authorization_session.h"
 #include "mongo/db/catalog/document_validation.h"
+#include "mongo/db/concurrency/write_conflict_exception.h"
 #include "mongo/db/db_raii.h"
 #include "mongo/db/dbhelpers.h"
 #include "mongo/db/index/index_descriptor.h"
@@ -1064,13 +1065,15 @@ bool MigrationDestinationManager::_applyMigrateOp(OperationContext* opCtx,
                 uassertStatusOK(rs->goingToDelete(fullObj));
             }
 
-            deleteObjects(opCtx,
-                          autoColl.getCollection(),
-                          _nss,
-                          id,
-                          true /* justOne */,
-                          false /* god */,
-                          true /* fromMigrate */);
+            writeConflictRetry(opCtx, "transferModsDeletes", _nss.ns(), [&] {
+                deleteObjects(opCtx,
+                              autoColl.getCollection(),
+                              _nss,
+                              id,
+                              true /* justOne */,
+                              false /* god */,
+                              true /* fromMigrate */);
+            });
 
             *lastOpApplied = repl::ReplClientInfo::forClient(opCtx->getClient()).getLastOp();
             didAnything = true;
@@ -1116,7 +1119,9 @@ bool MigrationDestinationManager::_applyMigrateOp(OperationContext* opCtx,
             }
 
             // We are in write lock here, so sure we aren't killing
-            Helpers::upsert(opCtx, _nss.ns(), updatedDoc, true);
+            writeConflictRetry(opCtx, "transferModsUpdates", _nss.ns(), [&] {
+                Helpers::upsert(opCtx, _nss.ns(), updatedDoc, true);
+            });
 
             *lastOpApplied = repl::ReplClientInfo::forClient(opCtx->getClient()).getLastOp();
             didAnything = true;