SERVER-42390 Allow documents in a sharded collection to not contain all shard key fields

15 files changed, 446 insertions, 310 deletions

diff --git a/jstests/sharding/bulk_insert.js b/jstests/sharding/bulk_insert.js
index 00e03fe8019..f602da597e0 100644
--- a/jstests/sharding/bulk_insert.js
+++ b/jstests/sharding/bulk_insert.js
@@ -63,13 +63,13 @@ assert.eq(2, collUn.find().itcount());
 assert.commandWorked(collDi.insert(inserts));
 assert.eq(2, collDi.find().itcount());
 
-jsTest.log("Bulk insert (no COE) with mongos error...");
+jsTest.log("Bulk insert (no COE) to single shard...");
 
 resetColls();
 var inserts = [{ukey: 0}, {hello: "world"}, {ukey: 1}];
 
-assert.writeError(collSh.insert(inserts));
-assert.eq(1, collSh.find().itcount());
+assert.commandWorked(collSh.insert(inserts));
+assert.eq(3, collSh.find().itcount());
 
 jsTest.log("Bulk insert (no COE) with mongod error...");
 
@@ -85,7 +85,7 @@ assert.eq(1, collUn.find().itcount());
 assert.writeError(collDi.insert(inserts));
 assert.eq(1, collDi.find().itcount());
 
-jsTest.log("Bulk insert (no COE) with mongod and mongos error...");
+jsTest.log("Bulk insert (no COE) with mongod error...");
 
 resetColls();
 var inserts = [{ukey: 0}, {ukey: 0}, {ukey: 1}, {hello: "world"}];
@@ -116,7 +116,7 @@ assert.eq(2, collUn.find().itcount());
 assert.commandWorked(collDi.insert(inserts));
 assert.eq(2, collDi.find().itcount());
 
-jsTest.log("Bulk insert to second shard (no COE) with mongos error...");
+jsTest.log("Bulk insert to second shard (no COE) on second shard...");
 
 resetColls();
 var inserts = [
@@ -126,8 +126,8 @@ var inserts = [
     {hello: "world"}
 ];
 
-assert.writeError(collSh.insert(inserts));
-assert.eq(3, collSh.find().itcount());
+assert.commandWorked(collSh.insert(inserts));
+assert.eq(4, collSh.find().itcount());
 
 jsTest.log("Bulk insert to second shard (no COE) with mongod error...");
 
@@ -143,7 +143,7 @@ assert.eq(4, collUn.find().itcount());
 assert.writeError(collDi.insert(inserts));
 assert.eq(4, collDi.find().itcount());
 
-jsTest.log("Bulk insert to third shard (no COE) with mongod and mongos error...");
+jsTest.log("Bulk insert to third shard (no COE) with mongod error...");
 
 resetColls();
 var inserts =
@@ -165,14 +165,6 @@ assert.eq(5, collDi.find().itcount());
 // CONTINUE-ON-ERROR
 //
 
-jsTest.log("Bulk insert (yes COE) with mongos error...");
-
-resetColls();
-var inserts = [{ukey: 0}, {hello: "world"}, {ukey: 1}];
-
-assert.writeError(collSh.insert(inserts, 1));  // COE
-assert.eq(2, collSh.find().itcount());
-
 jsTest.log("Bulk insert (yes COE) with mongod error...");
 
 resetColls();
@@ -187,18 +179,17 @@ assert.eq(2, collUn.find().itcount());
 assert.writeError(collDi.insert(inserts, 1));
 assert.eq(2, collDi.find().itcount());
 
-jsTest.log("Bulk insert to third shard (yes COE) with mongod and mongos error...");
+jsTest.log("Bulk insert to third shard (yes COE) with mongod error...");
 
 resetColls();
 var inserts =
     [{ukey: 0}, {ukey: 1}, {ukey: -2}, {ukey: -3}, {ukey: 4}, {ukey: 4}, {hello: "world"}];
 
-// Last error here is mongos error
+// Extra insert goes through
 res = assert.writeError(collSh.insert(inserts, 1));
-assert(!isDupKeyError(res.getWriteErrorAt(res.getWriteErrorCount() - 1).errmsg), res.toString());
-assert.eq(5, collSh.find().itcount());
+assert.eq(6, res.nInserted, res.toString());
+assert.eq(6, collSh.find().itcount());
 
-// Extra insert goes through, since mongos error "doesn't count"
 res = assert.writeError(collUn.insert(inserts, 1));
 assert.eq(6, res.nInserted, res.toString());
 assert.eq(6, collUn.find().itcount());
@@ -207,27 +198,6 @@ res = assert.writeError(collDi.insert(inserts, 1));
 assert.eq(6, res.nInserted, res.toString());
 assert.eq(6, collDi.find().itcount());
 
-jsTest.log("Bulk insert to third shard (yes COE) with mongod and mongos error " +
-           "(mongos error first)...");
-
-resetColls();
-var inserts =
-    [{ukey: 0}, {ukey: 1}, {ukey: -2}, {ukey: -3}, {hello: "world"}, {ukey: 4}, {ukey: 4}];
-
-// Last error here is mongos error
-res = assert.writeError(collSh.insert(inserts, 1));
-assert(isDupKeyError(res.getWriteErrorAt(res.getWriteErrorCount() - 1).errmsg), res.toString());
-assert.eq(5, collSh.find().itcount());
-
-// Extra insert goes through, since mongos error "doesn't count"
-res = assert.writeError(collUn.insert(inserts, 1));
-assert(isDupKeyError(res.getWriteErrorAt(res.getWriteErrorCount() - 1).errmsg), res.toString());
-assert.eq(6, collUn.find().itcount());
-
-res = assert.writeError(collDi.insert(inserts, 1));
-assert(isDupKeyError(res.getWriteErrorAt(0).errmsg), res.toString());
-assert.eq(6, collDi.find().itcount());
-
 //
 // Test when WBL has to be invoked mid-insert
 //
diff --git a/jstests/sharding/covered_shard_key_indexes.js b/jstests/sharding/covered_shard_key_indexes.js
index 2926fea3b1b..773a3d0927b 100644
--- a/jstests/sharding/covered_shard_key_indexes.js
+++ b/jstests/sharding/covered_shard_key_indexes.js
@@ -129,9 +129,9 @@ assert.commandWorked(st.shard0.getCollection(coll.toString()).insert({_id: "bad
 // Index without shard key query - not covered but succeeds
 assert.commandWorked(coll.ensureIndex({c: 1}));
 var explain = coll.find({c: true}).explain(true).executionStats;
-assert.eq(0, explain.nReturned);
+assert.eq(1, explain.nReturned);
 assert.eq(1, explain.totalDocsExamined);
-assert.eq(1, getChunkSkips(explain.executionStages.shards[0].executionStages));
+assert.eq(0, getChunkSkips(explain.executionStages.shards[0].executionStages));
 
 // Index with shard key query - covered and succeeds and returns result
 //
diff --git a/jstests/sharding/extract_shard_key_values.js b/jstests/sharding/extract_shard_key_values.js
new file mode 100644
index 00000000000..c3a17ea8174
--- /dev/null
+++ b/jstests/sharding/extract_shard_key_values.js
@@ -0,0 +1,176 @@
+//
+// Tests that documents in a sharded collection with missing shard key fields are treated as if they
+// contain an explicit null value for any missing fields.
+//
+// @tags: [requires_find_command]
+
+(function() {
+'use strict';
+
+const st = new ShardingTest({shards: 2});
+const mongos = st.s0;
+const primaryShard = st.shard0.shardName;
+const secondaryShard = st.shard1.shardName;
+const kDbName = 'db';
+const kCollName = 'foo';
+const kNsName = kDbName + '.' + kCollName;
+const kOldKeyDoc = {
+    a: 1
+};
+const kNewKeyDoc = {
+    a: 1,
+    b: 1
+};
+
+function orphanDocCount() {
+    // Since count() includes orphaned documents while find({}).itcount() excludes them, their
+    // difference corresponds to the number of orphaned documents in 'db.foo'.
+    return mongos.getCollection(kNsName).count() - mongos.getCollection(kNsName).find({}).itcount();
+}
+
+function dropAndReshardColl() {
+    assert.commandWorked(mongos.getDB(kDbName).runCommand({drop: kCollName}));
+    assert.commandWorked(mongos.adminCommand({shardCollection: kNsName, key: kOldKeyDoc}));
+    assert.commandWorked(mongos.getCollection(kNsName).createIndex(kNewKeyDoc));
+
+    // Insert six documents such that three correspond to the old shard key and three correspond to
+    // the new shard key. Verify that there are no orphaned documents in 'db.foo'.
+    assert.commandWorked(mongos.getCollection(kNsName).insert({a: 1}));
+    assert.commandWorked(mongos.getCollection(kNsName).insert({a: 10}));
+    assert.commandWorked(mongos.getCollection(kNsName).insert({a: null}));
+    assert.commandWorked(mongos.getCollection(kNsName).insert({a: 1, b: 1}));
+    assert.commandWorked(mongos.getCollection(kNsName).insert({a: 10, b: 1}));
+    assert.commandWorked(mongos.getCollection(kNsName).insert({a: null, b: 1}));
+    assert.eq(0, orphanDocCount());
+}
+
+function isOwnedByShard(shardName, doc) {
+    let isOwned = false;
+    mongos.getCollection(kNsName)
+        .find(doc)
+        .explain('executionStats')
+        .executionStats.executionStages.shards.forEach((shard) => {
+            if (shard.shardName.localeCompare(shardName) === 0) {
+                isOwned = (1 === shard.nReturned);
+            }
+        });
+    return isOwned;
+}
+
+function isOwnedByPrimaryShard(doc) {
+    return isOwnedByShard(primaryShard, doc);
+}
+
+function isOwnedBySecondaryShard(doc) {
+    return isOwnedByShard(secondaryShard, doc);
+}
+
+assert.commandWorked(mongos.adminCommand({enableSharding: kDbName}));
+st.ensurePrimaryShard(kDbName, primaryShard);
+
+jsTestLog('********** ORPHAN FILTERING **********');
+
+dropAndReshardColl();
+
+// Verify that moving a chunk to the secondary shard produces two orphaned documents in 'db.foo' as
+// a result of migrating documents {a: 10} and {a: 10, b: 1}.
+assert.commandWorked(mongos.adminCommand({split: kNsName, middle: {a: 5}}));
+assert.commandWorked(mongos.adminCommand({moveChunk: kNsName, find: {a: 5}, to: secondaryShard}));
+assert.lte(2, orphanDocCount());
+
+// Verify that refining the shard key produces no additional orphaned documents in 'db.foo'.
+assert.commandWorked(mongos.adminCommand({refineCollectionShardKey: kNsName, key: kNewKeyDoc}));
+assert.lte(2, orphanDocCount());
+
+jsTestLog('********** REQUEST TARGETING **********');
+
+dropAndReshardColl();
+
+// Ensure that there exist two chunks belonging to 'db.foo' covering the entire key range.
+//
+// Chunk 1: {a: MinKey, b: MinKey} -->> {a: 5, b: MinKey} (belongs to the primary shard)
+// Chunk 2: {a: 5, b: MinKey} -->> {a: MaxKey, b: MaxKey} (belongs to the secondary shard)
+assert.commandWorked(mongos.adminCommand({split: kNsName, middle: {a: 5}}));
+assert.commandWorked(mongos.adminCommand({moveChunk: kNsName, find: {a: 5}, to: secondaryShard}));
+assert.commandWorked(mongos.adminCommand({refineCollectionShardKey: kNsName, key: kNewKeyDoc}));
+
+// Verify that chunk 1 owns documents {a: 1}, {a: null}, {a: 1, b: 1}, {a: null, b: 1} while chunk 2
+// owns documents {a: 10} and {a: 10, b: 1}.
+assert(isOwnedByPrimaryShard({a: 1, b: {$exists: false}}));
+assert(isOwnedBySecondaryShard({a: 10, b: {$exists: false}}));
+assert(isOwnedByPrimaryShard({a: null, b: {$exists: false}}));
+assert(isOwnedByPrimaryShard({a: 1, b: 1}));
+assert(isOwnedBySecondaryShard({a: 10, b: 1}));
+assert(isOwnedByPrimaryShard({a: null, b: 1}));
+
+// Verify that find targets shards without treating missing shard key fields as null values.
+let docsArr = mongos.getCollection(kNsName).find({b: 1}, {_id: 0}).sort({a: 1}).toArray();
+assert.eq(3, docsArr.length);
+assert.eq({a: null, b: 1}, docsArr[0]);
+assert.eq({a: 1, b: 1}, docsArr[1]);
+assert.eq({a: 10, b: 1}, docsArr[2]);
+
+// Verify that count targets shards without treating missing shard key fields as null values.
+assert.eq(3, mongos.getCollection(kNsName).count({b: 1}));
+
+// Verify that distinct targets shards without treating missing shard key fields as null values.
+const valuesArr = mongos.getCollection(kNsName).distinct('a').sort();
+assert.eq(3, valuesArr.length);
+assert.eq(1, valuesArr[0]);
+assert.eq(10, valuesArr[1]);
+assert.eq(null, valuesArr[2]);
+
+// Verify that insert targets shards as if missing shard key fields were null values.
+assert.commandWorked(mongos.getCollection(kNsName).insert({b: 10}));
+docsArr = mongos.getCollection(kNsName).find({b: 10}).toArray();
+assert(isOwnedByPrimaryShard({b: 10}));
+assert(!isOwnedBySecondaryShard({b: 10}));
+
+// Verify that delete targets shards without treating missing shard key fields as null values.
+assert.commandWorked(mongos.getCollection(kNsName).insert({a: 10, b: 10}));
+assert(!isOwnedByPrimaryShard({a: 10, b: 10}));
+assert(isOwnedBySecondaryShard({a: 10, b: 10}));
+
+assert.commandWorked(mongos.getCollection(kNsName).remove({b: 10}));
+docsArr = mongos.getCollection(kNsName).find({b: 10}, {_id: 0}).toArray();
+assert.eq(0, docsArr.length);
+
+// Verify that a query-style update targets shards without treating missing shard key fields as null
+// values.
+assert.commandWorked(mongos.getCollection(kNsName).update({b: 1}, {$set: {c: 1}}, {multi: true}));
+docsArr = mongos.getCollection(kNsName).find({c: 1}, {_id: 0}).sort({a: 1}).toArray();
+assert.eq(3, docsArr.length);
+assert.eq({a: null, b: 1, c: 1}, docsArr[0]);
+assert.eq({a: 1, b: 1, c: 1}, docsArr[1]);
+assert.eq({a: 10, b: 1, c: 1}, docsArr[2]);
+
+// Verify that a replacement update targets shards while treating missing shard keys as null values.
+
+// Insert documents all with {d: 1} so they're matched by the update query.
+assert.commandWorked(mongos.getCollection(kNsName).insert({a: -100, b: 1, c: 1, d: 1}));
+assert.commandWorked(mongos.getCollection(kNsName).insert({a: 0, b: 1, c: 2, d: 1}));
+assert.commandWorked(mongos.getCollection(kNsName).insert({a: 100, b: 1, c: 3, d: 1}));
+
+// Update via a query that's missing the shard key, in order to force the targeting logic to fall
+// back to the replacement document.
+
+// Need to start a session to change the shard key.
+const session = st.s.startSession({retryWrites: true});
+const sessionDB = session.getDatabase(kDbName);
+const sessionColl = sessionDB[kCollName];
+
+assert.commandWorked(sessionColl.update({d: 1}, {b: 1, c: 4, d: 1}));
+docsArr = sessionColl.find({c: 4, d: 1}, {_id: 0}).toArray();
+assert.eq(1, docsArr.length);
+assert.eq({b: 1, c: 4, d: 1}, docsArr[0]);
+assert(isOwnedByPrimaryShard({b: 1, c: 4, d: 1}));
+assert(!isOwnedBySecondaryShard({b: 1, c: 4, d: 1}));
+
+// Verify that an upsert targets shards without treating missing shard key fields as null values.
+// This implies that upsert still requires the entire shard key to be specified in the query.
+assert.writeErrorWithCode(
+    mongos.getCollection(kNsName).update({b: 1}, {$set: {c: 2}}, {upsert: true}),
+    ErrorCodes.ShardKeyNotFound);
+
+st.stop();
+})();
\ No newline at end of file
diff --git a/jstests/sharding/features1.js b/jstests/sharding/features1.js
index 92e5ff5b468..25308f47788 100644
--- a/jstests/sharding/features1.js
+++ b/jstests/sharding/features1.js
@@ -110,11 +110,10 @@ assert.commandWorked(db.foo8.insert({a: 1}));
 assert.commandFailed(s.s0.adminCommand({shardcollection: "test.foo8", key: {a: 1}}),
                      "non-empty collection");
 
-// ---- can't shard non-empty collection with null values in shard key ----
+// ---- can shard non-empty collection with null values in shard key ----
 assert.commandWorked(db.foo9.insert({b: 1}));
 assert.commandWorked(db.foo9.createIndex({a: 1}));
-assert.commandFailed(s.s0.adminCommand({shardcollection: "test.foo9", key: {a: 1}}),
-                     "entry with null value");
+assert.commandWorked(s.s0.adminCommand({shardcollection: "test.foo9", key: {a: 1}}));
 
 // --- listDatabases ---
 var r = db.getMongo().getDBs();
diff --git a/jstests/sharding/libs/update_shard_key_helpers.js b/jstests/sharding/libs/update_shard_key_helpers.js
index 2260372bc6f..178a09b33da 100644
--- a/jstests/sharding/libs/update_shard_key_helpers.js
+++ b/jstests/sharding/libs/update_shard_key_helpers.js
@@ -409,6 +409,42 @@ function assertCanUpdatePartialShardKey(st,
     sessionDB.foo.drop();
 }
 
+function assertCanDoReplacementUpdateWhereShardKeyMissingFields(st,
+                                                                kDbName,
+                                                                ns,
+                                                                session,
+                                                                sessionDB,
+                                                                inTxn,
+                                                                isFindAndModify,
+                                                                queries,
+                                                                updates,
+                                                                upsert,
+                                                                pipelineUpdateResult) {
+    assertCanUpdatePartialShardKey(st,
+                                   kDbName,
+                                   ns,
+                                   session,
+                                   sessionDB,
+                                   inTxn,
+                                   isFindAndModify,
+                                   queries,
+                                   updates,
+                                   upsert,
+                                   pipelineUpdateResult);
+}
+
+function assertCanUnsetSKField(
+    st, kDbName, ns, session, sessionDB, inTxn, isFindAndModify, query, update, upsert) {
+    assertCanUpdatePrimitiveShardKey(
+        st, kDbName, ns, session, sessionDB, inTxn, isFindAndModify, query, update, upsert);
+}
+
+function assertCanUnsetSKFieldUsingPipeline(
+    st, kDbName, ns, session, sessionDB, inTxn, isFindAndModify, query, update, upsert) {
+    assertCanUpdatePrimitiveShardKey(
+        st, kDbName, ns, session, sessionDB, inTxn, isFindAndModify, query, update, upsert);
+}
+
 function assertCannotUpdate_id(st,
                                kDbName,
                                ns,
@@ -483,23 +519,6 @@ function assertCannotUpdate_idDottedPath(st,
     sessionDB.foo.drop();
 }
 
-function assertCannotDoReplacementUpdateWhereShardKeyMissingFields(
-    st, kDbName, ns, session, sessionDB, inTxn, isFindAndModify, query, update) {
-    let docsToInsert =
-        [{"x": 4, "y": 3}, {"x": 100, "y": 50}, {"x": 300, "y": 80}, {"x": 500, "y": 600}];
-    shardCollectionMoveChunks(
-        st, kDbName, ns, {"x": 1, "y": 1}, docsToInsert, {"x": 100, "y": 50}, {"x": 300, "y": 80});
-    cleanupOrphanedDocs(st, ns);
-
-    if (isFindAndModify) {
-        runFindAndModifyCmdFail(st, kDbName, session, sessionDB, inTxn, query, update);
-    } else {
-        runUpdateCmdFail(st, kDbName, session, sessionDB, inTxn, query, update, false);
-    }
-
-    sessionDB.foo.drop();
-}
-
 function assertCannotUpdateWithMultiTrue(
     st, kDbName, ns, session, sessionDB, inTxn, query, update, pipelineUpdateResult) {
     let docsToInsert = [{"x": 4, "a": 3}, {"x": 100}, {"x": 300, "a": 3}, {"x": 500, "a": 6}];
@@ -527,59 +546,6 @@ function assertCannotUpdateSKToArray(
     sessionDB.foo.drop();
 }
 
-function assertCannotUnsetSKField(
-    st, kDbName, ns, session, sessionDB, inTxn, isFindAndModify, query, update) {
-    // Updates to the shard key cannot $unset a shard key field from a doc
-    let docsToInsert = [{"x": 4, "a": 3}, {"x": 100}, {"x": 300, "a": 3}, {"x": 500, "a": 6}];
-    shardCollectionMoveChunks(st, kDbName, ns, {"x": 1}, docsToInsert, {"x": 100}, {"x": 300});
-    cleanupOrphanedDocs(st, ns);
-
-    if (isFindAndModify) {
-        runFindAndModifyCmdFail(st, kDbName, session, sessionDB, inTxn, query, update);
-    } else {
-        runUpdateCmdFail(st, kDbName, session, sessionDB, inTxn, query, update, false);
-    }
-
-    sessionDB.foo.drop();
-}
-
-function assertCannotUnsetSKFieldUsingPipeline(st,
-                                               kDbName,
-                                               ns,
-                                               session,
-                                               sessionDB,
-                                               inTxn,
-                                               isFindAndModify,
-                                               query,
-                                               update,
-                                               pipelineUpdateResult) {
-    // Updates to the shard key cannot $project out a shard key field from a doc
-    let docsToInsert =
-        [{"x": 4, "y": 3}, {"x": 100, "y": 50}, {"x": 300, "y": 80}, {"x": 500, "y": 600}];
-    let splitDoc = {"x": 100, "y": 50};
-    shardCollectionMoveChunks(
-        st, kDbName, ns, {"x": 1, "y": 1}, docsToInsert, splitDoc, {"x": 300, "y": 80});
-    cleanupOrphanedDocs(st, ns);
-
-    if (isFindAndModify) {
-        runFindAndModifyCmdFail(
-            st, kDbName, session, sessionDB, inTxn, query, update, false, pipelineUpdateResult);
-    } else {
-        runUpdateCmdFail(st,
-                         kDbName,
-                         session,
-                         sessionDB,
-                         inTxn,
-                         query,
-                         update,
-                         false,
-                         null,
-                         pipelineUpdateResult);
-    }
-
-    sessionDB.foo.drop();
-}
-
 // Shard key updates are allowed in bulk ops if the update doesn't cause the doc to move shards
 function assertCanUpdateInBulkOpWhenDocsRemainOnSameShard(
     st, kDbName, ns, session, sessionDB, inTxn, ordered) {
diff --git a/jstests/sharding/missing_key.js b/jstests/sharding/missing_key.js
deleted file mode 100644
index e6b04623fad..00000000000
--- a/jstests/sharding/missing_key.js
+++ /dev/null
@@ -1,42 +0,0 @@
-// Test that the shardCollection command fails when a preexisting document lacks a shard key field.
-// SERVER-8772
-(function() {
-'use strict';
-
-var st = new ShardingTest({shards: 1});
-
-var db = st.s.getDB('testDb');
-var coll = db.testColl;
-
-assert.commandWorked(coll.insert({x: 1, z: 1}));
-assert.commandWorked(coll.insert({y: 1, z: 1}));
-
-assert.commandWorked(db.adminCommand({enableSharding: 'testDb'}));
-
-/**
- * Assert that the shardCollection command fails, with a preexisting index on the provided
- * 'shardKey'.
- */
-function assertInvalidShardKey(shardKey) {
-    // Manually create a shard key index.
-    coll.dropIndexes();
-    coll.ensureIndex(shardKey);
-
-    // Ensure that the shard key index identifies 'x' as present in one document and absent in
-    // the other.
-    assert.eq(1, coll.find({x: 1}).hint(shardKey).itcount());
-    assert.eq(1, coll.find({x: {$exists: false}}).hint(shardKey).itcount());
-
-    // Assert that the shardCollection command fails with the provided 'shardKey'.
-    assert.commandFailed(db.adminCommand({shardCollection: 'testDb.testColl', key: shardKey}),
-                         'shardCollection should have failed on key ' + tojson(shardKey));
-}
-
-// Test single, compound, and hashed shard keys.
-assertInvalidShardKey({x: 1});
-assertInvalidShardKey({x: 1, y: 1});
-assertInvalidShardKey({y: 1, x: 1});
-assertInvalidShardKey({x: 'hashed'});
-
-st.stop();
-})();
diff --git a/jstests/sharding/mongos_validate_writes.js b/jstests/sharding/mongos_validate_writes.js
index f920992799c..8f08b56608e 100644
--- a/jstests/sharding/mongos_validate_writes.js
+++ b/jstests/sharding/mongos_validate_writes.js
@@ -13,7 +13,6 @@ var staleMongosA = st.s1;
 var staleMongosB = st.s2;
 
 var admin = mongos.getDB("admin");
-var config = mongos.getDB("config");
 var coll = mongos.getCollection("foo.bar");
 var staleCollA = staleMongosA.getCollection(coll + "");
 var staleCollB = staleMongosB.getCollection(coll + "");
@@ -39,9 +38,6 @@ st.shardColl(coll, {b: 1}, {b: 0}, {b: 1}, coll.getDB(), true);
 // Make sure that we can successfully insert, even though we have stale state
 assert.commandWorked(staleCollA.insert({b: "b"}));
 
-// Make sure we unsuccessfully insert with old info
-assert.writeError(staleCollB.insert({a: "a"}));
-
 // Change the collection sharding state
 coll.drop();
 coll.ensureIndex({c: 1});
diff --git a/jstests/sharding/prefix_shard_key.js b/jstests/sharding/prefix_shard_key.js
index b0e37e91bba..be6f41fed34 100644
--- a/jstests/sharding/prefix_shard_key.js
+++ b/jstests/sharding/prefix_shard_key.js
@@ -42,14 +42,8 @@ assert.throws(function() {
 // create usable index
 assert.commandWorked(coll.ensureIndex({num: 1, x: 1}));
 
-// usable index, but doc with empty 'num' value, so still should throw
+// usable index, doc with empty 'num' value
 assert.commandWorked(coll.insert({x: -5}));
-assert.throws(function() {
-    s.adminCommand({shardCollection: coll.getFullName(), key: {num: 1}});
-});
-
-// remove the bad doc.  now should finally succeed
-assert.commandWorked(coll.remove({x: -5}));
 assert.commandWorked(s.s0.adminCommand({shardCollection: coll.getFullName(), key: {num: 1}}));
 
 // make sure extra index is not created
diff --git a/jstests/sharding/refine_collection_shard_key_basic.js b/jstests/sharding/refine_collection_shard_key_basic.js
index 2cbc4790edd..e29fd6b3368 100644
--- a/jstests/sharding/refine_collection_shard_key_basic.js
+++ b/jstests/sharding/refine_collection_shard_key_basic.js
@@ -101,31 +101,31 @@ function validateCRUDAfterRefine() {
     assert.eq([{a: 10, b: 10, c: 10, d: 10}],
               sessionDB.getCollection(kCollName).find({a: 10}, {_id: 0}).toArray());
 
-    // The full shard key is required when inserting documents.
-    assert.writeErrorWithCode(sessionDB.getCollection(kCollName).insert({a: 1, b: 1}),
-                              ErrorCodes.ShardKeyNotFound);
-    assert.writeErrorWithCode(sessionDB.getCollection(kCollName).insert({a: -1, b: -1}),
-                              ErrorCodes.ShardKeyNotFound);
+    // A write with the incomplete shard key is treated as if the missing values are null.
+
+    assert.commandWorked(sessionDB.getCollection(kCollName).insert({a: 1, b: 1}));
+    assert.commandWorked(sessionDB.getCollection(kCollName).insert({a: -1, b: -1}));
+
+    assert.neq(null, sessionDB.getCollection(kCollName).findOne({a: 1, b: 1, c: null, d: null}));
+    assert.neq(null, sessionDB.getCollection(kCollName).findOne({a: -1, b: -1, c: null, d: null}));
+
+    // Full shard key writes work properly.
     assert.commandWorked(sessionDB.getCollection(kCollName).insert({a: 1, b: 1, c: 1, d: 1}));
     assert.commandWorked(sessionDB.getCollection(kCollName).insert({a: -1, b: -1, c: -1, d: -1}));
 
-    // The full shard key is required when updating documents.
-    assert.writeErrorWithCode(
-        sessionDB.getCollection(kCollName).update({a: 1, b: 1}, {$set: {b: 2}}), 31025);
-    assert.writeErrorWithCode(
-        sessionDB.getCollection(kCollName).update({a: -1, b: -1}, {$set: {b: 2}}), 31025);
+    // The full shard key is not required when updating documents.
     assert.commandWorked(
-        sessionDB.getCollection(kCollName).update({a: 1, b: 1, c: 1, d: 1}, {$set: {b: 2}}));
+        sessionDB.getCollection(kCollName).update({a: 1, b: 1, c: 1}, {$set: {b: 2}}));
     assert.commandWorked(
-        sessionDB.getCollection(kCollName).update({a: -1, b: -1, c: -1, d: -1}, {$set: {b: 4}}));
+        sessionDB.getCollection(kCollName).update({a: -1, b: -1, c: -1}, {$set: {b: 4}}));
 
-    assert.eq(2, sessionDB.getCollection(kCollName).findOne({a: 1}).b);
-    assert.eq(4, sessionDB.getCollection(kCollName).findOne({a: -1}).b);
+    assert.eq(2, sessionDB.getCollection(kCollName).findOne({c: 1}).b);
+    assert.eq(4, sessionDB.getCollection(kCollName).findOne({c: -1}).b);
 
     // Versioned reads against secondaries should work as expected.
     mongos.setReadPref("secondary");
-    assert.eq(2, sessionDB.getCollection(kCollName).findOne({a: 1}).b);
-    assert.eq(4, sessionDB.getCollection(kCollName).findOne({a: -1}).b);
+    assert.eq(2, sessionDB.getCollection(kCollName).findOne({c: 1}).b);
+    assert.eq(4, sessionDB.getCollection(kCollName).findOne({c: -1}).b);
     mongos.setReadPref(null);
 
     // The full shard key is required when removing documents.
@@ -139,6 +139,11 @@ function validateCRUDAfterRefine() {
     assert.commandWorked(sessionDB.getCollection(kCollName).remove({a: 5, b: 5, c: 5, d: 5}, true));
     assert.commandWorked(
         sessionDB.getCollection(kCollName).remove({a: 10, b: 10, c: 10, d: 10}, true));
+    assert.commandWorked(
+        sessionDB.getCollection(kCollName).remove({a: 1, b: 1, c: null, d: null}, true));
+    assert.commandWorked(
+        sessionDB.getCollection(kCollName).remove({a: -1, b: -1, c: null, d: null}, true));
+
     assert.eq(null, sessionDB.getCollection(kCollName).findOne());
 }
 
@@ -408,15 +413,14 @@ assert.commandWorked(
 validateConfigCollections({_id: 1, aKey: 1}, oldEpoch);
 validateConfigChangelog(1);
 
-// Should fail because only an index with missing or incomplete shard key entries exists for new
-// shard key {_id: 1, aKey: 1}.
+// Should work because an index with missing or incomplete shard key entries exists for new shard
+// key {_id: 1, aKey: 1} and these entries are treated as null values.
 dropAndReshardColl({_id: 1});
 assert.commandWorked(mongos.getCollection(kNsName).createIndex({_id: 1, aKey: 1}));
 assert.commandWorked(mongos.getCollection(kNsName).insert({_id: 12345}));
 
-assert.commandFailedWithCode(
-    mongos.adminCommand({refineCollectionShardKey: kNsName, key: {_id: 1, aKey: 1}}),
-    ErrorCodes.OperationFailed);
+assert.commandWorked(
+    mongos.adminCommand({refineCollectionShardKey: kNsName, key: {_id: 1, aKey: 1}}));
 
 // Should fail because new shard key {aKey: 1} is not a prefix of current shard key {_id: 1,
 // aKey: 1}.
@@ -462,7 +466,7 @@ oldEpoch = mongos.getCollection(kConfigCollections).findOne({_id: kNsName}).last
 assert.commandWorked(
     mongos.adminCommand({refineCollectionShardKey: kNsName, key: {_id: 1, aKey: 1}}));
 validateConfigCollections({_id: 1, aKey: 1}, oldEpoch);
-validateConfigChangelog(2);
+validateConfigChangelog(3);
 
 // Should work because a 'useful' index exists for new shard key {a: 1, b.c: 1}. NOTE: We are
 // explicitly verifying that refineCollectionShardKey works with a dotted field.
@@ -473,7 +477,7 @@ oldEpoch = mongos.getCollection(kConfigCollections).findOne({_id: kNsName}).last
 assert.commandWorked(
     mongos.adminCommand({refineCollectionShardKey: kNsName, key: {a: 1, 'b.c': 1}}));
 validateConfigCollections({a: 1, 'b.c': 1}, oldEpoch);
-validateConfigChangelog(3);
+validateConfigChangelog(4);
 
 assert.commandWorked(mongos.getDB(kDbName).dropDatabase());
 
diff --git a/jstests/sharding/server_status_crud_metrics.js b/jstests/sharding/server_status_crud_metrics.js
index 6ffefe87b71..20b0274ae9d 100644
--- a/jstests/sharding/server_status_crud_metrics.js
+++ b/jstests/sharding/server_status_crud_metrics.js
@@ -31,16 +31,14 @@ assert.commandWorked(testDB.coll.update({_id: 1}, {$set: {a: 2}}, {multi: false}
 
 // Should increment the metric because we broadcast by _id, even though the update subsequently
 // fails on the individual shard.
-assert.commandFailedWithCode(testDB.coll.update({_id: 1}, {$set: {x: 2}}, {multi: false}),
-                             [ErrorCodes.ImmutableField, 31025]);
 assert.commandFailedWithCode(
     testDB.coll.update({_id: 1}, {$set: {x: 2, $invalidField: 4}}, {multi: false}),
     ErrorCodes.DollarPrefixedFieldName);
 
 let mongosServerStatus = testDB.adminCommand({serverStatus: 1});
 
-// Verify that the above four updates incremented the metric counter.
-assert.eq(4, mongosServerStatus.metrics.query.updateOneOpStyleBroadcastWithExactIDCount);
+// Verify that the above three updates incremented the metric counter.
+assert.eq(3, mongosServerStatus.metrics.query.updateOneOpStyleBroadcastWithExactIDCount);
 
 // Shouldn't increment the metric when {multi:true}.
 assert.commandWorked(testDB.coll.update({_id: 1}, {$set: {a: 3}}, {multi: true}));
@@ -71,8 +69,8 @@ assert.commandFailedWithCode(
 
 mongosServerStatus = testDB.adminCommand({serverStatus: 1});
 
-// Verify that only the first four upserts incremented the metric counter.
-assert.eq(4, mongosServerStatus.metrics.query.updateOneOpStyleBroadcastWithExactIDCount);
+// Verify that only the first three upserts incremented the metric counter.
+assert.eq(3, mongosServerStatus.metrics.query.updateOneOpStyleBroadcastWithExactIDCount);
 
 st.stop();
 })();
diff --git a/jstests/sharding/update_compound_shard_key.js b/jstests/sharding/update_compound_shard_key.js
index 46e28ed597f..8b54153d367 100644
--- a/jstests/sharding/update_compound_shard_key.js
+++ b/jstests/sharding/update_compound_shard_key.js
@@ -10,7 +10,7 @@ load("jstests/sharding/libs/update_shard_key_helpers.js");
 const st = new ShardingTest({mongos: 1, shards: 3});
 const kDbName = 'update_compound_sk';
 const ns = kDbName + '.coll';
-const session = st.s.startSession();
+const session = st.s.startSession({retryWrites: true});
 const sessionDB = session.getDatabase(kDbName);
 
 assert.commandWorked(st.s0.adminCommand({enableSharding: kDbName}));
@@ -40,7 +40,7 @@ assert.commandWorked(
 cleanupOrphanedDocs(st, ns);
 
 function assertUpdateWorked(query, update, isUpsert, _id) {
-    const res = st.s.getDB(kDbName).coll.update(query, update, {upsert: isUpsert});
+    const res = sessionDB.coll.update(query, update, {upsert: isUpsert});
     assert.commandWorked(res);
     assert.eq(0, res.nUpserted);
     assert.eq(1, res.nMatched);
@@ -108,15 +108,9 @@ assertUpdateWorkedWithNoMatchingDoc({x: 10}, {x: 10, y: 3, z: 3, a: 5}, false);
 assertUpdateWorkedWithNoMatchingDoc({x: 100, y: 55, a: 15}, {x: 100, y: 55, z: 3, a: 6}, false);
 assertUpdateWorkedWithNoMatchingDoc({x: 11, _id: 3}, {x: 11, y: 3, z: 3, a: 7}, false);
 
-// Partial shard key in query can target a single shard, but fails while attempting to
-// modify shard key value.
-assert.commandFailedWithCode(
-    st.s.getDB(kDbName).coll.update(
-        {x: 100, y: 50, a: 5}, {x: 100, y: 55, z: 3, a: 1}, {upsert: false}),
-    [31025]);
-assert.commandFailedWithCode(
-    st.s.getDB(kDbName).coll.update({x: 4, z: 3}, {x: 4, y: 3, z: 4, a: 1}, {upsert: false}),
-    [31025]);
+// Partial shard key in query can target a single shard, and succeeds in updating shard key value.
+assertUpdateWorkedWithNoMatchingDoc({x: 400, y: 50, a: 5}, {x: 100, y: 55, z: 3, a: 1}, false);
+assertUpdateWorked({x: 4, z: 3}, {x: 4, y: 3, z: 4, a: 1}, false, 0);
 
 // Full shard key in query, matches no document.
 assertUpdateWorkedWithNoMatchingDoc({x: 4, y: 0, z: 0}, {x: 1110, y: 55, z: 3, a: 111}, false);
@@ -135,11 +129,9 @@ assertUpdateWorkedWithNoMatchingDoc({_id: 1}, {x: 110, y: 55, z: 3, a: 110}, fal
 assertUpdateWorked({_id: 0, y: 3}, {z: 3, x: 4, y: 3, a: 2}, false, 0);
 assertUpdateWorked({_id: 0}, {z: 3, x: 4, y: 3, replStyle: 2}, false, 0);
 
-// When query matches a doc and fails to update because shard key needs to be updated.
-assert.commandFailedWithCode(
-    st.s.getDB(kDbName).coll.update({}, {x: 110, y: 55, z: 3, a: 110}, false), 31025);
-assert.commandFailedWithCode(
-    st.s.getDB(kDbName).coll.update({_id: 2}, {x: 110, y: 55, z: 3, a: 110}, false), 31025);
+// When query matches a doc and updates the shard key.
+assertUpdateWorked({}, {x: 110, y: 55, z: 3, a: 110}, false, 2);
+assertUpdateWorked({_id: 2}, {x: 110, y: 55, z: 3, a: 110}, false, 2);
 
 //
 // Test upsert-specific behaviours.
@@ -237,15 +229,9 @@ assertUpdateWorkedWithNoMatchingDoc({x: -1, y: 0}, {"$set": {z: 3, y: 110, a: 91
 assertUpdateWorked({x: 4, z: 3}, {"$set": {opStyle: 3}}, false, 0);
 assertUpdateWorked({x: 4, _id: 0, z: 3}, {"$set": {y: 3, x: 4, z: 3, opStyle: 4}}, false, 0);
 
-// Partial shard key in query can target a single shard, but fails while attempting to modify
-// shard key value.
-assert.commandFailedWithCode(
-    st.s.getDB(kDbName).coll.update(
-        {_id: 1, x: 100, z: 3, a: 5}, {"$set": {y: 55, a: 11}}, {upsert: false}),
-    [31025]);
-assert.commandFailedWithCode(st.s.getDB(kDbName).coll.update(
-                                 {x: 4, z: 3}, {"$set": {x: 4, y: 3, z: 4, a: 1}}, {upsert: false}),
-                             [31025]);
+// Partial shard key in query can target a single shard and updates the shard key.
+assertUpdateWorked({_id: 1, x: 100, z: 3, a: 5}, {"$set": {y: 55, a: 11}}, false, 1);
+assertUpdateWorked({x: 4, z: 3}, {"$set": {x: 4, y: 3, z: 4, a: 1}}, false, 0);
 
 // Test upsert-specific behaviours.
 
@@ -300,16 +286,16 @@ assert.commandFailedWithCode(
 [false, true].forEach(function(isUpsert) {
     // Full shard key in query.
     assertUpdateWorked(
-        {_id: 0, x: 4, z: 3, y: 3}, [{$addFields: {pipelineUpdate: isUpsert}}], isUpsert, 0);
+        {_id: 0, x: 4, z: 4, y: 3}, [{$addFields: {pipelineUpdate: isUpsert}}], isUpsert, 0);
     assert.eq(1,
               st.s.getDB(kDbName)
-                  .coll.find({_id: 0, x: 4, z: 3, y: 3, pipelineUpdate: isUpsert})
+                  .coll.find({_id: 0, x: 4, z: 4, y: 3, pipelineUpdate: isUpsert})
                   .itcount());
     assertUpdateWorkedWithNoMatchingDoc(
-        {_id: 15, x: 44, z: 3, y: 3}, [{$addFields: {pipelineUpdate: true}}], isUpsert);
+        {_id: 15, x: 44, z: 4, y: 3}, [{$addFields: {pipelineUpdate: true}}], isUpsert);
     assert.eq(isUpsert ? 1 : 0,
               st.s.getDB(kDbName)
-                  .coll.find({_id: 15, x: 44, z: 3, y: 3, pipelineUpdate: true})
+                  .coll.find({_id: 15, x: 44, z: 4, y: 3, pipelineUpdate: true})
                   .itcount());
 
     assertUpdateWorkedWithNoMatchingDoc(
@@ -355,7 +341,7 @@ assertUpdateWorkedWithNoMatchingDoc({_id: 14, z: 4, x: 3}, [{$addFields: {foo: 4
 // value.
 assertUpdateWorkedWithNoMatchingDoc(
     {x: 46, z: 4}, [{$addFields: {y: 10, pipelineUpdateNoOp: false}}], false);
-assertUpdateWorked({x: 4, z: 3}, [{$addFields: {pipelineUpdateDoc: false}}], false, 0);
+assertUpdateWorked({x: 4, z: 4}, [{$addFields: {pipelineUpdateDoc: false}}], false, 0);
 
 // Partial shard key in query cannot target a single shard.
 assert.commandFailedWithCode(
diff --git a/jstests/sharding/update_replace_id.js b/jstests/sharding/update_replace_id.js
index 0cd19ef1d88..837b1f97c98 100644
--- a/jstests/sharding/update_replace_id.js
+++ b/jstests/sharding/update_replace_id.js
@@ -10,6 +10,8 @@
  * These special cases are allowed because mongoD always propagates the _id of an existing document
  * into its replacement, and in the case of an upsert will use the value of _id from the query
  * filter.
+ *
+ * @tags: [requires_find_command]
  */
 (function() {
 load("jstests/libs/profiler.js");  // For profilerHas*OrThrow helper functions.
@@ -148,35 +150,28 @@ function runReplacementUpdateTestsForCompoundShardKey() {
     assert.docEq(mongosColl.find({_id: 101}).itcount(), 0);
 
     // Verify that an update whose query contains an exact match on _id but whose replacement
-    // doc does not contain all other shard key fields will be rejected by mongoS.
-    writeRes = assert.commandFailedWithCode(
-        mongosColl.update({_id: -100, a: -100}, {msg: "update_failed_missing_shard_key_field"}),
-        ErrorCodes.ShardKeyNotFound);
+    // doc does not contain all other shard key fields will be targeted as if the missing shard
+    // key values are null, but will write the replacement document as-is.
 
-    // Check that the existing document remains unchanged, and that the update did not reach
-    // either shard per their respective profilers.
-    assert.docEq(mongosColl.find({_id: -100, a: -100}).toArray(),
-                 [{_id: -100, a: -100, msg: "update_extracted_id_from_query"}]);
-    profilerHasZeroMatchingEntriesOrThrow({
-        profileDB: shard0DB,
-        filter: {op: "update", "command.u.msg": "update_failed_missing_shard_key_field"}
-    });
-    profilerHasZeroMatchingEntriesOrThrow({
-        profileDB: shard1DB,
-        filter: {op: "update", "command.u.msg": "update_failed_missing_shard_key_field"}
-    });
+    // Need to start a session to change the shard key.
+    const session = st.s.startSession({retryWrites: true});
+    const sessionDB = session.getDatabase(jsTestName());
+    const sessionColl = sessionDB.test;
+
+    sessionColl.insert({_id: -99, a: null, msg: "not_updated"});
+
+    assert.commandWorked(
+        sessionColl.update({_id: -99}, {_id: -99, msg: "update_missing_shard_key_field"}));
+
+    assert.docEq(sessionColl.find({_id: -99}).toArray(),
+                 [{_id: -99, msg: "update_missing_shard_key_field"}]);
 
     // Verify that an upsert whose query contains an exact match on _id but whose replacement
-    // document does not contain all other shard key fields will be rejected by mongoS, since it
-    // does not contain an exact shard key match.
-    writeRes = assert.commandFailedWithCode(
-        mongosColl.update({_id: 200, a: 200}, {msg: "upsert_targeting_failed"}, {upsert: true}),
-        ErrorCodes.ShardKeyNotFound);
-    profilerHasZeroMatchingEntriesOrThrow(
-        {profileDB: shard0DB, filter: {op: "update", "command.u.msg": "upsert_targeting_failed"}});
-    profilerHasZeroMatchingEntriesOrThrow(
-        {profileDB: shard1DB, filter: {op: "update", "command.u.msg": "upsert_targeting_failed"}});
-    assert.eq(mongosColl.find({_id: 200, a: 200}).itcount(), 0);
+    // document does not contain all other shard key fields will work properly.
+    assert.commandWorked(
+        sessionColl.update({_id: -100, a: -100}, {msg: "upsert_targeting_worked"}, {upsert: true}));
+    assert.eq(mongosColl.find({_id: -100, a: -100}).itcount(), 0);
+    assert.eq(mongosColl.find({msg: "upsert_targeting_worked"}).itcount(), 1);
 }
 
 // Shard the test collection on {_id: 1, a: 1}, split it into two chunks, and migrate one of
diff --git a/jstests/sharding/update_shard_key_doc_moves_shards.js b/jstests/sharding/update_shard_key_doc_moves_shards.js
index 9567b807b1e..a2d1c23f29f 100644
--- a/jstests/sharding/update_shard_key_doc_moves_shards.js
+++ b/jstests/sharding/update_shard_key_doc_moves_shards.js
@@ -162,6 +162,17 @@ changeShardKeyOptions.forEach(function(updateConfig) {
                                    [{"$set": {"x": 30}}, {"$set": {"x": 600}}],
                                    upsert);
 
+    assertCanUnsetSKField(st,
+                          kDbName,
+                          ns,
+                          session,
+                          sessionDB,
+                          runInTxn,
+                          isFindAndModify,
+                          {"x": 300},
+                          {"$unset": {"x": 1}},
+                          upsert);
+
     // Failure cases. These tests do not take 'upsert' as an option so we do not need to test
     // them for both upsert true and false.
     if (!upsert) {
@@ -177,10 +188,6 @@ changeShardKeyOptions.forEach(function(updateConfig) {
             st, kDbName, ns, session, sessionDB, runInTxn, isFindAndModify, {"x": 300}, {
                 "$set": {"x": [30]}
             });
-        assertCannotUnsetSKField(
-            st, kDbName, ns, session, sessionDB, runInTxn, isFindAndModify, {"x": 300}, {
-                "$unset": {"x": 1}
-            });
 
         if (!isFindAndModify) {
             assertCannotUpdateWithMultiTrue(
@@ -236,6 +243,9 @@ changeShardKeyOptions.forEach(function(updateConfig) {
                                    [{"x": 30, "y": 80}, {"x": 600, "y": 3}],
                                    upsert);
 
+    assertCanUnsetSKField(
+        st, kDbName, ns, session, sessionDB, runInTxn, isFindAndModify, {"x": 300}, {}, upsert);
+
     // Failure cases. These tests do not take 'upsert' as an option so we do not need to test
     // them for both upsert true and false.
     if (!upsert) {
@@ -255,8 +265,6 @@ changeShardKeyOptions.forEach(function(updateConfig) {
             st, kDbName, ns, session, sessionDB, runInTxn, isFindAndModify, {"x": 300}, {
                 "x": [30]
             });
-        assertCannotUnsetSKField(
-            st, kDbName, ns, session, sessionDB, runInTxn, isFindAndModify, {"x": 300}, {});
     }
 });
 
diff --git a/jstests/sharding/update_shard_key_doc_on_same_shard.js b/jstests/sharding/update_shard_key_doc_on_same_shard.js
index 00f2aa23435..d2bfe047e2e 100644
--- a/jstests/sharding/update_shard_key_doc_on_same_shard.js
+++ b/jstests/sharding/update_shard_key_doc_on_same_shard.js
@@ -79,6 +79,8 @@ assertCanUpdatePartialShardKey(st,
                                [{"x": 300, "y": 80}, {"x": 4, "y": 3}],
                                [{"$set": {"x": 600}}, {"$set": {"x": 30}}],
                                false);
+assertCanUnsetSKField(
+    st, kDbName, ns, session, sessionDB, false, false, {"x": 300}, {"$unset": {"x": 1}}, false);
 
 // upsert : true
 assertCanUpdatePrimitiveShardKey(st,
@@ -111,6 +113,8 @@ assertCanUpdatePartialShardKey(st,
                                [{"x": 300, "y": 80}, {"x": 4, "y": 3}],
                                [{"$set": {"x": 600}}, {"$set": {"x": 30}}],
                                true);
+assertCanUnsetSKField(
+    st, kDbName, ns, session, sessionDB, false, false, {"x": 300}, {"$unset": {"x": 1}}, true);
 
 // failing cases
 assertCannotUpdate_id(
@@ -122,8 +126,6 @@ assertCannotUpdateWithMultiTrue(
     st, kDbName, ns, session, sessionDB, false, {"x": 300}, {"$set": {"x": 600}});
 assertCannotUpdateSKToArray(
     st, kDbName, ns, session, sessionDB, false, false, {"x": 300}, {"$set": {"x": [300]}});
-assertCannotUnsetSKField(
-    st, kDbName, ns, session, sessionDB, false, false, {"x": 300}, {"$unset": {"x": 1}});
 
 // Replacement updates
 
@@ -158,6 +160,19 @@ assertCanUpdatePartialShardKey(st,
                                [{"x": 300, "y": 80}, {"x": 4, "y": 3}],
                                [{"x": 600, "y": 80}, {"x": 30, "y": 3}],
                                false);
+assertCanDoReplacementUpdateWhereShardKeyMissingFields(
+    st, kDbName, ns, session, sessionDB, false, false, {"x": 300, "y": 80}, {"x": 600}, false);
+// Shard key fields are missing in query.
+assertCanDoReplacementUpdateWhereShardKeyMissingFields(st,
+                                                       kDbName,
+                                                       ns,
+                                                       session,
+                                                       sessionDB,
+                                                       false,
+                                                       false,
+                                                       {"x": 300},
+                                                       {"x": 600, "y": 80, "a": 2},
+                                                       false);
 
 // upsert : true
 assertCanUpdatePrimitiveShardKey(st,
@@ -190,6 +205,19 @@ assertCanUpdatePartialShardKey(st,
                                [{"x": 300, "y": 80}, {"x": 4, "y": 3}],
                                [{"x": 600, "y": 80}, {"x": 30, "y": 3}],
                                true);
+assertCanDoReplacementUpdateWhereShardKeyMissingFields(
+    st, kDbName, ns, session, sessionDB, false, false, {"x": 300, "y": 80}, {"x": 600}, true);
+// Shard key fields are missing in query.
+assertCanDoReplacementUpdateWhereShardKeyMissingFields(st,
+                                                       kDbName,
+                                                       ns,
+                                                       session,
+                                                       sessionDB,
+                                                       false,
+                                                       false,
+                                                       {"x": 300},
+                                                       {"x": 600, "y": 80, "a": 2},
+                                                       true);
 
 // failing cases
 assertCannotUpdate_id(
@@ -197,11 +225,6 @@ assertCannotUpdate_id(
 assertCannotUpdate_idDottedPath(
     st, kDbName, ns, session, sessionDB, false, false, {"_id.a": 300}, {"_id": {"a": 600}});
 assertCannotUpdateWithMultiTrue(st, kDbName, ns, session, sessionDB, false, {"x": 300}, {"x": 600});
-assertCannotDoReplacementUpdateWhereShardKeyMissingFields(
-    st, kDbName, ns, session, sessionDB, false, false, {"x": 300, "y": 80}, {"x": 600});
-// Shard key fields are missing in query.
-assertCannotDoReplacementUpdateWhereShardKeyMissingFields(
-    st, kDbName, ns, session, sessionDB, false, false, {"x": 300}, {"x": 600, "y": 80, "a": 2});
 assertCannotUpdateSKToArray(
     st, kDbName, ns, session, sessionDB, false, false, {"x": 300}, {"x": [300]});
 
@@ -238,6 +261,8 @@ assertCanUpdatePartialShardKey(st,
                                [{"x": 300, "y": 80}, {"x": 4, "y": 3}],
                                [{"$set": {"x": 600}}, {"$set": {"x": 30}}],
                                false);
+assertCanUnsetSKField(
+    st, kDbName, ns, session, sessionDB, false, true, {"x": 300}, {"$unset": {"x": 1}}, false);
 
 // upsert : true
 assertCanUpdatePrimitiveShardKey(st,
@@ -270,6 +295,8 @@ assertCanUpdatePartialShardKey(st,
                                [{"x": 300, "y": 80}, {"x": 4, "y": 3}],
                                [{"$set": {"x": 600}}, {"$set": {"x": 30}}],
                                true);
+assertCanUnsetSKField(
+    st, kDbName, ns, session, sessionDB, false, true, {"x": 300}, {"$unset": {"x": 1}}, true);
 
 // failing cases
 assertCannotUpdate_id(
@@ -279,8 +306,6 @@ assertCannotUpdate_idDottedPath(st, kDbName, ns, session, sessionDB, false, true
 });
 assertCannotUpdateSKToArray(
     st, kDbName, ns, session, sessionDB, false, true, {"x": 300}, {"$set": {"x": [300]}});
-assertCannotUnsetSKField(
-    st, kDbName, ns, session, sessionDB, false, true, {"x": 300}, {"$unset": {"x": 1}});
 
 // Replacement style findAndModify
 
@@ -315,6 +340,19 @@ assertCanUpdatePartialShardKey(st,
                                [{"x": 300, "y": 80}, {"x": 4, "y": 3}],
                                [{"x": 600, "y": 80}, {"x": 30, "y": 3}],
                                false);
+assertCanDoReplacementUpdateWhereShardKeyMissingFields(
+    st, kDbName, ns, session, sessionDB, false, true, {"x": 300, "y": 80}, {"x": 600}, false);
+// Shard key fields are missing in query.
+assertCanDoReplacementUpdateWhereShardKeyMissingFields(st,
+                                                       kDbName,
+                                                       ns,
+                                                       session,
+                                                       sessionDB,
+                                                       false,
+                                                       true,
+                                                       {"x": 300},
+                                                       {"x": 600, "y": 80, "a": 2},
+                                                       false);
 
 // upsert: true
 assertCanUpdatePrimitiveShardKey(st,
@@ -347,16 +385,24 @@ assertCanUpdatePartialShardKey(st,
                                [{"x": 300, "y": 80}, {"x": 4, "y": 3}],
                                [{"x": 600, "y": 80}, {"x": 30, "y": 3}],
                                true);
+assertCanDoReplacementUpdateWhereShardKeyMissingFields(
+    st, kDbName, ns, session, sessionDB, false, true, {"x": 300, "y": 80}, {"x": 600}, true);
+// Shard key fields are missing in query.
+assertCanDoReplacementUpdateWhereShardKeyMissingFields(st,
+                                                       kDbName,
+                                                       ns,
+                                                       session,
+                                                       sessionDB,
+                                                       false,
+                                                       true,
+                                                       {"x": 300},
+                                                       {"x": 600, "y": 80, "a": 2},
+                                                       true);
 
 // failing cases
 assertCannotUpdate_id(st, kDbName, ns, session, sessionDB, false, true, {"_id": 300}, {"_id": 600});
 assertCannotUpdate_idDottedPath(
     st, kDbName, ns, session, sessionDB, false, true, {"_id.a": 300}, {"_id": {"a": 600}});
-assertCannotDoReplacementUpdateWhereShardKeyMissingFields(
-    st, kDbName, ns, session, sessionDB, false, true, {"x": 300, "y": 80}, {"x": 600});
-// Shard key fields are missing in query.
-assertCannotDoReplacementUpdateWhereShardKeyMissingFields(
-    st, kDbName, ns, session, sessionDB, false, true, {"x": 300}, {"x": 600, "y": 80, "a": 2});
 assertCannotUpdateSKToArray(
     st, kDbName, ns, session, sessionDB, false, true, {"x": 300}, {"x": [300]});
 
@@ -410,6 +456,8 @@ assertCanUpdatePartialShardKey(st,
                                [{"x": 300, "y": 80}, {"x": 4, "y": 3}],
                                [{"$set": {"x": 600}}, {"$set": {"x": 30}}],
                                false);
+assertCanUnsetSKField(
+    st, kDbName, ns, session, sessionDB, true, false, {"x": 300}, {"$unset": {"x": 1}}, false);
 
 // upsert : true
 assertCanUpdatePrimitiveShardKey(st,
@@ -442,6 +490,8 @@ assertCanUpdatePartialShardKey(st,
                                [{"x": 300, "y": 80}, {"x": 4, "y": 3}],
                                [{"$set": {"x": 600}}, {"$set": {"x": 30}}],
                                true);
+assertCanUnsetSKField(
+    st, kDbName, ns, session, sessionDB, true, false, {"x": 300}, {"$unset": {"x": 1}}, true);
 
 // failing cases
 assertCannotUpdate_id(
@@ -453,8 +503,6 @@ assertCannotUpdateWithMultiTrue(
     st, kDbName, ns, session, sessionDB, true, {"x": 300}, {"$set": {"x": 600}});
 assertCannotUpdateSKToArray(
     st, kDbName, ns, session, sessionDB, true, false, {"x": 300}, {"$set": {"x": [300]}});
-assertCannotUnsetSKField(
-    st, kDbName, ns, session, sessionDB, true, false, {"x": 300}, {"$unset": {"x": 1}});
 
 // Replacement updates
 
@@ -489,6 +537,19 @@ assertCanUpdatePartialShardKey(st,
                                [{"x": 300, "y": 80}, {"x": 4, "y": 3}],
                                [{"x": 600, "y": 80}, {"x": 30, "y": 3}],
                                false);
+assertCanDoReplacementUpdateWhereShardKeyMissingFields(
+    st, kDbName, ns, session, sessionDB, true, false, {"x": 300, "y": 80}, {"x": 600}, false);
+// Shard key fields are missing in query.
+assertCanDoReplacementUpdateWhereShardKeyMissingFields(st,
+                                                       kDbName,
+                                                       ns,
+                                                       session,
+                                                       sessionDB,
+                                                       true,
+                                                       false,
+                                                       {"x": 300},
+                                                       {"x": 600, "y": 80, "a": 2},
+                                                       false);
 
 // upsert : true
 assertCanUpdatePrimitiveShardKey(st,
@@ -521,18 +582,25 @@ assertCanUpdatePartialShardKey(st,
                                [{"x": 300, "y": 80}, {"x": 4, "y": 3}],
                                [{"x": 600, "y": 80}, {"x": 30, "y": 3}],
                                true);
+assertCanDoReplacementUpdateWhereShardKeyMissingFields(
+    st, kDbName, ns, session, sessionDB, true, false, {"x": 300, "y": 80}, {"x": 600}, true);
+// Shard key fields are missing in query.
+assertCanDoReplacementUpdateWhereShardKeyMissingFields(st,
+                                                       kDbName,
+                                                       ns,
+                                                       session,
+                                                       sessionDB,
+                                                       true,
+                                                       false,
+                                                       {"x": 300},
+                                                       {"x": 600, "y": 80, "a": 2},
+                                                       true);
 
 // failing cases
 assertCannotUpdate_id(st, kDbName, ns, session, sessionDB, true, false, {"_id": 300}, {"_id": 600});
 assertCannotUpdate_idDottedPath(
     st, kDbName, ns, session, sessionDB, true, false, {"_id.a": 300}, {"_id": {"a": 600}});
 assertCannotUpdateWithMultiTrue(st, kDbName, ns, session, sessionDB, true, {"x": 300}, {"x": 600});
-assertCannotDoReplacementUpdateWhereShardKeyMissingFields(
-    st, kDbName, ns, session, sessionDB, true, false, {"x": 300, "y": 80}, {"x": 600});
-// Shard key fields are missing in query.
-assertCannotDoReplacementUpdateWhereShardKeyMissingFields(
-    st, kDbName, ns, session, sessionDB, true, false, {"x": 300}, {"x": 600, "y": 80, "a": 2});
-
 assertCannotUpdateSKToArray(
     st, kDbName, ns, session, sessionDB, true, false, {"x": 300}, {"x": [300]});
 
@@ -569,6 +637,8 @@ assertCanUpdatePartialShardKey(st,
                                [{"x": 300, "y": 80}, {"x": 4, "y": 3}],
                                [{"$set": {"x": 600}}, {"$set": {"x": 30}}],
                                false);
+assertCanUnsetSKField(
+    st, kDbName, ns, session, sessionDB, true, true, {"x": 300}, {"$unset": {"x": 1}}, false);
 
 // upsert : true
 assertCanUpdatePrimitiveShardKey(st,
@@ -601,6 +671,8 @@ assertCanUpdatePartialShardKey(st,
                                [{"x": 300, "y": 80}, {"x": 4, "y": 3}],
                                [{"$set": {"x": 600}}, {"$set": {"x": 30}}],
                                true);
+assertCanUnsetSKField(
+    st, kDbName, ns, session, sessionDB, true, true, {"x": 300}, {"$unset": {"x": 1}}, true);
 
 // failing cases
 assertCannotUpdate_id(
@@ -609,8 +681,6 @@ assertCannotUpdate_idDottedPath(
     st, kDbName, ns, session, sessionDB, true, true, {"_id.a": 300}, {"$set": {"_id": {"a": 600}}});
 assertCannotUpdateSKToArray(
     st, kDbName, ns, session, sessionDB, true, true, {"x": 300}, {"$set": {"x": [300]}});
-assertCannotUnsetSKField(
-    st, kDbName, ns, session, sessionDB, true, true, {"x": 300}, {"$unset": {"x": 1}});
 
 // Replacement style findAndModify
 
@@ -645,6 +715,19 @@ assertCanUpdatePartialShardKey(st,
                                [{"x": 300, "y": 80}, {"x": 4, "y": 3}],
                                [{"x": 600, "y": 80}, {"x": 30, "y": 3}],
                                false);
+assertCanDoReplacementUpdateWhereShardKeyMissingFields(
+    st, kDbName, ns, session, sessionDB, true, true, {"x": 300, "y": 80}, {"x": 600}, false);
+// Shard key fields are missing in query.
+assertCanDoReplacementUpdateWhereShardKeyMissingFields(st,
+                                                       kDbName,
+                                                       ns,
+                                                       session,
+                                                       sessionDB,
+                                                       true,
+                                                       true,
+                                                       {"x": 300},
+                                                       {"x": 600, "y": 80, "a": 2},
+                                                       false);
 
 // upsert : true
 assertCanUpdatePrimitiveShardKey(st,
@@ -677,16 +760,16 @@ assertCanUpdatePartialShardKey(st,
                                [{"x": 300, "y": 80}, {"x": 4, "y": 3}],
                                [{"x": 600, "y": 80}, {"x": 30, "y": 3}],
                                true);
+assertCanDoReplacementUpdateWhereShardKeyMissingFields(
+    st, kDbName, ns, session, sessionDB, true, true, {"x": 300, "y": 80}, {"x": 600}, true);
+// Shard key fields are missing in query.
+assertCanDoReplacementUpdateWhereShardKeyMissingFields(
+    st, kDbName, ns, session, sessionDB, true, true, {"x": 300}, {"x": 600, "y": 80, "a": 2}, true);
 
 // failing cases
 assertCannotUpdate_id(st, kDbName, ns, session, sessionDB, true, true, {"_id": 300}, {"_id": 600});
 assertCannotUpdate_idDottedPath(
     st, kDbName, ns, session, sessionDB, true, true, {"_id.a": 300}, {"_id": {"a": 600}});
-assertCannotDoReplacementUpdateWhereShardKeyMissingFields(
-    st, kDbName, ns, session, sessionDB, true, true, {"x": 300, "y": 80}, {"x": 600});
-// Shard key fields are missing in query.
-assertCannotDoReplacementUpdateWhereShardKeyMissingFields(
-    st, kDbName, ns, session, sessionDB, true, true, {"x": 300}, {"x": 600, "y": 80, "a": 2});
 assertCannotUpdateSKToArray(
     st, kDbName, ns, session, sessionDB, true, true, {"x": 300}, {"x": [300]});
 
diff --git a/jstests/sharding/update_shard_key_pipeline_update.js b/jstests/sharding/update_shard_key_pipeline_update.js
index 9f1ff0082e0..cdde2f98bf2 100644
--- a/jstests/sharding/update_shard_key_pipeline_update.js
+++ b/jstests/sharding/update_shard_key_pipeline_update.js
@@ -85,6 +85,18 @@ changeShardKeyOptions.forEach(function(updateConfig) {
         upsert,
         [{"x": 600}, {"x": -4}]);
 
+    assertCanUnsetSKFieldUsingPipeline(st,
+                                       kDbName,
+                                       ns,
+                                       session,
+                                       sessionDB,
+                                       runInTxn,
+                                       isFindAndModify,
+                                       {"x": 300, "y": 80},
+                                       [{$project: {"y": 0}}],
+                                       upsert,
+                                       {"x": 300, "y": 80});
+
     // Failure cases. These tests do not take 'upsert' as an option so we do not need to test
     // them for both upsert true and false.
     if (!upsert) {
@@ -108,16 +120,6 @@ changeShardKeyOptions.forEach(function(updateConfig) {
                                         {"_id.a": 300},
                                         [{$set: {"_id": {"a": {$multiply: ["$_id.a", 2]}}}}],
                                         {"_id": {"a": 600}});
-        assertCannotUnsetSKFieldUsingPipeline(st,
-                                              kDbName,
-                                              ns,
-                                              session,
-                                              sessionDB,
-                                              runInTxn,
-                                              isFindAndModify,
-                                              {"x": 300, "y": 80},
-                                              [{$project: {"y": 0}}],
-                                              {"x": 300, "y": 80});
         if (!isFindAndModify) {
             assertCannotUpdateWithMultiTrue(st,
                                             kDbName,
@@ -186,6 +188,17 @@ changeShardKeyOptions.forEach(function(updateConfig) {
         [[{$set: {"x": {$multiply: ["$x", -1]}}}], [{$set: {"x": {$multiply: ["$x", 100]}}}]],
         upsert,
         [{"x": -300}, {"x": 400}]);
+    assertCanUnsetSKFieldUsingPipeline(st,
+                                       kDbName,
+                                       ns,
+                                       session,
+                                       sessionDB,
+                                       runInTxn,
+                                       isFindAndModify,
+                                       {"x": 300, "y": 80},
+                                       [{$project: {"y": 0}}],
+                                       upsert,
+                                       {"x": 300, "y": 80});
 
     // Failure cases. These tests do not take 'upsert' as an option so we do not need to test
     // them for both upsert true and false.
@@ -210,16 +223,6 @@ changeShardKeyOptions.forEach(function(updateConfig) {
                                         {"_id.a": 300},
                                         [{$set: {"_id": {"a": {$multiply: ["$_id.a", -1]}}}}],
                                         {"_id": {"a": -300}});
-        assertCannotUnsetSKFieldUsingPipeline(st,
-                                              kDbName,
-                                              ns,
-                                              session,
-                                              sessionDB,
-                                              runInTxn,
-                                              isFindAndModify,
-                                              {"x": 300, "y": 80},
-                                              [{$project: {"y": 0}}],
-                                              {"x": 300, "y": 80});
         if (!isFindAndModify) {
             assertCannotUpdateWithMultiTrue(st,
                                             kDbName,