SERVER-40435 A clearJumboChunk command to clear the jumbo flag

(cherry picked from commit 546e411b72cf6f75d24b304ce9219d1f3d3a4e4f)
(cherry picked from commit 380711ea7dba41cb372a12a8d5258fdc124a0d23)
(cherry picked from commit 7a4e7bb1274a382832a6144e214e658825ffa3cd)

7 files changed, 158 insertions, 0 deletions

diff --git a/jstests/auth/lib/commands_lib.js b/jstests/auth/lib/commands_lib.js
index 910c9ea6837..c98964c5bfd 100644
--- a/jstests/auth/lib/commands_lib.js
+++ b/jstests/auth/lib/commands_lib.js
@@ -5912,6 +5912,37 @@ var authCommandsLib = {
               },
           ]
         },
+        {
+          testname: "clearJumboFlag",
+          command: {clearJumboFlag: "test.x"},
+          skipUnlessSharded: true,
+          testcases: [
+              {
+                runOnDb: adminDbName,
+                roles: roles_clusterManager,
+                privileges:
+                    [{resource: {db: "test", collection: "x"}, actions: ["clearJumboFlag"]}],
+                expectFail: true
+              },
+              {runOnDb: firstDbName, roles: {}},
+              {runOnDb: secondDbName, roles: {}}
+          ]
+        },
+        {
+          testname: "_configsvrClearJumboFlag",
+          command:
+              {_configsvrClearJumboFlag: "x.y", epoch: ObjectId(), minKey: {x: 0}, maxKey: {x: 10}},
+          skipSharded: true,
+          expectFail: true,
+          testcases: [
+              {
+                runOnDb: adminDbName,
+                roles: {__system: 1},
+                privileges: [{resource: {cluster: true}, actions: ["internal"]}],
+                expectFail: true
+              },
+          ]
+        },
     ],
 
     /************* SHARED TEST LOGIC ****************/
diff --git a/jstests/core/views/views_all_commands.js b/jstests/core/views/views_all_commands.js
index a5d902c9ad9..fc2bc856dbe 100644
--- a/jstests/core/views/views_all_commands.js
+++ b/jstests/core/views/views_all_commands.js
@@ -74,6 +74,7 @@
         _configsvrBalancerStart: {skip: isAnInternalCommand},
         _configsvrBalancerStatus: {skip: isAnInternalCommand},
         _configsvrBalancerStop: {skip: isAnInternalCommand},
+        _configsvrClearJumboFlag: {skip: isAnInternalCommand},
         _configsvrCommitChunkMerge: {skip: isAnInternalCommand},
         _configsvrCommitChunkMigration: {skip: isAnInternalCommand},
         _configsvrCommitChunkSplit: {skip: isAnInternalCommand},
@@ -131,6 +132,13 @@
             skip: "Tested in views/views_sharded.js",
         },
         clearLog: {skip: isUnrelated},
+        clearJumboFlag: {
+            command: {clearJumboFlag: "test.view"},
+            skipStandalone: true,
+            isAdminCommand: true,
+            expectFailure: true,
+            expectedErrorCode: ErrorCodes.NamespaceNotSharded,
+        },
         clone: {skip: "Tested in replsets/cloneDb.js"},
         cloneCollection: {skip: "Tested in noPassthroughWithMongod/clonecollection.js"},
         cloneCollectionAsCapped: {
diff --git a/jstests/sharding/clear_jumbo.js b/jstests/sharding/clear_jumbo.js
new file mode 100644
index 00000000000..5c682df5db8
--- /dev/null
+++ b/jstests/sharding/clear_jumbo.js
@@ -0,0 +1,112 @@
+(function() {
+    "use strict";
+
+    let st = new ShardingTest({shards: 2});
+
+    assert.commandWorked(st.s.adminCommand({enableSharding: 'test'}));
+    st.ensurePrimaryShard('test', st.shard0.shardName);
+    assert.commandWorked(
+        st.s.adminCommand({addShardToZone: st.shard1.shardName, zone: 'finalDestination'}));
+
+    ////////////////////////////////////////////////////////////////////////////
+    // Ranged shard key
+    assert.commandWorked(st.s.adminCommand({shardCollection: 'test.range', key: {x: 1}}));
+    assert.commandWorked(st.s.adminCommand({split: 'test.range', middle: {x: 0}}));
+
+    let chunkColl = st.s.getDB('config').chunks;
+    assert.commandWorked(chunkColl.update({ns: 'test.range', min: {x: 0}}, {$set: {jumbo: true}}));
+
+    let jumboChunk = chunkColl.findOne({ns: 'test.range', min: {x: 0}});
+    assert(jumboChunk.jumbo, tojson(jumboChunk));
+    let jumboMajorVersionBefore = jumboChunk.lastmod.getTime();
+
+    // Target non-jumbo chunk should not affect real jumbo chunk.
+    assert.commandWorked(st.s.adminCommand({clearJumboFlag: 'test.range', find: {x: -1}}));
+    jumboChunk = chunkColl.findOne({ns: 'test.range', min: {x: 0}});
+    assert(jumboChunk.jumbo, tojson(jumboChunk));
+    assert.eq(jumboMajorVersionBefore, jumboChunk.lastmod.getTime());
+
+    // Target real jumbo chunk should bump version.
+    assert.commandWorked(st.s.adminCommand({clearJumboFlag: 'test.range', find: {x: 1}}));
+    jumboChunk = chunkColl.findOne({ns: 'test.range', min: {x: 0}});
+    assert(!jumboChunk.jumbo, tojson(jumboChunk));
+    assert.lt(jumboMajorVersionBefore, jumboChunk.lastmod.getTime());
+
+    ////////////////////////////////////////////////////////////////////////////
+    // Hashed shard key
+    assert.commandWorked(st.s.adminCommand(
+        {shardCollection: 'test.hashed', key: {x: 'hashed'}, numInitialChunks: 2}));
+    assert.commandWorked(chunkColl.update({ns: 'test.hashed', min: {x: 0}}, {$set: {jumbo: true}}));
+    jumboChunk = chunkColl.findOne({ns: 'test.hashed', min: {x: 0}});
+    assert(jumboChunk.jumbo, tojson(jumboChunk));
+    jumboMajorVersionBefore = jumboChunk.lastmod.getTime();
+
+    // Target non-jumbo chunk should not affect real jumbo chunk.
+    let unrelatedChunk = chunkColl.findOne({ns: 'test.hashed', min: {x: MinKey}});
+    assert.commandWorked(st.s.adminCommand(
+        {clearJumboFlag: 'test.hashed', bounds: [unrelatedChunk.min, unrelatedChunk.max]}));
+    jumboChunk = chunkColl.findOne({ns: 'test.hashed', min: {x: 0}});
+    assert(jumboChunk.jumbo, tojson(jumboChunk));
+    assert.eq(jumboMajorVersionBefore, jumboChunk.lastmod.getTime());
+
+    // Target real jumbo chunk should bump version.
+    assert.commandWorked(st.s.adminCommand(
+        {clearJumboFlag: 'test.hashed', bounds: [jumboChunk.min, jumboChunk.max]}));
+    jumboChunk = chunkColl.findOne({ns: 'test.hashed', min: {x: 0}});
+    assert(!jumboChunk.jumbo, tojson(jumboChunk));
+    assert.lt(jumboMajorVersionBefore, jumboChunk.lastmod.getTime());
+
+    ////////////////////////////////////////////////////////////////////////////
+    // Balancer with jumbo chunks behavior
+    // Forces a jumbo chunk to be on a wrong zone but balancer shouldn't be able to move it until
+    // jumbo flag is cleared.
+
+    st.stopBalancer();
+    assert.commandWorked(chunkColl.update({ns: 'test.range', min: {x: 0}}, {$set: {jumbo: true}}));
+    assert.commandWorked(st.s.adminCommand({
+        updateZoneKeyRange: 'test.range',
+        min: {x: 0},
+        max: {x: MaxKey},
+        zone: 'finalDestination'
+    }));
+
+    let chunk = chunkColl.findOne({ns: 'test.range', min: {x: 0}});
+    assert(chunk.jumbo, tojson(chunk));
+    assert.eq(st.shard0.shardName, chunk.shard);
+
+    st._configServers.forEach((conn) => {
+        conn.adminCommand({
+            configureFailPoint: 'overrideBalanceRoundInterval',
+            mode: 'alwaysOn',
+            data: {intervalMs: 200}
+        });
+    });
+
+    let waitForBalancerToRun = function() {
+        let lastRoundNumber =
+            assert.commandWorked(st.s.adminCommand({balancerStatus: 1})).numBalancerRounds;
+        st.startBalancer();
+
+        assert.soon(function() {
+            let roundNumber =
+                assert.commandWorked(st.s.adminCommand({balancerStatus: 1})).numBalancerRounds;
+            return roundNumber > lastRoundNumber;
+        });
+
+        st.stopBalancer();
+    };
+
+    waitForBalancerToRun();
+
+    chunk = chunkColl.findOne({ns: 'test.range', min: {x: 0}});
+    assert.eq(st.shard0.shardName, chunk.shard);
+
+    assert.commandWorked(st.s.adminCommand({clearJumboFlag: 'test.range', find: {x: 0}}));
+
+    waitForBalancerToRun();
+
+    chunk = chunkColl.findOne({ns: 'test.range', min: {x: 0}});
+    assert.eq(st.shard1.shardName, chunk.shard);
+
+    st.stop();
+})();
diff --git a/jstests/sharding/database_and_shard_versioning_all_commands.js b/jstests/sharding/database_and_shard_versioning_all_commands.js
index 65fdb8ab2e4..d91d00ae53f 100644
--- a/jstests/sharding/database_and_shard_versioning_all_commands.js
+++ b/jstests/sharding/database_and_shard_versioning_all_commands.js
@@ -63,6 +63,7 @@
         balancerStatus: {skip: "not on a user database"},
         balancerStop: {skip: "not on a user database"},
         buildInfo: {skip: "executes locally on mongos (not sent to any remote node)"},
+        clearJumboFlag: {skip: "does not forward command to primary shard"},
         clearLog: {skip: "executes locally on mongos (not sent to any remote node)"},
         collMod: {
             sendsDbVersion: true,
diff --git a/jstests/sharding/safe_secondary_reads_drop_recreate.js b/jstests/sharding/safe_secondary_reads_drop_recreate.js
index 9024d479d9f..f8dc59f4d52 100644
--- a/jstests/sharding/safe_secondary_reads_drop_recreate.js
+++ b/jstests/sharding/safe_secondary_reads_drop_recreate.js
@@ -42,6 +42,7 @@
         _configsvrBalancerStart: {skip: "primary only"},
         _configsvrBalancerStatus: {skip: "primary only"},
         _configsvrBalancerStop: {skip: "primary only"},
+        _configsvrClearJumboFlag: {skip: "primary only"},
         _configsvrCommitChunkMerge: {skip: "primary only"},
         _configsvrCommitChunkMigration: {skip: "primary only"},
         _configsvrCommitChunkSplit: {skip: "primary only"},
@@ -92,6 +93,7 @@
         checkShardingIndex: {skip: "primary only"},
         cleanupOrphaned: {skip: "primary only"},
         clearLog: {skip: "does not return user data"},
+        clearJumboFlag: {skip: "primary only"},
         clone: {skip: "primary only"},
         cloneCollection: {skip: "primary only"},
         cloneCollectionAsCapped: {skip: "primary only"},
diff --git a/jstests/sharding/safe_secondary_reads_single_migration_suspend_range_deletion.js b/jstests/sharding/safe_secondary_reads_single_migration_suspend_range_deletion.js
index a96831be95f..c16bf388fad 100644
--- a/jstests/sharding/safe_secondary_reads_single_migration_suspend_range_deletion.js
+++ b/jstests/sharding/safe_secondary_reads_single_migration_suspend_range_deletion.js
@@ -49,6 +49,7 @@
         _configsvrBalancerStart: {skip: "primary only"},
         _configsvrBalancerStatus: {skip: "primary only"},
         _configsvrBalancerStop: {skip: "primary only"},
+        _configsvrClearJumboFlag: {skip: "primary only"},
         _configsvrCommitChunkMerge: {skip: "primary only"},
         _configsvrCommitChunkMigration: {skip: "primary only"},
         _configsvrCommitChunkSplit: {skip: "primary only"},
@@ -104,6 +105,7 @@
         captrunc: {skip: "primary only"},
         checkShardingIndex: {skip: "primary only"},
         cleanupOrphaned: {skip: "primary only"},
+        clearJumboFlag: {skip: "primary only"},
         clearLog: {skip: "does not return user data"},
         clone: {skip: "primary only"},
         cloneCollection: {skip: "primary only"},
diff --git a/jstests/sharding/safe_secondary_reads_single_migration_waitForDelete.js b/jstests/sharding/safe_secondary_reads_single_migration_waitForDelete.js
index dda3c35e981..d71e6ef308d 100644
--- a/jstests/sharding/safe_secondary_reads_single_migration_waitForDelete.js
+++ b/jstests/sharding/safe_secondary_reads_single_migration_waitForDelete.js
@@ -42,6 +42,7 @@
         _configsvrBalancerStart: {skip: "primary only"},
         _configsvrBalancerStatus: {skip: "primary only"},
         _configsvrBalancerStop: {skip: "primary only"},
+        _configsvrClearJumboFlag: {skip: "primary only"},
         _configsvrCommitChunkMerge: {skip: "primary only"},
         _configsvrCommitChunkMigration: {skip: "primary only"},
         _configsvrCommitChunkSplit: {skip: "primary only"},
@@ -92,6 +93,7 @@
         captrunc: {skip: "primary only"},
         checkShardingIndex: {skip: "primary only"},
         cleanupOrphaned: {skip: "primary only"},
+        clearJumboFlag: {skip: "primary only"},
         clearLog: {skip: "does not return user data"},
         clone: {skip: "primary only"},
         cloneCollection: {skip: "primary only"},