SERVER-62065 Introduce the 'repairShardedCollectionChunksHistory' command

19 files changed, 487 insertions, 147 deletions

diff --git a/jstests/core/views/views_all_commands.js b/jstests/core/views/views_all_commands.js
index ad017402d5b..1e160b8099b 100644
--- a/jstests/core/views/views_all_commands.js
+++ b/jstests/core/views/views_all_commands.js
@@ -90,6 +90,7 @@
         _configsvrMovePrimary: {skip: isAnInternalCommand},
         _configsvrRemoveShard: {skip: isAnInternalCommand},
         _configsvrRemoveShardFromZone: {skip: isAnInternalCommand},
+        _configsvrRepairShardedCollectionChunksHistory: {skip: isAnInternalCommand},
         _configsvrShardCollection: {skip: isAnInternalCommand},
         _configsvrUpdateZoneKeyRange: {skip: isAnInternalCommand},
         _cpuProfilerStart: {skip: isAnInternalCommand},
@@ -461,6 +462,13 @@
         ],
         repairCursor: {command: {repairCursor: "view"}, expectFailure: true},
         repairDatabase: {command: {repairDatabase: 1}},
+        repairShardedCollectionChunksHistory: {
+            command: {repairShardedCollectionChunksHistory: "test.view"},
+            skipStandalone: true,
+            isAdminCommand: true,
+            expectFailure: true,
+            expectedErrorCode: ErrorCodes.NamespaceNotFound,
+        },
         replSetAbortPrimaryCatchUp: {skip: isUnrelated},
         replSetElect: {skip: isUnrelated},
         replSetFreeze: {skip: isUnrelated},
diff --git a/jstests/multiVersion/fcv_chunk_history.js b/jstests/multiVersion/fcv_chunk_history.js
index 90beb36a2c4..624528258d8 100644
--- a/jstests/multiVersion/fcv_chunk_history.js
+++ b/jstests/multiVersion/fcv_chunk_history.js
@@ -12,8 +12,11 @@
         shards: 1,
     });
 
-    let configPrimaryAdminDB = st.configRS.getPrimary().getDB("admin");
-    let shardPrimaryAdminDB = st.rs0.getPrimary().getDB("admin");
+    let configPrimary = st.configRS.getPrimary();
+    let configPrimaryAdminDB = configPrimary.getDB("admin");
+    let shardPrimary = st.rs0.getPrimary();
+    let shardPrimaryAdminDB = shardPrimary.getDB("admin");
+    let shardPrimaryConfigDB = shardPrimary.getDB("config");
 
     // Change FCV to last stable so chunks will not have histories.
     assert.commandWorked(st.s.adminCommand({setFeatureCompatibilityVersion: "3.6"}));
@@ -28,6 +31,8 @@
     assert.commandWorked(
         st.s.adminCommand({shardCollection: testDB.foo.getFullName(), key: {a: 1}}));
     assert.commandWorked(st.s.adminCommand({split: testDB.foo.getFullName(), middle: {a: 0}}));
+    assert.commandWorked(st.s.adminCommand({split: testDB.foo.getFullName(), middle: {a: -1000}}));
+    assert.commandWorked(st.s.adminCommand({split: testDB.foo.getFullName(), middle: {a: +1000}}));
 
     assert.writeOK(st.s.getDB("test1").foo.insert({_id: "id1", a: 1}));
     assert.neq(null, st.s.getDB("test1").foo.findOne({_id: "id1", a: 1}));
@@ -35,40 +40,90 @@
     assert.writeOK(st.s.getDB("test1").foo.insert({_id: "id2", a: -1}));
     assert.neq(null, st.s.getDB("test1").foo.findOne({_id: "id2", a: -1}));
 
-    // Make sure chunks do not have history when FCV <= 3.6.
+    // Make sure chunks do not have history when FCV is 3.6.
     let chunks = st.s.getDB("config").getCollection("chunks").find({ns: "test1.foo"}).toArray();
-    assert.eq(chunks.length, 2);
+    assert.eq(chunks.length, 4);
     chunks.forEach((chunk) => {
         assert.neq(null, chunk);
         assert(!chunk.hasOwnProperty("history"), "test1.foo has a history before upgrade");
+        assert(!chunk.hasOwnProperty("historyIsAt40"),
+               "test1.foo has a historyIsAt40 before upgrade");
+    });
+    chunks = shardPrimaryConfigDB.getCollection("cache.chunks.test1.foo").find().toArray();
+    assert.eq(chunks.length, 4);
+    chunks.forEach((chunk) => {
+        assert.neq(null, chunk);
+        assert(!chunk.hasOwnProperty("history"),
+               "test1.foo has a history on the shard before upgrade");
     });
 
-    // Set FCV to latest.
+    // Set FCV to 4.0.
     assert.commandWorked(st.s.adminCommand({setFeatureCompatibilityVersion: "4.0"}));
     checkFCV(configPrimaryAdminDB, latestFCV);
     checkFCV(shardPrimaryAdminDB, latestFCV);
 
     // Make sure chunks for test1.foo were given history after upgrade.
     chunks = st.s.getDB("config").getCollection("chunks").find({ns: "test1.foo"}).toArray();
-    assert.eq(chunks.length, 2);
+    assert.eq(chunks.length, 4);
     chunks.forEach((chunk) => {
         assert.neq(null, chunk);
         assert(chunk.hasOwnProperty("history"), "test1.foo does not have a history after upgrade");
+        assert(chunk.hasOwnProperty("historyIsAt40"),
+               "test1.foo does not have a historyIsAt40 after upgrade");
+    });
+    chunks = shardPrimaryConfigDB.getCollection("cache.chunks.test1.foo").find().toArray();
+    assert.eq(chunks.length, 4);
+    chunks.forEach((chunk) => {
+        assert.neq(null, chunk);
+        assert(chunk.hasOwnProperty("history"),
+               "test1.foo does not have a history on the shard after upgrade");
     });
 
-    // Set FCV to last-stable.
+    // Set FCV to 3.6.
     assert.commandWorked(st.s.adminCommand({setFeatureCompatibilityVersion: "3.6"}));
     checkFCV(configPrimaryAdminDB, "3.6");
     checkFCV(shardPrimaryAdminDB, "3.6");
 
-    // Make sure history was removed when FCV changed to <= 3.6.
+    // Make sure history was removed from the config server entries when FCV changed to 3.6.
     chunks = st.s.getDB("config").getCollection("chunks").find({ns: "test1.foo"}).toArray();
-    assert.eq(chunks.length, 2);
+    assert.eq(chunks.length, 4);
     chunks.forEach((chunk) => {
         assert.neq(null, chunk);
         assert(!chunk.hasOwnProperty("history"), "test1.foo has a history after downgrade");
+        assert(!chunk.hasOwnProperty("historyIsAt40"),
+               "test1.foo has a historyIsAt40 after downgrade");
     });
 
-    st.stop();
+    // Set FCV back to 4.0 in order to test partially-upgraded chunk histories due to SERVER-62065
+    assert.commandWorked(st.s.adminCommand({setFeatureCompatibilityVersion: "4.0"}));
+    checkFCV(configPrimaryAdminDB, latestFCV);
+    checkFCV(shardPrimaryAdminDB, latestFCV);
+
+    // Manually clear the 'historyIsAt40' field from the config server and the history entries from
+    // the shards' cache collections in order to simulate a wrong upgrade due to SERVER-62065
+    assert.writeOK(st.s.getDB("config").chunks.update(
+        {ns: 'test1.foo'}, {'$unset': {historyIsAt40: ''}}, {multi: true}));
+    assert.writeOK(shardPrimaryConfigDB.getCollection("cache.chunks.test1.foo")
+                       .update({}, {'$unset': {history: ''}}, {multi: true}));
 
-})();
\ No newline at end of file
+    assert.commandWorked(st.s.adminCommand({repairShardedCollectionChunksHistory: 'test1.foo'}));
+
+    // Make sure chunks for test1.foo were given history after repair.
+    chunks = st.s.getDB("config").getCollection("chunks").find({ns: "test1.foo"}).toArray();
+    assert.eq(chunks.length, 4);
+    chunks.forEach((chunk) => {
+        assert.neq(null, chunk);
+        assert(chunk.hasOwnProperty("history"), "test1.foo does not have a history after repair");
+        assert(chunk.hasOwnProperty("historyIsAt40"),
+               "test1.foo does not have a historyIsAt40 after repair");
+    });
+    chunks = shardPrimaryConfigDB.getCollection("cache.chunks.test1.foo").find().toArray();
+    assert.eq(chunks.length, 4);
+    chunks.forEach((chunk) => {
+        assert.neq(null, chunk);
+        assert(chunk.hasOwnProperty("history"),
+               "test1.foo does not have a history on the shard after repair");
+    });
+
+    st.stop();
+})();
diff --git a/jstests/replsets/db_reads_while_recovering_all_commands.js b/jstests/replsets/db_reads_while_recovering_all_commands.js
index 6bcdce6d723..d6152ed659d 100644
--- a/jstests/replsets/db_reads_while_recovering_all_commands.js
+++ b/jstests/replsets/db_reads_while_recovering_all_commands.js
@@ -49,6 +49,7 @@
         _configsvrRefineCollectionShardKey: {skip: isPrimaryOnly},
         _configsvrRemoveShard: {skip: isPrimaryOnly},
         _configsvrRemoveShardFromZone: {skip: isPrimaryOnly},
+        _configsvrRepairShardedCollectionChunksHistory: {skip: isPrimaryOnly},
         _configsvrShardCollection: {skip: isPrimaryOnly},
         _configsvrUpdateZoneKeyRange: {skip: isPrimaryOnly},
         _flushDatabaseCacheUpdates: {skip: isPrimaryOnly},
@@ -275,6 +276,7 @@
         renameCollection: {skip: isPrimaryOnly},
         repairCursor: {skip: isNotAUserDataRead},
         repairDatabase: {skip: isNotAUserDataRead},
+        repairShardedCollectionChunksHistory: {skip: isPrimaryOnly},
         replSetAbortPrimaryCatchUp: {skip: isNotAUserDataRead},
         replSetElect: {skip: isNotAUserDataRead},
         replSetFreeze: {skip: isNotAUserDataRead},
diff --git a/jstests/sharding/database_and_shard_versioning_all_commands.js b/jstests/sharding/database_and_shard_versioning_all_commands.js
index d11d50b5449..410748a9136 100644
--- a/jstests/sharding/database_and_shard_versioning_all_commands.js
+++ b/jstests/sharding/database_and_shard_versioning_all_commands.js
@@ -416,6 +416,7 @@
                 assert(mongosConn.getDB(dbName).getCollection(collName + "_renamed").drop());
             }
         },
+        repairShardedCollectionChunksHistory: {skip: "always targets the config server"},
         replSetGetStatus: {skip: "not supported in mongos"},
         resetError: {skip: "not on a user database"},
         restartCatalog: {skip: "not on a user database"},
diff --git a/jstests/sharding/safe_secondary_reads_drop_recreate.js b/jstests/sharding/safe_secondary_reads_drop_recreate.js
index f07a38ee68d..8f69d1e0b3d 100644
--- a/jstests/sharding/safe_secondary_reads_drop_recreate.js
+++ b/jstests/sharding/safe_secondary_reads_drop_recreate.js
@@ -285,6 +285,7 @@
         renameCollection: {skip: "primary only"},
         repairCursor: {skip: "does not return user data"},
         repairDatabase: {skip: "does not return user data"},
+        repairShardedCollectionChunksHistory: {skip: "does not return user data"},
         replSetAbortPrimaryCatchUp: {skip: "does not return user data"},
         replSetElect: {skip: "does not return user data"},
         replSetFreeze: {skip: "does not return user data"},
diff --git a/jstests/sharding/safe_secondary_reads_single_migration_suspend_range_deletion.js b/jstests/sharding/safe_secondary_reads_single_migration_suspend_range_deletion.js
index e819bf50574..bda38ce0268 100644
--- a/jstests/sharding/safe_secondary_reads_single_migration_suspend_range_deletion.js
+++ b/jstests/sharding/safe_secondary_reads_single_migration_suspend_range_deletion.js
@@ -334,6 +334,7 @@
         renameCollection: {skip: "primary only"},
         repairCursor: {skip: "does not return user data"},
         repairDatabase: {skip: "does not return user data"},
+        repairShardedCollectionChunksHistory: {skip: "does not return user data"},
         replSetAbortPrimaryCatchUp: {skip: "does not return user data"},
         replSetElect: {skip: "does not return user data"},
         replSetFreeze: {skip: "does not return user data"},
diff --git a/jstests/sharding/safe_secondary_reads_single_migration_waitForDelete.js b/jstests/sharding/safe_secondary_reads_single_migration_waitForDelete.js
index fa7bbefdff8..57420e9e841 100644
--- a/jstests/sharding/safe_secondary_reads_single_migration_waitForDelete.js
+++ b/jstests/sharding/safe_secondary_reads_single_migration_waitForDelete.js
@@ -291,6 +291,7 @@
         renameCollection: {skip: "primary only"},
         repairCursor: {skip: "does not return user data"},
         repairDatabase: {skip: "does not return user data"},
+        repairShardedCollectionChunksHistory: {skip: "does not return user data"},
         replSetAbortPrimaryCatchUp: {skip: "does not return user data"},
         replSetElect: {skip: "does not return user data"},
         replSetFreeze: {skip: "does not return user data"},
diff --git a/src/mongo/db/commands/set_feature_compatibility_version_command.cpp b/src/mongo/db/commands/set_feature_compatibility_version_command.cpp
index a74b9374089..d702fc6d2c0 100644
--- a/src/mongo/db/commands/set_feature_compatibility_version_command.cpp
+++ b/src/mongo/db/commands/set_feature_compatibility_version_command.cpp
@@ -248,9 +248,8 @@ public:
 
                     for (const auto& coll : collections) {
                         if (!coll.getDropped()) {
-                            uassertStatusOK(
-                                ShardingCatalogManager::get(opCtx)->upgradeChunksHistory(
-                                    opCtx, coll.getNs(), coll.getEpoch(), clusterTime));
+                            ShardingCatalogManager::get(opCtx)->upgradeChunksHistory(
+                                opCtx, coll.getNs(), coll.getEpoch(), clusterTime);
                         }
                     }
                 }
@@ -344,12 +343,12 @@ public:
 
                     for (const auto& coll : collections) {
                         if (!coll.getDropped()) {
-                            uassertStatusOK(
-                                ShardingCatalogManager::get(opCtx)->downgradeChunksHistory(
-                                    opCtx, coll.getNs(), coll.getEpoch()));
+                            ShardingCatalogManager::get(opCtx)->downgradeChunksHistory(
+                                opCtx, coll.getNs());
                         }
                     }
                 }
+
                 // Now that new metadata are written out to disk flush the local in memory state.
                 Grid::get(opCtx)->catalogCache()->purgeAllDatabases();
             }
diff --git a/src/mongo/db/s/SConscript b/src/mongo/db/s/SConscript
index d8c920756f5..e50953cca4c 100644
--- a/src/mongo/db/s/SConscript
+++ b/src/mongo/db/s/SConscript
@@ -242,6 +242,7 @@ env.Library(
         'config/configsvr_move_primary_command.cpp',
         'config/configsvr_remove_shard_command.cpp',
         'config/configsvr_remove_shard_from_zone_command.cpp',
+        'config/configsvr_repair_sharded_collection_chunks_history_command.cpp',
         'config/configsvr_shard_collection_command.cpp',
         'config/configsvr_split_chunk_command.cpp',
         'config/configsvr_update_zone_key_range_command.cpp',
@@ -255,9 +256,9 @@ env.Library(
         'move_chunk_command.cpp',
         'move_primary_command.cpp',
         'set_shard_version_command.cpp',
-        'shardsvr_shard_collection.cpp',
         'sharding_server_status.cpp',
         'sharding_state_command.cpp',
+        'shardsvr_shard_collection.cpp',
         'split_chunk_command.cpp',
         'split_vector_command.cpp',
         'unset_sharding_command.cpp',
diff --git a/src/mongo/db/s/config/configsvr_repair_sharded_collection_chunks_history_command.cpp b/src/mongo/db/s/config/configsvr_repair_sharded_collection_chunks_history_command.cpp
new file mode 100644
index 00000000000..138c66d4385
--- /dev/null
+++ b/src/mongo/db/s/config/configsvr_repair_sharded_collection_chunks_history_command.cpp
@@ -0,0 +1,163 @@
+
+/**
+ *    Copyright (C) 2021 MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kSharding
+
+#include "mongo/platform/basic.h"
+
+#include "mongo/db/auth/action_type.h"
+#include "mongo/db/auth/authorization_session.h"
+#include "mongo/db/auth/privilege.h"
+#include "mongo/db/commands.h"
+#include "mongo/db/commands/feature_compatibility_version.h"
+#include "mongo/db/commands/feature_compatibility_version_parser.h"
+#include "mongo/db/logical_clock.h"
+#include "mongo/db/namespace_string.h"
+#include "mongo/db/s/config/sharding_catalog_manager.h"
+#include "mongo/s/grid.h"
+#include "mongo/util/log.h"
+
+namespace mongo {
+namespace {
+
+class ConfigSvrRepairShardedCollectionChunksHistoryCommand : public BasicCommand {
+public:
+    ConfigSvrRepairShardedCollectionChunksHistoryCommand()
+        : BasicCommand("_configsvrRepairShardedCollectionChunksHistory") {}
+
+    std::string help() const override {
+        return "Internal command, which is exported by the sharding config server. Do not call "
+               "directly.";
+    }
+
+    AllowedOnSecondary secondaryAllowed(ServiceContext*) const override {
+        return AllowedOnSecondary::kNever;
+    }
+
+    bool adminOnly() const override {
+        return true;
+    }
+
+    bool supportsWriteConcern(const BSONObj& cmd) const override {
+        return true;
+    }
+
+    std::string parseNs(const std::string& unusedDbName, const BSONObj& cmdObj) const override {
+        return CommandHelpers::parseNsFullyQualified(cmdObj);
+    }
+
+    Status checkAuthForCommand(Client* client,
+                               const std::string& dbname,
+                               const BSONObj& cmdObj) const override {
+        if (!AuthorizationSession::get(client)->isAuthorizedForActionsOnResource(
+                ResourcePattern::forClusterResource(), ActionType::internal)) {
+            return Status(ErrorCodes::Unauthorized, "Unauthorized");
+        }
+        return Status::OK();
+    }
+
+    bool run(OperationContext* opCtx,
+             const std::string& unusedDbName,
+             const BSONObj& cmdObj,
+             BSONObjBuilder& result) override {
+        uassert(ErrorCodes::IllegalOperation,
+                "_configsvrRepairShardedCollectionChunksHistory can only be run on config servers",
+                serverGlobalParams.clusterRole == ClusterRole::ConfigServer);
+
+        // Set the operation context read concern level to local for reads into the config database.
+        repl::ReadConcernArgs::get(opCtx) =
+            repl::ReadConcernArgs(repl::ReadConcernLevel::kLocalReadConcern);
+
+        uassert(ErrorCodes::InvalidOptions,
+                str::stream() << "_configsvrRepairShardedCollectionChunksHistory must be called with majority writeConcern, got "
+                              << cmdObj,
+                opCtx->getWriteConcern().wMode == WriteConcernOptions::kMajority);
+
+        const NamespaceString nss{parseNs(unusedDbName, cmdObj)};
+
+        // Ensure that the feature compatibility version stays stable while the command is executing
+        Lock::SharedLock lk(opCtx->lockState(), FeatureCompatibilityVersion::fcvLock);
+        uassert(620650,
+                str::stream() << "Collection history entries can only be repaired when the server "
+                                 "is in feature compatibility version "
+                              << FeatureCompatibilityVersionParser::kVersion40,
+                serverGlobalParams.featureCompatibility.getVersion() ==
+                    ServerGlobalParams::FeatureCompatibility::Version::kFullyUpgradedTo40);
+
+        auto const catalogClient = Grid::get(opCtx)->catalogClient();
+        auto collection =
+            uassertStatusOK(
+                catalogClient->getCollection(opCtx, nss, repl::ReadConcernLevel::kLocalReadConcern))
+                .value;
+
+        if (cmdObj["force"].booleanSafe()) {
+            LOG(0) << "Resetting the 'historyIsAt40' field for all chunks in collection "
+                   << nss.ns() << " in order to force all chunks' history to get recreated";
+
+            BatchedCommandRequest request([&] {
+                write_ops::Update updateOp(ChunkType::ConfigNS);
+                updateOp.setUpdates({[&] {
+                    write_ops::UpdateOpEntry entry;
+                    entry.setQ(BSON("ns" << nss.ns()));
+                    entry.setU(BSON("$unset" << BSON(ChunkType::historyIsAt40() << "")));
+                    entry.setUpsert(false);
+                    entry.setMulti(true);
+                    return entry;
+                }()});
+                return updateOp;
+            }());
+            request.setWriteConcern(ShardingCatalogClient::kLocalWriteConcern.toBSON());
+
+            const auto configShard = Grid::get(opCtx)->shardRegistry()->getConfigShard();
+            auto response = configShard->runBatchWriteCommand(opCtx,
+                                                              Shard::kDefaultConfigCommandTimeout,
+                                                              request,
+                                                              Shard::RetryPolicy::kIdempotent);
+            uassertStatusOK(response.toStatus());
+
+            uassert(ErrorCodes::Error(620651),
+                    str::stream() << "No chunks found for collection " << nss.ns(),
+                    response.getN() > 0);
+        }
+
+        auto validAfter = LogicalClock::get(opCtx)->getClusterTime().asTimestamp();
+
+        ShardingCatalogManager::get(opCtx)->upgradeChunksHistory(
+            opCtx, nss, collection.getEpoch(), validAfter);
+
+        Grid::get(opCtx)->catalogCache()->invalidateShardedCollection(nss);
+
+        return true;
+    }
+
+} configSvrRepairShardedCollectionChunksHistoryCommand;
+
+}  // namespace
+}  // namespace mongo
diff --git a/src/mongo/db/s/config/sharding_catalog_manager.h b/src/mongo/db/s/config/sharding_catalog_manager.h
index b2f8b278165..0a257d4274f 100644
--- a/src/mongo/db/s/config/sharding_catalog_manager.h
+++ b/src/mongo/db/s/config/sharding_catalog_manager.h
@@ -434,17 +434,15 @@ public:
     /**
      * Upgrade the chunk metadata to include the history field.
      */
-    Status upgradeChunksHistory(OperationContext* opCtx,
-                                const NamespaceString& nss,
-                                const OID& collectionEpoch,
-                                const Timestamp validAfter);
+    void upgradeChunksHistory(OperationContext* opCtx,
+                              const NamespaceString& nss,
+                              const OID& collectionEpoch,
+                              const Timestamp validAfter);
 
     /**
      * Remove the history field from the chunk metadata.
      */
-    Status downgradeChunksHistory(OperationContext* opCtx,
-                                  const NamespaceString& nss,
-                                  const OID& collectionEpoch);
+    void downgradeChunksHistory(OperationContext* opCtx, const NamespaceString& nss);
 
 private:
     /**
diff --git a/src/mongo/db/s/config/sharding_catalog_manager_chunk_operations.cpp b/src/mongo/db/s/config/sharding_catalog_manager_chunk_operations.cpp
index c5e703e8129..056d676d273 100644
--- a/src/mongo/db/s/config/sharding_catalog_manager_chunk_operations.cpp
+++ b/src/mongo/db/s/config/sharding_catalog_manager_chunk_operations.cpp
@@ -43,7 +43,9 @@
 #include "mongo/db/dbdirectclient.h"
 #include "mongo/db/namespace_string.h"
 #include "mongo/db/operation_context.h"
+#include "mongo/db/repl/repl_client_info.h"
 #include "mongo/db/server_parameters.h"
+#include "mongo/db/write_concern.h"
 #include "mongo/rpc/get_status_from_command_result.h"
 #include "mongo/s/catalog/sharding_catalog_client.h"
 #include "mongo/s/catalog/type_chunk.h"
@@ -1126,140 +1128,123 @@ StatusWith<ChunkType> ShardingCatalogManager::_findChunkOnConfig(OperationContex
     return ChunkType::fromConfigBSON(origChunks.front());
 }
 
-Status ShardingCatalogManager::upgradeChunksHistory(OperationContext* opCtx,
-                                                    const NamespaceString& nss,
-                                                    const OID& collectionEpoch,
-                                                    const Timestamp validAfter) {
-    auto const configShard = Grid::get(opCtx)->shardRegistry()->getConfigShard();
+void ShardingCatalogManager::upgradeChunksHistory(OperationContext* opCtx,
+                                                  const NamespaceString& nss,
+                                                  const OID& collectionEpoch,
+                                                  const Timestamp validAfter) {
     auto const catalogClient = Grid::get(opCtx)->catalogClient();
+    const auto shardRegistry = Grid::get(opCtx)->shardRegistry();
 
     // Take _kChunkOpLock in exclusive mode to prevent concurrent chunk splits, merges, and
     // migrations.
     Lock::ExclusiveLock lk(opCtx->lockState(), _kChunkOpLock);
 
-    auto findResponse =
-        configShard->exhaustiveFindOnConfig(opCtx,
-                                            ReadPreferenceSetting{ReadPreference::PrimaryOnly},
-                                            repl::ReadConcernLevel::kLocalReadConcern,
-                                            ChunkType::ConfigNS,
-                                            BSON("ns" << nss.ns()),
-                                            BSONObj(),
-                                            boost::none);
-    if (!findResponse.isOK()) {
-        return findResponse.getStatus();
-    }
-
-    const auto chunksVector = std::move(findResponse.getValue().docs);
-    if (chunksVector.empty()) {
-        return {ErrorCodes::IncompatibleShardingMetadata,
-                str::stream() << "Tried to find chunks for collection '" << nss.ns()
-                              << ", but found no chunks"};
-    }
-
-    const auto currentCollectionVersion = _findCollectionVersion(opCtx, nss, collectionEpoch);
-    if (!currentCollectionVersion.isOK()) {
-        return currentCollectionVersion.getStatus();
-    }
+    auto const configShard = shardRegistry->getConfigShard();
+    const auto chunksVector = [&] {
+        auto findChunksResponse = uassertStatusOK(
+            configShard->exhaustiveFindOnConfig(opCtx,
+                                                ReadPreferenceSetting{ReadPreference::PrimaryOnly},
+                                                repl::ReadConcernLevel::kLocalReadConcern,
+                                                ChunkType::ConfigNS,
+                                                BSON("ns" << nss.ns()),
+                                                BSONObj(),
+                                                boost::none));
+        uassert(ErrorCodes::Error(620652),
+                str::stream() << "No chunks found for collection " << nss.ns(),
+                !findChunksResponse.docs.empty());
+        return std::move(findChunksResponse.docs);
+    }();
 
-    // Bump the version.
-    auto newCollectionVersion = ChunkVersion(currentCollectionVersion.getValue().majorVersion() + 1,
-                                             0,
-                                             currentCollectionVersion.getValue().epoch());
+    const auto currentCollectionVersion =
+        uassertStatusOK(_findCollectionVersion(opCtx, nss, collectionEpoch));
 
-    stdx::unordered_set<ShardId, ShardId::Hasher> bumpedShards;
+    // Bump the major version in order to be guaranteed to trigger refresh on every shard
+    ChunkVersion newCollectionVersion(
+        currentCollectionVersion.majorVersion() + 1, 0, currentCollectionVersion.epoch());
+    std::set<ShardId> changedShardIds;
     for (const auto& chunk : chunksVector) {
-        auto swChunk = ChunkType::fromConfigBSON(chunk);
-        if (!swChunk.isOK()) {
-            return swChunk.getStatus();
+        auto upgradeChunk = uassertStatusOK(ChunkType::fromConfigBSON(chunk));
+        bool historyIsAt40 = chunk[ChunkType::historyIsAt40()].booleanSafe();
+        if (historyIsAt40) {
+            uassert(
+                ErrorCodes::Error(620653),
+                str::stream() << "Chunk " << upgradeChunk.getName() << " in collection " << nss.ns()
+                              << " indicates that it has been upgraded to version 4.0, but is "
+                                 "missing the history field. This indicates a corrupted routing "
+                                 "table and requires a manual intervention to be fixed.",
+                !upgradeChunk.getHistory().empty());
+            continue;
         }
-        auto& upgradeChunk = swChunk.getValue();
-
-        if (upgradeChunk.getHistory().empty()) {
-            // Bump the version for only one chunk per shard to satisfy the requirement imposed by
-            // SERVER-33356
-            const auto& shardId = upgradeChunk.getShard();
-            if (!bumpedShards.count(shardId)) {
-                upgradeChunk.setVersion(newCollectionVersion);
-                newCollectionVersion.incMajor();
-                bumpedShards.emplace(shardId);
-            }
-
-            // Construct the fresh history.
-            upgradeChunk.setHistory({ChunkHistory{validAfter, upgradeChunk.getShard()}});
-
-            // Run the update.
-            auto status =
-                catalogClient->updateConfigDocument(opCtx,
-                                                    ChunkType::ConfigNS,
-                                                    BSON(ChunkType::name(upgradeChunk.getName())),
-                                                    upgradeChunk.toConfigBSON(),
-                                                    false,
-                                                    ShardingCatalogClient::kLocalWriteConcern);
-
-            if (!status.isOK()) {
-                return status.getStatus();
-            }
-        }
-    }
-
-    return Status::OK();
-}
-
-Status ShardingCatalogManager::downgradeChunksHistory(OperationContext* opCtx,
-                                                      const NamespaceString& nss,
-                                                      const OID& collectionEpoch) {
-    auto const configShard = Grid::get(opCtx)->shardRegistry()->getConfigShard();
-    auto const catalogClient = Grid::get(opCtx)->catalogClient();
-
-    // Take _kChunkOpLock in exclusive mode to prevent concurrent chunk splits, merges, and
-    // migrations.
-    //
-    Lock::ExclusiveLock lk(opCtx->lockState(), _kChunkOpLock);
-
-    auto findResponse =
-        configShard->exhaustiveFindOnConfig(opCtx,
-                                            ReadPreferenceSetting{ReadPreference::PrimaryOnly},
-                                            repl::ReadConcernLevel::kLocalReadConcern,
-                                            ChunkType::ConfigNS,
-                                            BSON("ns" << nss.ns()),
-                                            BSONObj(),
-                                            boost::none);
-    if (!findResponse.isOK()) {
-        return findResponse.getStatus();
-    }
 
-    const auto chunksVector = std::move(findResponse.getValue().docs);
-    if (chunksVector.empty()) {
-        return {ErrorCodes::IncompatibleShardingMetadata,
-                str::stream() << "Tried to find chunks for collection '" << nss.ns()
-                              << ", but found no chunks"};
-    }
+        upgradeChunk.setVersion(newCollectionVersion);
+        newCollectionVersion.incMinor();
+        changedShardIds.emplace(upgradeChunk.getShard());
 
-    for (const auto& chunk : chunksVector) {
-        auto swChunk = ChunkType::fromConfigBSON(chunk);
-        if (!swChunk.isOK()) {
-            return swChunk.getStatus();
-        }
-        auto& downgradeChunk = swChunk.getValue();
+        // Construct the fresh history.
+        upgradeChunk.setHistory({ChunkHistory{validAfter, upgradeChunk.getShard()}});
 
-        // Clear the history.
-        downgradeChunk.setHistory({});
+        // Set the 'historyIsAt40' field so that it gets skipped if the command is re-run
+        BSONObjBuilder chunkObjBuilder(upgradeChunk.toConfigBSON());
+        chunkObjBuilder.appendBool(ChunkType::historyIsAt40(), true);
 
         // Run the update.
-        auto status =
+        uassertStatusOK(
             catalogClient->updateConfigDocument(opCtx,
                                                 ChunkType::ConfigNS,
-                                                BSON(ChunkType::name(downgradeChunk.getName())),
-                                                downgradeChunk.toConfigBSON(),
+                                                BSON(ChunkType::name(upgradeChunk.getName())),
+                                                chunkObjBuilder.obj(),
                                                 false,
-                                                ShardingCatalogClient::kLocalWriteConcern);
-
-        if (!status.isOK()) {
-            return status.getStatus();
-        }
+                                                ShardingCatalogClient::kLocalWriteConcern));
+    }
+
+    // Wait for the writes to become majority committed so that the subsequent shard refreshes can
+    // see them
+    const auto clientOpTime = repl::ReplClientInfo::forClient(opCtx->getClient()).getLastOp();
+    WriteConcernResult unusedWCResult;
+    uassertStatusOK(waitForWriteConcern(
+        opCtx, clientOpTime, ShardingCatalogClient::kMajorityWriteConcern, &unusedWCResult));
+
+    for (const auto& shardId : changedShardIds) {
+        auto shard = uassertStatusOK(shardRegistry->getShard(opCtx, shardId));
+        uassertStatusOK(
+            Shard::CommandResponse::getEffectiveStatus(shard->runCommandWithFixedRetryAttempts(
+                opCtx,
+                ReadPreferenceSetting{ReadPreference::PrimaryOnly},
+                "admin",
+                BSON("_flushRoutingTableCacheUpdates" << nss.ns()),
+                Shard::RetryPolicy::kIdempotent)));
     }
+}
 
-    return Status::OK();
+void ShardingCatalogManager::downgradeChunksHistory(OperationContext* opCtx,
+                                                    const NamespaceString& nss) {
+    // Take _kChunkOpLock in exclusive mode to prevent concurrent chunk splits, merges, and
+    // migrations.
+    Lock::ExclusiveLock lk(opCtx->lockState(), _kChunkOpLock);
+
+    BatchedCommandRequest request([&] {
+        write_ops::Update updateOp(ChunkType::ConfigNS);
+        updateOp.setUpdates({[&] {
+            write_ops::UpdateOpEntry entry;
+            entry.setQ(BSON("ns" << nss.ns()));
+            entry.setU(BSON(
+                "$unset" << BSON(ChunkType::history() << "" << ChunkType::historyIsAt40() << "")));
+            entry.setUpsert(false);
+            entry.setMulti(true);
+            return entry;
+        }()});
+        return updateOp;
+    }());
+    request.setWriteConcern(ShardingCatalogClient::kLocalWriteConcern.toBSON());
+
+    auto const configShard = Grid::get(opCtx)->shardRegistry()->getConfigShard();
+    auto response = configShard->runBatchWriteCommand(
+        opCtx, Shard::kDefaultConfigCommandTimeout, request, Shard::RetryPolicy::kIdempotent);
+    uassertStatusOK(response.toStatus());
+
+    uassert(ErrorCodes::Error(620654),
+            str::stream() << "No chunks found for collection " << nss.ns(),
+            response.getN() > 0);
 }
 
 StatusWith<ChunkVersion> ShardingCatalogManager::_findCollectionVersion(
diff --git a/src/mongo/db/s/flush_routing_table_cache_updates_command.cpp b/src/mongo/db/s/flush_routing_table_cache_updates_command.cpp
index d518d63bd93..48682d632cd 100644
--- a/src/mongo/db/s/flush_routing_table_cache_updates_command.cpp
+++ b/src/mongo/db/s/flush_routing_table_cache_updates_command.cpp
@@ -57,9 +57,9 @@ namespace {
 class FlushRoutingTableCacheUpdatesCmd final
     : public TypedCommand<FlushRoutingTableCacheUpdatesCmd> {
 public:
-    using Request = _flushRoutingTableCacheUpdates;
+    using Request = FlushRoutingTableCacheUpdates;
 
-    // Support deprecated name 'forceRoutingTableRefresh' for backwards compatibility with 3.6.0.
+    // Support deprecated name 'forceRoutingTableRefresh' for backwards compatibility with 3.6
     FlushRoutingTableCacheUpdatesCmd()
         : TypedCommand<FlushRoutingTableCacheUpdatesCmd>(Request::kCommandName,
                                                          "forceRoutingTableRefresh") {}
@@ -106,11 +106,12 @@ public:
             uassertStatusOK(shardingState->canAcceptShardedCommands());
 
             uassert(ErrorCodes::IllegalOperation,
-                    "Can't issue _flushRoutingTableCacheUpdates from 'eval'",
+                    str::stream() << "Can't issue " << Request::kCommandName << " from 'eval'",
                     !opCtx->getClient()->isInDirectClient());
 
             uassert(ErrorCodes::IllegalOperation,
-                    "Can't call _flushRoutingTableCacheUpdates if in read-only mode",
+                    str::stream() << "Can't call " << Request::kCommandName
+                                  << " if in read-only mode",
                     !storageGlobalParams.readOnly);
 
             auto& oss = OperationShardingState::get(opCtx);
diff --git a/src/mongo/s/catalog/type_chunk.cpp b/src/mongo/s/catalog/type_chunk.cpp
index cb64bca6300..a258cb62da1 100644
--- a/src/mongo/s/catalog/type_chunk.cpp
+++ b/src/mongo/s/catalog/type_chunk.cpp
@@ -57,6 +57,7 @@ const BSONField<bool> ChunkType::jumbo("jumbo");
 const BSONField<Date_t> ChunkType::lastmod("lastmod");
 const BSONField<OID> ChunkType::epoch("lastmodEpoch");
 const BSONField<BSONObj> ChunkType::history("history");
+const BSONField<bool> ChunkType::historyIsAt40("historyIsAt40");
 
 namespace {
 
diff --git a/src/mongo/s/catalog/type_chunk.h b/src/mongo/s/catalog/type_chunk.h
index 7406fdb5d58..d67484b32b8 100644
--- a/src/mongo/s/catalog/type_chunk.h
+++ b/src/mongo/s/catalog/type_chunk.h
@@ -187,6 +187,7 @@ public:
     static const BSONField<Date_t> lastmod;
     static const BSONField<OID> epoch;
     static const BSONField<BSONObj> history;
+    static const BSONField<bool> historyIsAt40;
 
     ChunkType();
     ChunkType(NamespaceString nss, ChunkRange range, ChunkVersion version, ShardId shardId);
diff --git a/src/mongo/s/client/shard.h b/src/mongo/s/client/shard.h
index 8919a69b1d1..14ebaef99c9 100644
--- a/src/mongo/s/client/shard.h
+++ b/src/mongo/s/client/shard.h
@@ -41,11 +41,11 @@
 #include "mongo/db/repl/read_concern_args.h"
 #include "mongo/executor/remote_command_response.h"
 #include "mongo/s/shard_id.h"
+#include "mongo/s/write_ops/batched_command_request.h"
+#include "mongo/s/write_ops/batched_command_response.h"
 
 namespace mongo {
 
-class BatchedCommandRequest;
-class BatchedCommandResponse;
 class OperationContext;
 class RemoteCommandTargeter;
 
diff --git a/src/mongo/s/commands/SConscript b/src/mongo/s/commands/SConscript
index e7e9b60e78d..7e13a38dac8 100644
--- a/src/mongo/s/commands/SConscript
+++ b/src/mongo/s/commands/SConscript
@@ -86,6 +86,7 @@ env.Library(
         'cluster_reindex_cmd.cpp',
         'cluster_remove_shard_cmd.cpp',
         'cluster_remove_shard_from_zone_cmd.cpp',
+        'cluster_repair_sharded_collection_chunks_history_cmd.cpp',
         'cluster_repl_set_get_status_cmd.cpp',
         'cluster_reset_error_cmd.cpp',
         'cluster_restart_catalog_command.cpp',
diff --git a/src/mongo/s/commands/cluster_repair_sharded_collection_chunks_history_cmd.cpp b/src/mongo/s/commands/cluster_repair_sharded_collection_chunks_history_cmd.cpp
new file mode 100644
index 00000000000..22f9ccf7525
--- /dev/null
+++ b/src/mongo/s/commands/cluster_repair_sharded_collection_chunks_history_cmd.cpp
@@ -0,0 +1,122 @@
+
+/**
+ *    Copyright (C) 2021 MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kSharding
+
+#include "mongo/platform/basic.h"
+
+#include "mongo/db/audit.h"
+#include "mongo/db/auth/action_set.h"
+#include "mongo/db/auth/action_type.h"
+#include "mongo/db/auth/authorization_manager.h"
+#include "mongo/db/auth/authorization_session.h"
+#include "mongo/db/client.h"
+#include "mongo/db/commands.h"
+#include "mongo/s/grid.h"
+
+namespace mongo {
+namespace {
+
+class RepairShardedCollectionChunksHistoryCommand : public BasicCommand {
+public:
+    RepairShardedCollectionChunksHistoryCommand()
+        : BasicCommand("repairShardedCollectionChunksHistory") {}
+
+    AllowedOnSecondary secondaryAllowed(ServiceContext*) const override {
+        return AllowedOnSecondary::kAlways;
+    }
+
+    bool adminOnly() const override {
+        return true;
+    }
+
+    bool supportsWriteConcern(const BSONObj& cmd) const override {
+        return false;
+    }
+
+    std::string help() const override {
+        return "Administrative command to repair the effects of SERVER-62065. If the collection "
+               "has been upgraded through a cluster comprised of binaries which do not contain "
+               "this command, the chunks cache collections on the shards will miss history "
+               "entries. This command will correct that and will mark such collections as "
+               "correctly repaired, so that a subsequent invocation will not cause any changes to "
+               "the routing information. In rare cases where the history entries are missing due "
+               "to corrupted restore, the 'force:true' parameter can be passed which will force "
+               "all history entries to be re-added.";
+    }
+
+    // The command intentionally uses the permission control of split/mergeChunks since it only
+    // modifies the contents of chunk entries and increments the collection/shard versions without
+    // causing any data placement changes
+    Status checkAuthForCommand(Client* client,
+                               const std::string& dbname,
+                               const BSONObj& cmdObj) const override {
+        if (!AuthorizationSession::get(client)->isAuthorizedForActionsOnResource(
+                ResourcePattern::forExactNamespace(NamespaceString(parseNs(dbname, cmdObj))),
+                ActionType::splitChunk)) {
+            return Status(ErrorCodes::Unauthorized, "Unauthorized");
+        }
+        return Status::OK();
+    }
+
+    std::string parseNs(const std::string& unusedDbName, const BSONObj& cmdObj) const override {
+        return CommandHelpers::parseNsFullyQualified(cmdObj);
+    }
+
+    bool run(OperationContext* opCtx,
+             const std::string& unusedDbName,
+             const BSONObj& cmdObj,
+             BSONObjBuilder& result) override {
+        const NamespaceString nss{parseNs(unusedDbName, cmdObj)};
+
+        BSONObjBuilder cmdBuilder(
+            BSON("_configsvrRepairShardedCollectionChunksHistory" << nss.ns()));
+        if (cmdObj["force"].booleanSafe())
+            cmdBuilder.appendBool("force", true);
+
+        auto configShard = Grid::get(opCtx)->shardRegistry()->getConfigShard();
+        auto cmdResponse = uassertStatusOK(configShard->runCommandWithFixedRetryAttempts(
+            opCtx,
+            ReadPreferenceSetting{ReadPreference::PrimaryOnly},
+            "admin",
+            cmdBuilder.obj(),
+            Shard::RetryPolicy::kIdempotent));
+        uassertStatusOK(cmdResponse.commandStatus);
+
+        // Append any return value from the response, which the config server returned
+        CommandHelpers::filterCommandReplyForPassthrough(cmdResponse.response, &result);
+
+        return true;
+    }
+
+} repairShardedCollectionChunksHistoryCommand;
+
+}  // namespace
+}  // namespace mongo
diff --git a/src/mongo/s/request_types/flush_routing_table_cache_updates.idl b/src/mongo/s/request_types/flush_routing_table_cache_updates.idl
index 07981295bac..e5a27a58126 100644
--- a/src/mongo/s/request_types/flush_routing_table_cache_updates.idl
+++ b/src/mongo/s/request_types/flush_routing_table_cache_updates.idl
@@ -26,8 +26,6 @@
 # it in the license file.
 #
 
-# _flushRoutingTableCacheUpdates IDL File
-
 global:
     cpp_namespace: "mongo"
 
@@ -36,6 +34,7 @@ imports:
 
 commands:
     _flushRoutingTableCacheUpdates:
+        cpp_name: FlushRoutingTableCacheUpdates
         description: "An internal command to wait for the last routing table cache refresh for a particular namespace to be persisted to disk"
         strict: true
         namespace: type