SERVER-68511 MovePrimary update of `config.databases` entry must use dotted fields notation

2 files changed, 123 insertions, 20 deletions

diff --git a/jstests/multiVersion/genericSetFCVUsage/move_primary_setFCV.js b/jstests/multiVersion/genericSetFCVUsage/move_primary_setFCV.js
new file mode 100644
index 00000000000..34eea8fedb9
--- /dev/null
+++ b/jstests/multiVersion/genericSetFCVUsage/move_primary_setFCV.js
@@ -0,0 +1,84 @@
+/**
+ * Test that `movePrimary` works for databases created under a different FCV
+ */
+(function() {
+
+"use strict";
+
+let st = new ShardingTest({shards: 2, mongos: 1, rs: {nodes: 1}});
+
+const mongos = st.s;
+const kBeforeDowngradingDbName = 'createdBeforeDowngrading';
+const kBeforeUpgradingDbName = 'createdBeforeUpgrading';
+const shard0 = st.shard0.shardName;
+const shard1 = st.shard1.shardName;
+
+const createdBeforeDowngradingDB = mongos.getDB(kBeforeDowngradingDbName);
+const createdBeforeUpgradingDB = mongos.getDB(kBeforeUpgradingDbName);
+const fcvValues = [lastLTSFCV, lastContinuousFCV];
+
+function testMovePrimary(db) {
+    const dbName = db.getName();
+    //  The following pipeline update modifies the config.databases entry to simulate its database
+    //  version field order as having come from running {setFeatureCompatibility: "5.0"} as part of
+    //  upgrading from MongoDB 4.4. See SERVER-68511 for more details of the original issue.
+    mongos.getDB('config').databases.update({_id: dbName}, [{
+                                                $replaceWith: {
+                                                    $mergeObjects: [
+                                                        "$$ROOT",
+                                                        {
+                                                            version: {
+                                                                uuid: "$version.uuid",
+                                                                lastMod: "$version.lastMod",
+                                                                timestamp: "$version.timestamp"
+                                                            }
+                                                        }
+                                                    ]
+                                                }
+                                            }]);
+
+    const currentPrimary = mongos.getDB('config').databases.findOne({_id: dbName}).primary;
+    const newPrimary = currentPrimary == shard0 ? shard1 : shard0;
+    assert.eq(db.coll.countDocuments({}), 1);
+    assert.commandWorked(mongos.adminCommand({movePrimary: dbName, to: newPrimary}));
+    assert.eq(newPrimary, mongos.getDB('config').databases.findOne({_id: dbName}).primary);
+    assert.eq(db.coll.countDocuments({}), 1);
+}
+
+for (var i = 0; i < fcvValues.length; i++) {
+    // Latest FCV
+    assert.commandWorked(mongos.adminCommand({setFeatureCompatibilityVersion: latestFCV}));
+
+    // Create database `createdBeforeDowngrading` under latest FCV
+    assert.commandWorked(
+        mongos.adminCommand({enableSharding: kBeforeDowngradingDbName, primaryShard: shard0}));
+    assert.commandWorked(createdBeforeDowngradingDB.coll.insert({_id: 'foo'}));
+
+    // Downgrade FCV
+    assert.commandWorked(mongos.adminCommand({setFeatureCompatibilityVersion: fcvValues[i]}));
+
+    // Make sure movePrimary works for `createdBeforeDowngrading`
+    testMovePrimary(createdBeforeDowngradingDB);
+
+    // Create database `createdBeforeUpgrading` under downgraded FCV
+    assert.commandWorked(mongos.adminCommand({setFeatureCompatibilityVersion: latestFCV}));
+    assert.commandWorked(
+        mongos.adminCommand({enableSharding: kBeforeUpgradingDbName, primaryShard: shard0}));
+    assert.commandWorked(createdBeforeUpgradingDB.coll.insert({_id: 'foo'}));
+
+    // Upgrade FCV
+    assert.commandWorked(mongos.adminCommand({setFeatureCompatibilityVersion: latestFCV}));
+
+    // Make sure movePrimary works (again) for `createdBeforeDowngrading`
+    testMovePrimary(createdBeforeDowngradingDB);
+
+    // Make sure movePrimary works for `createdBeforeUpgrading`
+    testMovePrimary(createdBeforeUpgradingDB);
+
+    // Drop databases for next round
+    assert.commandWorked(createdBeforeDowngradingDB.dropDatabase());
+    assert.commandWorked(createdBeforeUpgradingDB.dropDatabase());
+}
+
+st.stop();
+})();
diff --git a/src/mongo/db/s/move_primary_source_manager.cpp b/src/mongo/db/s/move_primary_source_manager.cpp
index 5292e7ce66d..dfb6e44ed80 100644
--- a/src/mongo/db/s/move_primary_source_manager.cpp
+++ b/src/mongo/db/s/move_primary_source_manager.cpp
@@ -329,23 +329,26 @@ Status MovePrimarySourceManager::commitOnConfig(OperationContext* opCtx) {
 Status MovePrimarySourceManager::_commitOnConfig(OperationContext* opCtx) {
     auto const configShard = Grid::get(opCtx)->shardRegistry()->getConfigShard();
 
-    auto findResponse = uassertStatusOK(
-        configShard->exhaustiveFindOnConfig(opCtx,
-                                            ReadPreferenceSetting{ReadPreference::PrimaryOnly},
-                                            repl::ReadConcernLevel::kMajorityReadConcern,
-                                            NamespaceString::kConfigDatabasesNamespace,
-                                            BSON(DatabaseType::kNameFieldName << _dbname),
-                                            BSON(DatabaseType::kNameFieldName << -1),
-                                            1));
-
-    const auto databasesVector = std::move(findResponse.docs);
-    uassert(ErrorCodes::IncompatibleShardingMetadata,
-            str::stream() << "Tried to find max database version for database '" << _dbname
-                          << "', but found no databases",
-            !databasesVector.empty());
-
-    const auto dbType =
-        DatabaseType::parse(IDLParserContext("DatabaseType"), databasesVector.front());
+    auto getDatabaseEntry = [&]() {
+        auto findResponse = uassertStatusOK(
+            configShard->exhaustiveFindOnConfig(opCtx,
+                                                ReadPreferenceSetting{ReadPreference::PrimaryOnly},
+                                                repl::ReadConcernLevel::kMajorityReadConcern,
+                                                NamespaceString::kConfigDatabasesNamespace,
+                                                BSON(DatabaseType::kNameFieldName << _dbname),
+                                                BSON(DatabaseType::kNameFieldName << -1),
+                                                1));
+
+        const auto databasesVector = std::move(findResponse.docs);
+        uassert(ErrorCodes::IncompatibleShardingMetadata,
+                str::stream() << "Tried to find max database version for database '" << _dbname
+                              << "', but found no databases",
+                !databasesVector.empty());
+
+        return DatabaseType::parse(IDLParserContext("DatabaseType"), databasesVector.front());
+    };
+
+    const auto dbType = getDatabaseEntry();
 
     if (dbType.getPrimary() == _toShard) {
         return Status::OK();
@@ -358,9 +361,17 @@ Status MovePrimarySourceManager::_commitOnConfig(OperationContext* opCtx) {
 
     newDbType.setVersion(currentDatabaseVersion.makeUpdated());
 
-    auto const updateQuery =
-        BSON(DatabaseType::kNameFieldName << _dbname << DatabaseType::kVersionFieldName
-                                          << currentDatabaseVersion.toBSON());
+    auto const updateQuery = [&] {
+        BSONObjBuilder queryBuilder;
+        queryBuilder.append(DatabaseType::kNameFieldName, _dbname);
+        // Include the version in the update filter to be resilient to potential network retries and
+        // delayed messages.
+        for (auto [fieldName, elem] : currentDatabaseVersion.toBSON()) {
+            auto dottedFieldName = DatabaseType::kVersionFieldName + "." + fieldName;
+            queryBuilder.appendAs(elem, dottedFieldName);
+        }
+        return queryBuilder.obj();
+    }();
 
     auto updateStatus = Grid::get(opCtx)->catalogClient()->updateConfigDocument(
         opCtx,
@@ -379,6 +390,14 @@ Status MovePrimarySourceManager::_commitOnConfig(OperationContext* opCtx) {
         return updateStatus.getStatus();
     }
 
+    const auto updatedDbType = getDatabaseEntry();
+    tassert(6851100,
+            "Error committing movePrimary: database version went backwards",
+            updatedDbType.getVersion() > currentDatabaseVersion);
+    uassert(6851101,
+            "Error committing movePrimary: update of `config.databases` failed",
+            updatedDbType.getPrimary() != _fromShard);
+
     return Status::OK();
 }