SERVER-31805 rollbackViaRefetchNoUUID resyncs uuids correctly

(cherry picked from commit aa8b6f7657450d537cc14a77371dcd8742018a28)

6 files changed, 263 insertions, 38 deletions

diff --git a/src/mongo/db/catalog/coll_mod.cpp b/src/mongo/db/catalog/coll_mod.cpp
index 2100adf8164..4d5a4bac55c 100644
--- a/src/mongo/db/catalog/coll_mod.cpp
+++ b/src/mongo/db/catalog/coll_mod.cpp
@@ -647,6 +647,11 @@ void updateUUIDSchemaVersion(OperationContext* opCtx, bool upgrade) {
 
         _updateDatabaseUUIDSchemaVersion(opCtx, dbName, dbToCollToUUID[dbName], upgrade);
     }
+
+    std::string upgradeStr = upgrade ? "upgrade" : "downgrade";
+    log() << "Finished updating UUID schema version for " << upgradeStr
+          << ", waiting for all UUIDs to be committed.";
+
     const WriteConcernOptions writeConcern(WriteConcernOptions::kMajority,
                                            WriteConcernOptions::SyncMode::UNSET,
                                            /*timeout*/ INT_MAX);
diff --git a/src/mongo/db/repl/rollback_test_fixture.cpp b/src/mongo/db/repl/rollback_test_fixture.cpp
index 83e2ed8687a..4724fc9da8d 100644
--- a/src/mongo/db/repl/rollback_test_fixture.cpp
+++ b/src/mongo/db/repl/rollback_test_fixture.cpp
@@ -239,18 +239,28 @@ RollbackResyncsCollectionOptionsTest::RollbackSourceWithCollectionOptions::getCo
 
 void RollbackResyncsCollectionOptionsTest::resyncCollectionOptionsTest(
     CollectionOptions localCollOptions, BSONObj remoteCollOptionsObj) {
+    resyncCollectionOptionsTest(localCollOptions,
+                                remoteCollOptionsObj,
+                                BSON("collMod"
+                                     << "coll"
+                                     << "noPadding"
+                                     << false),
+                                "coll");
+}
+void RollbackResyncsCollectionOptionsTest::resyncCollectionOptionsTest(
+    CollectionOptions localCollOptions,
+    BSONObj remoteCollOptionsObj,
+    BSONObj collModCmd,
+    std::string collName) {
     createOplog(_opCtx.get());
 
     auto dbName = "test";
-    auto collName = "coll";
     auto nss = NamespaceString(dbName, collName);
 
     auto coll = _createCollection(_opCtx.get(), nss.toString(), localCollOptions);
     auto commonOperation =
         std::make_pair(BSON("ts" << Timestamp(Seconds(1), 0) << "h" << 1LL), RecordId(1));
 
-    // 'collMod' operation used to trigger metadata re-sync.
-    BSONObj collModCmd = BSON("collMod" << collName << "noPadding" << false);
     auto collectionModificationOperation =
         makeCommandOp(Timestamp(Seconds(2), 0), coll->uuid(), nss.toString(), collModCmd, 2);
 
diff --git a/src/mongo/db/repl/rollback_test_fixture.h b/src/mongo/db/repl/rollback_test_fixture.h
index c24ba1bcffa..c31aeed617e 100644
--- a/src/mongo/db/repl/rollback_test_fixture.h
+++ b/src/mongo/db/repl/rollback_test_fixture.h
@@ -172,9 +172,9 @@ private:
  * 'remoteCollOptionsObj': the collection options object that the sync source will respond with to
  * the rollback node when it fetches collection metadata.
  *
- * A collMod operation with a 'noPadding' argument is used to trigger a collection metadata resync,
- * since the rollback of collMod operations does not take into account the actual command object. It
- * simply re-syncs all the collection options.
+ * If no command is provided, a collMod operation with a 'noPadding' argument is used to trigger a
+ * collection metadata resync, since the rollback of collMod operations does not take into account
+ * the actual command object. It simply re-syncs all the collection options.
  */
 class RollbackResyncsCollectionOptionsTest : public RollbackTest {
 
@@ -194,6 +194,10 @@ class RollbackResyncsCollectionOptionsTest : public RollbackTest {
 
 public:
     void resyncCollectionOptionsTest(CollectionOptions localCollOptions,
+                                     BSONObj remoteCollOptionsObj,
+                                     BSONObj collModCmd,
+                                     std::string collName);
+    void resyncCollectionOptionsTest(CollectionOptions localCollOptions,
                                      BSONObj remoteCollOptionsObj);
 };
 
diff --git a/src/mongo/db/repl/rs_rollback.cpp b/src/mongo/db/repl/rs_rollback.cpp
index a0ffbedc496..79566d71cd3 100644
--- a/src/mongo/db/repl/rs_rollback.cpp
+++ b/src/mongo/db/repl/rs_rollback.cpp
@@ -1186,7 +1186,7 @@ void rollback_internal::syncFixUp(OperationContext* opCtx,
             wuow.commit();
 
             LOG(1) << "Resynced collection metadata for collection: " << nss << ", UUID: " << uuid
-                   << ", with: " << redact(options.toBSON())
+                   << ", with: " << redact(info)
                    << ", to: " << redact(cce->getCollectionOptions(opCtx).toBSON());
         }
 
diff --git a/src/mongo/db/repl/rs_rollback_no_uuid.cpp b/src/mongo/db/repl/rs_rollback_no_uuid.cpp
index d76cba479ac..b26a9908228 100644
--- a/src/mongo/db/repl/rs_rollback_no_uuid.cpp
+++ b/src/mongo/db/repl/rs_rollback_no_uuid.cpp
@@ -153,6 +153,44 @@ void FixUpInfo::addIndexToDrop(const NamespaceString& nss, const DocID& doc) {
     }
 }
 
+namespace {
+
+bool canRollBackCollMod(BSONObj obj) {
+
+    // If there is only one field, then the collMod is a UUID upgrade/downgrade collMod.
+    if (obj.nFields() == 1) {
+        return true;
+    }
+    for (auto field : obj) {
+        // Example collMod obj
+        // o:{
+        //       collMod : "x",
+        //       validationLevel : "off",
+        //       index: {
+        //                  name: "indexName_1",
+        //                  expireAfterSeconds: 600
+        //              }
+        //    }
+
+        const auto modification = field.fieldNameStringData();
+        if (modification == "collMod") {
+            continue;  // Skips the command name. The first field in the obj will be the
+                       // command name.
+        }
+
+        if (modification == "validator" || modification == "validationAction" ||
+            modification == "validationLevel" || modification == "usePowerOf2Sizes" ||
+            modification == "noPadding") {
+            continue;
+        }
+
+        // Some collMod fields cannot be rolled back, such as the index field.
+        return false;
+    }
+    return true;
+}
+
+}  // namespace
 
 Status rollback_internal_no_uuid::updateFixUpInfoFromLocalOplogEntry(FixUpInfo& fixUpInfo,
                                                                      const BSONObj& ourObj) {
@@ -274,36 +312,14 @@ Status rollback_internal_no_uuid::updateFixUpInfoFromLocalOplogEntry(FixUpInfo&
             severe() << message << redact(obj);
             throw RSFatalException(message);
         } else if (cmdname == "collMod") {
-            const auto ns = NamespaceString(cmd->parseNs(nss.db().toString(), obj));
-            for (auto field : obj) {
-                // Example collMod obj
-                // o:{
-                //       collMod : "x",
-                //       validationLevel : "off",
-                //       index: {
-                //                  name: "indexName_1",
-                //                  expireAfterSeconds: 600
-                //              }
-                //    }
-
-                const auto modification = field.fieldNameStringData();
-                if (modification == cmdname) {
-                    continue;  // Skips the command name. The first field in the obj will be the
-                               // command name.
-                }
-
-                if (modification == "validator" || modification == "validationAction" ||
-                    modification == "validationLevel" || modification == "usePowerOf2Sizes" ||
-                    modification == "noPadding") {
-                    fixUpInfo.collectionsToResyncMetadata.insert(ns.ns());
-                    continue;
-                }
-                // Some collMod fields cannot be rolled back, such as the index field.
-                string message = "Cannot roll back a collMod command: ";
-                severe() << message << redact(obj);
-                throw RSFatalException(message);
+            if (canRollBackCollMod(obj)) {
+                const auto ns = NamespaceString(cmd->parseNs(nss.db().toString(), obj));
+                fixUpInfo.collectionsToResyncMetadata.insert(ns.ns());
+                return Status::OK();
             }
-            return Status::OK();
+            string message = "Cannot roll back a collMod command: ";
+            severe() << message << redact(obj);
+            throw RSFatalException(message);
         } else if (cmdname == "applyOps") {
 
             if (first.type() != Array) {
@@ -533,7 +549,8 @@ void syncFixUp(OperationContext* opCtx,
             // Updates the collection flags.
             if (auto optionsField = info["options"]) {
                 if (optionsField.type() != Object) {
-                    throw RSFatalException(str::stream() << "Failed to parse options " << info
+                    throw RSFatalException(str::stream() << "Failed to parse options "
+                                                         << redact(info)
                                                          << ": expected 'options' to be an "
                                                          << "Object, got "
                                                          << typeName(optionsField.type()));
@@ -545,7 +562,6 @@ void syncFixUp(OperationContext* opCtx,
                                                          << ": "
                                                          << status.toString());
                 }
-                // TODO(SERVER-27992): Set options.uuid.
             } else {
                 // Use default options.
             }
@@ -566,7 +582,62 @@ void syncFixUp(OperationContext* opCtx,
             cce->updateValidator(
                 opCtx, options.validator, options.validationLevel, options.validationAction);
 
+            OptionalCollectionUUID originalLocalUUID = collection->uuid();
+            OptionalCollectionUUID remoteUUID = boost::none;
+            if (auto infoField = info["info"]) {
+                if (infoField.type() != Object) {
+                    throw RSFatalException(str::stream() << "Failed to parse collection info "
+                                                         << redact(info)
+                                                         << ": expected 'info' to be an "
+                                                         << "Object, got "
+                                                         << typeName(infoField.type()));
+                }
+                auto infoFieldObj = infoField.Obj();
+                if (infoFieldObj.hasField("uuid")) {
+                    remoteUUID = boost::make_optional(UUID::parse(infoFieldObj));
+                }
+            }
+
+            // If the local collection has a UUID, it must match the remote UUID. If the sync source
+            // has no UUID or they do not match, we remove the local UUID and allow the 'collMod'
+            // operation on the sync source to add the UUID back.
+            if (originalLocalUUID) {
+                if (!remoteUUID) {
+                    log() << "Removing UUID " << originalLocalUUID.get() << " from " << nss.ns()
+                          << " because sync source had no UUID for namespace.";
+                    cce->removeUUID(opCtx);
+                    collection->refreshUUID(opCtx);
+                } else if (originalLocalUUID.get() != remoteUUID.get()) {
+                    log() << "Removing UUID " << originalLocalUUID.get() << " from " << nss.ns()
+                          << " because sync source had different UUID (" << remoteUUID.get()
+                          << ") for collection.";
+                    cce->removeUUID(opCtx);
+                    collection->refreshUUID(opCtx);
+                }
+            } else if (remoteUUID &&
+                       (serverGlobalParams.featureCompatibility.getVersion() ==
+                        ServerGlobalParams::FeatureCompatibility::Version::kDowngradingTo34)) {
+                // If we are in the process of downgrading, and we have no UUID but the sync source
+                // does, then it is possible we will not see a 'collMod' during RECOVERING to add
+                // back in the UUID from the sync source. In that case, we add the sync source's
+                // UUID here.
+                invariant(!originalLocalUUID);
+                log() << "Assigning UUID " << remoteUUID.get() << " to " << nss.ns()
+                      << " because we had no UUID, the sync source had a UUID, and we were in the "
+                         "middle of downgrade.";
+                cce->addUUID(opCtx, remoteUUID.get(), collection);
+                collection->refreshUUID(opCtx);
+            }
+
             wuow.commit();
+
+            auto originalLocalUUIDString =
+                (originalLocalUUID) ? originalLocalUUID.get().toString() : "no UUID";
+            auto remoteUuidString = (remoteUUID) ? remoteUUID.get().toString() : "no UUID";
+            LOG(1) << "Resynced collection metadata for collection: " << nss
+                   << ", original local UUID: " << originalLocalUUIDString
+                   << ", remote UUID: " << remoteUuidString << ", with: " << redact(info)
+                   << ", to: " << redact(cce->getCollectionOptions(opCtx).toBSON());
         }
 
         // Since we read from the sync source to retrieve the metadata of the
diff --git a/src/mongo/db/repl/rs_rollback_no_uuid_test.cpp b/src/mongo/db/repl/rs_rollback_no_uuid_test.cpp
index 28676e85075..07b767548eb 100644
--- a/src/mongo/db/repl/rs_rollback_no_uuid_test.cpp
+++ b/src/mongo/db/repl/rs_rollback_no_uuid_test.cpp
@@ -1213,6 +1213,141 @@ TEST_F(RollbackResyncsCollectionOptionsTest, ChangingTempStatusAlsoChangesOtherC
     resyncCollectionOptionsTest(localCollOptions, remoteCollOptionsObj);
 }
 
+TEST_F(RollbackResyncsCollectionOptionsTest, EmptyCollModResyncsCollectionMetadata) {
+    CollectionOptions localCollOptions;
+    localCollOptions.validator = BSON("x" << BSON("$exists" << 1));
+    localCollOptions.validationLevel = "moderate";
+    localCollOptions.validationAction = "warn";
+
+    BSONObj remoteCollOptionsObj = BSONObj();
+
+    resyncCollectionOptionsTest(localCollOptions,
+                                remoteCollOptionsObj,
+                                BSON("collMod"
+                                     << "coll"),
+                                "coll");
+}
+
+void resyncInconsistentUUIDsTest(OperationContext* opCtx,
+                                 ReplicationCoordinator* coordinator,
+                                 ReplicationProcess* replicationProcess,
+                                 OptionalCollectionUUID localUUID,
+                                 OptionalCollectionUUID remoteUUID,
+                                 OptionalCollectionUUID expectedUUIDAfterRollback) {
+    createOplog(opCtx);
+    CollectionOptions localOptions;
+    localOptions.uuid = localUUID;
+    RollbackTest::_createCollection(opCtx, "test.t", localOptions);
+    auto commonOperation =
+        std::make_pair(BSON("ts" << Timestamp(Seconds(1), 0) << "h" << 1LL), RecordId(1));
+    BSONObj collModCmd = BSON("collMod"
+                              << "t"
+                              << "noPadding"
+                              << false);
+    auto collectionModificationOperation =
+        RollbackTest::makeCommandOp(Timestamp(Seconds(2), 0), boost::none, "test.t", collModCmd, 2);
+
+    class RollbackSourceLocal : public RollbackSourceMock {
+    public:
+        RollbackSourceLocal(std::unique_ptr<OplogInterface> oplog, OptionalCollectionUUID uuid)
+            : RollbackSourceMock(std::move(oplog)), called(false), _uuid(uuid) {}
+
+        // Remote collection options are empty.
+        StatusWith<BSONObj> getCollectionInfo(const NamespaceString& nss) const {
+            called = true;
+            if (_uuid) {
+                return BSON("options" << BSONObj() << "info" << BSON("uuid" << _uuid.get()));
+            } else {
+                return BSON("options" << BSONObj());
+            }
+        }
+        mutable bool called;
+
+    private:
+        OptionalCollectionUUID _uuid;
+    };
+
+    RollbackSourceLocal rollbackSource(std::unique_ptr<OplogInterface>(new OplogInterfaceMock({
+                                           commonOperation,
+                                       })),
+                                       remoteUUID);
+
+    ASSERT_OK(
+        syncRollbackNoUUID(opCtx,
+                           OplogInterfaceMock({collectionModificationOperation, commonOperation}),
+                           rollbackSource,
+                           {},
+                           coordinator,
+                           replicationProcess));
+    ASSERT_TRUE(rollbackSource.called);
+
+    // Make sure the collection options are correct.
+    AutoGetCollectionForReadCommand autoColl(opCtx, NamespaceString("test.t"));
+    auto collAfterRollbackOptions =
+        autoColl.getCollection()->getCatalogEntry()->getCollectionOptions(opCtx);
+    BSONObjBuilder expectedOptionsBob;
+    if (expectedUUIDAfterRollback) {
+        expectedUUIDAfterRollback.get().appendToBuilder(&expectedOptionsBob, "uuid");
+    }
+    ASSERT_BSONOBJ_EQ(expectedOptionsBob.obj(), collAfterRollbackOptions.toBSON());
+}
+
+TEST_F(RollbackResyncsCollectionOptionsTest, LocalUUIDGetsRemovedOnConflict) {
+    resyncInconsistentUUIDsTest(_opCtx.get(),
+                                _coordinator,
+                                _replicationProcess.get(),
+                                UUID::gen(),
+                                UUID::gen(),
+                                boost::none);
+}
+
+TEST_F(RollbackResyncsCollectionOptionsTest, LocalUUIDWithNoRemoteUUIDGetsRemoved) {
+    resyncInconsistentUUIDsTest(_opCtx.get(),
+                                _coordinator,
+                                _replicationProcess.get(),
+                                UUID::gen(),
+                                boost::none,
+                                boost::none);
+}
+
+TEST_F(RollbackResyncsCollectionOptionsTest, RemoteUUIDWithNoLocalUUIDGetsAddedWhileDowngrading) {
+    serverGlobalParams.featureCompatibility.setVersion(
+        ServerGlobalParams::FeatureCompatibility::Version::kDowngradingTo34);
+    auto remoteUuid = UUID::gen();
+    resyncInconsistentUUIDsTest(
+        _opCtx.get(), _coordinator, _replicationProcess.get(), boost::none, remoteUuid, remoteUuid);
+}
+
+TEST_F(RollbackResyncsCollectionOptionsTest,
+       RemoteUUIDWithNoLocalUUIDDoesNotGetAddedWhileUpgrading) {
+    serverGlobalParams.featureCompatibility.setVersion(
+        ServerGlobalParams::FeatureCompatibility::Version::kUpgradingTo36);
+    resyncInconsistentUUIDsTest(_opCtx.get(),
+                                _coordinator,
+                                _replicationProcess.get(),
+                                boost::none,
+                                UUID::gen(),
+                                boost::none);
+}
+
+TEST_F(RollbackResyncsCollectionOptionsTest,
+       RemoteUUIDWithNoLocalUUIDDoesNotGetAddedWhileDowngraded) {
+    serverGlobalParams.featureCompatibility.setVersion(
+        ServerGlobalParams::FeatureCompatibility::Version::kFullyDowngradedTo34);
+    resyncInconsistentUUIDsTest(_opCtx.get(),
+                                _coordinator,
+                                _replicationProcess.get(),
+                                boost::none,
+                                UUID::gen(),
+                                boost::none);
+}
+
+TEST_F(RollbackResyncsCollectionOptionsTest, SameUUIDIsNotChanged) {
+    UUID uuid = UUID::gen();
+    resyncInconsistentUUIDsTest(
+        _opCtx.get(), _coordinator, _replicationProcess.get(), uuid, uuid, uuid);
+}
+
 TEST_F(RSRollbackTest, RollbackCollectionModificationCommandInvalidCollectionOptions) {
     createOplog(_opCtx.get());
     auto commonOperation =