SERVER-60509 onReplicationRollback should crash on failure

19 files changed, 66 insertions, 51 deletions

diff --git a/src/mongo/db/auth/auth_op_observer.cpp b/src/mongo/db/auth/auth_op_observer.cpp
index 867d2c1b598..bce25a68154 100644
--- a/src/mongo/db/auth/auth_op_observer.cpp
+++ b/src/mongo/db/auth/auth_op_observer.cpp
@@ -222,8 +222,8 @@ void AuthOpObserver::onEmptyCapped(OperationContext* opCtx,
         ->logOp(opCtx, "c", cmdNss, cmdObj, nullptr);
 }
 
-void AuthOpObserver::onReplicationRollback(OperationContext* opCtx,
-                                           const RollbackObserverInfo& rbInfo) noexcept {
+void AuthOpObserver::_onReplicationRollback(OperationContext* opCtx,
+                                            const RollbackObserverInfo& rbInfo) {
     // Invalidate any in-memory auth data if necessary.
     const auto& rollbackNamespaces = rbInfo.rollbackNamespaces;
     if (rollbackNamespaces.count(AuthorizationManager::versionCollectionNamespace) == 1 ||
diff --git a/src/mongo/db/auth/auth_op_observer.h b/src/mongo/db/auth/auth_op_observer.h
index 53c343c62be..6d9c6ef354e 100644
--- a/src/mongo/db/auth/auth_op_observer.h
+++ b/src/mongo/db/auth/auth_op_observer.h
@@ -195,9 +195,6 @@ public:
     void onTransactionAbort(OperationContext* opCtx,
                             boost::optional<OplogSlot> abortOplogEntryOpTime) final {}
 
-    void onReplicationRollback(OperationContext* opCtx,
-                               const RollbackObserverInfo& rbInfo) noexcept;
-
     void onMajorityCommitPointUpdate(ServiceContext* service,
                                      const repl::OpTime& newCommitPoint) final {}
 
@@ -205,6 +202,9 @@ public:
     static BSONObj getDocumentKey(OperationContext* opCtx,
                                   NamespaceString const& nss,
                                   BSONObj const& doc);
+
+private:
+    void _onReplicationRollback(OperationContext* opCtx, const RollbackObserverInfo& rbInfo);
 };
 
 }  // namespace mongo
diff --git a/src/mongo/db/fcv_op_observer.cpp b/src/mongo/db/fcv_op_observer.cpp
index 7b5ca4074f7..7b2a2b899f3 100644
--- a/src/mongo/db/fcv_op_observer.cpp
+++ b/src/mongo/db/fcv_op_observer.cpp
@@ -186,8 +186,8 @@ void FcvOpObserver::onDelete(OperationContext* opCtx,
     }
 }
 
-void FcvOpObserver::onReplicationRollback(OperationContext* opCtx,
-                                          const RollbackObserverInfo& rbInfo) noexcept {
+void FcvOpObserver::_onReplicationRollback(OperationContext* opCtx,
+                                           const RollbackObserverInfo& rbInfo) {
     // Ensures the in-memory and on-disk FCV states are consistent after a rollback.
     const auto query = BSON("_id" << multiversion::kParameterName);
     const auto swFcv = repl::StorageInterface::get(opCtx)->findById(
diff --git a/src/mongo/db/fcv_op_observer.h b/src/mongo/db/fcv_op_observer.h
index 8e967bb2ec2..7a9e021ce81 100644
--- a/src/mongo/db/fcv_op_observer.h
+++ b/src/mongo/db/fcv_op_observer.h
@@ -65,9 +65,6 @@ public:
                   StmtId stmtId,
                   const OplogDeleteEntryArgs& args) final;
 
-    void onReplicationRollback(OperationContext* opCtx,
-                               const RollbackObserverInfo& rbInfo) noexcept final;
-
     // Noop overrides.
 
     void onCreateIndex(OperationContext* opCtx,
@@ -210,6 +207,8 @@ private:
      * document and on commit, updates the server parameter.
      */
     static void _onInsertOrUpdate(OperationContext* opCtx, const BSONObj& doc);
+
+    void _onReplicationRollback(OperationContext* opCtx, const RollbackObserverInfo& rbInfo) final;
 };
 
 }  // namespace mongo
diff --git a/src/mongo/db/free_mon/free_mon_op_observer.cpp b/src/mongo/db/free_mon/free_mon_op_observer.cpp
index 2b1c504b359..534f643cc62 100644
--- a/src/mongo/db/free_mon/free_mon_op_observer.cpp
+++ b/src/mongo/db/free_mon/free_mon_op_observer.cpp
@@ -152,8 +152,8 @@ void FreeMonOpObserver::onDelete(OperationContext* opCtx,
     }
 }
 
-void FreeMonOpObserver::onReplicationRollback(OperationContext* opCtx,
-                                              const RollbackObserverInfo& rbInfo) noexcept {
+void FreeMonOpObserver::_onReplicationRollback(OperationContext* opCtx,
+                                               const RollbackObserverInfo& rbInfo) {
     // Invalidate any in-memory auth data if necessary.
     const auto& rollbackNamespaces = rbInfo.rollbackNamespaces;
     if (rollbackNamespaces.count(NamespaceString::kServerConfigurationNamespace) == 1) {
diff --git a/src/mongo/db/free_mon/free_mon_op_observer.h b/src/mongo/db/free_mon/free_mon_op_observer.h
index ad0add0d94e..9e291f2afdd 100644
--- a/src/mongo/db/free_mon/free_mon_op_observer.h
+++ b/src/mongo/db/free_mon/free_mon_op_observer.h
@@ -195,11 +195,11 @@ public:
     void onTransactionAbort(OperationContext* opCtx,
                             boost::optional<OplogSlot> abortOplogEntryOpTime) final {}
 
-    void onReplicationRollback(OperationContext* opCtx,
-                               const RollbackObserverInfo& rbInfo) noexcept;
-
     void onMajorityCommitPointUpdate(ServiceContext* service,
                                      const repl::OpTime& newCommitPoint) final {}
+
+private:
+    void _onReplicationRollback(OperationContext* opCtx, const RollbackObserverInfo& rbInfo);
 };
 
 }  // namespace mongo
diff --git a/src/mongo/db/op_observer.h b/src/mongo/db/op_observer.h
index a8c3bc38e0a..dbbf505ae07 100644
--- a/src/mongo/db/op_observer.h
+++ b/src/mongo/db/op_observer.h
@@ -477,9 +477,19 @@ public:
      *
      * This method is only applicable to the "rollback to a stable timestamp" algorithm, and is not
      * called when using any other rollback algorithm i.e "rollback via refetch".
+     *
+     * This function will call the private virtual '_onReplicationRollback' method. Any exceptions
+     * thrown indicates rollback failure that may have led us to some inconsistent on-disk or memory
+     * state, so we crash instead.
      */
-    virtual void onReplicationRollback(OperationContext* opCtx,
-                                       const RollbackObserverInfo& rbInfo) noexcept = 0;
+    void onReplicationRollback(OperationContext* opCtx,
+                               const RollbackObserverInfo& rbInfo) noexcept {
+        try {
+            _onReplicationRollback(opCtx, rbInfo);
+        } catch (const DBException& ex) {
+            fassert(6050902, ex.toStatus());
+        }
+    };
 
     /**
      * Called when the majority commit point is updated by replication.
@@ -493,6 +503,10 @@ public:
 
     struct Times;
 
+private:
+    virtual void _onReplicationRollback(OperationContext* opCtx,
+                                        const RollbackObserverInfo& rbInfo) = 0;
+
 protected:
     class ReservedTimes;
 };
diff --git a/src/mongo/db/op_observer_impl.cpp b/src/mongo/db/op_observer_impl.cpp
index 25382c37265..234d7c3d0df 100644
--- a/src/mongo/db/op_observer_impl.cpp
+++ b/src/mongo/db/op_observer_impl.cpp
@@ -1657,8 +1657,8 @@ void OpObserverImpl::onTransactionAbort(OperationContext* opCtx,
     logCommitOrAbortForPreparedTransaction(opCtx, &oplogEntry, DurableTxnStateEnum::kAborted);
 }
 
-void OpObserverImpl::onReplicationRollback(OperationContext* opCtx,
-                                           const RollbackObserverInfo& rbInfo) noexcept {
+void OpObserverImpl::_onReplicationRollback(OperationContext* opCtx,
+                                            const RollbackObserverInfo& rbInfo) {
     // Reset the key manager cache.
     auto validator = LogicalTimeValidator::get(opCtx);
     if (validator) {
diff --git a/src/mongo/db/op_observer_impl.h b/src/mongo/db/op_observer_impl.h
index ec589614801..57b7c591c83 100644
--- a/src/mongo/db/op_observer_impl.h
+++ b/src/mongo/db/op_observer_impl.h
@@ -212,8 +212,6 @@ public:
                               size_t numberOfPreImagesToWrite) final;
     void onTransactionAbort(OperationContext* opCtx,
                             boost::optional<OplogSlot> abortOplogEntryOpTime) final;
-    void onReplicationRollback(OperationContext* opCtx,
-                               const RollbackObserverInfo& rbInfo) noexcept final;
     void onMajorityCommitPointUpdate(ServiceContext* service,
                                      const repl::OpTime& newCommitPoint) final {}
 
@@ -255,6 +253,7 @@ private:
         OperationContext* opCtx,
         const std::vector<repl::ReplOperation>& stmts,
         const repl::OpTime& prepareOrCommitOptime) {}
+    void _onReplicationRollback(OperationContext* opCtx, const RollbackObserverInfo& rbInfo) final;
 };
 
 extern const OperationContext::Decoration<boost::optional<OpObserverImpl::DocumentKey>>
diff --git a/src/mongo/db/op_observer_noop.h b/src/mongo/db/op_observer_noop.h
index ff13628943d..b7a39363d7e 100644
--- a/src/mongo/db/op_observer_noop.h
+++ b/src/mongo/db/op_observer_noop.h
@@ -171,10 +171,12 @@ public:
                               size_t numberOfPreImagesToWrite) override{};
     void onTransactionAbort(OperationContext* opCtx,
                             boost::optional<OplogSlot> abortOplogEntryOpTime) override{};
-    void onReplicationRollback(OperationContext* opCtx,
-                               const RollbackObserverInfo& rbInfo) noexcept override {}
     void onMajorityCommitPointUpdate(ServiceContext* service,
                                      const repl::OpTime& newCommitPoint) override {}
+
+private:
+    void _onReplicationRollback(OperationContext* opCtx,
+                                const RollbackObserverInfo& rbInfo) override {}
 };
 
 }  // namespace mongo
diff --git a/src/mongo/db/op_observer_registry.h b/src/mongo/db/op_observer_registry.h
index 6f6accb06bc..c14123c3e5f 100644
--- a/src/mongo/db/op_observer_registry.h
+++ b/src/mongo/db/op_observer_registry.h
@@ -401,12 +401,6 @@ public:
             o->onTransactionAbort(opCtx, abortOplogEntryOpTime);
     }
 
-    void onReplicationRollback(OperationContext* opCtx,
-                               const RollbackObserverInfo& rbInfo) noexcept override {
-        for (auto& o : _observers)
-            o->onReplicationRollback(opCtx, rbInfo);
-    }
-
     void onMajorityCommitPointUpdate(ServiceContext* service,
                                      const repl::OpTime& newCommitPoint) override {
         for (auto& o : _observers)
@@ -414,6 +408,12 @@ public:
     }
 
 private:
+    void _onReplicationRollback(OperationContext* opCtx,
+                                const RollbackObserverInfo& rbInfo) override {
+        for (auto& o : _observers)
+            o->onReplicationRollback(opCtx, rbInfo);
+    }
+
     static repl::OpTime _getOpTimeToReturn(const std::vector<repl::OpTime>& times) {
         if (times.empty()) {
             return repl::OpTime{};
diff --git a/src/mongo/db/repl/primary_only_service_op_observer.h b/src/mongo/db/repl/primary_only_service_op_observer.h
index 586c4711061..9e9d1bd69d6 100644
--- a/src/mongo/db/repl/primary_only_service_op_observer.h
+++ b/src/mongo/db/repl/primary_only_service_op_observer.h
@@ -197,13 +197,13 @@ public:
     void onTransactionAbort(OperationContext* opCtx,
                             boost::optional<OplogSlot> abortOplogEntryOpTime) final {}
 
-    void onReplicationRollback(OperationContext* opCtx,
-                               const RollbackObserverInfo& rbInfo) noexcept final{};
-
     void onMajorityCommitPointUpdate(ServiceContext* service,
                                      const repl::OpTime& newCommitPoint) final {}
 
 private:
+    void _onReplicationRollback(OperationContext* opCtx,
+                                const RollbackObserverInfo& rbInfo) final{};
+
     PrimaryOnlyServiceRegistry* _registry;
 };
 
diff --git a/src/mongo/db/repl/tenant_migration_donor_op_observer.h b/src/mongo/db/repl/tenant_migration_donor_op_observer.h
index 3bc56608215..fe240e8f503 100644
--- a/src/mongo/db/repl/tenant_migration_donor_op_observer.h
+++ b/src/mongo/db/repl/tenant_migration_donor_op_observer.h
@@ -194,11 +194,12 @@ public:
     void onTransactionAbort(OperationContext* opCtx,
                             boost::optional<OplogSlot> abortOplogEntryOpTime) final {}
 
-    void onReplicationRollback(OperationContext* opCtx,
-                               const RollbackObserverInfo& rbInfo) noexcept final{};
-
     void onMajorityCommitPointUpdate(ServiceContext* service,
                                      const repl::OpTime& newCommitPoint) final;
+
+private:
+    void _onReplicationRollback(OperationContext* opCtx, const RollbackObserverInfo& rbInfo) final {
+    }
 };
 
 }  // namespace repl
diff --git a/src/mongo/db/repl/tenant_migration_recipient_op_observer.h b/src/mongo/db/repl/tenant_migration_recipient_op_observer.h
index 3399759138a..791fd7eeccd 100644
--- a/src/mongo/db/repl/tenant_migration_recipient_op_observer.h
+++ b/src/mongo/db/repl/tenant_migration_recipient_op_observer.h
@@ -196,11 +196,12 @@ public:
     void onTransactionAbort(OperationContext* opCtx,
                             boost::optional<OplogSlot> abortOplogEntryOpTime) final {}
 
-    void onReplicationRollback(OperationContext* opCtx,
-                               const RollbackObserverInfo& rbInfo) noexcept final {}
-
     void onMajorityCommitPointUpdate(ServiceContext* service,
                                      const repl::OpTime& newCommitPoint) final {}
+
+private:
+    void _onReplicationRollback(OperationContext* opCtx, const RollbackObserverInfo& rbInfo) final {
+    }
 };
 
 }  // namespace repl
diff --git a/src/mongo/db/s/config_server_op_observer.cpp b/src/mongo/db/s/config_server_op_observer.cpp
index 4bc0616ddab..fe73090a366 100644
--- a/src/mongo/db/s/config_server_op_observer.cpp
+++ b/src/mongo/db/s/config_server_op_observer.cpp
@@ -95,8 +95,8 @@ repl::OpTime ConfigServerOpObserver::onDropCollection(OperationContext* opCtx,
     return {};
 }
 
-void ConfigServerOpObserver::onReplicationRollback(OperationContext* opCtx,
-                                                   const RollbackObserverInfo& rbInfo) noexcept {
+void ConfigServerOpObserver::_onReplicationRollback(OperationContext* opCtx,
+                                                    const RollbackObserverInfo& rbInfo) {
     if (rbInfo.configServerConfigVersionRolledBack) {
         // Throw out any cached information related to the cluster ID.
         ShardingCatalogManager::get(opCtx)->discardCachedConfigDatabaseInitializationState();
diff --git a/src/mongo/db/s/config_server_op_observer.h b/src/mongo/db/s/config_server_op_observer.h
index b9f12804a2d..53e54132952 100644
--- a/src/mongo/db/s/config_server_op_observer.h
+++ b/src/mongo/db/s/config_server_op_observer.h
@@ -197,13 +197,11 @@ public:
     void onTransactionAbort(OperationContext* opCtx,
                             boost::optional<OplogSlot> abortOplogEntryOpTime) override {}
 
-    void onReplicationRollback(OperationContext* opCtx,
-                               const RollbackObserverInfo& rbInfo) noexcept;
-
     void onMajorityCommitPointUpdate(ServiceContext* service,
                                      const repl::OpTime& newCommitPoint) override;
 
 private:
+    void _onReplicationRollback(OperationContext* opCtx, const RollbackObserverInfo& rbInfo);
     void _registerTopologyTimeTickPoint(Timestamp newTopologyTime);
     void _tickTopologyTimeIfNecessary(ServiceContext* service, Timestamp newCommitPointTime);
 
diff --git a/src/mongo/db/s/resharding/resharding_op_observer.h b/src/mongo/db/s/resharding/resharding_op_observer.h
index 941e1aa3c59..4b786ee2796 100644
--- a/src/mongo/db/s/resharding/resharding_op_observer.h
+++ b/src/mongo/db/s/resharding/resharding_op_observer.h
@@ -217,11 +217,12 @@ public:
     void onTransactionAbort(OperationContext* opCtx,
                             boost::optional<OplogSlot> abortOplogEntryOpTime) override {}
 
-    void onReplicationRollback(OperationContext* opCtx,
-                               const RollbackObserverInfo& rbInfo) noexcept override {}
-
     void onMajorityCommitPointUpdate(ServiceContext* service,
                                      const repl::OpTime& newCommitPoint) override {}
+
+private:
+    void _onReplicationRollback(OperationContext* opCtx,
+                                const RollbackObserverInfo& rbInfo) override {}
 };
 
 }  // namespace mongo
diff --git a/src/mongo/db/s/shard_server_op_observer.cpp b/src/mongo/db/s/shard_server_op_observer.cpp
index a8f83636cf0..d483a8184ed 100644
--- a/src/mongo/db/s/shard_server_op_observer.cpp
+++ b/src/mongo/db/s/shard_server_op_observer.cpp
@@ -641,8 +641,8 @@ void ShardServerOpObserver::onCollMod(OperationContext* opCtx,
     abortOngoingMigrationIfNeeded(opCtx, nss);
 };
 
-void ShardServerOpObserver::onReplicationRollback(OperationContext* opCtx,
-                                                  const RollbackObserverInfo& rbInfo) noexcept {
+void ShardServerOpObserver::_onReplicationRollback(OperationContext* opCtx,
+                                                   const RollbackObserverInfo& rbInfo) {
     if (rbInfo.rollbackNamespaces.find(NamespaceString::kCollectionCriticalSectionsNamespace) !=
         rbInfo.rollbackNamespaces.end()) {
         RecoverableCriticalSectionService::get(opCtx)->recoverRecoverableCriticalSections(opCtx);
diff --git a/src/mongo/db/s/shard_server_op_observer.h b/src/mongo/db/s/shard_server_op_observer.h
index 7447298320b..34fbb1a6ebf 100644
--- a/src/mongo/db/s/shard_server_op_observer.h
+++ b/src/mongo/db/s/shard_server_op_observer.h
@@ -196,11 +196,11 @@ public:
     void onTransactionAbort(OperationContext* opCtx,
                             boost::optional<OplogSlot> abortOplogEntryOpTime) override {}
 
-    void onReplicationRollback(OperationContext* opCtx,
-                               const RollbackObserverInfo& rbInfo) noexcept;
-
     void onMajorityCommitPointUpdate(ServiceContext* service,
                                      const repl::OpTime& newCommitPoint) override {}
+
+private:
+    void _onReplicationRollback(OperationContext* opCtx, const RollbackObserverInfo& rbInfo);
 };
 
 }  // namespace mongo