SERVER-59382: Enforce non-critical facets not entering ActiveFault state

1 files changed, 43 insertions, 14 deletions

diff --git a/src/mongo/db/process_health/fault_state_machine_test.cpp b/src/mongo/db/process_health/fault_state_machine_test.cpp
index c1d3c5efc59..8a4b639d576 100644
--- a/src/mongo/db/process_health/fault_state_machine_test.cpp
+++ b/src/mongo/db/process_health/fault_state_machine_test.cpp
@@ -214,10 +214,6 @@ TEST_F(FaultManagerTest, OneFacetIsResolved) {
 DEATH_TEST_F(FaultManagerTest, TransitionsToActiveFaultAfterTimeoutFromTransientFault, "Fatal") {
     RAIIServerParameterControllerForTest _controller{"featureFlagHealthMonitoring", true};
     auto faultFacetType = FaultFacetType::kMock1;
-    auto config = test::getConfigWithDisabledPeriodicChecks();
-    auto activeFaultDuration = Milliseconds(100);
-    config->setActiveFaultDurationForTests(activeFaultDuration);
-    resetManager(std::move(config));
 
     registerMockHealthObserver(faultFacetType, [] { return 1.1; });
     auto initialHealthCheckFuture = manager().startPeriodicHealthChecks();
@@ -227,47 +223,80 @@ DEATH_TEST_F(FaultManagerTest, TransitionsToActiveFaultAfterTimeoutFromTransient
     manager().acceptTest(HealthCheckStatus(faultFacetType, 1.0, "error"));
     ASSERT(manager().getFaultState() == FaultState::kTransientFault);
 
-    advanceTime(activeFaultDuration);
+    advanceTime(Seconds(kActiveFaultDurationSecs));
     waitForTransitionIntoState(FaultState::kActiveFault);
 }
 
-DEATH_TEST_F(FaultManagerTest, TransitionsToActiveFaultAfterTimeoutFromStartupCheck, "Fatal") {
+TEST_F(FaultManagerTest,
+       NonCriticalFacetDoesNotTransitionToActiveFaultAfterTimeoutFromTransientFault) {
     RAIIServerParameterControllerForTest _controller{"featureFlagHealthMonitoring", true};
     auto faultFacetType = FaultFacetType::kMock1;
     auto config = test::getConfigWithDisabledPeriodicChecks();
-    auto activeFaultDuration = Milliseconds(100);
-    config->setActiveFaultDurationForTests(activeFaultDuration);
+    config->setIntensityForType(faultFacetType, HealthObserverIntensityEnum::kNonCritical);
     resetManager(std::move(config));
 
     registerMockHealthObserver(faultFacetType, [] { return 1.1; });
     auto initialHealthCheckFuture = manager().startPeriodicHealthChecks();
+    manager().acceptTest(HealthCheckStatus(faultFacetType));
+    ASSERT(manager().getFaultState() == FaultState::kOk);
+
+    manager().acceptTest(HealthCheckStatus(faultFacetType, 1.0, "error"));
+    ASSERT(manager().getFaultState() == FaultState::kTransientFault);
+
+    advanceTime(Seconds(kActiveFaultDurationSecs));
+    // Should be enough time to move to Active fault if we were going to crash.
+    sleepFor(Seconds(1));
+    ASSERT(manager().getFaultState() == FaultState::kTransientFault);
+}
+
+DEATH_TEST_F(FaultManagerTest, TransitionsToActiveFaultAfterTimeoutFromStartupCheck, "Fatal") {
+    RAIIServerParameterControllerForTest _controller{"featureFlagHealthMonitoring", true};
+    auto faultFacetType = FaultFacetType::kMock1;
+
+    registerMockHealthObserver(faultFacetType, [] { return 1.1; });
+    auto initialHealthCheckFuture = manager().startPeriodicHealthChecks();
     manager().acceptTest(HealthCheckStatus(faultFacetType, 1.0, "error"));
     ASSERT(manager().getFaultState() == FaultState::kStartupCheck);
 
-    advanceTime(activeFaultDuration);
+    advanceTime(Seconds(kActiveFaultDurationSecs));
     waitForTransitionIntoState(FaultState::kActiveFault);
 }
 
-TEST_F(FaultManagerTest, DoesNotTransitionToActiveFaultIfResolved) {
+TEST_F(FaultManagerTest,
+       NonCriticalFacetDoesNotTransitionToActiveFaultAfterTimeoutFromStartupCheck) {
     RAIIServerParameterControllerForTest _controller{"featureFlagHealthMonitoring", true};
     auto faultFacetType = FaultFacetType::kMock1;
     auto config = test::getConfigWithDisabledPeriodicChecks();
-    auto activeFaultDuration = Milliseconds(100);
-    config->setActiveFaultDurationForTests(activeFaultDuration);
+    config->setIntensityForType(faultFacetType, HealthObserverIntensityEnum::kNonCritical);
     resetManager(std::move(config));
 
     registerMockHealthObserver(faultFacetType, [] { return 1.1; });
     auto initialHealthCheckFuture = manager().startPeriodicHealthChecks();
+    manager().acceptTest(HealthCheckStatus(faultFacetType, 1.0, "error"));
+    ASSERT(manager().getFaultState() == FaultState::kStartupCheck);
+
+    advanceTime(Seconds(kActiveFaultDurationSecs) * 10);
+    // Should be enough time to move to Active fault if we were going to crash.
+    sleepFor(Seconds(1));
+    ASSERT(manager().getFaultState() == FaultState::kStartupCheck);
+}
+
+TEST_F(FaultManagerTest, DoesNotTransitionToActiveFaultIfResolved) {
+    RAIIServerParameterControllerForTest _controller{"featureFlagHealthMonitoring", true};
+    auto faultFacetType = FaultFacetType::kMock1;
+
+    registerMockHealthObserver(faultFacetType, [] { return 1.1; });
+    auto initialHealthCheckFuture = manager().startPeriodicHealthChecks();
     manager().acceptTest(HealthCheckStatus(faultFacetType));
     ASSERT(manager().getFaultState() == FaultState::kOk);
 
     manager().acceptTest(HealthCheckStatus(faultFacetType, 1.0, "error"));
     ASSERT(manager().getFaultState() == FaultState::kTransientFault);
 
-    advanceTime(activeFaultDuration / 2);
+    advanceTime(Seconds(kActiveFaultDurationSecs / 2));
     manager().acceptTest(HealthCheckStatus(faultFacetType));
 
-    advanceTime(activeFaultDuration);
+    advanceTime(Seconds(kActiveFaultDurationSecs));
 
     ASSERT(manager().getFaultState() == FaultState::kOk);
 }