SERVER-59396 Adds server parameter `healthMonitoring`

11 files changed, 193 insertions, 7 deletions

diff --git a/jstests/sharding/health_monitor/set_parameter_health_monitor_intensity.js b/jstests/sharding/health_monitor/set_parameter_health_monitor_intensity.js
new file mode 100644
index 00000000000..a751d9b2ed8
--- /dev/null
+++ b/jstests/sharding/health_monitor/set_parameter_health_monitor_intensity.js
@@ -0,0 +1,29 @@
+
+
+(function() {
+'use strict';
+
+var st = new ShardingTest({
+    mongos: [{
+        setParameter: {
+            healthMonitoring: tojson({dns: "off", ldap: "critical"}),
+        }
+    }],
+    shards: 1,
+});
+
+var result = st.s0.adminCommand({"getParameter": 1, "healthMonitoring": 1});
+print(tojson(result));
+assert.eq(result.dns, "off");
+assert.eq(result.ldap, "critical");
+
+assert.commandFailed(st.s0.adminCommand({"setParameter": 1, healthMonitoring: {dns: "INVALID"}}));
+assert.commandFailed(st.s0.adminCommand({"setParameter": 1, healthMonitoring: {invalid: "off"}}));
+
+assert.commandWorked(
+    st.s0.adminCommand({"setParameter": 1, healthMonitoring: {dns: 'non-critical', ldap: 'off'}}));
+var result = assert.commandWorked(st.adminCommand({"getParameter": 1, healthMonitoring: 1}));
+print(tojson(result));
+assert.eq(result.dns, "non-critical");
+assert.eq(result.ldap, "off");
+}());
diff --git a/src/mongo/db/process_health/SConscript b/src/mongo/db/process_health/SConscript
index a57ff1a2d00..61395478ea9 100644
--- a/src/mongo/db/process_health/SConscript
+++ b/src/mongo/db/process_health/SConscript
@@ -10,7 +10,9 @@ env.Library(
         'fault_facet_impl.cpp',
         'fault_impl.cpp',
         'fault_manager.cpp',
-	'fault_manager_config.cpp',
+        'fault_manager_config.cpp',
+        'health_monitoring_server_parameters.idl',
+        'health_monitoring_server_parameters.cpp',
         'health_observer_base.cpp',
         'health_observer_registration.cpp',
     ],
diff --git a/src/mongo/db/process_health/fault_manager.cpp b/src/mongo/db/process_health/fault_manager.cpp
index 9976afed448..0f7833c9f8a 100644
--- a/src/mongo/db/process_health/fault_manager.cpp
+++ b/src/mongo/db/process_health/fault_manager.cpp
@@ -260,7 +260,8 @@ bool FaultManager::hasCriticalFacet(const FaultInternal* fault) const {
     const auto& facets = fault->getFacets();
     for (const auto& facet : facets) {
         auto facetType = facet->getType();
-        if (_config->getHealthObserverIntensity(facetType) == HealthObserverIntensity::kCritical)
+        if (_config->getHealthObserverIntensity(facetType) ==
+            HealthObserverIntensityEnum::kCritical)
             return true;
     }
     return false;
diff --git a/src/mongo/db/process_health/fault_manager.h b/src/mongo/db/process_health/fault_manager.h
index 823bf724350..2bf518da975 100644
--- a/src/mongo/db/process_health/fault_manager.h
+++ b/src/mongo/db/process_health/fault_manager.h
@@ -34,6 +34,7 @@
 #include "mongo/db/process_health/fault_facet.h"
 #include "mongo/db/process_health/fault_facet_container.h"
 #include "mongo/db/process_health/fault_manager_config.h"
+#include "mongo/db/process_health/health_monitoring_server_parameters_gen.h"
 #include "mongo/db/process_health/health_observer.h"
 #include "mongo/db/service_context.h"
 #include "mongo/executor/task_executor.h"
diff --git a/src/mongo/db/process_health/fault_manager_config.h b/src/mongo/db/process_health/fault_manager_config.h
index 0cdf6fd7279..d127abc20a4 100644
--- a/src/mongo/db/process_health/fault_manager_config.h
+++ b/src/mongo/db/process_health/fault_manager_config.h
@@ -30,9 +30,11 @@
 
 #include <ostream>
 
+#include "mongo/db/process_health/health_monitoring_server_parameters_gen.h"
 #include "mongo/platform/basic.h"
 #include "mongo/util/duration.h"
 
+
 namespace mongo {
 namespace process_health {
 
@@ -75,13 +77,26 @@ enum class HealthObserverIntensity {
 /**
  * Types of health observers available.
  */
-enum class FaultFacetType { kMock1 = 0, kMock2, kLdap };
+enum class FaultFacetType { kMock1 = 0, kMock2, kLdap, kDns };
 
 
 class FaultManagerConfig {
 public:
-    HealthObserverIntensity getHealthObserverIntensity(FaultFacetType type) {
-        return HealthObserverIntensity::kCritical;
+    HealthObserverIntensityEnum getHealthObserverIntensity(FaultFacetType type) {
+        auto intensities = getHealthObserverIntensities();
+        switch (type) {
+            case FaultFacetType::kLdap:
+                return intensities->_data->getLdap();
+            case FaultFacetType::kDns:
+                return intensities->_data->getDns();
+            // TODO: update this function with additional fault facets when they are added
+            case FaultFacetType::kMock1:
+                return HealthObserverIntensityEnum::kCritical;
+            case FaultFacetType::kMock2:
+                return HealthObserverIntensityEnum::kCritical;
+            default:
+                MONGO_UNREACHABLE;
+        }
     }
     Milliseconds getActiveFaultDuration() {
         return kActiveFaultDuration;
@@ -91,6 +106,12 @@ protected:
     // If the server persists in TransientFault for more than this duration
     // it will move to the ActiveFault state and terminate.
     static inline const auto kActiveFaultDuration = Seconds(120);
+
+private:
+    static HealthMonitoringIntensitiesServerParameter* getHealthObserverIntensities() {
+        return ServerParameterSet::getGlobal()->get<HealthMonitoringIntensitiesServerParameter>(
+            "healthMonitoring");
+    }
 };
 
 }  // namespace process_health
diff --git a/src/mongo/db/process_health/fault_manager_test.cpp b/src/mongo/db/process_health/fault_manager_test.cpp
index bb7f6f92691..50cb1de29f8 100644
--- a/src/mongo/db/process_health/fault_manager_test.cpp
+++ b/src/mongo/db/process_health/fault_manager_test.cpp
@@ -36,6 +36,7 @@ namespace mongo {
 
 namespace process_health {
 
+using test::FaultManagerTest;
 using test::FaultManagerTestImpl;
 
 namespace {
@@ -45,6 +46,15 @@ TEST(FaultManagerTest, Registration) {
     ASSERT_TRUE(FaultManager::get(serviceCtx.get()));
 }
 
+// Tests the default health observer intensity of non-critical
+TEST_F(FaultManagerTest, GetHealthObserverIntensity) {
+    auto config = manager().getConfig();
+    ASSERT(config->getHealthObserverIntensity(FaultFacetType::kLdap) ==
+           HealthObserverIntensityEnum::kNonCritical);
+    ASSERT(config->getHealthObserverIntensity(FaultFacetType::kDns) ==
+           HealthObserverIntensityEnum::kNonCritical);
+}
+
 }  // namespace
 }  // namespace process_health
 }  // namespace mongo
diff --git a/src/mongo/db/process_health/fault_manager_test_suite.h b/src/mongo/db/process_health/fault_manager_test_suite.h
index 0fe28e99497..2258168ebeb 100644
--- a/src/mongo/db/process_health/fault_manager_test_suite.h
+++ b/src/mongo/db/process_health/fault_manager_test_suite.h
@@ -175,7 +175,7 @@ public:
                 return;
             sleepFor(kSleepTime);
         }
-        ASSERT(false);
+        invariant(false);
     }
 
     static inline const Milliseconds kCheckTimeIncrement{100};
diff --git a/src/mongo/db/process_health/health_monitoring_server_parameters.cpp b/src/mongo/db/process_health/health_monitoring_server_parameters.cpp
new file mode 100644
index 00000000000..66d0e5fd204
--- /dev/null
+++ b/src/mongo/db/process_health/health_monitoring_server_parameters.cpp
@@ -0,0 +1,55 @@
+/**
+ *    Copyright (C) 2020-present MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#include "mongo/bson/json.h"
+#include "mongo/db/process_health/health_monitoring_server_parameters_gen.h"
+#include "mongo/db/process_health/health_observer.h"
+
+
+namespace mongo {
+
+Status HealthMonitoringIntensitiesServerParameter::setFromString(const std::string& value) {
+    *_data = HealthObserverIntensities::parse(
+        IDLParserErrorContext("health monitoring intensities"), fromjson(value));
+    return Status::OK();
+}
+
+Status HealthMonitoringIntensitiesServerParameter::set(const BSONElement& newValueElement) {
+    *_data = HealthObserverIntensities::parse(
+        IDLParserErrorContext("health monitoring intensities"), newValueElement.Obj());
+    return Status::OK();
+}
+
+void HealthMonitoringIntensitiesServerParameter::append(OperationContext*,
+                                                        BSONObjBuilder& b,
+                                                        const std::string& name) {
+    _data->serialize(&b);
+}
+
+}  // namespace mongo
diff --git a/src/mongo/db/process_health/health_monitoring_server_parameters.idl b/src/mongo/db/process_health/health_monitoring_server_parameters.idl
new file mode 100644
index 00000000000..1222242a812
--- /dev/null
+++ b/src/mongo/db/process_health/health_monitoring_server_parameters.idl
@@ -0,0 +1,66 @@
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the Server Side Public License, version 1,
+# as published by MongoDB, Inc.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# Server Side Public License for more details.
+#
+# You should have received a copy of the Server Side Public License
+# along with this program. If not, see
+# <http://www.mongodb.com/licensing/server-side-public-license>.
+#
+# As a special exception, the copyright holders give permission to link the
+# code of portions of this program with the OpenSSL library under certain
+# conditions as described in each individual source file and distribute
+# linked combinations including the program with the OpenSSL library. You
+# must comply with the Server Side Public License in all respects for
+# all of the code used other than as permitted herein. If you modify file(s)
+# with this exception, you may extend this exception to your version of the
+# file(s), but you are not obligated to do so. If you do not wish to do so,
+# delete this exception statement from your version. If you delete this
+# exception statement from all source files in the program, then also delete
+# it in the license file.
+#
+
+# Feature flag for fault facet support.
+
+global:
+    cpp_namespace: "mongo"
+
+imports:
+    - "mongo/idl/basic_types.idl"
+
+enums:
+    HealthObserverIntensity:
+        description: "Enum representing the intensity of a health observer."
+        type: string
+        values:
+            kOff: "off"
+            kCritical: "critical"
+            kNonCritical: "non-critical"
+
+structs:
+    HealthObserverIntensities:
+        description: "A struct representing the health observer intensities."
+        strict: true
+        fields:
+            dns:
+                description: "Intensity of DNS fault facet"
+                type: HealthObserverIntensity
+                default: kNonCritical
+            ldap:
+                description: "Intensity of LDAP fault facet"
+                type: HealthObserverIntensity
+                default: kNonCritical
+
+server_parameters:
+    healthMonitoring:
+        set_at: ["startup", "runtime"]
+        description: "A server parameter for specifying the intensity of fault facets."
+        cpp_class:
+            name: "HealthMonitoringIntensitiesServerParameter"
+            data: "synchronized_value<HealthObserverIntensities>"
+            override_set: true
diff --git a/src/mongo/db/process_health/health_observer_test.cpp b/src/mongo/db/process_health/health_observer_test.cpp
index 6f995359a46..630f2f8d4a1 100644
--- a/src/mongo/db/process_health/health_observer_test.cpp
+++ b/src/mongo/db/process_health/health_observer_test.cpp
@@ -226,7 +226,7 @@ TEST_F(FaultManagerTest, TransitionsToActiveFaultAfterTimeout) {
     waitForTransitionIntoState(FaultState::kTransientFault);
     ASSERT_TRUE(manager().getFaultState() == FaultState::kTransientFault);
     advanceTime(manager().getConfig()->getActiveFaultDuration() + Milliseconds(1));
-    assertSoon([this]() { return manager().getFaultState() == FaultState::kActiveFault; });
+    waitForTransitionIntoState(FaultState::kActiveFault);
 }
 
 TEST_F(FaultManagerTest, DoesNotTransitionToActiveFaultIfResolved) {
diff --git a/src/mongo/s/SConscript b/src/mongo/s/SConscript
index d1978d76ae6..f03c01109c1 100644
--- a/src/mongo/s/SConscript
+++ b/src/mongo/s/SConscript
@@ -449,6 +449,7 @@ env.Library(
         '$BUILD_DIR/mongo/db/logical_session_cache_impl',
         '$BUILD_DIR/mongo/db/logical_time_metadata_hook',
         '$BUILD_DIR/mongo/db/pipeline/process_interface/mongos_process_interface_factory',
+        '$BUILD_DIR/mongo/db/process_health/fault_manager',
         '$BUILD_DIR/mongo/db/read_write_concern_defaults',
         '$BUILD_DIR/mongo/db/server_options',
         '$BUILD_DIR/mongo/db/server_options_base',