SERVER-59360 Health observer registration and basic mock class

18 files changed, 634 insertions, 11 deletions

diff --git a/src/mongo/db/process_health/SConscript b/src/mongo/db/process_health/SConscript
index b2fc96d9889..00241d2a62e 100644
--- a/src/mongo/db/process_health/SConscript
+++ b/src/mongo/db/process_health/SConscript
@@ -9,6 +9,8 @@ env.Library(
     source=[
         'fault_impl.cpp',
         'fault_manager.cpp',
+        'health_observer_base.cpp',
+        'health_observer_registration.cpp',
     ],
     LIBDEPS=[
         '$BUILD_DIR/mongo/base',
@@ -23,6 +25,7 @@ env.CppUnitTest(
     source=[
         'fault_impl_test.cpp',
         'fault_facet_test.cpp',
+        'health_observer_test.cpp',
         'fault_manager_test.cpp',
     ],
     LIBDEPS=[
diff --git a/src/mongo/db/process_health/fault.h b/src/mongo/db/process_health/fault.h
index 6dbb2358772..8a5588ec7d4 100644
--- a/src/mongo/db/process_health/fault.h
+++ b/src/mongo/db/process_health/fault.h
@@ -31,6 +31,7 @@
 #include <memory>
 
 #include "mongo/bson/bsonobjbuilder.h"
+#include "mongo/db/process_health/fault_facet_container.h"
 #include "mongo/util/duration.h"
 #include "mongo/util/uuid.h"
 
@@ -41,7 +42,7 @@ namespace process_health {
  * Detailed description of the current fault.
  * @see FaultManager for more details.
  */
-class Fault {
+class Fault : public std::enable_shared_from_this<Fault> {
     Fault(const Fault&) = delete;
     Fault& operator=(const Fault&) = delete;
 
@@ -85,6 +86,14 @@ public:
 
 using FaultConstPtr = std::shared_ptr<const Fault>;
 
+/**
+ * Internal Fault interface that has accessors to manage Facets this Fault owns.
+ */
+class FaultInternal : public Fault, public FaultFacetContainer {
+public:
+    ~FaultInternal() override = default;
+};
+
 
 }  // namespace process_health
 }  // namespace mongo
diff --git a/src/mongo/db/process_health/fault_facet.h b/src/mongo/db/process_health/fault_facet.h
index 8598260dad6..7d87f8797ff 100644
--- a/src/mongo/db/process_health/fault_facet.h
+++ b/src/mongo/db/process_health/fault_facet.h
@@ -38,10 +38,12 @@ namespace process_health {
  * The instance is created and deleted by the fault observer when a fault
  * condition is detected or resolved.
  */
-class FaultFacet {
+class FaultFacet : public std::enable_shared_from_this<FaultFacet> {
 public:
     virtual ~FaultFacet() = default;
 
+    virtual FaultFacetType getType() const = 0;
+
     /**
      * The interface used to communicate with the Fault instance that
      * owns all facets.
@@ -51,5 +53,7 @@ public:
     virtual HealthCheckStatus getStatus() const = 0;
 };
 
+using FaultFacetPtr = std::shared_ptr<FaultFacet>;
+
 }  // namespace process_health
 }  // namespace mongo
diff --git a/src/mongo/db/process_health/fault_facet_container.h b/src/mongo/db/process_health/fault_facet_container.h
new file mode 100644
index 00000000000..88e46ef4d7e
--- /dev/null
+++ b/src/mongo/db/process_health/fault_facet_container.h
@@ -0,0 +1,94 @@
+/**
+ *    Copyright (C) 2021-present MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+#pragma once
+
+#include <memory>
+
+#include "mongo/db/process_health/fault_facet.h"
+
+namespace mongo {
+namespace process_health {
+
+/**
+ * Interface for the container of Fault facets.
+ */
+class FaultFacetContainer {
+public:
+    /**
+     * We do not allow the facets added to this container to be immediately deleted. This
+     * is the minimal lifetime before a fully resolved facet could be deleted.
+     */
+    static constexpr Milliseconds kMinimalFacetLifetimeToDelete = Milliseconds(10000);
+
+    virtual ~FaultFacetContainer() = default;
+
+    virtual std::vector<FaultFacetPtr> getFacets() const = 0;
+
+    /**
+     * Checks that a Facet of a given type already exists and returns it.
+     */
+    virtual boost::optional<FaultFacetPtr> getFaultFacet(FaultFacetType type) = 0;
+
+    /**
+     * Getter that takes a create callback in case the facet of a given type is missing.
+     * We do not have a separate create factory interface to avoid having the registration
+     * mechanism for those factories, which is not necessary.
+     *
+     * @param createCb The callback is invoked only if the facet of this type does not exist.
+     */
+    virtual FaultFacetPtr getOrCreateFaultFacet(FaultFacetType type,
+                                                std::function<FaultFacetPtr()> createCb) = 0;
+
+    /**
+     * Performs necessary actions to delete all resolved facets with lifetime of
+     * at least kMinimalFacetLifetimeToDelete.
+     *
+     * The interface for deleting facets is not provided because the container should
+     * garbage collect them.
+     */
+    virtual void garbageCollectResolvedFacets() = 0;
+};
+
+using FaultFacetContainerPtr = std::shared_ptr<FaultFacetContainer>;
+
+/**
+ * Interface to get or create a FaultFacetContainer.
+ * The implementor of this interface owns the singleton instance.
+ */
+class FaultFacetContainerFactory {
+public:
+    virtual ~FaultFacetContainerFactory() = default;
+
+    virtual boost::optional<FaultFacetContainerPtr> getFaultFacetContainer() = 0;
+
+    virtual FaultFacetContainerPtr getOrCreateFaultFacetContainer() = 0;
+};
+
+}  // namespace process_health
+}  // namespace mongo
diff --git a/src/mongo/db/process_health/fault_facet_mock.h b/src/mongo/db/process_health/fault_facet_mock.h
index 500c8046036..ec1af225545 100644
--- a/src/mongo/db/process_health/fault_facet_mock.h
+++ b/src/mongo/db/process_health/fault_facet_mock.h
@@ -49,6 +49,10 @@ public:
 
     ~FaultFacetMock() = default;
 
+    FaultFacetType getType() const override {
+        return FaultFacetType::kMock;
+    }
+
     HealthCheckStatus getStatus() const override {
         double severity;
         _callback(&severity);
@@ -69,7 +73,6 @@ public:
                                             now - _activeFaultTime,
                                             now - _startTime);
 
-
         return healthCheckStatus;
     }
 
diff --git a/src/mongo/db/process_health/fault_impl.cpp b/src/mongo/db/process_health/fault_impl.cpp
index 1a0a9db9f32..1cbdaa5ffd8 100644
--- a/src/mongo/db/process_health/fault_impl.cpp
+++ b/src/mongo/db/process_health/fault_impl.cpp
@@ -32,8 +32,10 @@
 namespace mongo {
 namespace process_health {
 
-FaultImpl::FaultImpl(ServiceContext* svcCtx)
-    : _svcCtx(svcCtx), _startTime(_svcCtx->getFastClockSource()->now()) {
+FaultImpl::FaultImpl(ServiceContext* svcCtx, Milliseconds minimalGarbageCollectTimeout)
+    : _svcCtx(svcCtx),
+      _minimalGarbageCollectTimeout(minimalGarbageCollectTimeout),
+      _startTime(_svcCtx->getFastClockSource()->now()) {
     invariant(svcCtx);  // Will crash before this line, just for readability.
 }
 
@@ -53,6 +55,50 @@ Milliseconds FaultImpl::getDuration() const {
     return Milliseconds(_svcCtx->getFastClockSource()->now() - _startTime);
 }
 
+std::vector<FaultFacetPtr> FaultImpl::getFacets() const {
+    auto lk = stdx::lock_guard(_mutex);
+    std::vector<FaultFacetPtr> result(_facets.begin(), _facets.end());
+    return result;
+}
+
+boost::optional<FaultFacetPtr> FaultImpl::getFaultFacet(FaultFacetType type) {
+    auto lk = stdx::lock_guard(_mutex);
+    auto it = std::find_if(_facets.begin(), _facets.end(), [type](const FaultFacetPtr& facet) {
+        return facet->getType() == type;
+    });
+    if (it == _facets.end()) {
+        return {};
+    }
+    return *it;
+}
+
+FaultFacetPtr FaultImpl::getOrCreateFaultFacet(FaultFacetType type,
+                                               std::function<FaultFacetPtr()> createCb) {
+    auto lk = stdx::lock_guard(_mutex);
+    auto it = std::find_if(_facets.begin(), _facets.end(), [type](const FaultFacetPtr& facet) {
+        return facet->getType() == type;
+    });
+    if (it == _facets.end()) {
+        auto facet = createCb();
+        _facets.push_back(facet);
+        return facet;
+    }
+    return *it;
+}
+
+void FaultImpl::garbageCollectResolvedFacets() {
+    auto lk = stdx::lock_guard(_mutex);
+    _facets.erase(std::remove_if(_facets.begin(),
+                                 _facets.end(),
+                                 [this](const FaultFacetPtr& facet) {
+                                     auto status = facet->getStatus();
+                                     // Remove if severity is zero and duration > threshold.
+                                     return HealthCheckStatus::isResolved(status.getSeverity()) &&
+                                         status.getDuration() >= _minimalGarbageCollectTimeout;
+                                 }),
+                  _facets.end());
+}
+
 void FaultImpl::appendDescription(BSONObjBuilder* builder) const {}
 
 }  // namespace process_health
diff --git a/src/mongo/db/process_health/fault_impl.h b/src/mongo/db/process_health/fault_impl.h
index be0e4fc09e1..6afc09a7ed9 100644
--- a/src/mongo/db/process_health/fault_impl.h
+++ b/src/mongo/db/process_health/fault_impl.h
@@ -32,6 +32,7 @@
 
 #include "mongo/db/service_context.h"
 #include "mongo/util/clock_source.h"
+#include "mongo/util/duration.h"
 #include "mongo/util/timer.h"
 
 namespace mongo {
@@ -41,9 +42,11 @@ namespace process_health {
  * Internal implementation of the Fault class.
  * @see Fault
  */
-class FaultImpl : public Fault {
+class FaultImpl : public FaultInternal {
 public:
-    explicit FaultImpl(ServiceContext* svcCtx);
+    explicit FaultImpl(ServiceContext* svcCtx,
+                       Milliseconds minimalGarbageCollectTimeout =
+                           FaultFacetContainer::kMinimalFacetLifetimeToDelete);
 
     ~FaultImpl() override = default;
 
@@ -59,10 +62,29 @@ public:
 
     void appendDescription(BSONObjBuilder* builder) const override;
 
+    // FaultFacetContainer interface.
+
+    std::vector<FaultFacetPtr> getFacets() const override;
+
+    boost::optional<FaultFacetPtr> getFaultFacet(FaultFacetType type) override;
+
+    FaultFacetPtr getOrCreateFaultFacet(FaultFacetType type,
+                                        std::function<FaultFacetPtr()> createCb) override;
+
+    void garbageCollectResolvedFacets() override;
+
 private:
-    ServiceContext* const _svcCtx;
     const UUID _id = UUID::gen();
+
+    ServiceContext* const _svcCtx;
+    // A resolved instance of facet can be garbage collected only after this timeout.
+    const Milliseconds _minimalGarbageCollectTimeout;
     const Date_t _startTime;
+
+    mutable Mutex _mutex = MONGO_MAKE_LATCH(HierarchicalAcquisitionLevel(0), "FaultImpl::_mutex");
+    // We don't need a map by type because we expect to have only few facets.
+    // Linear search is much faster, we want to avoid any lock contention here.
+    std::deque<FaultFacetPtr> _facets;
 };
 
 }  // namespace process_health
diff --git a/src/mongo/db/process_health/fault_impl_test.cpp b/src/mongo/db/process_health/fault_impl_test.cpp
index 047ff5d0029..67a1bd33257 100644
--- a/src/mongo/db/process_health/fault_impl_test.cpp
+++ b/src/mongo/db/process_health/fault_impl_test.cpp
@@ -82,6 +82,38 @@ TEST_F(FaultImplTest, SeverityLevelHelpersWork) {
     ASSERT_TRUE(HealthCheckStatus::isActiveFault(faultyFacet.getStatus().getSeverity()));
 }
 
+TEST_F(FaultImplTest, FindFacetByType) {
+    ASSERT_EQ(0, fault().getFacets().size());
+    ASSERT_FALSE(fault().getFaultFacet(FaultFacetType::kMock));
+
+    fault().getOrCreateFaultFacet(FaultFacetType::kMock, [this] {
+        return std::make_shared<FaultFacetMock>(svcCtx(), [](double* severity) { *severity = 0; });
+    });
+    auto facet = fault().getFaultFacet(FaultFacetType::kMock);
+    ASSERT_TRUE(facet);
+    auto status = (*facet)->getStatus();
+    ASSERT_EQ(FaultFacetType::kMock, status.getType());
+}
+
+TEST_F(FaultImplTest, CanCreateAndGarbageCollectFacetsAfterTimeout) {
+    ASSERT_EQ(0, fault().getFacets().size());
+    fault().getOrCreateFaultFacet(FaultFacetType::kMock, [this] {
+        return std::make_shared<FaultFacetMock>(svcCtx(), [](double* severity) { *severity = 0; });
+    });
+    // New facet was added successfully.
+    ASSERT_EQ(1, fault().getFacets().size());
+
+    // The facet has severity of 0 but it cannot be garbage collected because the timeout did not
+    // pass yet.
+    fault().garbageCollectResolvedFacets();
+    ASSERT_EQ(1, fault().getFacets().size());
+
+    // After the time was advanced by the minimum timeout the GC works.
+    clockSource().advance(FaultFacetContainer::kMinimalFacetLifetimeToDelete);
+    fault().garbageCollectResolvedFacets();
+    ASSERT_EQ(0, fault().getFacets().size());
+}
+
 }  // namespace
 }  // namespace process_health
 }  // namespace mongo
diff --git a/src/mongo/db/process_health/fault_manager.cpp b/src/mongo/db/process_health/fault_manager.cpp
index d7cf6acaedd..7d287a881f2 100644
--- a/src/mongo/db/process_health/fault_manager.cpp
+++ b/src/mongo/db/process_health/fault_manager.cpp
@@ -31,6 +31,7 @@
 
 #include "mongo/db/process_health/fault_manager.h"
 
+#include "mongo/db/process_health/fault_impl.h"
 #include "mongo/logv2/log.h"
 
 namespace mongo {
@@ -41,14 +42,14 @@ namespace {
 
 const auto sFaultManager = ServiceContext::declareDecoration<std::unique_ptr<FaultManager>>();
 
-}  // namespace
-
 ServiceContext::ConstructorActionRegisterer faultManagerRegisterer{
     "FaultManagerRegisterer", [](ServiceContext* svcCtx) {
         auto faultManager = std::make_unique<FaultManager>(svcCtx);
         FaultManager::set(svcCtx, std::move(faultManager));
     }};
 
+}  // namespace
+
 
 FaultManager* FaultManager::get(ServiceContext* svcCtx) {
     return sFaultManager(svcCtx).get();
@@ -73,6 +74,23 @@ boost::optional<FaultConstPtr> FaultManager::activeFault() const {
     return {};
 }
 
+boost::optional<FaultFacetContainerPtr> FaultManager::getFaultFacetContainer() {
+    auto lk = stdx::lock_guard(_mutex);
+    if (!_fault) {
+        return {};
+    }
+    return std::static_pointer_cast<FaultFacetContainer>(_fault);
+}
+
+FaultFacetContainerPtr FaultManager::getOrCreateFaultFacetContainer() {
+    auto lk = stdx::lock_guard(_mutex);
+    if (!_fault) {
+        // Create a new one.
+        _fault = std::make_shared<FaultImpl>(_svcCtx);
+    }
+    return std::static_pointer_cast<FaultFacetContainer>(_fault);
+}
+
 void FaultManager::healthCheck() {}
 
 Status FaultManager::transitionToState(FaultState newState) {
diff --git a/src/mongo/db/process_health/fault_manager.h b/src/mongo/db/process_health/fault_manager.h
index 20b2dc8cede..eb036ac3a83 100644
--- a/src/mongo/db/process_health/fault_manager.h
+++ b/src/mongo/db/process_health/fault_manager.h
@@ -31,6 +31,8 @@
 #include <memory>
 
 #include "mongo/db/process_health/fault.h"
+#include "mongo/db/process_health/fault_facet.h"
+#include "mongo/db/process_health/fault_facet_container.h"
 #include "mongo/db/service_context.h"
 #include "mongo/platform/mutex.h"
 
@@ -65,7 +67,7 @@ enum class FaultState {
  * If an active fault state persists, FaultManager puts the server into quiesce
  * mode and shuts it down.
  */
-class FaultManager {
+class FaultManager : protected FaultFacetContainerFactory {
     FaultManager(const FaultManager&) = delete;
     FaultManager& operator=(const FaultManager&) = delete;
 
@@ -90,6 +92,13 @@ protected:
 
     virtual Status transitionToState(FaultState newState);
 
+    // Protected interface FaultFacetContainerFactory implementation.
+
+    // The interface FaultFacetContainerFactory is implemented by the member '_fault'.
+    boost::optional<FaultFacetContainerPtr> getFaultFacetContainer() override;
+
+    FaultFacetContainerPtr getOrCreateFaultFacetContainer() override;
+
 private:
     Status _transitionToKOk();
     Status _transitionToKTransientFault();
@@ -97,6 +106,10 @@ private:
 
     ServiceContext* const _svcCtx;
 
+    mutable Mutex _mutex =
+        MONGO_MAKE_LATCH(HierarchicalAcquisitionLevel(0), "FaultManager::_mutex");
+    std::shared_ptr<FaultInternal> _fault;
+
     mutable Mutex _stateMutex =
         MONGO_MAKE_LATCH(HierarchicalAcquisitionLevel(0), "ProcessHealthFaultManager::_stateMutex");
     FaultState _currentState = FaultState::kStartupCheck;
diff --git a/src/mongo/db/process_health/health_check_status.h b/src/mongo/db/process_health/health_check_status.h
index b24de01455d..432a1c23ebb 100644
--- a/src/mongo/db/process_health/health_check_status.h
+++ b/src/mongo/db/process_health/health_check_status.h
@@ -29,6 +29,7 @@
 #pragma once
 
 #include <memory>
+#include <ostream>
 
 #include "mongo/base/string_data.h"
 #include "mongo/bson/bsonobjbuilder.h"
diff --git a/src/mongo/db/process_health/health_observer.h b/src/mongo/db/process_health/health_observer.h
index d70f146d427..89b2823850a 100644
--- a/src/mongo/db/process_health/health_observer.h
+++ b/src/mongo/db/process_health/health_observer.h
@@ -28,6 +28,7 @@
  */
 #pragma once
 
+#include "mongo/db/process_health/fault_facet.h"
 
 namespace mongo {
 namespace process_health {
@@ -42,6 +43,14 @@ public:
     virtual ~HealthObserver() = default;
 
     /**
+     * Health observer of this type is unique and can only create the fault facet
+     * of the same type.
+     *
+     * @return FaultFacetType of this health observer.
+     */
+    virtual FaultFacetType getType() const = 0;
+
+    /**
      * Triggers health check.
      * It should be safe to invoke this method arbitrary often, the implementation
      * should prorate the invocations to avoid DoS.
diff --git a/src/mongo/db/process_health/health_observer_base.cpp b/src/mongo/db/process_health/health_observer_base.cpp
new file mode 100644
index 00000000000..b2aa6567f58
--- /dev/null
+++ b/src/mongo/db/process_health/health_observer_base.cpp
@@ -0,0 +1,42 @@
+/**
+ *    Copyright (C) 2021-present MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#include "mongo/db/process_health/health_observer_base.h"
+
+#include "mongo/db/service_context.h"
+
+namespace mongo {
+namespace process_health {
+
+HealthObserverBase::HealthObserverBase(ServiceContext* svcCtx) : _svcCtx(svcCtx) {}
+
+void HealthObserverBase::periodicCheck() {}
+
+}  // namespace process_health
+}  // namespace mongo
diff --git a/src/mongo/db/process_health/health_observer_base.h b/src/mongo/db/process_health/health_observer_base.h
new file mode 100644
index 00000000000..82cb3a0f668
--- /dev/null
+++ b/src/mongo/db/process_health/health_observer_base.h
@@ -0,0 +1,55 @@
+/**
+ *    Copyright (C) 2021-present MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+#pragma once
+
+#include "mongo/db/process_health/health_observer.h"
+
+#include "mongo/db/service_context.h"
+
+namespace mongo {
+namespace process_health {
+
+/**
+ * Interface to conduct periodic health checks.
+ * Every instance of health observer is wired internally to update the state of the FaultManager
+ * when a problem is detected.
+ */
+class HealthObserverBase : public HealthObserver {
+public:
+    HealthObserverBase(ServiceContext* svcCtx);
+    virtual ~HealthObserverBase() = default;
+
+    void periodicCheck() override;
+
+protected:
+    ServiceContext* const _svcCtx;
+};
+
+}  // namespace process_health
+}  // namespace mongo
diff --git a/src/mongo/db/process_health/health_observer_mock.h b/src/mongo/db/process_health/health_observer_mock.h
new file mode 100644
index 00000000000..516b47ee433
--- /dev/null
+++ b/src/mongo/db/process_health/health_observer_mock.h
@@ -0,0 +1,54 @@
+/**
+ *    Copyright (C) 2021-present MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+#pragma once
+
+#include "mongo/db/process_health/health_observer_base.h"
+
+namespace mongo {
+namespace process_health {
+
+/**
+ * Interface to conduct periodic health checks.
+ * Every instance of health observer is wired internally to update the state of the FaultManager
+ * when a problem is detected.
+ */
+class HealthObserverMock : public HealthObserverBase {
+public:
+    HealthObserverMock(ServiceContext* svcCtx) : HealthObserverBase(svcCtx) {}
+    virtual ~HealthObserverMock() = default;
+
+    FaultFacetType getType() const override {
+        return FaultFacetType::kMock;
+    }
+
+    void periodicCheck() override {}
+};
+
+}  // namespace process_health
+}  // namespace mongo
diff --git a/src/mongo/db/process_health/health_observer_registration.cpp b/src/mongo/db/process_health/health_observer_registration.cpp
new file mode 100644
index 00000000000..17e10b0c529
--- /dev/null
+++ b/src/mongo/db/process_health/health_observer_registration.cpp
@@ -0,0 +1,72 @@
+/**
+ *    Copyright (C) 2021-present MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#include "mongo/db/process_health/health_observer_registration.h"
+
+namespace mongo {
+namespace process_health {
+
+namespace {
+
+const auto sRegistration =
+    ServiceContext::declareDecoration<std::unique_ptr<HealthObserverRegistration>>();
+
+ServiceContext::ConstructorActionRegisterer healthObserverRegisterer{
+    "healthObserverRegisterer", [](ServiceContext* svcCtx) {
+        auto healthObserver = std::make_unique<HealthObserverRegistration>(svcCtx);
+        auto& instance = sRegistration(svcCtx);
+        instance = std::move(healthObserver);
+    }};
+
+}  // namespace
+
+HealthObserverRegistration* HealthObserverRegistration::get(ServiceContext* svcCtx) {
+    return sRegistration(svcCtx).get();
+}
+
+HealthObserverRegistration::HealthObserverRegistration(ServiceContext* svcCtx) : _svcCtx(svcCtx) {}
+
+void HealthObserverRegistration::registerObserverFactory(
+    std::function<std::unique_ptr<HealthObserver>(ServiceContext* svcCtx)> factoryCallback) {
+    auto lk = stdx::lock_guard(_mutex);
+    _factories.push_back(std::move(factoryCallback));
+}
+
+std::vector<std::unique_ptr<HealthObserver>> HealthObserverRegistration::instantiateAllObservers()
+    const {
+    std::vector<std::unique_ptr<HealthObserver>> result;
+    auto lk = stdx::lock_guard(_mutex);
+    for (auto& cb : _factories) {
+        result.push_back(cb(_svcCtx));
+    }
+    return result;
+}
+
+}  // namespace process_health
+}  // namespace mongo
diff --git a/src/mongo/db/process_health/health_observer_registration.h b/src/mongo/db/process_health/health_observer_registration.h
new file mode 100644
index 00000000000..332c2d12854
--- /dev/null
+++ b/src/mongo/db/process_health/health_observer_registration.h
@@ -0,0 +1,74 @@
+/**
+ *    Copyright (C) 2021-present MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+#pragma once
+
+#include <functional>
+#include <vector>
+
+#include "mongo/db/process_health/health_observer.h"
+
+#include "mongo/db/service_context.h"
+#include "mongo/platform/mutex.h"
+
+namespace mongo {
+namespace process_health {
+
+/**
+ * Registration mechanism for all health observers.
+ */
+class HealthObserverRegistration {
+public:
+    static HealthObserverRegistration* get(ServiceContext* svcCtx);
+
+    explicit HealthObserverRegistration(ServiceContext* svcCtx);
+
+    /**
+     * Registers a factory method, which will be invoked later to instantiate the observer.
+     *
+     * @param factoryCallback creates observer instance when invoked.
+     */
+    void registerObserverFactory(
+        std::function<std::unique_ptr<HealthObserver>(ServiceContext* svcCtx)> factoryCallback);
+
+    /**
+     * Invokes all registered factories and returns new instances.
+     * The ownership of all observers is transferred to the invoker.
+     */
+    std::vector<std::unique_ptr<HealthObserver>> instantiateAllObservers() const;
+
+private:
+    ServiceContext* const _svcCtx;
+
+    mutable Mutex _mutex =
+        MONGO_MAKE_LATCH(HierarchicalAcquisitionLevel(0), "HealthObserverRegistration::_mutex");
+    std::vector<std::function<std::unique_ptr<HealthObserver>(ServiceContext* svcCtx)>> _factories;
+};
+
+}  // namespace process_health
+}  // namespace mongo
diff --git a/src/mongo/db/process_health/health_observer_test.cpp b/src/mongo/db/process_health/health_observer_test.cpp
new file mode 100644
index 00000000000..6c7e97763b8
--- /dev/null
+++ b/src/mongo/db/process_health/health_observer_test.cpp
@@ -0,0 +1,72 @@
+/**
+ *    Copyright (C) 2021-present MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#include "mongo/db/process_health/health_observer.h"
+
+#include "mongo/db/process_health/health_observer_mock.h"
+#include "mongo/db/process_health/health_observer_registration.h"
+#include "mongo/db/service_context.h"
+#include "mongo/unittest/unittest.h"
+
+namespace mongo {
+
+namespace process_health {
+
+namespace {
+
+class HealthObserverTest : public unittest::Test {
+public:
+    void setUp() override {
+        _svcCtx = ServiceContext::make();
+    }
+
+    void registerMock() {
+        HealthObserverRegistration* reg = HealthObserverRegistration::get(_svcCtx.get());
+        reg->registerObserverFactory(
+            [](ServiceContext* svcCtx) { return std::make_unique<HealthObserverMock>(svcCtx); });
+    }
+
+    HealthObserverRegistration* registration() {
+        return HealthObserverRegistration::get(_svcCtx.get());
+    }
+
+private:
+    ServiceContext::UniqueServiceContext _svcCtx;
+};
+
+TEST_F(HealthObserverTest, Registration) {
+    registerMock();
+    auto allObservers = registration()->instantiateAllObservers();
+    ASSERT_EQ(1, allObservers.size());
+    ASSERT_EQ(FaultFacetType::kMock, allObservers[0]->getType());
+}
+
+}  // namespace
+}  // namespace process_health
+}  // namespace mongo