/** * Copyright (C) 2021-present MongoDB, Inc. * * This program is free software: you can redistribute it and/or modify * it under the terms of the Server Side Public License, version 1, * as published by MongoDB, Inc. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * Server Side Public License for more details. * * You should have received a copy of the Server Side Public License * along with this program. If not, see * . * * As a special exception, the copyright holders give permission to link the * code of portions of this program with the OpenSSL library under certain * conditions as described in each individual source file and distribute * linked combinations including the program with the OpenSSL library. You * must comply with the Server Side Public License in all respects for * all of the code used other than as permitted herein. If you modify file(s) * with this exception, you may extend this exception to your version of the * file(s), but you are not obligated to do so. If you do not wish to do so, * delete this exception statement from your version. If you delete this * exception statement from all source files in the program, then also delete * it in the license file. */ #pragma once #include #include "mongo/db/process_health/fault_manager.h" #include "mongo/db/concurrency/locker_noop_client_observer.h" #include "mongo/db/process_health/health_observer_mock.h" #include "mongo/db/process_health/health_observer_registration.h" #include "mongo/executor/network_interface_factory.h" #include "mongo/executor/thread_pool_task_executor_test_fixture.h" #include "mongo/idl/server_parameter_test_util.h" #include "mongo/unittest/unittest.h" #include "mongo/util/concurrency/thread_pool.h" #include "mongo/util/tick_source_mock.h" namespace mongo { using executor::TaskExecutor; using executor::ThreadPoolExecutorTest; namespace process_health { namespace test { static inline std::unique_ptr getConfigWithDisabledPeriodicChecks() { auto config = std::make_unique(); config->disablePeriodicChecksForTests(); return config; } /** * Test wrapper class for FaultManager that has access to protected methods * for testing. */ class FaultManagerTestImpl : public FaultManager { public: FaultManagerTestImpl(ServiceContext* svcCtx, std::shared_ptr taskExecutor, std::unique_ptr config) : FaultManager(svcCtx, taskExecutor, [&config]() -> std::unique_ptr { if (config) return std::move(config); else return getConfigWithDisabledPeriodicChecks(); }(), [](std::string cause) { // In tests, do not crash. LOGV2(5936606, "Fault manager progress monitor triggered the termination", "cause"_attr = cause); }) {} void healthCheckTest(HealthObserver* observer, CancellationToken token) { healthCheck(observer, token); } void schedulePeriodicHealthCheckThreadTest() { schedulePeriodicHealthCheckThread(); } std::vector getHealthObserversTest() { return getHealthObservers(); } FaultPtr getOrCreateFaultTest() { return getOrCreateFault(); } Fault& getFault() { FaultPtr fault = FaultManager::getFault(); invariant(fault); return *(static_cast(fault.get())); } void progressMonitorCheckTest(std::function crashCb) { progressMonitorCheckForTests(crashCb); } const FaultManagerConfig& getConfigTest() { return getConfig(); } FaultState acceptTest(const HealthCheckStatus& message) { return accept(message); } void scheduleNextImmediateCheckForTest(HealthObserver* observer) { scheduleNextHealthCheck(observer, CancellationToken::uncancelable(), true); } }; /** * Test suite for fault manager. */ class FaultManagerTest : public unittest::Test { public: void setUp() override { HealthObserverRegistration::resetObserverFactoriesForTest(); createServiceContextIfNeeded(); bumpUpLogging(); resetManager(); } void createServiceContextIfNeeded() { if (!_svcCtx) { // Reset only once because the Ldap connection reaper is running asynchronously // and is using the simulated clock, which should not go out of scope. _svcCtx = ServiceContext::make(); _svcCtx->setFastClockSource(std::make_unique()); _svcCtx->setPreciseClockSource(std::make_unique()); _svcCtx->setTickSource(std::make_unique>()); _svcCtx->registerClientObserver( std::make_unique()); advanceTime(Seconds(100)); } } void bumpUpLogging() { logv2::LogManager::global().getGlobalSettings().setMinimumLoggedSeverity( mongo::logv2::LogComponent::kProcessHealth, logv2::LogSeverity::Debug(3)); logv2::LogManager::global().getGlobalSettings().setMinimumLoggedSeverity( mongo::logv2::LogComponent::kAccessControl, logv2::LogSeverity::Debug(3)); } void tearDown() override { LOGV2(6007905, "Clean up test resources"); // Shutdown the executor before the context is deleted. resetManager(); } void constructTaskExecutor() { if (_executor) { _executor->shutdown(); _executor->join(); } auto network = std::shared_ptr( executor::makeNetworkInterface("FaultManagerTest").release()); ThreadPool::Options options; auto pool = std::make_unique(options); _executor = std::make_unique(std::move(pool), std::move(network)); } void resetManager(std::unique_ptr config = nullptr) { constructTaskExecutor(); FaultManager::set( _svcCtx.get(), std::make_unique(_svcCtx.get(), _executor, std::move(config))); } void registerMockHealthObserver(FaultFacetType mockType, std::function getSeverityCallback, Milliseconds timeout) { HealthObserverRegistration::registerObserverFactory( [mockType, getSeverityCallback, timeout](ServiceContext* svcCtx) { return std::make_unique( mockType, svcCtx, getSeverityCallback, timeout); }); } void registerMockHealthObserver(FaultFacetType mockType, std::function getSeverityCallback) { registerMockHealthObserver(mockType, getSeverityCallback, Milliseconds(Seconds(30))); } template void scheduleNextImmediateCheck(FaultFacetType type) { auto& obsrv = observer(type); manager().scheduleNextImmediateCheckForTest(&obsrv); } template void registerHealthObserver() { HealthObserverRegistration::registerObserverFactory( [](ServiceContext* svcCtx) { return std::make_unique(svcCtx); }); } FaultManagerTestImpl& manager() { return *static_cast(FaultManager::get(_svcCtx.get())); } ClockSourceMock& clockSource() { return *static_cast(_svcCtx->getFastClockSource()); } ServiceContext* svcCtx() const { return _svcCtx.get(); } TickSourceMock& tickSource() { return *static_cast*>(_svcCtx->getTickSource()); } template Observer& observer(FaultFacetType type) { std::vector observers = manager().getHealthObserversTest(); ASSERT_TRUE(!observers.empty()); auto it = std::find_if(observers.begin(), observers.end(), [type](const HealthObserver* o) { return o->getType() == type; }); ASSERT_TRUE(it != observers.end()); return *static_cast(*it); } HealthObserverBase::PeriodicHealthCheckContext checkContext() { HealthObserverBase::PeriodicHealthCheckContext ctx{CancellationToken::uncancelable(), _executor}; return ctx; } template void advanceTime(Duration d) { clockSource().advance(d); static_cast(_svcCtx->getPreciseClockSource())->advance(d); tickSource().advance(d); } static inline const Seconds kWaitTimeout{10}; static inline const Milliseconds kSleepTime{1}; static inline const int kActiveFaultDurationSecs = 5; RAIIServerParameterControllerForTest serverParamController{"activeFaultDurationSecs", kActiveFaultDurationSecs}; void assertSoon(std::function predicate, Milliseconds timeout = kWaitTimeout) { Timer t; while (t.elapsed() < timeout) { if (predicate()) return; sleepFor(kSleepTime); } ASSERT(false); } bool hasFault() { return static_cast(manager().currentFault()); } void waitForFaultBeingResolved() { assertSoon([this]() { return !hasFault(); }); } void waitForFaultBeingCreated() { assertSoon([this]() { return hasFault(); }); } void waitForTransitionIntoState(FaultState state) { assertSoon([=]() { return manager().getFaultState() == state; }); } private: ServiceContext::UniqueServiceContext _svcCtx; std::shared_ptr _executor; }; } // namespace test } // namespace process_health } // namespace mongo