diff options
author | joshua <80741223+jlap199@users.noreply.github.com> | 2022-11-12 02:47:38 +0000 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2022-11-12 03:19:26 +0000 |
commit | 85b89ff203ec7a70e43af24831b35a3d53a259e4 (patch) | |
tree | 8c484e57d3da06c9aa30da7e79796f23526403a7 | |
parent | 2041a8a461f42c6fc1c257e87e88c6ae695a51a8 (diff) | |
download | mongo-85b89ff203ec7a70e43af24831b35a3d53a259e4.tar.gz |
SERVER-71065 Implement onTelemetryStoreSizeUpdate function
-rw-r--r-- | src/mongo/db/query/SConscript | 27 | ||||
-rw-r--r-- | src/mongo/db/query/query_knobs.idl | 8 | ||||
-rw-r--r-- | src/mongo/db/query/sbe_plan_cache.cpp | 85 | ||||
-rw-r--r-- | src/mongo/db/query/sbe_plan_cache_on_parameter_change.h | 2 | ||||
-rw-r--r-- | src/mongo/db/query/telemetry.cpp | 70 | ||||
-rw-r--r-- | src/mongo/db/query/telemetry.h | 14 | ||||
-rw-r--r-- | src/mongo/db/query/telemetry_util.cpp | 84 | ||||
-rw-r--r-- | src/mongo/db/query/telemetry_util.h | 67 | ||||
-rw-r--r-- | src/mongo/db/query/util/memory_util.cpp (renamed from src/mongo/util/memory_util.cpp) | 58 | ||||
-rw-r--r-- | src/mongo/db/query/util/memory_util.h (renamed from src/mongo/util/memory_util.h) | 11 | ||||
-rw-r--r-- | src/mongo/db/query/util/memory_util_test.cpp (renamed from src/mongo/util/memory_util_test.cpp) | 2 | ||||
-rw-r--r-- | src/mongo/util/SConscript | 12 |
12 files changed, 309 insertions, 131 deletions
diff --git a/src/mongo/db/query/SConscript b/src/mongo/db/query/SConscript index 5b21dd4243a..f3cce404031 100644 --- a/src/mongo/db/query/SConscript +++ b/src/mongo/db/query/SConscript @@ -90,6 +90,17 @@ env.Library( ) env.Library( + target='memory_util', + source=[ + 'util/memory_util.cpp', + ], + LIBDEPS=[ + '$BUILD_DIR/mongo/util/pcre_wrapper', + '$BUILD_DIR/mongo/util/processinfo', + ], +) + +env.Library( target="query_plan_cache", source=[ "classic_plan_cache.cpp", @@ -98,10 +109,8 @@ env.Library( "sbe_plan_cache.cpp", ], LIBDEPS=[ - "$BUILD_DIR/mongo/base", - "$BUILD_DIR/mongo/db/exec/sbe/query_sbe", - '$BUILD_DIR/mongo/util/memory_util', - "canonical_query", + "$BUILD_DIR/mongo/base", "$BUILD_DIR/mongo/db/exec/sbe/query_sbe", "canonical_query", + "memory_util" ], ) @@ -262,13 +271,13 @@ env.Library( 'query_feature_flags.idl', 'query_knobs.idl', 'sbe_plan_cache_on_parameter_change.cpp', + 'telemetry_util.cpp', ], LIBDEPS_PRIVATE=[ - '$BUILD_DIR/mongo/db/query/telemetry', '$BUILD_DIR/mongo/db/server_base', '$BUILD_DIR/mongo/db/service_context', - '$BUILD_DIR/mongo/util/memory_util', '$BUILD_DIR/mongo/util/pcre_wrapper', + 'memory_util', ], ) @@ -354,9 +363,10 @@ env.Library( ], LIBDEPS=[ '$BUILD_DIR/mongo/base', - "$BUILD_DIR/mongo/db/concurrency/lock_manager", - '$BUILD_DIR/mongo/util/memory_util', + '$BUILD_DIR/mongo/db/concurrency/lock_manager', '$BUILD_DIR/mongo/util/processinfo', + 'memory_util', + 'query_knobs', ], ) @@ -386,6 +396,7 @@ env.CppUnitTest( "killcursors_request_test.cpp", "lru_key_value_test.cpp", 'map_reduce_output_format_test.cpp', + "util/memory_util_test.cpp", "parsed_distinct_test.cpp", "plan_cache_indexability_test.cpp", "plan_cache_key_info_test.cpp", diff --git a/src/mongo/db/query/query_knobs.idl b/src/mongo/db/query/query_knobs.idl index 5c859d09c8e..c247bdabc47 100644 --- a/src/mongo/db/query/query_knobs.idl +++ b/src/mongo/db/query/query_knobs.idl @@ -32,7 +32,7 @@ global: - "mongo/db/query/cost_model/cost_model_on_update.h" - "mongo/db/query/ce_mode_parameter.h" - "mongo/db/query/sbe_plan_cache_on_parameter_change.h" - - "mongo/db/query/telemetry.h" + - "mongo/db/query/telemetry_util.h" - "mongo/platform/atomic_proxy.h" - "mongo/platform/atomic_word.h" @@ -966,12 +966,12 @@ server_parameters: 2. <number>(MB|GB), indicates the amount of memory in MB or GB. E.g.: 1.5GB, 100MB. The default value is 5%, which means 5% of the physical memory available to the process." set_at: [ startup, runtime ] - cpp_varname: "queryTelemetryCacheSize" + cpp_varname: "queryTelemetryStoreSize" cpp_vartype: synchronized_value<std::string> default: "5%" - on_update: telemetry_util::onTelemetryCacheSizeUpdate + on_update: telemetry_util::onTelemetryStoreSizeUpdate validator: - callback: telemetry_util::validateTelemetryCacheSize + callback: telemetry_util::validateTelemetryStoreSize # Note for adding additional query knobs: diff --git a/src/mongo/db/query/sbe_plan_cache.cpp b/src/mongo/db/query/sbe_plan_cache.cpp index 18d8b9fc5c9..1498fa28932 100644 --- a/src/mongo/db/query/sbe_plan_cache.cpp +++ b/src/mongo/db/query/sbe_plan_cache.cpp @@ -30,9 +30,9 @@ #include "mongo/db/query/sbe_plan_cache.h" +#include "mongo/db/query/util/memory_util.h" #include "mongo/db/server_options.h" #include "mongo/logv2/log.h" -#include "mongo/util/memory_util.h" #include "mongo/util/processinfo.h" #define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kQuery @@ -44,69 +44,23 @@ namespace { const auto sbePlanCacheDecoration = ServiceContext::declareDecoration<std::unique_ptr<sbe::PlanCache>>(); -size_t convertToSizeInBytes(const memory_util::MemorySize& memSize) { - constexpr size_t kBytesInMB = 1024 * 1024; - constexpr size_t kMBytesInGB = 1024; - - double sizeInMB = memSize.size; - - switch (memSize.units) { - case memory_util::MemoryUnits::kPercent: - sizeInMB *= ProcessInfo::getMemSizeMB() / 100.0; - break; - case memory_util::MemoryUnits::kMB: - break; - case memory_util::MemoryUnits::kGB: - sizeInMB *= kMBytesInGB; - break; - } - - return static_cast<size_t>(sizeInMB * kBytesInMB); -} - -/** - * Sets upper size limit on the PlanCache size to 500GB or 25% of the system's memory, whichever is - * smaller. - */ -size_t capPlanCacheSize(size_t planCacheSize) { - constexpr size_t kBytesInGB = 1024 * 1024 * 1024; - - // Maximum size of the plan cache expressed in bytes. - constexpr size_t kMaximumPlanCacheSize = 500 * kBytesInGB; - - // Maximum size of the plan cache expressed as a share of the memory available to the process. - const memory_util::MemorySize limitToProcessSize{25, memory_util::MemoryUnits::kPercent}; - const size_t limitToProcessSizeInBytes = convertToSizeInBytes(limitToProcessSize); - - // The size will be capped by the minimum of the two values defined above. - const size_t maxPlanCacheSize = std::min(kMaximumPlanCacheSize, limitToProcessSizeInBytes); - - if (planCacheSize > maxPlanCacheSize) { - planCacheSize = maxPlanCacheSize; - LOGV2_DEBUG(6007000, - 1, - "The plan cache size has been capped", - "maxPlanCacheSize"_attr = maxPlanCacheSize); - } - - return planCacheSize; -} - -size_t getPlanCacheSizeInBytes(const memory_util::MemorySize& memSize) { - size_t planCacheSize = convertToSizeInBytes(memSize); - uassert(5968001, - "Cache size must be at least 1KB * number of cores", - planCacheSize >= 1024 * ProcessInfo::getNumCores()); - return capPlanCacheSize(planCacheSize); -} class PlanCacheOnParamChangeUpdaterImpl final : public plan_cache_util::OnParamChangeUpdater { public: void updateCacheSize(ServiceContext* serviceCtx, memory_util::MemorySize memSize) final { if (feature_flags::gFeatureFlagSbeFull.isEnabledAndIgnoreFCV()) { - auto size = getPlanCacheSizeInBytes(memSize); + auto newSizeBytes = memory_util::getRequestedMemSizeInBytes(memSize); + auto cappedCacheSize = memory_util::capMemorySize(newSizeBytes /*requestedSizeBytes*/, + 500 /*maximumSizeGB*/, + 25 /*percentTotalSystemMemory*/); + if (cappedCacheSize < newSizeBytes) { + LOGV2_DEBUG(6007001, + 1, + "The plan cache size has been capped", + "cappedSize"_attr = cappedCacheSize); + } auto& globalPlanCache = sbePlanCacheDecoration(serviceCtx); - globalPlanCache->reset(size); + globalPlanCache->reset(cappedCacheSize); } } @@ -126,10 +80,19 @@ ServiceContext::ConstructorActionRegisterer planCacheRegisterer{ if (feature_flags::gFeatureFlagSbeFull.isEnabledAndIgnoreFCV()) { auto status = memory_util::MemorySize::parse(planCacheSize.get()); uassertStatusOK(status); - - auto size = getPlanCacheSizeInBytes(status.getValue()); + auto size = memory_util::getRequestedMemSizeInBytes(status.getValue()); + auto cappedCacheSize = memory_util::capMemorySize(size /*requestedSizeBytes*/, + 500 /*maximumSizeGB*/, + 25 /*percentTotalSystemMemory*/); + if (cappedCacheSize < size) { + LOGV2_DEBUG(6007000, + 1, + "The plan cache size has been capped", + "cappedSize"_attr = cappedCacheSize); + } auto& globalPlanCache = sbePlanCacheDecoration(serviceCtx); - globalPlanCache = std::make_unique<sbe::PlanCache>(size, ProcessInfo::getNumCores()); + globalPlanCache = + std::make_unique<sbe::PlanCache>(cappedCacheSize, ProcessInfo::getNumCores()); } }}; diff --git a/src/mongo/db/query/sbe_plan_cache_on_parameter_change.h b/src/mongo/db/query/sbe_plan_cache_on_parameter_change.h index 4135bb7f2b9..415b44d82c3 100644 --- a/src/mongo/db/query/sbe_plan_cache_on_parameter_change.h +++ b/src/mongo/db/query/sbe_plan_cache_on_parameter_change.h @@ -32,8 +32,8 @@ #include <string> #include "mongo/base/status.h" +#include "mongo/db/query/util/memory_util.h" #include "mongo/db/service_context.h" -#include "mongo/util/memory_util.h" namespace mongo::plan_cache_util { diff --git a/src/mongo/db/query/telemetry.cpp b/src/mongo/db/query/telemetry.cpp index 3a78512a9ab..468ebb6a396 100644 --- a/src/mongo/db/query/telemetry.cpp +++ b/src/mongo/db/query/telemetry.cpp @@ -33,8 +33,11 @@ #include "mongo/db/concurrency/lock_state.h" #include "mongo/db/concurrency/locker.h" #include "mongo/db/query/query_planner_params.h" -#include "mongo/util/memory_util.h" +#include "mongo/logv2/log.h" #include "mongo/util/processinfo.h" +#include <cstddef> + +#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kQuery namespace mongo { @@ -94,17 +97,48 @@ private: const auto telemetryStoreDecoration = ServiceContext::declareDecoration<std::unique_ptr<TelemetryStoreManager>>(); +class TelemetryOnParamChangeUpdaterImpl final : public telemetry_util::OnParamChangeUpdater { +public: + void updateCacheSize(ServiceContext* serviceCtx, memory_util::MemorySize memSize) final { + auto newSizeBytes = memory_util::getRequestedMemSizeInBytes(memSize); + auto cappedSize = memory_util::capMemorySize( + newSizeBytes /*requestedSize*/, 1 /*maximumSizeGB*/, 25 /*percentTotalSystemMemory*/); + + /* If capped size is less than requested size, the telemetry store has been capped at its + * upper limit*/ + if (cappedSize < newSizeBytes) { + LOGV2_DEBUG(7106503, + 1, + "The telemetry store size has been capped", + "cappedSize"_attr = cappedSize); + } + auto& telemetryStoreManager = telemetryStoreDecoration(serviceCtx); + auto&& [telemetryStore, resourceLock] = telemetryStoreManager->getTelemetryStore(); + telemetryStore->reset(cappedSize); + } +}; + + ServiceContext::ConstructorActionRegisterer telemetryStoreManagerRegisterer{ "TelemetryStoreManagerRegisterer", [](ServiceContext* serviceCtx) { - // TODO SERVER-71065 update telemetry cache size - (void)internalQueryDisableExclusionProjectionFastPath; - (void)queryTelemetrySamplingRate; - (void)queryTelemetryCacheSize; - - auto size = 100000; // getPlanCacheSizeInBytes(queryTelemetryCacheSize.get()); + telemetry_util::telemetryStoreOnParamChangeUpdater(serviceCtx) = + std::make_unique<TelemetryOnParamChangeUpdaterImpl>(); + auto status = memory_util::MemorySize::parse(queryTelemetryStoreSize.get()); + uassertStatusOK(status); + auto size = memory_util::getRequestedMemSizeInBytes(status.getValue()); + auto cappedStoreSize = memory_util::capMemorySize( + size /*requestedSizeBytes*/, 1 /*maximumSizeGB*/, 25 /*percentTotalSystemMemory*/); + /* If capped size is less than requested size, the telemetry store has been capped at its + * upper limit*/ + if (cappedStoreSize < size) { + LOGV2_DEBUG(7106502, + 1, + "The telemetry store size has been capped", + "cappedSize"_attr = cappedStoreSize); + } auto&& globalTelemetryStoreManager = telemetryStoreDecoration(serviceCtx); - globalTelemetryStoreManager = - std::make_unique<TelemetryStoreManager>(serviceCtx, size, ProcessInfo::getNumCores()); + globalTelemetryStoreManager = std::make_unique<TelemetryStoreManager>( + serviceCtx, cappedStoreSize, ProcessInfo::getNumCores()); }}; } // namespace @@ -118,22 +152,4 @@ std::unique_ptr<TelemetryStore> resetTelemetryStore(ServiceContext* serviceCtx) return telemetryStoreDecoration(serviceCtx)->resetTelemetryStore(); } -namespace telemetry_util { - -Status onTelemetryCacheSizeUpdate(const std::string& str) { - auto newSize = memory_util::MemorySize::parse(str); - if (!newSize.isOK()) { - return newSize.getStatus(); - } - - // TODO SERVER-71065 update telemetry cache size - - return Status::OK(); -} - -Status validateTelemetryCacheSize(const std::string& str, const boost::optional<TenantId>&) { - return memory_util::MemorySize::parse(str).getStatus(); -} - -} // namespace telemetry_util } // namespace mongo diff --git a/src/mongo/db/query/telemetry.h b/src/mongo/db/query/telemetry.h index 93e2581ad84..1e60df54e04 100644 --- a/src/mongo/db/query/telemetry.h +++ b/src/mongo/db/query/telemetry.h @@ -32,6 +32,7 @@ #include "mongo/base/status.h" #include "mongo/db/concurrency/d_concurrency.h" #include "mongo/db/query/partitioned_cache.h" +#include "mongo/db/query/util/memory_util.h" #include "mongo/db/service_context.h" namespace mongo { @@ -135,17 +136,4 @@ std::pair<TelemetryStore*, Lock::ResourceLock> getTelemetryStoreForRead(ServiceC std::unique_ptr<TelemetryStore> resetTelemetryStore(ServiceContext* serviceCtx); -namespace telemetry_util { - -/** - * Callback called on a change of telemetryCacheSize parameter. - */ -Status onTelemetryCacheSizeUpdate(const std::string& str); - -/** - * Callback called on validation of telemetryCacheSize parameter. - */ -Status validateTelemetryCacheSize(const std::string& str, const boost::optional<TenantId>&); - -} // namespace telemetry_util } // namespace mongo diff --git a/src/mongo/db/query/telemetry_util.cpp b/src/mongo/db/query/telemetry_util.cpp new file mode 100644 index 00000000000..dcce8d7452b --- /dev/null +++ b/src/mongo/db/query/telemetry_util.cpp @@ -0,0 +1,84 @@ +/** + * Copyright (C) 2022-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/query/telemetry_util.h" + +#include "mongo/base/status.h" +#include "mongo/db/concurrency/d_concurrency.h" +#include "mongo/db/query/partitioned_cache.h" +#include "mongo/db/query/util/memory_util.h" +#include "mongo/db/service_context.h" +#include "mongo/logv2/log.h" + + +#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kQuery + +namespace mongo::telemetry_util { + +namespace { +/** + * Given the current 'Client', returns a pointer to the 'ServiceContext' and an interface for + * updating the telemetry store. + */ +std::pair<ServiceContext*, OnParamChangeUpdater*> getUpdater(const Client& client) { + auto serviceCtx = client.getServiceContext(); + tassert(7106500, "ServiceContext must be non null", serviceCtx); + + auto updater = telemetryStoreOnParamChangeUpdater(serviceCtx).get(); + tassert(7106501, "Telemetry store size updater must be non null", updater); + return {serviceCtx, updater}; +} +} // namespace + + +Status onTelemetryStoreSizeUpdate(const std::string& str) { + auto newSize = memory_util::MemorySize::parse(str); + if (!newSize.isOK()) { + return newSize.getStatus(); + } + + // The client is nullptr if the parameter is supplied from the command line. In this case, we + // ignore the update event, the parameter will be processed when initializing the service + // context. + if (auto client = Client::getCurrent()) { + auto&& [serviceCtx, updater] = getUpdater(*client); + updater->updateCacheSize(serviceCtx, newSize.getValue()); + } + + return Status::OK(); +} + +Status validateTelemetryStoreSize(const std::string& str, const boost::optional<TenantId>&) { + return memory_util::MemorySize::parse(str).getStatus(); +} + +const Decorable<ServiceContext>::Decoration<std::unique_ptr<OnParamChangeUpdater>> + telemetryStoreOnParamChangeUpdater = + ServiceContext::declareDecoration<std::unique_ptr<OnParamChangeUpdater>>(); +} // namespace mongo::telemetry_util diff --git a/src/mongo/db/query/telemetry_util.h b/src/mongo/db/query/telemetry_util.h new file mode 100644 index 00000000000..0aedd95e26e --- /dev/null +++ b/src/mongo/db/query/telemetry_util.h @@ -0,0 +1,67 @@ +/** + * Copyright (C) 2022-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/base/status.h" +#include "mongo/db/concurrency/d_concurrency.h" +#include "mongo/db/query/partitioned_cache.h" +#include "mongo/db/query/util/memory_util.h" + + +namespace mongo::telemetry_util { + +Status onTelemetryStoreSizeUpdate(const std::string& str); + + +Status validateTelemetryStoreSize(const std::string& str, const boost::optional<TenantId>&); + +/** + * An interface used to modify the telemetry store when query setParameters are modified. This is + * done via an interface decorating the 'ServiceContext' in order to avoid a link-time dependency + * of the query knobs library on the telemetry code. + */ +class OnParamChangeUpdater { +public: + virtual ~OnParamChangeUpdater() = default; + + /** + * Resizes the telemetry store decorating 'serviceCtx' to the new size given by 'memSize'. If + * the new size is smaller than the old, cache entries are evicted in order to ensure the + * cache fits within the new size bound. + */ + virtual void updateCacheSize(ServiceContext* serviceCtx, memory_util::MemorySize memSize) = 0; +}; + +/** + * Decorated accessor to the 'OnParamChangeUpdater' stored in 'ServiceContext'. + */ +extern const Decorable<ServiceContext>::Decoration<std::unique_ptr<OnParamChangeUpdater>> + telemetryStoreOnParamChangeUpdater; +} // namespace mongo::telemetry_util diff --git a/src/mongo/util/memory_util.cpp b/src/mongo/db/query/util/memory_util.cpp index 28b5822c508..dbd4f23bee1 100644 --- a/src/mongo/util/memory_util.cpp +++ b/src/mongo/db/query/util/memory_util.cpp @@ -26,10 +26,14 @@ * exception statement from all source files in the program, then also delete * it in the license file. */ +#include "mongo/db/query/util/memory_util.h" +#include "mongo/logv2/log.h" +#include "mongo/util/pcre.h" +#include "mongo/util/processinfo.h" +#include <cstddef> -#include "mongo/util/memory_util.h" +#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kQuery -#include "mongo/util/pcre.h" namespace mongo::memory_util { @@ -67,4 +71,54 @@ StatusWith<MemorySize> MemorySize::parse(const std::string& str) { return MemorySize{size, statusWithUnit.getValue()}; } +size_t convertToSizeInBytes(const MemorySize& memSize) { + constexpr size_t kBytesInMB = 1024 * 1024; + constexpr size_t kMBytesInGB = 1024; + + double sizeInMB = memSize.size; + + switch (memSize.units) { + case MemoryUnits::kPercent: + sizeInMB *= ProcessInfo::getMemSizeMB() / 100.0; + break; + case MemoryUnits::kMB: + break; + case MemoryUnits::kGB: + sizeInMB *= kMBytesInGB; + break; + } + + return static_cast<size_t>(sizeInMB * kBytesInMB); +} + +size_t getRequestedMemSizeInBytes(const MemorySize& memSize) { + size_t planCacheSize = convertToSizeInBytes(memSize); + uassert(5968001, + "Cache size must be at least 1KB * number of cores", + planCacheSize >= 1024 * ProcessInfo::getNumCores()); + return planCacheSize; +} + +/** + * Sets upper limit on a storage structure's size. Either that structure's maximumSize or to + * percentage of the total system's memory (both known at call site), whichever is smaller. + */ +size_t capMemorySize(size_t requestedSizeBytes, + size_t maximumSizeGB, + double percentTotalSystemMemory) { + constexpr size_t kBytesInGB = 1024 * 1024 * 1024; + // Express maximum size in bytes. + const size_t maximumSizeBytes = maximumSizeGB * kBytesInGB; + const memory_util::MemorySize limitToProcessSize{percentTotalSystemMemory, + memory_util::MemoryUnits::kPercent}; + const size_t limitToProcessSizeInBytes = convertToSizeInBytes(limitToProcessSize); + + // The size will be capped by the minimum of the two values defined above. + const size_t upperLimit = std::min(maximumSizeBytes, limitToProcessSizeInBytes); + + if (requestedSizeBytes > upperLimit) { + requestedSizeBytes = upperLimit; + } + return requestedSizeBytes; +} } // namespace mongo::memory_util diff --git a/src/mongo/util/memory_util.h b/src/mongo/db/query/util/memory_util.h index d1452dbfee9..345780b4c84 100644 --- a/src/mongo/util/memory_util.h +++ b/src/mongo/db/query/util/memory_util.h @@ -31,6 +31,7 @@ #include <string> +#include "mongo/base/error_codes.h" #include "mongo/base/status_with.h" namespace mongo::memory_util { @@ -44,8 +45,6 @@ enum class MemoryUnits { kGB, }; -StatusWith<MemoryUnits> parseUnitString(const std::string& strUnit); - /** * Represents parsed memory size parameter. */ @@ -56,4 +55,12 @@ struct MemorySize { const MemoryUnits units; }; +StatusWith<MemoryUnits> parseUnitString(const std::string& strUnit); +size_t convertToSizeInBytes(const MemorySize& memSize); +size_t capMemorySize(size_t requestedSizeBytes, + size_t maximumSizeGB, + double percentTotalSystemMemory); +size_t getRequestedMemSizeInBytes(const MemorySize& memSize); + + } // namespace mongo::memory_util diff --git a/src/mongo/util/memory_util_test.cpp b/src/mongo/db/query/util/memory_util_test.cpp index d19ca82e89a..78f7b3098d6 100644 --- a/src/mongo/util/memory_util_test.cpp +++ b/src/mongo/db/query/util/memory_util_test.cpp @@ -27,7 +27,7 @@ * it in the license file. */ -#include "mongo/util/memory_util.h" +#include "mongo/db/query/util/memory_util.h" #include "mongo/unittest/unittest.h" diff --git a/src/mongo/util/SConscript b/src/mongo/util/SConscript index f4124ef733d..c267eb6a63f 100644 --- a/src/mongo/util/SConscript +++ b/src/mongo/util/SConscript @@ -622,16 +622,6 @@ env.Library( ], ) -env.Library( - target='memory_util', - source=[ - 'memory_util.cpp', - ], - LIBDEPS=[ - 'pcre_wrapper', - ], -) - env.Benchmark( target='hash_table_bm', source='hash_table_bm.cpp', @@ -766,7 +756,6 @@ icuEnv.CppUnitTest( 'lru_cache_test.cpp', 'md5_test.cpp', 'md5main.cpp', - 'memory_util_test.cpp', 'optional_util_test.cpp', 'out_of_line_executor_test.cpp', 'overloaded_visitor_test.cpp', @@ -814,7 +803,6 @@ icuEnv.CppUnitTest( 'icu', 'latch_analyzer' if get_option('use-diagnostic-latches') == 'on' else [], 'md5', - 'memory_util', 'pcre_util', 'pcre_wrapper', 'periodic_runner_impl', |