summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorjoshua <80741223+jlap199@users.noreply.github.com>2022-11-12 02:47:38 +0000
committerEvergreen Agent <no-reply@evergreen.mongodb.com>2022-11-12 03:19:26 +0000
commit85b89ff203ec7a70e43af24831b35a3d53a259e4 (patch)
tree8c484e57d3da06c9aa30da7e79796f23526403a7
parent2041a8a461f42c6fc1c257e87e88c6ae695a51a8 (diff)
downloadmongo-85b89ff203ec7a70e43af24831b35a3d53a259e4.tar.gz
SERVER-71065 Implement onTelemetryStoreSizeUpdate function
-rw-r--r--src/mongo/db/query/SConscript27
-rw-r--r--src/mongo/db/query/query_knobs.idl8
-rw-r--r--src/mongo/db/query/sbe_plan_cache.cpp85
-rw-r--r--src/mongo/db/query/sbe_plan_cache_on_parameter_change.h2
-rw-r--r--src/mongo/db/query/telemetry.cpp70
-rw-r--r--src/mongo/db/query/telemetry.h14
-rw-r--r--src/mongo/db/query/telemetry_util.cpp84
-rw-r--r--src/mongo/db/query/telemetry_util.h67
-rw-r--r--src/mongo/db/query/util/memory_util.cpp (renamed from src/mongo/util/memory_util.cpp)58
-rw-r--r--src/mongo/db/query/util/memory_util.h (renamed from src/mongo/util/memory_util.h)11
-rw-r--r--src/mongo/db/query/util/memory_util_test.cpp (renamed from src/mongo/util/memory_util_test.cpp)2
-rw-r--r--src/mongo/util/SConscript12
12 files changed, 309 insertions, 131 deletions
diff --git a/src/mongo/db/query/SConscript b/src/mongo/db/query/SConscript
index 5b21dd4243a..f3cce404031 100644
--- a/src/mongo/db/query/SConscript
+++ b/src/mongo/db/query/SConscript
@@ -90,6 +90,17 @@ env.Library(
)
env.Library(
+ target='memory_util',
+ source=[
+ 'util/memory_util.cpp',
+ ],
+ LIBDEPS=[
+ '$BUILD_DIR/mongo/util/pcre_wrapper',
+ '$BUILD_DIR/mongo/util/processinfo',
+ ],
+)
+
+env.Library(
target="query_plan_cache",
source=[
"classic_plan_cache.cpp",
@@ -98,10 +109,8 @@ env.Library(
"sbe_plan_cache.cpp",
],
LIBDEPS=[
- "$BUILD_DIR/mongo/base",
- "$BUILD_DIR/mongo/db/exec/sbe/query_sbe",
- '$BUILD_DIR/mongo/util/memory_util',
- "canonical_query",
+ "$BUILD_DIR/mongo/base", "$BUILD_DIR/mongo/db/exec/sbe/query_sbe", "canonical_query",
+ "memory_util"
],
)
@@ -262,13 +271,13 @@ env.Library(
'query_feature_flags.idl',
'query_knobs.idl',
'sbe_plan_cache_on_parameter_change.cpp',
+ 'telemetry_util.cpp',
],
LIBDEPS_PRIVATE=[
- '$BUILD_DIR/mongo/db/query/telemetry',
'$BUILD_DIR/mongo/db/server_base',
'$BUILD_DIR/mongo/db/service_context',
- '$BUILD_DIR/mongo/util/memory_util',
'$BUILD_DIR/mongo/util/pcre_wrapper',
+ 'memory_util',
],
)
@@ -354,9 +363,10 @@ env.Library(
],
LIBDEPS=[
'$BUILD_DIR/mongo/base',
- "$BUILD_DIR/mongo/db/concurrency/lock_manager",
- '$BUILD_DIR/mongo/util/memory_util',
+ '$BUILD_DIR/mongo/db/concurrency/lock_manager',
'$BUILD_DIR/mongo/util/processinfo',
+ 'memory_util',
+ 'query_knobs',
],
)
@@ -386,6 +396,7 @@ env.CppUnitTest(
"killcursors_request_test.cpp",
"lru_key_value_test.cpp",
'map_reduce_output_format_test.cpp',
+ "util/memory_util_test.cpp",
"parsed_distinct_test.cpp",
"plan_cache_indexability_test.cpp",
"plan_cache_key_info_test.cpp",
diff --git a/src/mongo/db/query/query_knobs.idl b/src/mongo/db/query/query_knobs.idl
index 5c859d09c8e..c247bdabc47 100644
--- a/src/mongo/db/query/query_knobs.idl
+++ b/src/mongo/db/query/query_knobs.idl
@@ -32,7 +32,7 @@ global:
- "mongo/db/query/cost_model/cost_model_on_update.h"
- "mongo/db/query/ce_mode_parameter.h"
- "mongo/db/query/sbe_plan_cache_on_parameter_change.h"
- - "mongo/db/query/telemetry.h"
+ - "mongo/db/query/telemetry_util.h"
- "mongo/platform/atomic_proxy.h"
- "mongo/platform/atomic_word.h"
@@ -966,12 +966,12 @@ server_parameters:
2. <number>(MB|GB), indicates the amount of memory in MB or GB. E.g.: 1.5GB, 100MB.
The default value is 5%, which means 5% of the physical memory available to the process."
set_at: [ startup, runtime ]
- cpp_varname: "queryTelemetryCacheSize"
+ cpp_varname: "queryTelemetryStoreSize"
cpp_vartype: synchronized_value<std::string>
default: "5%"
- on_update: telemetry_util::onTelemetryCacheSizeUpdate
+ on_update: telemetry_util::onTelemetryStoreSizeUpdate
validator:
- callback: telemetry_util::validateTelemetryCacheSize
+ callback: telemetry_util::validateTelemetryStoreSize
# Note for adding additional query knobs:
diff --git a/src/mongo/db/query/sbe_plan_cache.cpp b/src/mongo/db/query/sbe_plan_cache.cpp
index 18d8b9fc5c9..1498fa28932 100644
--- a/src/mongo/db/query/sbe_plan_cache.cpp
+++ b/src/mongo/db/query/sbe_plan_cache.cpp
@@ -30,9 +30,9 @@
#include "mongo/db/query/sbe_plan_cache.h"
+#include "mongo/db/query/util/memory_util.h"
#include "mongo/db/server_options.h"
#include "mongo/logv2/log.h"
-#include "mongo/util/memory_util.h"
#include "mongo/util/processinfo.h"
#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kQuery
@@ -44,69 +44,23 @@ namespace {
const auto sbePlanCacheDecoration =
ServiceContext::declareDecoration<std::unique_ptr<sbe::PlanCache>>();
-size_t convertToSizeInBytes(const memory_util::MemorySize& memSize) {
- constexpr size_t kBytesInMB = 1024 * 1024;
- constexpr size_t kMBytesInGB = 1024;
-
- double sizeInMB = memSize.size;
-
- switch (memSize.units) {
- case memory_util::MemoryUnits::kPercent:
- sizeInMB *= ProcessInfo::getMemSizeMB() / 100.0;
- break;
- case memory_util::MemoryUnits::kMB:
- break;
- case memory_util::MemoryUnits::kGB:
- sizeInMB *= kMBytesInGB;
- break;
- }
-
- return static_cast<size_t>(sizeInMB * kBytesInMB);
-}
-
-/**
- * Sets upper size limit on the PlanCache size to 500GB or 25% of the system's memory, whichever is
- * smaller.
- */
-size_t capPlanCacheSize(size_t planCacheSize) {
- constexpr size_t kBytesInGB = 1024 * 1024 * 1024;
-
- // Maximum size of the plan cache expressed in bytes.
- constexpr size_t kMaximumPlanCacheSize = 500 * kBytesInGB;
-
- // Maximum size of the plan cache expressed as a share of the memory available to the process.
- const memory_util::MemorySize limitToProcessSize{25, memory_util::MemoryUnits::kPercent};
- const size_t limitToProcessSizeInBytes = convertToSizeInBytes(limitToProcessSize);
-
- // The size will be capped by the minimum of the two values defined above.
- const size_t maxPlanCacheSize = std::min(kMaximumPlanCacheSize, limitToProcessSizeInBytes);
-
- if (planCacheSize > maxPlanCacheSize) {
- planCacheSize = maxPlanCacheSize;
- LOGV2_DEBUG(6007000,
- 1,
- "The plan cache size has been capped",
- "maxPlanCacheSize"_attr = maxPlanCacheSize);
- }
-
- return planCacheSize;
-}
-
-size_t getPlanCacheSizeInBytes(const memory_util::MemorySize& memSize) {
- size_t planCacheSize = convertToSizeInBytes(memSize);
- uassert(5968001,
- "Cache size must be at least 1KB * number of cores",
- planCacheSize >= 1024 * ProcessInfo::getNumCores());
- return capPlanCacheSize(planCacheSize);
-}
class PlanCacheOnParamChangeUpdaterImpl final : public plan_cache_util::OnParamChangeUpdater {
public:
void updateCacheSize(ServiceContext* serviceCtx, memory_util::MemorySize memSize) final {
if (feature_flags::gFeatureFlagSbeFull.isEnabledAndIgnoreFCV()) {
- auto size = getPlanCacheSizeInBytes(memSize);
+ auto newSizeBytes = memory_util::getRequestedMemSizeInBytes(memSize);
+ auto cappedCacheSize = memory_util::capMemorySize(newSizeBytes /*requestedSizeBytes*/,
+ 500 /*maximumSizeGB*/,
+ 25 /*percentTotalSystemMemory*/);
+ if (cappedCacheSize < newSizeBytes) {
+ LOGV2_DEBUG(6007001,
+ 1,
+ "The plan cache size has been capped",
+ "cappedSize"_attr = cappedCacheSize);
+ }
auto& globalPlanCache = sbePlanCacheDecoration(serviceCtx);
- globalPlanCache->reset(size);
+ globalPlanCache->reset(cappedCacheSize);
}
}
@@ -126,10 +80,19 @@ ServiceContext::ConstructorActionRegisterer planCacheRegisterer{
if (feature_flags::gFeatureFlagSbeFull.isEnabledAndIgnoreFCV()) {
auto status = memory_util::MemorySize::parse(planCacheSize.get());
uassertStatusOK(status);
-
- auto size = getPlanCacheSizeInBytes(status.getValue());
+ auto size = memory_util::getRequestedMemSizeInBytes(status.getValue());
+ auto cappedCacheSize = memory_util::capMemorySize(size /*requestedSizeBytes*/,
+ 500 /*maximumSizeGB*/,
+ 25 /*percentTotalSystemMemory*/);
+ if (cappedCacheSize < size) {
+ LOGV2_DEBUG(6007000,
+ 1,
+ "The plan cache size has been capped",
+ "cappedSize"_attr = cappedCacheSize);
+ }
auto& globalPlanCache = sbePlanCacheDecoration(serviceCtx);
- globalPlanCache = std::make_unique<sbe::PlanCache>(size, ProcessInfo::getNumCores());
+ globalPlanCache =
+ std::make_unique<sbe::PlanCache>(cappedCacheSize, ProcessInfo::getNumCores());
}
}};
diff --git a/src/mongo/db/query/sbe_plan_cache_on_parameter_change.h b/src/mongo/db/query/sbe_plan_cache_on_parameter_change.h
index 4135bb7f2b9..415b44d82c3 100644
--- a/src/mongo/db/query/sbe_plan_cache_on_parameter_change.h
+++ b/src/mongo/db/query/sbe_plan_cache_on_parameter_change.h
@@ -32,8 +32,8 @@
#include <string>
#include "mongo/base/status.h"
+#include "mongo/db/query/util/memory_util.h"
#include "mongo/db/service_context.h"
-#include "mongo/util/memory_util.h"
namespace mongo::plan_cache_util {
diff --git a/src/mongo/db/query/telemetry.cpp b/src/mongo/db/query/telemetry.cpp
index 3a78512a9ab..468ebb6a396 100644
--- a/src/mongo/db/query/telemetry.cpp
+++ b/src/mongo/db/query/telemetry.cpp
@@ -33,8 +33,11 @@
#include "mongo/db/concurrency/lock_state.h"
#include "mongo/db/concurrency/locker.h"
#include "mongo/db/query/query_planner_params.h"
-#include "mongo/util/memory_util.h"
+#include "mongo/logv2/log.h"
#include "mongo/util/processinfo.h"
+#include <cstddef>
+
+#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kQuery
namespace mongo {
@@ -94,17 +97,48 @@ private:
const auto telemetryStoreDecoration =
ServiceContext::declareDecoration<std::unique_ptr<TelemetryStoreManager>>();
+class TelemetryOnParamChangeUpdaterImpl final : public telemetry_util::OnParamChangeUpdater {
+public:
+ void updateCacheSize(ServiceContext* serviceCtx, memory_util::MemorySize memSize) final {
+ auto newSizeBytes = memory_util::getRequestedMemSizeInBytes(memSize);
+ auto cappedSize = memory_util::capMemorySize(
+ newSizeBytes /*requestedSize*/, 1 /*maximumSizeGB*/, 25 /*percentTotalSystemMemory*/);
+
+ /* If capped size is less than requested size, the telemetry store has been capped at its
+ * upper limit*/
+ if (cappedSize < newSizeBytes) {
+ LOGV2_DEBUG(7106503,
+ 1,
+ "The telemetry store size has been capped",
+ "cappedSize"_attr = cappedSize);
+ }
+ auto& telemetryStoreManager = telemetryStoreDecoration(serviceCtx);
+ auto&& [telemetryStore, resourceLock] = telemetryStoreManager->getTelemetryStore();
+ telemetryStore->reset(cappedSize);
+ }
+};
+
+
ServiceContext::ConstructorActionRegisterer telemetryStoreManagerRegisterer{
"TelemetryStoreManagerRegisterer", [](ServiceContext* serviceCtx) {
- // TODO SERVER-71065 update telemetry cache size
- (void)internalQueryDisableExclusionProjectionFastPath;
- (void)queryTelemetrySamplingRate;
- (void)queryTelemetryCacheSize;
-
- auto size = 100000; // getPlanCacheSizeInBytes(queryTelemetryCacheSize.get());
+ telemetry_util::telemetryStoreOnParamChangeUpdater(serviceCtx) =
+ std::make_unique<TelemetryOnParamChangeUpdaterImpl>();
+ auto status = memory_util::MemorySize::parse(queryTelemetryStoreSize.get());
+ uassertStatusOK(status);
+ auto size = memory_util::getRequestedMemSizeInBytes(status.getValue());
+ auto cappedStoreSize = memory_util::capMemorySize(
+ size /*requestedSizeBytes*/, 1 /*maximumSizeGB*/, 25 /*percentTotalSystemMemory*/);
+ /* If capped size is less than requested size, the telemetry store has been capped at its
+ * upper limit*/
+ if (cappedStoreSize < size) {
+ LOGV2_DEBUG(7106502,
+ 1,
+ "The telemetry store size has been capped",
+ "cappedSize"_attr = cappedStoreSize);
+ }
auto&& globalTelemetryStoreManager = telemetryStoreDecoration(serviceCtx);
- globalTelemetryStoreManager =
- std::make_unique<TelemetryStoreManager>(serviceCtx, size, ProcessInfo::getNumCores());
+ globalTelemetryStoreManager = std::make_unique<TelemetryStoreManager>(
+ serviceCtx, cappedStoreSize, ProcessInfo::getNumCores());
}};
} // namespace
@@ -118,22 +152,4 @@ std::unique_ptr<TelemetryStore> resetTelemetryStore(ServiceContext* serviceCtx)
return telemetryStoreDecoration(serviceCtx)->resetTelemetryStore();
}
-namespace telemetry_util {
-
-Status onTelemetryCacheSizeUpdate(const std::string& str) {
- auto newSize = memory_util::MemorySize::parse(str);
- if (!newSize.isOK()) {
- return newSize.getStatus();
- }
-
- // TODO SERVER-71065 update telemetry cache size
-
- return Status::OK();
-}
-
-Status validateTelemetryCacheSize(const std::string& str, const boost::optional<TenantId>&) {
- return memory_util::MemorySize::parse(str).getStatus();
-}
-
-} // namespace telemetry_util
} // namespace mongo
diff --git a/src/mongo/db/query/telemetry.h b/src/mongo/db/query/telemetry.h
index 93e2581ad84..1e60df54e04 100644
--- a/src/mongo/db/query/telemetry.h
+++ b/src/mongo/db/query/telemetry.h
@@ -32,6 +32,7 @@
#include "mongo/base/status.h"
#include "mongo/db/concurrency/d_concurrency.h"
#include "mongo/db/query/partitioned_cache.h"
+#include "mongo/db/query/util/memory_util.h"
#include "mongo/db/service_context.h"
namespace mongo {
@@ -135,17 +136,4 @@ std::pair<TelemetryStore*, Lock::ResourceLock> getTelemetryStoreForRead(ServiceC
std::unique_ptr<TelemetryStore> resetTelemetryStore(ServiceContext* serviceCtx);
-namespace telemetry_util {
-
-/**
- * Callback called on a change of telemetryCacheSize parameter.
- */
-Status onTelemetryCacheSizeUpdate(const std::string& str);
-
-/**
- * Callback called on validation of telemetryCacheSize parameter.
- */
-Status validateTelemetryCacheSize(const std::string& str, const boost::optional<TenantId>&);
-
-} // namespace telemetry_util
} // namespace mongo
diff --git a/src/mongo/db/query/telemetry_util.cpp b/src/mongo/db/query/telemetry_util.cpp
new file mode 100644
index 00000000000..dcce8d7452b
--- /dev/null
+++ b/src/mongo/db/query/telemetry_util.cpp
@@ -0,0 +1,84 @@
+/**
+ * Copyright (C) 2022-present MongoDB, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ * As a special exception, the copyright holders give permission to link the
+ * code of portions of this program with the OpenSSL library under certain
+ * conditions as described in each individual source file and distribute
+ * linked combinations including the program with the OpenSSL library. You
+ * must comply with the Server Side Public License in all respects for
+ * all of the code used other than as permitted herein. If you modify file(s)
+ * with this exception, you may extend this exception to your version of the
+ * file(s), but you are not obligated to do so. If you do not wish to do so,
+ * delete this exception statement from your version. If you delete this
+ * exception statement from all source files in the program, then also delete
+ * it in the license file.
+ */
+
+#include "mongo/db/query/telemetry_util.h"
+
+#include "mongo/base/status.h"
+#include "mongo/db/concurrency/d_concurrency.h"
+#include "mongo/db/query/partitioned_cache.h"
+#include "mongo/db/query/util/memory_util.h"
+#include "mongo/db/service_context.h"
+#include "mongo/logv2/log.h"
+
+
+#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kQuery
+
+namespace mongo::telemetry_util {
+
+namespace {
+/**
+ * Given the current 'Client', returns a pointer to the 'ServiceContext' and an interface for
+ * updating the telemetry store.
+ */
+std::pair<ServiceContext*, OnParamChangeUpdater*> getUpdater(const Client& client) {
+ auto serviceCtx = client.getServiceContext();
+ tassert(7106500, "ServiceContext must be non null", serviceCtx);
+
+ auto updater = telemetryStoreOnParamChangeUpdater(serviceCtx).get();
+ tassert(7106501, "Telemetry store size updater must be non null", updater);
+ return {serviceCtx, updater};
+}
+} // namespace
+
+
+Status onTelemetryStoreSizeUpdate(const std::string& str) {
+ auto newSize = memory_util::MemorySize::parse(str);
+ if (!newSize.isOK()) {
+ return newSize.getStatus();
+ }
+
+ // The client is nullptr if the parameter is supplied from the command line. In this case, we
+ // ignore the update event, the parameter will be processed when initializing the service
+ // context.
+ if (auto client = Client::getCurrent()) {
+ auto&& [serviceCtx, updater] = getUpdater(*client);
+ updater->updateCacheSize(serviceCtx, newSize.getValue());
+ }
+
+ return Status::OK();
+}
+
+Status validateTelemetryStoreSize(const std::string& str, const boost::optional<TenantId>&) {
+ return memory_util::MemorySize::parse(str).getStatus();
+}
+
+const Decorable<ServiceContext>::Decoration<std::unique_ptr<OnParamChangeUpdater>>
+ telemetryStoreOnParamChangeUpdater =
+ ServiceContext::declareDecoration<std::unique_ptr<OnParamChangeUpdater>>();
+} // namespace mongo::telemetry_util
diff --git a/src/mongo/db/query/telemetry_util.h b/src/mongo/db/query/telemetry_util.h
new file mode 100644
index 00000000000..0aedd95e26e
--- /dev/null
+++ b/src/mongo/db/query/telemetry_util.h
@@ -0,0 +1,67 @@
+/**
+ * Copyright (C) 2022-present MongoDB, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ * As a special exception, the copyright holders give permission to link the
+ * code of portions of this program with the OpenSSL library under certain
+ * conditions as described in each individual source file and distribute
+ * linked combinations including the program with the OpenSSL library. You
+ * must comply with the Server Side Public License in all respects for
+ * all of the code used other than as permitted herein. If you modify file(s)
+ * with this exception, you may extend this exception to your version of the
+ * file(s), but you are not obligated to do so. If you do not wish to do so,
+ * delete this exception statement from your version. If you delete this
+ * exception statement from all source files in the program, then also delete
+ * it in the license file.
+ */
+
+#pragma once
+
+#include "mongo/base/status.h"
+#include "mongo/db/concurrency/d_concurrency.h"
+#include "mongo/db/query/partitioned_cache.h"
+#include "mongo/db/query/util/memory_util.h"
+
+
+namespace mongo::telemetry_util {
+
+Status onTelemetryStoreSizeUpdate(const std::string& str);
+
+
+Status validateTelemetryStoreSize(const std::string& str, const boost::optional<TenantId>&);
+
+/**
+ * An interface used to modify the telemetry store when query setParameters are modified. This is
+ * done via an interface decorating the 'ServiceContext' in order to avoid a link-time dependency
+ * of the query knobs library on the telemetry code.
+ */
+class OnParamChangeUpdater {
+public:
+ virtual ~OnParamChangeUpdater() = default;
+
+ /**
+ * Resizes the telemetry store decorating 'serviceCtx' to the new size given by 'memSize'. If
+ * the new size is smaller than the old, cache entries are evicted in order to ensure the
+ * cache fits within the new size bound.
+ */
+ virtual void updateCacheSize(ServiceContext* serviceCtx, memory_util::MemorySize memSize) = 0;
+};
+
+/**
+ * Decorated accessor to the 'OnParamChangeUpdater' stored in 'ServiceContext'.
+ */
+extern const Decorable<ServiceContext>::Decoration<std::unique_ptr<OnParamChangeUpdater>>
+ telemetryStoreOnParamChangeUpdater;
+} // namespace mongo::telemetry_util
diff --git a/src/mongo/util/memory_util.cpp b/src/mongo/db/query/util/memory_util.cpp
index 28b5822c508..dbd4f23bee1 100644
--- a/src/mongo/util/memory_util.cpp
+++ b/src/mongo/db/query/util/memory_util.cpp
@@ -26,10 +26,14 @@
* exception statement from all source files in the program, then also delete
* it in the license file.
*/
+#include "mongo/db/query/util/memory_util.h"
+#include "mongo/logv2/log.h"
+#include "mongo/util/pcre.h"
+#include "mongo/util/processinfo.h"
+#include <cstddef>
-#include "mongo/util/memory_util.h"
+#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kQuery
-#include "mongo/util/pcre.h"
namespace mongo::memory_util {
@@ -67,4 +71,54 @@ StatusWith<MemorySize> MemorySize::parse(const std::string& str) {
return MemorySize{size, statusWithUnit.getValue()};
}
+size_t convertToSizeInBytes(const MemorySize& memSize) {
+ constexpr size_t kBytesInMB = 1024 * 1024;
+ constexpr size_t kMBytesInGB = 1024;
+
+ double sizeInMB = memSize.size;
+
+ switch (memSize.units) {
+ case MemoryUnits::kPercent:
+ sizeInMB *= ProcessInfo::getMemSizeMB() / 100.0;
+ break;
+ case MemoryUnits::kMB:
+ break;
+ case MemoryUnits::kGB:
+ sizeInMB *= kMBytesInGB;
+ break;
+ }
+
+ return static_cast<size_t>(sizeInMB * kBytesInMB);
+}
+
+size_t getRequestedMemSizeInBytes(const MemorySize& memSize) {
+ size_t planCacheSize = convertToSizeInBytes(memSize);
+ uassert(5968001,
+ "Cache size must be at least 1KB * number of cores",
+ planCacheSize >= 1024 * ProcessInfo::getNumCores());
+ return planCacheSize;
+}
+
+/**
+ * Sets upper limit on a storage structure's size. Either that structure's maximumSize or to
+ * percentage of the total system's memory (both known at call site), whichever is smaller.
+ */
+size_t capMemorySize(size_t requestedSizeBytes,
+ size_t maximumSizeGB,
+ double percentTotalSystemMemory) {
+ constexpr size_t kBytesInGB = 1024 * 1024 * 1024;
+ // Express maximum size in bytes.
+ const size_t maximumSizeBytes = maximumSizeGB * kBytesInGB;
+ const memory_util::MemorySize limitToProcessSize{percentTotalSystemMemory,
+ memory_util::MemoryUnits::kPercent};
+ const size_t limitToProcessSizeInBytes = convertToSizeInBytes(limitToProcessSize);
+
+ // The size will be capped by the minimum of the two values defined above.
+ const size_t upperLimit = std::min(maximumSizeBytes, limitToProcessSizeInBytes);
+
+ if (requestedSizeBytes > upperLimit) {
+ requestedSizeBytes = upperLimit;
+ }
+ return requestedSizeBytes;
+}
} // namespace mongo::memory_util
diff --git a/src/mongo/util/memory_util.h b/src/mongo/db/query/util/memory_util.h
index d1452dbfee9..345780b4c84 100644
--- a/src/mongo/util/memory_util.h
+++ b/src/mongo/db/query/util/memory_util.h
@@ -31,6 +31,7 @@
#include <string>
+#include "mongo/base/error_codes.h"
#include "mongo/base/status_with.h"
namespace mongo::memory_util {
@@ -44,8 +45,6 @@ enum class MemoryUnits {
kGB,
};
-StatusWith<MemoryUnits> parseUnitString(const std::string& strUnit);
-
/**
* Represents parsed memory size parameter.
*/
@@ -56,4 +55,12 @@ struct MemorySize {
const MemoryUnits units;
};
+StatusWith<MemoryUnits> parseUnitString(const std::string& strUnit);
+size_t convertToSizeInBytes(const MemorySize& memSize);
+size_t capMemorySize(size_t requestedSizeBytes,
+ size_t maximumSizeGB,
+ double percentTotalSystemMemory);
+size_t getRequestedMemSizeInBytes(const MemorySize& memSize);
+
+
} // namespace mongo::memory_util
diff --git a/src/mongo/util/memory_util_test.cpp b/src/mongo/db/query/util/memory_util_test.cpp
index d19ca82e89a..78f7b3098d6 100644
--- a/src/mongo/util/memory_util_test.cpp
+++ b/src/mongo/db/query/util/memory_util_test.cpp
@@ -27,7 +27,7 @@
* it in the license file.
*/
-#include "mongo/util/memory_util.h"
+#include "mongo/db/query/util/memory_util.h"
#include "mongo/unittest/unittest.h"
diff --git a/src/mongo/util/SConscript b/src/mongo/util/SConscript
index f4124ef733d..c267eb6a63f 100644
--- a/src/mongo/util/SConscript
+++ b/src/mongo/util/SConscript
@@ -622,16 +622,6 @@ env.Library(
],
)
-env.Library(
- target='memory_util',
- source=[
- 'memory_util.cpp',
- ],
- LIBDEPS=[
- 'pcre_wrapper',
- ],
-)
-
env.Benchmark(
target='hash_table_bm',
source='hash_table_bm.cpp',
@@ -766,7 +756,6 @@ icuEnv.CppUnitTest(
'lru_cache_test.cpp',
'md5_test.cpp',
'md5main.cpp',
- 'memory_util_test.cpp',
'optional_util_test.cpp',
'out_of_line_executor_test.cpp',
'overloaded_visitor_test.cpp',
@@ -814,7 +803,6 @@ icuEnv.CppUnitTest(
'icu',
'latch_analyzer' if get_option('use-diagnostic-latches') == 'on' else [],
'md5',
- 'memory_util',
'pcre_util',
'pcre_wrapper',
'periodic_runner_impl',