diff options
20 files changed, 213 insertions, 33 deletions
diff --git a/jstests/aggregation/sources/setWindowFields/memory_limit.js b/jstests/aggregation/sources/setWindowFields/memory_limit.js new file mode 100644 index 00000000000..532abb67be8 --- /dev/null +++ b/jstests/aggregation/sources/setWindowFields/memory_limit.js @@ -0,0 +1,78 @@ +/** + * Test that DocumentSourceSetWindowFields errors when using more than the perscribed amount of + * data. Memory checks are per node, so only test when the data is all in one place. + */ + +(function() { +"use strict"; + +load("jstests/libs/fixture_helpers.js"); // For FixtureHelpers.isMongos. +load("jstests/noPassthrough/libs/server_parameter_helpers.js"); // For setParameterOnAllHosts. +load("jstests/libs/discover_topology.js"); // For findNonConfigNodes. + +const featureEnabled = + assert.commandWorked(db.adminCommand({getParameter: 1, featureFlagWindowFunctions: 1})) + .featureFlagWindowFunctions.value; +if (!featureEnabled) { + jsTestLog("Skipping test because the window function feature flag is disabled"); + return; +} +const coll = db[jsTestName()]; +coll.drop(); + +// Test that we can set the memory limit. +setParameterOnAllHosts(DiscoverTopology.findNonConfigNodes(db.getMongo()), + "internalDocumentSourceSetWindowFieldsMaxMemoryBytes", + 1200); + +// Create a collection with enough documents in a single partition to go over the memory limit. +for (let i = 0; i < 10; i++) { + coll.insert({_id: i, partitionKey: 1, str: "str"}); +} + +assert.commandFailedWithCode(coll.runCommand({ + aggregate: coll.getName(), + pipeline: [{$setWindowFields: {sortBy: {partitionKey: 1}, output: {val: {$sum: "$_id"}}}}], + cursor: {} +}), + 5414201); + +// The same query passes with a higher memory limit. +setParameterOnAllHosts(DiscoverTopology.findNonConfigNodes(db.getMongo()), + "internalDocumentSourceSetWindowFieldsMaxMemoryBytes", + 3170); +assert.commandWorked(coll.runCommand({ + aggregate: coll.getName(), + pipeline: [{$setWindowFields: {sortBy: {partitionKey: 1}, output: {val: {$sum: "$_id"}}}}], + cursor: {} +})); +// The query passes with multiple partitions of the same size. +for (let i = 0; i < 10; i++) { + coll.insert({_id: i, partitionKey: 2, str: "str"}); +} +assert.commandWorked(coll.runCommand({ + aggregate: coll.getName(), + pipeline: [{ + $setWindowFields: + {sortBy: {partitionKey: 1}, partitionBy: "$partitionKey", output: {val: {$sum: "$_id"}}} + }], + cursor: {} +})); +// Test that the query fails with a window function that stores documents. +assert.commandFailedWithCode(coll.runCommand({ + aggregate: coll.getName(), + pipeline: [{ + $setWindowFields: { + sortBy: {partitionKey: 1}, + partitionBy: "$partitionKey", + output: {val: {$max: "$_id", window: {documents: [-9, 9]}}} + } + }], + cursor: {} +}), + 5414201); +// Reset limit for other tests. +setParameterOnAllHosts(DiscoverTopology.findNonConfigNodes(db.getMongo()), + "internalDocumentSourceSetWindowFieldsMaxMemoryBytes", + 100 * 1024 * 1024); +})(); diff --git a/jstests/noPassthrough/query_knobs_validation.js b/jstests/noPassthrough/query_knobs_validation.js index 7cc00544b08..f296017a3ac 100644 --- a/jstests/noPassthrough/query_knobs_validation.js +++ b/jstests/noPassthrough/query_knobs_validation.js @@ -37,6 +37,7 @@ const expectedParamDefaults = { internalDocumentSourceLookupCacheSizeBytes: 100 * 1024 * 1024, internalLookupStageIntermediateDocumentMaxSizeBytes: 100 * 1024 * 1024, internalDocumentSourceGroupMaxMemoryBytes: 100 * 1024 * 1024, + internalDocumentSourceSetWindowFieldsMaxMemoryBytes: 100 * 1024 * 1024, internalPipelineLengthLimit: 1000, internalQueryMaxJsEmitBytes: 100 * 1024 * 1024, internalQueryMaxPushBytes: 100 * 1024 * 1024, @@ -150,6 +151,10 @@ assertSetParameterSucceeds("internalDocumentSourceGroupMaxMemoryBytes", 11); assertSetParameterFails("internalDocumentSourceGroupMaxMemoryBytes", 0); assertSetParameterFails("internalDocumentSourceGroupMaxMemoryBytes", -1); +assertSetParameterSucceeds("internalDocumentSourceSetWindowFieldsMaxMemoryBytes", 11); +assertSetParameterFails("internalDocumentSourceSetWindowFieldsMaxMemoryBytes", 0); +assertSetParameterFails("internalDocumentSourceSetWindowFieldsMaxMemoryBytes", -1); + assertSetParameterSucceeds("internalQueryMaxJsEmitBytes", 10); assertSetParameterFails("internalQueryMaxJsEmitBytes", 0); assertSetParameterFails("internalQueryMaxJsEmitBytes", -1); diff --git a/src/mongo/db/pipeline/accumulator.h b/src/mongo/db/pipeline/accumulator.h index 496b5d07d38..98ee003e129 100644 --- a/src/mongo/db/pipeline/accumulator.h +++ b/src/mongo/db/pipeline/accumulator.h @@ -97,7 +97,7 @@ public: /// The name of the op as used in a serialization of the pipeline. virtual const char* getOpName() const = 0; - int memUsageForSorter() const { + int getMemUsage() const { dassert(_memUsageBytes != 0); // This would mean subclass didn't set it return _memUsageBytes; } diff --git a/src/mongo/db/pipeline/document_source_group.cpp b/src/mongo/db/pipeline/document_source_group.cpp index 7987265b028..51c7ccd3fdc 100644 --- a/src/mongo/db/pipeline/document_source_group.cpp +++ b/src/mongo/db/pipeline/document_source_group.cpp @@ -155,10 +155,10 @@ int DocumentSourceGroup::freeMemory() { int totalMemorySaved = 0; for (auto&& group : *_groups) { for (size_t i = 0; i < group.second.size(); i++) { - auto prevMemUsage = group.second[i]->memUsageForSorter(); + auto prevMemUsage = group.second[i]->getMemUsage(); group.second[i]->reduceMemoryConsumptionIfAble(); - auto memorySaved = prevMemUsage - group.second[i]->memUsageForSorter(); + auto memorySaved = prevMemUsage - group.second[i]->getMemUsage(); // Update the memory usage for this AccumulationStatement. _memoryTracker.accumStatementMemoryBytes[i].currentMemoryBytes -= memorySaved; // Update the memory usage for this group. @@ -581,8 +581,8 @@ DocumentSource::GetNextResult DocumentSourceGroup::initialize() { } else { for (size_t i = 0; i < group.size(); i++) { // subtract old mem usage. New usage added back after processing. - _memoryTracker.memoryUsageBytes -= group[i]->memUsageForSorter(); - oldAccumMemUsage[i] = group[i]->memUsageForSorter(); + _memoryTracker.memoryUsageBytes -= group[i]->getMemUsage(); + oldAccumMemUsage[i] = group[i]->getMemUsage(); } } @@ -594,9 +594,9 @@ DocumentSource::GetNextResult DocumentSourceGroup::initialize() { _accumulatedFields[i].expr.argument->evaluate(rootDocument, &pExpCtx->variables), _doingMerge); - _memoryTracker.memoryUsageBytes += group[i]->memUsageForSorter(); + _memoryTracker.memoryUsageBytes += group[i]->getMemUsage(); _memoryTracker.accumStatementMemoryBytes[i].currentMemoryBytes += - group[i]->memUsageForSorter() - oldAccumMemUsage[i]; + group[i]->getMemUsage() - oldAccumMemUsage[i]; } if (kDebugBuild && !storageGlobalParams.readOnly) { diff --git a/src/mongo/db/pipeline/document_source_set_window_fields.cpp b/src/mongo/db/pipeline/document_source_set_window_fields.cpp index aa096914906..a84db98aef3 100644 --- a/src/mongo/db/pipeline/document_source_set_window_fields.cpp +++ b/src/mongo/db/pipeline/document_source_set_window_fields.cpp @@ -37,6 +37,7 @@ #include "mongo/db/pipeline/document_source_sort.h" #include "mongo/db/pipeline/lite_parsed_document_source.h" #include "mongo/db/query/query_feature_flags_gen.h" +#include "mongo/db/query/query_knobs_gen.h" #include "mongo/util/visit_helper.h" using boost::intrusive_ptr; @@ -267,6 +268,7 @@ boost::intrusive_ptr<DocumentSource> DocumentSourceInternalSetWindowFields::crea } void DocumentSourceInternalSetWindowFields::initialize() { + _maxMemory = internalDocumentSourceSetWindowFieldsMaxMemoryBytes.load(); for (auto& wfs : _outputFields) { _executableOutputs[wfs.fieldName] = WindowFunctionExec::create(&_iterator, wfs); } @@ -290,8 +292,13 @@ DocumentSource::GetNextResult DocumentSourceInternalSetWindowFields::doGetNext() // Populate the output document with the result from each window function. MutableDocument addFieldsSpec; + size_t functionMemUsage = 0; for (auto&& [fieldName, function] : _executableOutputs) { addFieldsSpec.addField(fieldName, function->getNext()); + functionMemUsage += function->getApproximateSize(); + uassert(5414201, + "Exceeded memory limit in DocumentSourceSetWindowFields", + functionMemUsage + _iterator.getApproximateSize() < _maxMemory); } // Advance the iterator and handle partition/EOF edge cases. diff --git a/src/mongo/db/pipeline/document_source_set_window_fields.h b/src/mongo/db/pipeline/document_source_set_window_fields.h index d293418a5bc..d1b89076ce4 100644 --- a/src/mongo/db/pipeline/document_source_set_window_fields.h +++ b/src/mongo/db/pipeline/document_source_set_window_fields.h @@ -137,6 +137,7 @@ private: StringMap<std::unique_ptr<WindowFunctionExec>> _executableOutputs; bool _init = false; bool _eof = false; + size_t _maxMemory; }; } // namespace mongo diff --git a/src/mongo/db/pipeline/window_function/partition_iterator.cpp b/src/mongo/db/pipeline/window_function/partition_iterator.cpp index a9a6296b72f..ff7eb143a01 100644 --- a/src/mongo/db/pipeline/window_function/partition_iterator.cpp +++ b/src/mongo/db/pipeline/window_function/partition_iterator.cpp @@ -85,6 +85,8 @@ PartitionIterator::AdvanceResult PartitionIterator::advance() { getNextDocument(); if (_state == IteratorState::kAwaitingAdvanceToEOF) { _cache.clear(); + // Everything should be empty at this point. + _memUsageBytes = 0; _currentIndex = 0; _state = IteratorState::kAdvancedToEOF; return AdvanceResult::kEOF; @@ -105,6 +107,7 @@ PartitionIterator::AdvanceResult PartitionIterator::advance() { // In either of these states, there's no point in reading from the prior document source // because we've already hit EOF. _cache.clear(); + _memUsageBytes = 0; _currentIndex = 0; return AdvanceResult::kEOF; default: @@ -200,10 +203,14 @@ void PartitionIterator::getNextDocument() { advanceToNextPartition(); } else if (_expCtx->getValueComparator().compare(curKey, _partitionKey) != 0) { _nextPartition = NextPartitionState{std::move(doc), std::move(curKey)}; + _memUsageBytes += getNextPartitionStateSize(); _state = IteratorState::kAwaitingAdvanceToNext; - } else + } else { + _memUsageBytes += doc.getApproximateSize(); _cache.emplace_back(std::move(doc)); + } } else { + _memUsageBytes += doc.getApproximateSize(); _cache.emplace_back(std::move(doc)); _state = IteratorState::kIntraPartition; } diff --git a/src/mongo/db/pipeline/window_function/partition_iterator.h b/src/mongo/db/pipeline/window_function/partition_iterator.h index 5a6b34f91bb..65995414550 100644 --- a/src/mongo/db/pipeline/window_function/partition_iterator.h +++ b/src/mongo/db/pipeline/window_function/partition_iterator.h @@ -102,6 +102,15 @@ public: */ boost::optional<std::pair<int, int>> getEndpoints(const WindowBounds& bounds); + /** + * Returns the value in bytes of the data being stored by this partition iterator. Does not + * include the size of the constant size objects being held or the overhead of the data + * structures. + */ + auto getApproximateSize() const { + return _memUsageBytes; + } + private: /** * Retrieves the next document from the prior stage and updates the state accordingly. @@ -116,6 +125,8 @@ private: "Invalid call to PartitionIterator::advanceToNextPartition", _nextPartition != boost::none); _cache.clear(); + // Cache is cleared, and we are moving the _nextPartition value to different positions. + _memUsageBytes = getNextPartitionStateSize(); _cache.emplace_back(std::move(_nextPartition->_doc)); _partitionKey = std::move(_nextPartition->_partitionKey); _nextPartition.reset(); @@ -138,6 +149,17 @@ private: Value _partitionKey; }; boost::optional<NextPartitionState> _nextPartition; + size_t getNextPartitionStateSize() { + if (_nextPartition) { + return _nextPartition->_doc.getApproximateSize() + + _nextPartition->_partitionKey.getApproximateSize(); + } + return 0; + } + + // The value in bytes of the data being held. Does not include the size of the constant size + // data members or overhead of data structures. + size_t _memUsageBytes = 0; enum class IteratorState { // Default state, no documents have been pulled into the cache. diff --git a/src/mongo/db/pipeline/window_function/window_function.h b/src/mongo/db/pipeline/window_function/window_function.h index 9acf4aa9b70..d1d5d817d6a 100644 --- a/src/mongo/db/pipeline/window_function/window_function.h +++ b/src/mongo/db/pipeline/window_function/window_function.h @@ -51,9 +51,14 @@ public: virtual void remove(Value) = 0; virtual Value getValue() const = 0; virtual void reset() = 0; + size_t getApproximateSize() { + tassert(5414200, "_memUsageBytes not set for function", _memUsageBytes != 0); + return _memUsageBytes; + } protected: ExpressionContext* _expCtx; + size_t _memUsageBytes = 0; }; } // namespace mongo diff --git a/src/mongo/db/pipeline/window_function/window_function_add_to_set.h b/src/mongo/db/pipeline/window_function/window_function_add_to_set.h index ee1059aefd9..f9433ca6aa8 100644 --- a/src/mongo/db/pipeline/window_function/window_function_add_to_set.h +++ b/src/mongo/db/pipeline/window_function/window_function_add_to_set.h @@ -43,9 +43,12 @@ public: explicit WindowFunctionAddToSet(ExpressionContext* const expCtx) : WindowFunctionState(expCtx), - _values(_expCtx->getValueComparator().makeOrderedValueMultiset()) {} + _values(_expCtx->getValueComparator().makeOrderedValueMultiset()) { + _memUsageBytes = sizeof(*this); + } void add(Value value) override { + _memUsageBytes += value.getApproximateSize(); _values.insert(std::move(value)); } @@ -56,11 +59,13 @@ public: auto iter = _values.find(std::move(value)); tassert( 5423800, "Can't remove from an empty WindowFunctionAddToSet", iter != _values.end()); + _memUsageBytes -= iter->getApproximateSize(); _values.erase(iter); } void reset() override { _values.clear(); + _memUsageBytes = sizeof(*this); } Value getValue() const override { diff --git a/src/mongo/db/pipeline/window_function/window_function_exec.h b/src/mongo/db/pipeline/window_function/window_function_exec.h index 5217be555d8..9b0de7c7c5c 100644 --- a/src/mongo/db/pipeline/window_function/window_function_exec.h +++ b/src/mongo/db/pipeline/window_function/window_function_exec.h @@ -71,6 +71,11 @@ public: */ virtual void reset() = 0; + /** + * Returns how much memory the accumulators or window functions being held are using. + */ + virtual size_t getApproximateSize() const = 0; + protected: WindowFunctionExec(PartitionIterator* iter) : _iter(iter){}; @@ -101,6 +106,15 @@ public: return _function->getValue(); } + /** + * Return the byte size of the values being stored by this class. Does not include the constant + * size objects being held or the overhead of the data structures. + */ + size_t getApproximateSize() const final { + return _function->getApproximateSize() + _memUsageBytes; + } + + protected: boost::intrusive_ptr<Expression> _input; std::unique_ptr<WindowFunctionState> _function; @@ -113,8 +127,13 @@ protected: void addValue(Value v) { _function->add(v); _values.push(v); + _memUsageBytes += v.getApproximateSize(); } + // Track the byte size of the values being stored by this class. Does not include the constant + // size objects being held or the overhead of the data structures. + size_t _memUsageBytes = 0; + private: virtual void processDocumentsToUpperBound() = 0; virtual void removeDocumentsUnderLowerBound() = 0; diff --git a/src/mongo/db/pipeline/window_function/window_function_exec_derivative.h b/src/mongo/db/pipeline/window_function/window_function_exec_derivative.h index 1fdf4d31630..2d11036b1b2 100644 --- a/src/mongo/db/pipeline/window_function/window_function_exec_derivative.h +++ b/src/mongo/db/pipeline/window_function/window_function_exec_derivative.h @@ -73,6 +73,11 @@ public: Value getNext() final; void reset() final {} + // This executor does not store any documents as it processes. + size_t getApproximateSize() const final { + return 0; + } + private: boost::intrusive_ptr<Expression> _position; boost::intrusive_ptr<Expression> _time; diff --git a/src/mongo/db/pipeline/window_function/window_function_exec_first_last.h b/src/mongo/db/pipeline/window_function/window_function_exec_first_last.h index d7df7a485fb..69ec8d86e5c 100644 --- a/src/mongo/db/pipeline/window_function/window_function_exec_first_last.h +++ b/src/mongo/db/pipeline/window_function/window_function_exec_first_last.h @@ -37,6 +37,12 @@ namespace mongo { class WindowFunctionExecForEndpoint : public WindowFunctionExec { +public: + // Endpoint executors are constant size and don't hold any of the values passing through. + size_t getApproximateSize() const final { + return 0; + } + protected: WindowFunctionExecForEndpoint(PartitionIterator* iter, boost::intrusive_ptr<Expression> input, diff --git a/src/mongo/db/pipeline/window_function/window_function_exec_non_removable.h b/src/mongo/db/pipeline/window_function/window_function_exec_non_removable.h index 150253a6fcc..092abf99b5f 100644 --- a/src/mongo/db/pipeline/window_function/window_function_exec_non_removable.h +++ b/src/mongo/db/pipeline/window_function/window_function_exec_non_removable.h @@ -89,6 +89,10 @@ public: return _function->getValue(false); } + size_t getApproximateSize() const final { + return _function->getMemUsage(); + } + void reset() final { _initialized = false; _function->reset(); diff --git a/src/mongo/db/pipeline/window_function/window_function_exec_removable_document.h b/src/mongo/db/pipeline/window_function/window_function_exec_removable_document.h index 7d5a9a45fff..dc8327368ab 100644 --- a/src/mongo/db/pipeline/window_function/window_function_exec_removable_document.h +++ b/src/mongo/db/pipeline/window_function/window_function_exec_removable_document.h @@ -71,6 +71,7 @@ private: if (_values.size() == 0) { return; } + _memUsageBytes -= _values.front().getApproximateSize(); _function->remove(_values.front()); _values.pop(); } diff --git a/src/mongo/db/pipeline/window_function/window_function_min_max.h b/src/mongo/db/pipeline/window_function/window_function_min_max.h index 97f0f1792c0..65e64f304cd 100644 --- a/src/mongo/db/pipeline/window_function/window_function_min_max.h +++ b/src/mongo/db/pipeline/window_function/window_function_min_max.h @@ -44,10 +44,13 @@ public: explicit WindowFunctionMinMax(ExpressionContext* const expCtx) : WindowFunctionState(expCtx), - _values(_expCtx->getValueComparator().makeOrderedValueMultiset()) {} + _values(_expCtx->getValueComparator().makeOrderedValueMultiset()) { + _memUsageBytes = sizeof(*this); + } void add(Value value) final { _values.insert(std::move(value)); + _memUsageBytes += value.getApproximateSize(); } void remove(Value value) final { @@ -56,11 +59,13 @@ public: // which is what we want, to satisfy "remove() undoes add() when called in FIFO order". auto iter = _values.find(std::move(value)); tassert(5371400, "Can't remove from an empty WindowFunctionMinMax", iter != _values.end()); + _memUsageBytes -= iter->getApproximateSize(); _values.erase(iter); } void reset() final { _values.clear(); + _memUsageBytes = sizeof(*this); } Value getValue() const final { diff --git a/src/mongo/db/pipeline/window_function/window_function_push.h b/src/mongo/db/pipeline/window_function/window_function_push.h index 80b3dae49c9..4ed5b083996 100644 --- a/src/mongo/db/pipeline/window_function/window_function_push.h +++ b/src/mongo/db/pipeline/window_function/window_function_push.h @@ -43,35 +43,32 @@ public: return std::make_unique<WindowFunctionPush>(expCtx); } - explicit WindowFunctionPush(ExpressionContext* const expCtx) - : WindowFunctionState(expCtx), - _values( - _expCtx->getValueComparator().makeOrderedValueMultimap<ValueListConstIterator>()) {} + explicit WindowFunctionPush(ExpressionContext* const expCtx) : WindowFunctionState(expCtx) { + _memUsageBytes = sizeof(*this); + } void add(Value value) override { - _list.emplace_back(std::move(value)); - auto iter = std::prev(_list.end()); - _values.insert({*iter, iter}); + _memUsageBytes += value.getApproximateSize(); + _values.push_back(std::move(value)); } /** * This should only remove the first/lowest element in the window. */ void remove(Value value) override { - // The order of the key-value pairs whose keys compare equivalent is the order of insertion - // and does not change in std::multimap. So find() / erase() will remove the oldest equal - // element, which is what we want, to satisfy "remove() undoes add() when called in FIFO - // order". - auto iter = _values.find(std::move(value)); - tassert(5423801, "Can't remove from an empty WindowFunctionPush", iter != _values.end()); - // Erase the element from both '_values' and '_list'. - _list.erase(iter->second); - _values.erase(iter); + tassert(5423801, "Can't remove from an empty WindowFunctionPush", _values.size() != 0); + auto valToRemove = _values.front(); + tassert( + 5414202, + "Attempted to remove an element other than the first element from WindowFunctionPush", + _expCtx->getValueComparator().evaluate(valToRemove == value)); + _values.pop_front(); + _memUsageBytes -= value.getApproximateSize(); } void reset() override { _values.clear(); - _list.clear(); + _memUsageBytes = sizeof(*this); } Value getValue() const override { @@ -79,14 +76,11 @@ public: if (_values.empty()) return kDefault; - return Value{std::vector<Value>(_list.begin(), _list.end())}; + return Value{std::vector<Value>(_values.begin(), _values.end())}; } private: - ValueMultimap<ValueListConstIterator> _values; - // std::list makes sure that the order of the elements in the returned array is the order of - // insertion. - std::list<Value> _list; + std::deque<Value> _values; }; } // namespace mongo diff --git a/src/mongo/db/pipeline/window_function/window_function_stddev.h b/src/mongo/db/pipeline/window_function/window_function_stddev.h index bd2eed4a544..5d818c06479 100644 --- a/src/mongo/db/pipeline/window_function/window_function_stddev.h +++ b/src/mongo/db/pipeline/window_function/window_function_stddev.h @@ -41,7 +41,9 @@ protected: _m2(AccumulatorSum::create(expCtx)), _isSamp(isSamp), _count(0), - _nonfiniteValueCount(0) {} + _nonfiniteValueCount(0) { + _memUsageBytes = sizeof(*this); + } public: static Value getDefault() { @@ -78,6 +80,7 @@ public: void reset() { _m2->reset(); _sum->reset(); + _memUsageBytes = sizeof(*this); _count = 0; _nonfiniteValueCount = 0; } @@ -106,6 +109,7 @@ private: _count += quantity; _sum->process(Value{value.coerceToDouble() * quantity}, false); _m2->process(Value{x * x * quantity / (_count * (_count - quantity))}, false); + _memUsageBytes = sizeof(*this) + _sum->getMemUsage() + _m2->getMemUsage(); } // Std dev cannot make use of RemovableSum because of its specific handling of non-finite diff --git a/src/mongo/db/query/explain_common.cpp b/src/mongo/db/query/explain_common.cpp index 2818d332ea3..1bfe9da481a 100644 --- a/src/mongo/db/query/explain_common.cpp +++ b/src/mongo/db/query/explain_common.cpp @@ -63,6 +63,8 @@ void generateServerParameters(BSONObjBuilder* out) { out->appendNumber("internalQueryProhibitBlockingMergeOnMongoS", internalQueryProhibitBlockingMergeOnMongoS.load()); out->appendNumber("internalQueryMaxAddToSetBytes", internalQueryMaxAddToSetBytes.load()); + out->appendNumber("internalDocumentSourceSetWindowFieldsMaxMemoryBytes", + internalDocumentSourceSetWindowFieldsMaxMemoryBytes.load()); } bool appendIfRoom(const BSONObj& toAppend, StringData fieldName, BSONObjBuilder* out) { diff --git a/src/mongo/db/query/query_knobs.idl b/src/mongo/db/query/query_knobs.idl index 8d3527f035a..4f606a45fa1 100644 --- a/src/mongo/db/query/query_knobs.idl +++ b/src/mongo/db/query/query_knobs.idl @@ -301,6 +301,16 @@ server_parameters: expr: 100 * 1024 * 1024 validator: gt: 0 + + internalDocumentSourceSetWindowFieldsMaxMemoryBytes: + description: "Maximum size of the data that the $setWindowFields aggregation stage will cache in-memory before throwing an error." + set_at: [ startup, runtime ] + cpp_varname: "internalDocumentSourceSetWindowFieldsMaxMemoryBytes" + cpp_vartype: AtomicWord<long long> + default: + expr: 100 * 1024 * 1024 + validator: + gt: 0 internalInsertMaxBatchSize: description: "Maximum number of documents that we will insert in a single batch." |