diff options
author | Martin Neupauer <martin.neupauer@mongodb.com> | 2022-09-23 12:39:08 +0000 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2022-09-23 13:15:15 +0000 |
commit | 125607bc02dfe1815bcef4eac4cdd6246b6445a7 (patch) | |
tree | adef1e642b6a1ec555a0da3d865837f8c0910f81 | |
parent | 774c0ba3616556db2d034675e5979c05611d43d9 (diff) | |
download | mongo-125607bc02dfe1815bcef4eac4cdd6246b6445a7.tar.gz |
SERVER-68844 Improve VM
-rw-r--r-- | src/mongo/db/exec/sbe/values/arith_common.cpp | 32 | ||||
-rw-r--r-- | src/mongo/db/exec/sbe/values/arith_common.h | 24 | ||||
-rw-r--r-- | src/mongo/db/exec/sbe/values/value.h | 18 | ||||
-rw-r--r-- | src/mongo/db/exec/sbe/vm/arith.cpp | 124 | ||||
-rw-r--r-- | src/mongo/db/exec/sbe/vm/datetime.cpp | 20 | ||||
-rw-r--r-- | src/mongo/db/exec/sbe/vm/vm.cpp | 672 | ||||
-rw-r--r-- | src/mongo/db/exec/sbe/vm/vm.h | 650 |
7 files changed, 702 insertions, 838 deletions
diff --git a/src/mongo/db/exec/sbe/values/arith_common.cpp b/src/mongo/db/exec/sbe/values/arith_common.cpp index db920191534..358dcdb65e7 100644 --- a/src/mongo/db/exec/sbe/values/arith_common.cpp +++ b/src/mongo/db/exec/sbe/values/arith_common.cpp @@ -115,10 +115,10 @@ struct Multiplication { * standard numeric types and also operations on the Date type. */ template <typename Op> -std::tuple<bool, value::TypeTags, value::Value> genericArithmeticOp(value::TypeTags lhsTag, - value::Value lhsValue, - value::TypeTags rhsTag, - value::Value rhsValue) { +FastTuple<bool, value::TypeTags, value::Value> genericArithmeticOp(value::TypeTags lhsTag, + value::Value lhsValue, + value::TypeTags rhsTag, + value::Value rhsValue) { if (value::isNumber(lhsTag) && value::isNumber(rhsTag)) { switch (getWidestNumericalType(lhsTag, rhsTag)) { case value::TypeTags::NumberInt32: { @@ -247,24 +247,24 @@ std::tuple<bool, value::TypeTags, value::Value> genericArithmeticOp(value::TypeT return {false, value::TypeTags::Nothing, 0}; } -std::tuple<bool, value::TypeTags, value::Value> genericAdd(value::TypeTags lhsTag, - value::Value lhsValue, - value::TypeTags rhsTag, - value::Value rhsValue) { +FastTuple<bool, value::TypeTags, value::Value> genericAdd(value::TypeTags lhsTag, + value::Value lhsValue, + value::TypeTags rhsTag, + value::Value rhsValue) { return genericArithmeticOp<Addition>(lhsTag, lhsValue, rhsTag, rhsValue); } -std::tuple<bool, value::TypeTags, value::Value> genericSub(value::TypeTags lhsTag, - value::Value lhsValue, - value::TypeTags rhsTag, - value::Value rhsValue) { +FastTuple<bool, value::TypeTags, value::Value> genericSub(value::TypeTags lhsTag, + value::Value lhsValue, + value::TypeTags rhsTag, + value::Value rhsValue) { return genericArithmeticOp<Subtraction>(lhsTag, lhsValue, rhsTag, rhsValue); } -std::tuple<bool, value::TypeTags, value::Value> genericMul(value::TypeTags lhsTag, - value::Value lhsValue, - value::TypeTags rhsTag, - value::Value rhsValue) { +FastTuple<bool, value::TypeTags, value::Value> genericMul(value::TypeTags lhsTag, + value::Value lhsValue, + value::TypeTags rhsTag, + value::Value rhsValue) { return genericArithmeticOp<Multiplication>(lhsTag, lhsValue, rhsTag, rhsValue); } diff --git a/src/mongo/db/exec/sbe/values/arith_common.h b/src/mongo/db/exec/sbe/values/arith_common.h index 5f2ace987e7..4e7682d0a3b 100644 --- a/src/mongo/db/exec/sbe/values/arith_common.h +++ b/src/mongo/db/exec/sbe/values/arith_common.h @@ -32,16 +32,16 @@ namespace mongo::sbe::value { -std::tuple<bool, value::TypeTags, value::Value> genericAdd(value::TypeTags lhsTag, - value::Value lhsValue, - value::TypeTags rhsTag, - value::Value rhsValue); -std::tuple<bool, value::TypeTags, value::Value> genericSub(value::TypeTags lhsTag, - value::Value lhsValue, - value::TypeTags rhsTag, - value::Value rhsValue); -std::tuple<bool, value::TypeTags, value::Value> genericMul(value::TypeTags lhsTag, - value::Value lhsValue, - value::TypeTags rhsTag, - value::Value rhsValue); +FastTuple<bool, value::TypeTags, value::Value> genericAdd(value::TypeTags lhsTag, + value::Value lhsValue, + value::TypeTags rhsTag, + value::Value rhsValue); +FastTuple<bool, value::TypeTags, value::Value> genericSub(value::TypeTags lhsTag, + value::Value lhsValue, + value::TypeTags rhsTag, + value::Value rhsValue); +FastTuple<bool, value::TypeTags, value::Value> genericMul(value::TypeTags lhsTag, + value::Value lhsValue, + value::TypeTags rhsTag, + value::Value rhsValue); } // namespace mongo::sbe::value diff --git a/src/mongo/db/exec/sbe/values/value.h b/src/mongo/db/exec/sbe/values/value.h index 082ee4d3523..208156b9865 100644 --- a/src/mongo/db/exec/sbe/values/value.h +++ b/src/mongo/db/exec/sbe/values/value.h @@ -69,6 +69,22 @@ class TimeZoneDatabase; class JsFunction; namespace sbe { +/** + * Trivially copyable variation on a tuple theme. This allow us to return tuples through registers. + */ +template <typename...> +struct FastTuple; + +template <typename... Ts> +FastTuple(Ts...)->FastTuple<Ts...>; + +template <typename A, typename B, typename C> +struct FastTuple<A, B, C> { + A a; + B b; + C c; +}; + using FrameId = int64_t; using SpoolId = int64_t; @@ -1474,7 +1490,7 @@ inline T numericCast(TypeTags tag, Value val) noexcept { * TypeTag. In the case that a conversion is lossy, we return Nothing. */ template <typename T> -inline std::tuple<bool, value::TypeTags, value::Value> numericConvLossless( +inline FastTuple<bool, value::TypeTags, value::Value> numericConvLossless( T value, value::TypeTags targetTag) { switch (targetTag) { case value::TypeTags::NumberInt32: { diff --git a/src/mongo/db/exec/sbe/vm/arith.cpp b/src/mongo/db/exec/sbe/vm/arith.cpp index e75cd98bcfc..a325d31f676 100644 --- a/src/mongo/db/exec/sbe/vm/arith.cpp +++ b/src/mongo/db/exec/sbe/vm/arith.cpp @@ -188,8 +188,8 @@ struct Tanh { * computation of the respective trigonometric function. */ template <typename TrigFunction> -std::tuple<bool, value::TypeTags, value::Value> genericTrigonometricFun(value::TypeTags argTag, - value::Value argValue) { +FastTuple<bool, value::TypeTags, value::Value> genericTrigonometricFun(value::TypeTags argTag, + value::Value argValue) { if (value::isNumber(argTag)) { switch (argTag) { case value::TypeTags::NumberInt32: { @@ -395,7 +395,7 @@ void ByteCode::aggStdDevImpl(value::Array* arr, value::TypeTags rhsTag, value::V return setStdDevArray(newCountVal, newMeanVal, newM2Val, arr); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::aggStdDevFinalizeImpl( +FastTuple<bool, value::TypeTags, value::Value> ByteCode::aggStdDevFinalizeImpl( value::Value fieldValue, bool isSamp) { auto arr = value::getArrayView(fieldValue); @@ -423,10 +423,10 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::aggStdDevFinalizeImpl( return {true, value::TypeTags::NumberDouble, value::bitcastFrom<double>(stdDev)}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::genericDiv(value::TypeTags lhsTag, - value::Value lhsValue, - value::TypeTags rhsTag, - value::Value rhsValue) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::genericDiv(value::TypeTags lhsTag, + value::Value lhsValue, + value::TypeTags rhsTag, + value::Value rhsValue) { auto assertNonZero = [](bool nonZero) { uassert(4848401, "can't $divide by zero", nonZero); }; if (value::isNumber(lhsTag) && value::isNumber(rhsTag)) { @@ -464,10 +464,10 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::genericDiv(value::Type return {false, value::TypeTags::Nothing, 0}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::genericIDiv(value::TypeTags lhsTag, - value::Value lhsValue, - value::TypeTags rhsTag, - value::Value rhsValue) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::genericIDiv(value::TypeTags lhsTag, + value::Value lhsValue, + value::TypeTags rhsTag, + value::Value rhsValue) { auto assertNonZero = [](bool nonZero) { uassert(4848402, "can't $divide by zero", nonZero); }; if (value::isNumber(lhsTag) && value::isNumber(rhsTag)) { @@ -516,10 +516,10 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::genericIDiv(value::Typ return {false, value::TypeTags::Nothing, 0}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::genericMod(value::TypeTags lhsTag, - value::Value lhsValue, - value::TypeTags rhsTag, - value::Value rhsValue) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::genericMod(value::TypeTags lhsTag, + value::Value lhsValue, + value::TypeTags rhsTag, + value::Value rhsValue) { auto assertNonZero = [](bool nonZero) { uassert(4848403, "can't $mod by zero", nonZero); }; if (value::isNumber(lhsTag) && value::isNumber(rhsTag)) { @@ -557,7 +557,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::genericMod(value::Type return {false, value::TypeTags::Nothing, 0}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::genericNumConvert( +FastTuple<bool, value::TypeTags, value::Value> ByteCode::genericNumConvert( value::TypeTags lhsTag, value::Value lhsValue, value::TypeTags targetTag) { if (value::isNumber(lhsTag)) { switch (lhsTag) { @@ -577,8 +577,8 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::genericNumConvert( return {false, value::TypeTags::Nothing, 0}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::genericAbs(value::TypeTags operandTag, - value::Value operandValue) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::genericAbs(value::TypeTags operandTag, + value::Value operandValue) { switch (operandTag) { case value::TypeTags::NumberInt32: { auto operand = value::bitcastTo<int32_t>(operandValue); @@ -618,8 +618,8 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::genericAbs(value::Type } } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::genericCeil(value::TypeTags operandTag, - value::Value operandValue) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::genericCeil(value::TypeTags operandTag, + value::Value operandValue) { if (isNumber(operandTag)) { switch (operandTag) { case value::TypeTags::NumberDouble: { @@ -645,8 +645,8 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::genericCeil(value::Typ return {false, value::TypeTags::Nothing, 0}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::genericFloor(value::TypeTags operandTag, - value::Value operandValue) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::genericFloor(value::TypeTags operandTag, + value::Value operandValue) { if (isNumber(operandTag)) { switch (operandTag) { case value::TypeTags::NumberDouble: { @@ -672,8 +672,8 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::genericFloor(value::Ty return {false, value::TypeTags::Nothing, 0}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::genericTrunc(value::TypeTags operandTag, - value::Value operandValue) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::genericTrunc(value::TypeTags operandTag, + value::Value operandValue) { if (!isNumber(operandTag)) { return {false, value::TypeTags::Nothing, 0}; } @@ -701,8 +701,8 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::genericTrunc(value::Ty } } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::genericExp(value::TypeTags operandTag, - value::Value operandValue) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::genericExp(value::TypeTags operandTag, + value::Value operandValue) { switch (operandTag) { case value::TypeTags::NumberDouble: { auto result = exp(value::bitcastTo<double>(operandValue)); @@ -725,8 +725,8 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::genericExp(value::Type } } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::genericLn(value::TypeTags operandTag, - value::Value operandValue) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::genericLn(value::TypeTags operandTag, + value::Value operandValue) { switch (operandTag) { case value::TypeTags::NumberDouble: { auto operand = value::bitcastTo<double>(operandValue); @@ -767,8 +767,8 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::genericLn(value::TypeT } } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::genericLog10(value::TypeTags operandTag, - value::Value operandValue) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::genericLog10(value::TypeTags operandTag, + value::Value operandValue) { switch (operandTag) { case value::TypeTags::NumberDouble: { auto operand = value::bitcastTo<double>(operandValue); @@ -808,8 +808,8 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::genericLog10(value::Ty } } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::genericSqrt(value::TypeTags operandTag, - value::Value operandValue) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::genericSqrt(value::TypeTags operandTag, + value::Value operandValue) { switch (operandTag) { case value::TypeTags::NumberDouble: { auto operand = value::bitcastTo<double>(operandValue); @@ -883,40 +883,40 @@ std::pair<value::TypeTags, value::Value> ByteCode::compare3way(value::TypeTags l return value::compareValue(lhsTag, lhsValue, rhsTag, rhsValue, comparator); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::genericAcos(value::TypeTags argTag, - value::Value argValue) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::genericAcos(value::TypeTags argTag, + value::Value argValue) { return genericTrigonometricFun<Acos>(argTag, argValue); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::genericAcosh(value::TypeTags argTag, - value::Value argValue) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::genericAcosh(value::TypeTags argTag, + value::Value argValue) { return genericTrigonometricFun<Acosh>(argTag, argValue); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::genericAsin(value::TypeTags argTag, - value::Value argValue) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::genericAsin(value::TypeTags argTag, + value::Value argValue) { return genericTrigonometricFun<Asin>(argTag, argValue); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::genericAsinh(value::TypeTags argTag, - value::Value argValue) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::genericAsinh(value::TypeTags argTag, + value::Value argValue) { return genericTrigonometricFun<Asinh>(argTag, argValue); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::genericAtan(value::TypeTags argTag, - value::Value argValue) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::genericAtan(value::TypeTags argTag, + value::Value argValue) { return genericTrigonometricFun<Atan>(argTag, argValue); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::genericAtanh(value::TypeTags argTag, - value::Value argValue) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::genericAtanh(value::TypeTags argTag, + value::Value argValue) { return genericTrigonometricFun<Atanh>(argTag, argValue); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::genericAtan2(value::TypeTags argTag1, - value::Value argValue1, - value::TypeTags argTag2, - value::Value argValue2) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::genericAtan2(value::TypeTags argTag1, + value::Value argValue1, + value::TypeTags argTag2, + value::Value argValue2) { if (value::isNumber(argTag1) && value::isNumber(argTag2)) { switch (getWidestNumericalType(argTag1, argTag2)) { case value::TypeTags::NumberInt32: @@ -939,17 +939,17 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::genericAtan2(value::Ty return {false, value::TypeTags::Nothing, 0}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::genericCos(value::TypeTags argTag, - value::Value argValue) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::genericCos(value::TypeTags argTag, + value::Value argValue) { return genericTrigonometricFun<Cos>(argTag, argValue); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::genericCosh(value::TypeTags argTag, - value::Value argValue) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::genericCosh(value::TypeTags argTag, + value::Value argValue) { return genericTrigonometricFun<Cosh>(argTag, argValue); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::genericDegreesToRadians( +FastTuple<bool, value::TypeTags, value::Value> ByteCode::genericDegreesToRadians( value::TypeTags argTag, value::Value argValue) { if (value::isNumber(argTag)) { switch (argTag) { @@ -972,7 +972,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::genericDegreesToRadian return {false, value::TypeTags::Nothing, 0}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::genericRadiansToDegrees( +FastTuple<bool, value::TypeTags, value::Value> ByteCode::genericRadiansToDegrees( value::TypeTags argTag, value::Value argValue) { if (value::isNumber(argTag)) { switch (argTag) { @@ -995,23 +995,23 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::genericRadiansToDegree return {false, value::TypeTags::Nothing, 0}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::genericSin(value::TypeTags argTag, - value::Value argValue) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::genericSin(value::TypeTags argTag, + value::Value argValue) { return genericTrigonometricFun<Sin>(argTag, argValue); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::genericSinh(value::TypeTags argTag, - value::Value argValue) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::genericSinh(value::TypeTags argTag, + value::Value argValue) { return genericTrigonometricFun<Sinh>(argTag, argValue); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::genericTan(value::TypeTags argTag, - value::Value argValue) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::genericTan(value::TypeTags argTag, + value::Value argValue) { return genericTrigonometricFun<Tan>(argTag, argValue); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::genericTanh(value::TypeTags argTag, - value::Value argValue) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::genericTanh(value::TypeTags argTag, + value::Value argValue) { return genericTrigonometricFun<Tanh>(argTag, argValue); } diff --git a/src/mongo/db/exec/sbe/vm/datetime.cpp b/src/mongo/db/exec/sbe/vm/datetime.cpp index 84839a4a2d4..710e83508b3 100644 --- a/src/mongo/db/exec/sbe/vm/datetime.cpp +++ b/src/mongo/db/exec/sbe/vm/datetime.cpp @@ -122,12 +122,12 @@ struct DayOfWeek { * This is a simple dayOf operation templated by the Op parameter. */ template <typename Op> -std::tuple<bool, value::TypeTags, value::Value> genericDayOfOp(value::TypeTags timezoneDBTag, - value::Value timezoneDBValue, - value::TypeTags dateTag, - value::Value dateValue, - value::TypeTags timezoneTag, - value::Value timezoneValue) { +FastTuple<bool, value::TypeTags, value::Value> genericDayOfOp(value::TypeTags timezoneDBTag, + value::Value timezoneDBValue, + value::TypeTags dateTag, + value::Value dateValue, + value::TypeTags timezoneTag, + value::Value timezoneValue) { // Get date. if (dateTag != value::TypeTags::Date && dateTag != value::TypeTags::Timestamp && dateTag != value::TypeTags::ObjectId && dateTag != value::TypeTags::bsonObjectId) { @@ -149,10 +149,10 @@ std::tuple<bool, value::TypeTags, value::Value> genericDayOfOp(value::TypeTags t int32_t result; Op::doOperation(date, timezone, result); - return {false, value::TypeTags::NumberInt32, value::bitcastTo<int32_t>(result)}; + return {false, value::TypeTags::NumberInt32, value::bitcastFrom<int32_t>(result)}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::genericDayOfYear( +FastTuple<bool, value::TypeTags, value::Value> ByteCode::genericDayOfYear( value::TypeTags timezoneDBTag, value::Value timezoneDBValue, value::TypeTags dateTag, @@ -163,7 +163,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::genericDayOfYear( timezoneDBTag, timezoneDBValue, dateTag, dateValue, timezoneTag, timezoneValue); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::genericDayOfMonth( +FastTuple<bool, value::TypeTags, value::Value> ByteCode::genericDayOfMonth( value::TypeTags timezoneDBTag, value::Value timezoneDBValue, value::TypeTags dateTag, @@ -174,7 +174,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::genericDayOfMonth( timezoneDBTag, timezoneDBValue, dateTag, dateValue, timezoneTag, timezoneValue); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::genericDayOfWeek( +FastTuple<bool, value::TypeTags, value::Value> ByteCode::genericDayOfWeek( value::TypeTags timezoneDBTag, value::Value timezoneDBValue, value::TypeTags dateTag, diff --git a/src/mongo/db/exec/sbe/vm/vm.cpp b/src/mongo/db/exec/sbe/vm/vm.cpp index 7a6790380ea..193d0a0e437 100644 --- a/src/mongo/db/exec/sbe/vm/vm.cpp +++ b/src/mongo/db/exec/sbe/vm/vm.cpp @@ -158,25 +158,15 @@ int Instruction::stackOffset[Instruction::Tags::lastInstruction] = { 0, // applyClassicMatcher }; -namespace { -template <typename T> -T readFromMemory(const uint8_t* ptr) noexcept { - static_assert(!IsEndian<T>::value); - - T val; - memcpy(&val, ptr, sizeof(T)); - return val; +void ByteCode::growAndResize() noexcept { + invariant(_argStackTop == _argStackEnd); + auto oldSize = (_argStackEnd - _argStack) / sizeOfElement; + auto newSize = oldSize * 3 / 2; + _argStack = reinterpret_cast<uint8_t*>(mongoRealloc(_argStack, newSize * sizeOfElement)); + _argStackEnd = _argStack + newSize * sizeOfElement; + _argStackTop = _argStack + oldSize * sizeOfElement; } -template <typename T> -size_t writeToMemory(uint8_t* ptr, const T val) noexcept { - static_assert(!IsEndian<T>::value); - - memcpy(ptr, &val, sizeof(T)); - return sizeof(T); -} -} // namespace - std::string CodeFragment::toString() const { std::ostringstream ss; auto pcPointer = _instrs.data(); @@ -720,52 +710,10 @@ void CodeFragment::appendJumpNothing(int jumpOffset) { offset += writeToMemory(offset, jumpOffset); } -void ByteCode::Stack::growAndResize(size_t newSize) { - auto currentCapacity = capacity(); - if (newSize <= currentCapacity) { - _size = newSize; - return; - } - - auto newCapacity = newSize; - if (newCapacity > kMaxCapacity) { - uasserted(6040901, - str::stream() << "Requested capacity of " << newCapacity - << " elements exceeds the maximum capacity of " << kMaxCapacity); - return; - } - - if (currentCapacity >= kMaxCapacity / 2) { - newCapacity = kMaxCapacity; - } else if (2 * currentCapacity > newCapacity) { - newCapacity = 2 * currentCapacity; - } - - try { - auto numSegments = (_size + ElementsPerSegment - 1) / ElementsPerSegment; - auto numNewSegments = (newCapacity + ElementsPerSegment - 1) / ElementsPerSegment; - newCapacity = numNewSegments * ElementsPerSegment; - - auto newSegments = std::make_unique<StackSegment[]>(numNewSegments); - - if (_segments.get() != nullptr && numSegments > 0) { - memcpy(newSegments.get(), _segments.get(), numSegments * sizeof(StackSegment)); - } - - _segments = std::move(newSegments); - _capacity = newCapacity; - _size = newSize; - } catch (std::bad_alloc&) { - uasserted(6040902, - str::stream() << "Unable to allocate requested capacity of " << newCapacity - << " elements"); - } -} - -std::tuple<bool, value::TypeTags, value::Value> ByteCode::getField(value::TypeTags objTag, - value::Value objValue, - value::TypeTags fieldTag, - value::Value fieldValue) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::getField(value::TypeTags objTag, + value::Value objValue, + value::TypeTags fieldTag, + value::Value fieldValue) { if (!value::isString(fieldTag)) { return {false, value::TypeTags::Nothing, 0}; } @@ -775,9 +723,9 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::getField(value::TypeTa return getField(objTag, objValue, fieldStr); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::getField(value::TypeTags objTag, - value::Value objValue, - StringData fieldStr) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::getField(value::TypeTags objTag, + value::Value objValue, + StringData fieldStr) { if (MONGO_unlikely(failOnPoisonedFieldLookup.shouldFail())) { uassert(4623399, "Lookup of $POISON", fieldStr != "POISON"); } @@ -804,10 +752,10 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::getField(value::TypeTa return {false, value::TypeTags::Nothing, 0}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::getElement(value::TypeTags arrTag, - value::Value arrValue, - value::TypeTags idxTag, - value::Value idxValue) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::getElement(value::TypeTags arrTag, + value::Value arrValue, + value::TypeTags idxTag, + value::Value idxValue) { // We need to ensure that 'size_t' is wide enough to store 32-bit index. static_assert(sizeof(size_t) >= sizeof(int32_t), "size_t must be at least 32-bits"); @@ -898,7 +846,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::getElement(value::Type } } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::getFieldOrElement( +FastTuple<bool, value::TypeTags, value::Value> ByteCode::getFieldOrElement( value::TypeTags objTag, value::Value objValue, value::TypeTags fieldTag, @@ -1057,13 +1005,14 @@ void ByteCode::traverseFInArray(const CodeFragment* code, int64_t position, bool // Transfer the ownership to the lambda pushStack(ownInput, tagInput, valInput); input.reset(); - return runLambdaInternal(code, position); + runLambdaInternal(code, position); + return; } pushStack(false, value::TypeTags::Boolean, value::bitcastFrom<bool>(false)); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::setField() { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::setField() { auto [newOwn, newTag, newVal] = moveFromStack(0); auto [fieldOwn, fieldTag, fieldVal] = getFromStack(1); // Consider using a moveFromStack optimization. @@ -1164,8 +1113,8 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::setField() { return {false, value::TypeTags::Nothing, 0}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::getArraySize(value::TypeTags tag, - value::Value val) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::getArraySize(value::TypeTags tag, + value::Value val) { size_t result = 0; switch (tag) { @@ -1190,10 +1139,10 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::getArraySize(value::Ty return {false, value::TypeTags::NumberInt64, value::bitcastFrom<int64_t>(result)}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::aggSum(value::TypeTags accTag, - value::Value accValue, - value::TypeTags fieldTag, - value::Value fieldValue) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::aggSum(value::TypeTags accTag, + value::Value accValue, + value::TypeTags fieldTag, + value::Value fieldValue) { // Skip aggregation step if we don't have the input. if (fieldTag == value::TypeTags::Nothing) { auto [tag, val] = value::copyValue(accTag, accValue); @@ -1209,7 +1158,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::aggSum(value::TypeTags return genericAdd(accTag, accValue, fieldTag, fieldValue); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinAggDoubleDoubleSum( +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinAggDoubleDoubleSum( ArityType arity) { auto [_, fieldTag, fieldValue] = getFromStack(1); @@ -1244,7 +1193,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinAggDoubleDouble // This function is necessary because 'aggDoubleDoubleSum()' result is 'Array' type but we need // to produce a scalar value out of it. -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinDoubleDoubleSumFinalize( +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinDoubleDoubleSumFinalize( ArityType arity) { auto [_, fieldTag, fieldValue] = getFromStack(0); auto arr = value::getArrayView(fieldValue); @@ -1314,7 +1263,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinDoubleDoubleSum } } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinDoubleDoublePartialSumFinalize( +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinDoubleDoublePartialSumFinalize( ArityType arity) { auto [_, fieldTag, fieldValue] = getFromStack(0); @@ -1391,7 +1340,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinDoubleDoublePar return {true, tag, val}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinAggStdDev(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinAggStdDev(ArityType arity) { auto [_, fieldTag, fieldValue] = getFromStack(1); // Move the incoming accumulator state from the stack. Given that we are now the owner of the // state we are free to do any in-place update as we see fit. @@ -1420,25 +1369,24 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinAggStdDev(Arity return {true, accTag, accValue}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinStdDevPopFinalize( - ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinStdDevPopFinalize(ArityType arity) { auto [_, fieldTag, fieldValue] = getFromStack(0); return aggStdDevFinalizeImpl(fieldValue, false /* isSamp */); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinStdDevSampFinalize( +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinStdDevSampFinalize( ArityType arity) { auto [_, fieldTag, fieldValue] = getFromStack(0); return aggStdDevFinalizeImpl(fieldValue, true /* isSamp */); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::aggMin(value::TypeTags accTag, - value::Value accValue, - value::TypeTags fieldTag, - value::Value fieldValue, - CollatorInterface* collator) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::aggMin(value::TypeTags accTag, + value::Value accValue, + value::TypeTags fieldTag, + value::Value fieldValue, + CollatorInterface* collator) { // Skip aggregation step if we don't have the input. if (fieldTag == value::TypeTags::Nothing) { auto [tag, val] = value::copyValue(accTag, accValue); @@ -1463,11 +1411,11 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::aggMin(value::TypeTags } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::aggMax(value::TypeTags accTag, - value::Value accValue, - value::TypeTags fieldTag, - value::Value fieldValue, - CollatorInterface* collator) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::aggMax(value::TypeTags accTag, + value::Value accValue, + value::TypeTags fieldTag, + value::Value fieldValue, + CollatorInterface* collator) { // Skip aggregation step if we don't have the input. if (fieldTag == value::TypeTags::Nothing) { auto [tag, val] = value::copyValue(accTag, accValue); @@ -1491,10 +1439,10 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::aggMax(value::TypeTags } } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::aggFirst(value::TypeTags accTag, - value::Value accValue, - value::TypeTags fieldTag, - value::Value fieldValue) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::aggFirst(value::TypeTags accTag, + value::Value accValue, + value::TypeTags fieldTag, + value::Value fieldValue) { // Skip aggregation step if we don't have the input. if (fieldTag == value::TypeTags::Nothing) { auto [tag, val] = value::copyValue(accTag, accValue); @@ -1512,10 +1460,10 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::aggFirst(value::TypeTa return {true, tag, val}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::aggLast(value::TypeTags accTag, - value::Value accValue, - value::TypeTags fieldTag, - value::Value fieldValue) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::aggLast(value::TypeTags accTag, + value::Value accValue, + value::TypeTags fieldTag, + value::Value fieldValue) { // Skip aggregation step if we don't have the input. if (fieldTag == value::TypeTags::Nothing) { auto [tag, val] = value::copyValue(accTag, accValue); @@ -1539,7 +1487,7 @@ bool hasSeparatorAt(size_t idx, StringData input, StringData separator) { input.substr(idx, separator.size()) == separator; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinSplit(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinSplit(ArityType arity) { auto [ownedSeparator, tagSeparator, valSeparator] = getFromStack(1); auto [ownedInput, tagInput, valInput] = getFromStack(0); @@ -1573,7 +1521,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinSplit(ArityType return {true, tag, val}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinDropFields(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinDropFields(ArityType arity) { auto [ownedSeparator, tagInObj, valInObj] = getFromStack(0); // We operate only on objects. @@ -1630,7 +1578,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinDropFields(Arit return {true, tag, val}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinNewArray(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinNewArray(ArityType arity) { auto [tag, val] = value::makeNewArray(); value::ValueGuard guard{tag, val}; @@ -1649,7 +1597,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinNewArray(ArityT return {true, tag, val}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinKeepFields(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinKeepFields(ArityType arity) { auto [ownedInObj, tagInObj, valInObj] = getFromStack(0); // We operate only on objects. @@ -1705,8 +1653,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinKeepFields(Arit return {true, tag, val}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinNewArrayFromRange( - ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinNewArrayFromRange(ArityType arity) { auto [tag, val] = value::makeNewArray(); value::ValueGuard guard{tag, val}; @@ -1754,7 +1701,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinNewArrayFromRan return {true, tag, val}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinNewObj(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinNewObj(ArityType arity) { std::vector<value::TypeTags> typeTags; std::vector<value::Value> values; std::vector<std::string> names; @@ -1797,8 +1744,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinNewObj(ArityTyp return {true, tag, val}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinKeyStringToString( - ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinKeyStringToString(ArityType arity) { auto [owned, tagInKey, valInKey] = getFromStack(0); // We operate only on keys. @@ -1813,7 +1759,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinKeyStringToStri return {true, tagStr, valStr}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::genericNewKeyString( +FastTuple<bool, value::TypeTags, value::Value> ByteCode::genericNewKeyString( ArityType arity, CollatorInterface* collator) { auto [_, tagVersion, valVersion] = getFromStack(0); auto [__, tagOrdering, valOrdering] = getFromStack(1); @@ -2002,14 +1948,14 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::genericNewKeyString( value::bitcastFrom<KeyString::Value*>(new KeyString::Value(kb.release()))}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinNewKeyString(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinNewKeyString(ArityType arity) { tassert(6333000, str::stream() << "Unsupported number of arguments passed to ks(): " << arity, arity >= 3 && arity <= Ordering::kMaxCompoundIndexKeys + 3); return genericNewKeyString(arity); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinCollNewKeyString(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinCollNewKeyString(ArityType arity) { tassert(6511500, str::stream() << "Unsupported number of arguments passed to collKs(): " << arity, arity >= 4 && arity <= Ordering::kMaxCompoundIndexKeys + 4); @@ -2022,7 +1968,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinCollNewKeyStrin return genericNewKeyString(arity - 1u, collator); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinAbs(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinAbs(ArityType arity) { invariant(arity == 1); auto [_, tagOperand, valOperand] = getFromStack(0); @@ -2030,7 +1976,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinAbs(ArityType a return genericAbs(tagOperand, valOperand); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinCeil(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinCeil(ArityType arity) { invariant(arity == 1); auto [_, tagOperand, valOperand] = getFromStack(0); @@ -2038,7 +1984,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinCeil(ArityType return genericCeil(tagOperand, valOperand); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinFloor(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinFloor(ArityType arity) { invariant(arity == 1); auto [_, tagOperand, valOperand] = getFromStack(0); @@ -2046,7 +1992,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinFloor(ArityType return genericFloor(tagOperand, valOperand); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinTrunc(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinTrunc(ArityType arity) { invariant(arity == 1); auto [_, tagOperand, valOperand] = getFromStack(0); @@ -2054,7 +2000,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinTrunc(ArityType return genericTrunc(tagOperand, valOperand); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinExp(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinExp(ArityType arity) { invariant(arity == 1); auto [_, tagOperand, valOperand] = getFromStack(0); @@ -2062,7 +2008,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinExp(ArityType a return genericExp(tagOperand, valOperand); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinLn(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinLn(ArityType arity) { invariant(arity == 1); auto [_, tagOperand, valOperand] = getFromStack(0); @@ -2070,7 +2016,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinLn(ArityType ar return genericLn(tagOperand, valOperand); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinLog10(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinLog10(ArityType arity) { invariant(arity == 1); auto [_, tagOperand, valOperand] = getFromStack(0); @@ -2078,7 +2024,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinLog10(ArityType return genericLog10(tagOperand, valOperand); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinSqrt(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinSqrt(ArityType arity) { invariant(arity == 1); auto [_, tagOperand, valOperand] = getFromStack(0); @@ -2086,7 +2032,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinSqrt(ArityType return genericSqrt(tagOperand, valOperand); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinAddToArray(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinAddToArray(ArityType arity) { auto [ownAgg, tagAgg, valAgg] = getFromStack(0); auto [tagField, valField] = moveOwnedFromStack(1); value::ValueGuard guardField{tagField, valField}; @@ -2114,7 +2060,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinAddToArray(Arit // The value being accumulated is an SBE array that contains an integer and the accumulated array, // where the integer is the total size in bytes of the elements in the array. -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinAddToArrayCapped(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinAddToArrayCapped(ArityType arity) { auto [ownArr, tagArr, valArr] = getFromStack(0); auto [tagNewElem, valNewElem] = moveOwnedFromStack(1); value::ValueGuard guardNewElem{tagNewElem, valNewElem}; @@ -2181,7 +2127,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinAddToArrayCappe return {ownArr, tagArr, valArr}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinMergeObjects(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinMergeObjects(ArityType arity) { auto [_, tagField, valField] = getFromStack(1); // Move the incoming accumulator state from the stack. Given that we are now the owner of the // state we are free to do any in-place update as we see fit. @@ -2239,7 +2185,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinMergeObjects(Ar return {true, tagAgg, valAgg}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinAddToSet(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinAddToSet(ArityType arity) { auto [ownAgg, tagAgg, valAgg] = getFromStack(0); auto [tagField, valField] = moveOwnedFromStack(1); value::ValueGuard guardField{tagField, valField}; @@ -2265,7 +2211,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinAddToSet(ArityT return {ownAgg, tagAgg, valAgg}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::addToSetCappedImpl( +FastTuple<bool, value::TypeTags, value::Value> ByteCode::addToSetCappedImpl( value::TypeTags tagNewElem, value::Value valNewElem, int32_t sizeCap, @@ -2331,7 +2277,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::addToSetCappedImpl( return {ownArr, tagArr, valArr}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinAddToSetCapped(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinAddToSetCapped(ArityType arity) { auto [tagNewElem, valNewElem] = moveOwnedFromStack(1); value::ValueGuard guardNewElem{tagNewElem, valNewElem}; auto [_, tagSizeCap, valSizeCap] = getFromStack(2); @@ -2347,7 +2293,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinAddToSetCapped( tagNewElem, valNewElem, value::bitcastTo<int32_t>(valSizeCap), nullptr /*collator*/); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinCollAddToSet(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinCollAddToSet(ArityType arity) { auto [ownAgg, tagAgg, valAgg] = getFromStack(0); auto [ownColl, tagColl, valColl] = getFromStack(1); auto [tagField, valField] = moveOwnedFromStack(2); @@ -2381,7 +2327,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinCollAddToSet(Ar return {ownAgg, tagAgg, valAgg}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinCollAddToSetCapped( +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinCollAddToSetCapped( ArityType arity) { auto [_1, tagColl, valColl] = getFromStack(1); auto [tagNewElem, valNewElem] = moveOwnedFromStack(2); @@ -2403,7 +2349,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinCollAddToSetCap value::getCollatorView(valColl)); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinRunJsPredicate(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinRunJsPredicate(ArityType arity) { invariant(arity == 2); auto [predicateOwned, predicateType, predicateValue] = getFromStack(0); @@ -2429,7 +2375,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinRunJsPredicate( return {false, value::TypeTags::Boolean, value::bitcastFrom<bool>(predicateResult)}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinReplaceOne(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinReplaceOne(ArityType arity) { invariant(arity == 3); auto [ownedInputStr, typeTagInputStr, valueInputStr] = getFromStack(0); @@ -2469,7 +2415,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinReplaceOne(Arit return {true, outputStrTypeTag, outputStrValue}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinDoubleDoubleSum(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinDoubleDoubleSum(ArityType arity) { invariant(arity >= 1); value::TypeTags resultTag = value::TypeTags::NumberInt32; @@ -2569,17 +2515,17 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinDoubleDoubleSum */ using DateFn = std::function<Date_t( TimeZone, long long, long long, long long, long long, long long, long long, long long)>; -std::tuple<bool, value::TypeTags, value::Value> builtinDateHelper( +FastTuple<bool, value::TypeTags, value::Value> builtinDateHelper( DateFn computeDateFn, - std::tuple<bool, value::TypeTags, value::Value> tzdb, - std::tuple<bool, value::TypeTags, value::Value> yearOrWeekYear, - std::tuple<bool, value::TypeTags, value::Value> monthOrWeek, - std::tuple<bool, value::TypeTags, value::Value> day, - std::tuple<bool, value::TypeTags, value::Value> hour, - std::tuple<bool, value::TypeTags, value::Value> minute, - std::tuple<bool, value::TypeTags, value::Value> second, - std::tuple<bool, value::TypeTags, value::Value> millisecond, - std::tuple<bool, value::TypeTags, value::Value> timezone) { + FastTuple<bool, value::TypeTags, value::Value> tzdb, + FastTuple<bool, value::TypeTags, value::Value> yearOrWeekYear, + FastTuple<bool, value::TypeTags, value::Value> monthOrWeek, + FastTuple<bool, value::TypeTags, value::Value> day, + FastTuple<bool, value::TypeTags, value::Value> hour, + FastTuple<bool, value::TypeTags, value::Value> minute, + FastTuple<bool, value::TypeTags, value::Value> second, + FastTuple<bool, value::TypeTags, value::Value> millisecond, + FastTuple<bool, value::TypeTags, value::Value> timezone) { auto [ownedTzdb, typeTagTzdb, valueTzdb] = tzdb; auto [ownedYearOrWeekYear, typeTagYearOrWeekYear, valueYearOrWeekYear] = yearOrWeekYear; @@ -2617,7 +2563,7 @@ std::tuple<bool, value::TypeTags, value::Value> builtinDateHelper( return {false, value::TypeTags::Date, value::bitcastFrom<int64_t>(date.asInt64())}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinDate(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinDate(ArityType arity) { auto timeZoneDBTuple = getFromStack(0); auto yearTuple = getFromStack(1); auto monthTuple = getFromStack(2); @@ -2650,7 +2596,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinDate(ArityType timezoneTuple); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinDateDiff(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinDateDiff(ArityType arity) { invariant(arity == 5 || arity == 6); // 6th parameter is 'startOfWeek'. auto [timezoneDBOwn, timezoneDBTag, timezoneDBValue] = getFromStack(0); @@ -2710,7 +2656,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinDateDiff(ArityT return {false, value::TypeTags::NumberInt64, value::bitcastFrom<int64_t>(result)}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinDateWeekYear(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinDateWeekYear(ArityType arity) { auto timeZoneDBTuple = getFromStack(0); auto yearTuple = getFromStack(1); auto weekTuple = getFromStack(2); @@ -2743,7 +2689,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinDateWeekYear(Ar timezoneTuple); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinDateToParts(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinDateToParts(ArityType arity) { auto [timezoneDBOwn, timezoneDBTag, timezoneDBVal] = getFromStack(0); if (timezoneDBTag != value::TypeTags::timeZoneDB) { return {false, value::TypeTags::Nothing, 0}; @@ -2782,7 +2728,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinDateToParts(Ari return {true, dateObjTag, dateObjVal}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinIsoDateToParts(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinIsoDateToParts(ArityType arity) { auto [timezoneDBOwn, timezoneDBTag, timezoneDBVal] = getFromStack(0); if (timezoneDBTag != value::TypeTags::timeZoneDB) { return {false, value::TypeTags::Nothing, 0}; @@ -2821,7 +2767,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinIsoDateToParts( return {true, dateObjTag, dateObjVal}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinDayOfYear(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinDayOfYear(ArityType arity) { invariant(arity == 3); auto [timezoneDBOwn, timezoneDBTag, timezoneDBValue] = getFromStack(0); @@ -2831,7 +2777,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinDayOfYear(Arity timezoneDBTag, timezoneDBValue, dateTag, dateValue, timezoneTag, timezoneValue); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinDayOfMonth(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinDayOfMonth(ArityType arity) { invariant(arity == 3); auto [timezoneDBOwn, timezoneDBTag, timezoneDBValue] = getFromStack(0); @@ -2841,7 +2787,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinDayOfMonth(Arit timezoneDBTag, timezoneDBValue, dateTag, dateValue, timezoneTag, timezoneValue); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinDayOfWeek(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinDayOfWeek(ArityType arity) { invariant(arity == 3); auto [timezoneDBOwn, timezoneDBTag, timezoneDBValue] = getFromStack(0); @@ -2851,7 +2797,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinDayOfWeek(Arity timezoneDBTag, timezoneDBValue, dateTag, dateValue, timezoneTag, timezoneValue); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinBitTestPosition(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinBitTestPosition(ArityType arity) { invariant(arity == 3); auto [ownedMask, maskTag, maskValue] = getFromStack(0); @@ -2913,7 +2859,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinBitTestPosition bitTestBehavior == BitTestBehavior::AllClear)}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinBitTestZero(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinBitTestZero(ArityType arity) { invariant(arity == 2); auto [maskOwned, maskTag, maskValue] = getFromStack(0); auto [inputOwned, inputTag, inputValue] = getFromStack(1); @@ -2929,7 +2875,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinBitTestZero(Ari return {false, value::TypeTags::Boolean, value::bitcastFrom<bool>(result)}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinBitTestMask(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinBitTestMask(ArityType arity) { invariant(arity == 2); auto [maskOwned, maskTag, maskValue] = getFromStack(0); auto [inputOwned, inputTag, inputValue] = getFromStack(1); @@ -2945,7 +2891,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinBitTestMask(Ari return {false, value::TypeTags::Boolean, value::bitcastFrom<bool>(result)}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinBsonSize(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinBsonSize(ArityType arity) { auto [_, tagOperand, valOperand] = getFromStack(0); if (tagOperand == value::TypeTags::Object) { @@ -2961,7 +2907,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinBsonSize(ArityT return {false, value::TypeTags::Nothing, 0}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinToUpper(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinToUpper(ArityType arity) { auto [_, operandTag, operandVal] = getFromStack(0); if (value::isString(operandTag)) { @@ -2974,7 +2920,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinToUpper(ArityTy return {false, value::TypeTags::Nothing, 0}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinToLower(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinToLower(ArityType arity) { auto [_, operandTag, operandVal] = getFromStack(0); if (value::isString(operandTag)) { @@ -2987,7 +2933,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinToLower(ArityTy return {false, value::TypeTags::Nothing, 0}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinCoerceToString(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinCoerceToString(ArityType arity) { auto [operandOwn, operandTag, operandVal] = getFromStack(0); if (value::isString(operandTag)) { @@ -3047,82 +2993,82 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinCoerceToString( return {false, value::TypeTags::Nothing, 0}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinAcos(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinAcos(ArityType arity) { auto [_, operandTag, operandValue] = getFromStack(0); return genericAcos(operandTag, operandValue); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinAcosh(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinAcosh(ArityType arity) { auto [_, operandTag, operandValue] = getFromStack(0); return genericAcosh(operandTag, operandValue); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinAsin(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinAsin(ArityType arity) { auto [_, operandTag, operandValue] = getFromStack(0); return genericAsin(operandTag, operandValue); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinAsinh(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinAsinh(ArityType arity) { auto [_, operandTag, operandValue] = getFromStack(0); return genericAsinh(operandTag, operandValue); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinAtan(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinAtan(ArityType arity) { auto [_, operandTag, operandValue] = getFromStack(0); return genericAtan(operandTag, operandValue); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinAtanh(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinAtanh(ArityType arity) { auto [_, operandTag, operandValue] = getFromStack(0); return genericAtanh(operandTag, operandValue); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinAtan2(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinAtan2(ArityType arity) { auto [owned1, operandTag1, operandValue1] = getFromStack(0); auto [owned2, operandTag2, operandValue2] = getFromStack(1); return genericAtan2(operandTag1, operandValue1, operandTag2, operandValue2); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinCos(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinCos(ArityType arity) { auto [_, operandTag, operandValue] = getFromStack(0); return genericCos(operandTag, operandValue); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinCosh(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinCosh(ArityType arity) { auto [_, operandTag, operandValue] = getFromStack(0); return genericCosh(operandTag, operandValue); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinDegreesToRadians(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinDegreesToRadians(ArityType arity) { auto [_, operandTag, operandValue] = getFromStack(0); return genericDegreesToRadians(operandTag, operandValue); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinRadiansToDegrees(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinRadiansToDegrees(ArityType arity) { auto [_, operandTag, operandValue] = getFromStack(0); return genericRadiansToDegrees(operandTag, operandValue); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinSin(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinSin(ArityType arity) { auto [_, operandTag, operandValue] = getFromStack(0); return genericSin(operandTag, operandValue); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinSinh(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinSinh(ArityType arity) { auto [_, operandTag, operandValue] = getFromStack(0); return genericSinh(operandTag, operandValue); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinTan(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinTan(ArityType arity) { auto [_, operandTag, operandValue] = getFromStack(0); return genericTan(operandTag, operandValue); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinTanh(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinTanh(ArityType arity) { auto [_, operandTag, operandValue] = getFromStack(0); return genericTanh(operandTag, operandValue); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinRound(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinRound(ArityType arity) { invariant(arity == 1); auto [owned, tag, val] = getFromStack(0); @@ -3161,7 +3107,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinRound(ArityType } } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinConcat(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinConcat(ArityType arity) { StringBuilder result; for (ArityType idx = 0; idx < arity; ++idx) { auto [_, tag, value] = getFromStack(idx); @@ -3221,7 +3167,7 @@ std::pair<value::TypeTags, value::Value> ByteCode::genericIsMember(value::TypeTa return genericIsMember(lhsTag, lhsVal, rhsTag, rhsVal, collator); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinIsMember(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinIsMember(ArityType arity) { invariant(arity == 2); auto [ownedInput, inputTag, inputVal] = getFromStack(0); @@ -3231,7 +3177,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinIsMember(ArityT return {false, resultTag, resultVal}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinCollIsMember(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinCollIsMember(ArityType arity) { invariant(arity == 3); auto [ownedColl, collTag, collVal] = getFromStack(0); @@ -3244,7 +3190,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinCollIsMember(Ar return {false, resultTag, resultVal}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinIndexOfBytes(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinIndexOfBytes(ArityType arity) { auto [strOwn, strTag, strVal] = getFromStack(0); auto [substrOwn, substrTag, substrVal] = getFromStack(1); if ((!value::isString(strTag)) || (!value::isString(substrTag))) { @@ -3292,7 +3238,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinIndexOfBytes(Ar return {false, value::TypeTags::NumberInt32, value::bitcastFrom<int32_t>(-1)}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinIndexOfCP(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinIndexOfCP(ArityType arity) { auto [strOwn, strTag, strVal] = getFromStack(0); auto [substrOwn, substrTag, substrVal] = getFromStack(1); if ((!value::isString(strTag)) || (!value::isString(substrTag))) { @@ -3363,7 +3309,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinIndexOfCP(Arity return {false, value::TypeTags::NumberInt32, value::bitcastFrom<int32_t>(-1)}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinIsTimeUnit(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinIsTimeUnit(ArityType arity) { invariant(arity == 1); auto [timeUnitOwn, timeUnitTag, timeUnitValue] = getFromStack(0); if (!value::isString(timeUnitTag)) { @@ -3375,7 +3321,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinIsTimeUnit(Arit isValidTimeUnit(value::getStringView(timeUnitTag, timeUnitValue)))}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinIsDayOfWeek(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinIsDayOfWeek(ArityType arity) { invariant(arity == 1); auto [dayOfWeekOwn, dayOfWeekTag, dayOfWeekValue] = getFromStack(0); if (!value::isString(dayOfWeekTag)) { @@ -3387,7 +3333,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinIsDayOfWeek(Ari isValidDayOfWeek(value::getStringView(dayOfWeekTag, dayOfWeekValue)))}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinIsTimezone(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinIsTimezone(ArityType arity) { auto [timezoneDBOwn, timezoneDBTag, timezoneDBVal] = getFromStack(0); if (timezoneDBTag != value::TypeTags::timeZoneDB) { return {false, value::TypeTags::Nothing, 0}; @@ -3405,7 +3351,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinIsTimezone(Arit } namespace { -std::tuple<bool, value::TypeTags, value::Value> setUnion( +FastTuple<bool, value::TypeTags, value::Value> setUnion( const std::vector<value::TypeTags>& argTags, const std::vector<value::Value>& argVals, const CollatorInterface* collator = nullptr) { @@ -3429,7 +3375,7 @@ std::tuple<bool, value::TypeTags, value::Value> setUnion( return {true, resTag, resVal}; } -std::tuple<bool, value::TypeTags, value::Value> setIntersection( +FastTuple<bool, value::TypeTags, value::Value> setIntersection( const std::vector<value::TypeTags>& argTags, const std::vector<value::Value>& argVals, const CollatorInterface* collator = nullptr) { @@ -3492,7 +3438,7 @@ value::ValueSetType valueToSetHelper(const value::TypeTags& tag, return setValues; } -std::tuple<bool, value::TypeTags, value::Value> setDifference( +FastTuple<bool, value::TypeTags, value::Value> setDifference( value::TypeTags lhsTag, value::Value lhsVal, value::TypeTags rhsTag, @@ -3518,7 +3464,7 @@ std::tuple<bool, value::TypeTags, value::Value> setDifference( return {true, resTag, resVal}; } -std::tuple<bool, value::TypeTags, value::Value> setEquals( +FastTuple<bool, value::TypeTags, value::Value> setEquals( const std::vector<value::TypeTags>& argTags, const std::vector<value::Value>& argVals, const CollatorInterface* collator = nullptr) { @@ -3535,7 +3481,7 @@ std::tuple<bool, value::TypeTags, value::Value> setEquals( } } // namespace -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinCollSetUnion(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinCollSetUnion(ArityType arity) { invariant(arity >= 1); auto [_, collTag, collVal] = getFromStack(0); @@ -3558,7 +3504,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinCollSetUnion(Ar return setUnion(argTags, argVals, value::getCollatorView(collVal)); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinSetUnion(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinSetUnion(ArityType arity) { std::vector<value::TypeTags> argTags; std::vector<value::Value> argVals; @@ -3575,7 +3521,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinSetUnion(ArityT return setUnion(argTags, argVals); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinCollSetIntersection( +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinCollSetIntersection( ArityType arity) { invariant(arity >= 1); @@ -3600,7 +3546,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinCollSetIntersec return setIntersection(argTags, argVals, value::getCollatorView(collVal)); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinSetIntersection(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinSetIntersection(ArityType arity) { std::vector<value::TypeTags> argTags; std::vector<value::Value> argVals; @@ -3617,8 +3563,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinSetIntersection return setIntersection(argTags, argVals); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinCollSetDifference( - ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinCollSetDifference(ArityType arity) { invariant(arity == 3); auto [_, collTag, collVal] = getFromStack(0); @@ -3636,7 +3581,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinCollSetDifferen return setDifference(lhsTag, lhsVal, rhsTag, rhsVal, value::getCollatorView(collVal)); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinCollSetEquals(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinCollSetEquals(ArityType arity) { invariant(arity >= 3); auto [_, collTag, collVal] = getFromStack(0); @@ -3660,7 +3605,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinCollSetEquals(A return setEquals(argTags, argVals, value::getCollatorView(collVal)); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinSetDifference(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinSetDifference(ArityType arity) { invariant(arity == 2); auto [lhsOwned, lhsTag, lhsVal] = getFromStack(0); @@ -3673,7 +3618,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinSetDifference(A return setDifference(lhsTag, lhsVal, rhsTag, rhsVal); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinSetEquals(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinSetEquals(ArityType arity) { invariant(arity >= 2); std::vector<value::TypeTags> argTags; @@ -3703,11 +3648,11 @@ namespace { * - isMatch: Boolean flag to mark if the caller function is $regexMatch, in which case the result * returned is true/false. */ -std::tuple<bool, value::TypeTags, value::Value> pcreNextMatch(pcre::Regex* pcre, - StringData inputString, - uint32_t& startBytePos, - uint32_t& codePointPos, - bool isMatch) { +FastTuple<bool, value::TypeTags, value::Value> pcreNextMatch(pcre::Regex* pcre, + StringData inputString, + uint32_t& startBytePos, + uint32_t& codePointPos, + bool isMatch) { pcre::MatchData m = pcre->matchView(inputString, {}, startBytePos); if (!m && m.error() != pcre::Errc::ERROR_NOMATCH) { LOGV2_ERROR(5073414, @@ -3766,7 +3711,7 @@ std::tuple<bool, value::TypeTags, value::Value> pcreNextMatch(pcre::Regex* pcre, * A helper function with common logic for $regexMatch and $regexFind functions. Both extract only * the first match to a regular expression, but return different result objects. */ -std::tuple<bool, value::TypeTags, value::Value> genericPcreRegexSingleMatch( +FastTuple<bool, value::TypeTags, value::Value> genericPcreRegexSingleMatch( value::TypeTags typeTagPcreRegex, value::Value valuePcreRegex, value::TypeTags typeTagInputStr, @@ -3816,7 +3761,7 @@ std::pair<value::TypeTags, value::Value> collComparisonKey(value::TypeTags tag, } // namespace -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinRegexCompile(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinRegexCompile(ArityType arity) { invariant(arity == 2); auto [patternOwned, patternTypeTag, patternValue] = getFromStack(0); @@ -3837,7 +3782,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinRegexCompile(Ar return {true, pcreTag, pcreValue}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinRegexMatch(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinRegexMatch(ArityType arity) { invariant(arity == 2); auto [ownedPcreRegex, tagPcreRegex, valPcreRegex] = getFromStack(0); auto [ownedInputStr, tagInputStr, valInputStr] = getFromStack(1); @@ -3863,7 +3808,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinRegexMatch(Arit return genericPcreRegexSingleMatch(tagPcreRegex, valPcreRegex, tagInputStr, valInputStr, true); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinRegexFind(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinRegexFind(ArityType arity) { invariant(arity == 2); auto [ownedPcreRegex, typeTagPcreRegex, valuePcreRegex] = getFromStack(0); auto [ownedInputStr, typeTagInputStr, valueInputStr] = getFromStack(1); @@ -3872,7 +3817,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinRegexFind(Arity typeTagPcreRegex, valuePcreRegex, typeTagInputStr, valueInputStr, false); } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinRegexFindAll(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinRegexFindAll(ArityType arity) { invariant(arity == 2); auto [ownedPcre, typeTagPcreRegex, valuePcreRegex] = getFromStack(0); auto [ownedStr, typeTagInputStr, valueInputStr] = getFromStack(1); @@ -3931,7 +3876,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinRegexFindAll(Ar return {true, arrTag, arrVal}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinShardFilter(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinShardFilter(ArityType arity) { invariant(arity == 2); auto [ownedFilter, filterTag, filterValue] = getFromStack(0); @@ -3956,7 +3901,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinShardFilter(Ari value::getShardFiltererView(filterValue)->keyBelongsToMe(keyAsUnownedBson))}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinShardHash(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinShardHash(ArityType arity) { invariant(arity == 1); auto [ownedShardKey, shardKeyTag, shardKeyValue] = getFromStack(0); @@ -3971,7 +3916,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinShardHash(Arity return {false, value::TypeTags::NumberInt64, value::bitcastFrom<decltype(hashVal)>(hashVal)}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinExtractSubArray(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinExtractSubArray(ArityType arity) { // We need to ensure that 'size_t' is wide enough to store 32-bit index. static_assert(sizeof(size_t) >= sizeof(int32_t), "size_t must be at least 32-bits"); @@ -4084,7 +4029,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinExtractSubArray return {true, resultTag, resultValue}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinIsArrayEmpty(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinIsArrayEmpty(ArityType arity) { invariant(arity == 1); auto [arrayOwned, arrayType, arrayValue] = getFromStack(0); @@ -4105,7 +4050,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinIsArrayEmpty(Ar } } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinHasNullBytes(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinHasNullBytes(ArityType arity) { invariant(arity == 1); auto [strOwned, strType, strValue] = getFromStack(0); @@ -4119,7 +4064,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinHasNullBytes(Ar return {false, value::TypeTags::Boolean, value::bitcastFrom<bool>(hasNullBytes)}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinGetRegexPattern(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinGetRegexPattern(ArityType arity) { invariant(arity == 1); auto [regexOwned, regexType, regexValue] = getFromStack(0); @@ -4133,7 +4078,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinGetRegexPattern return {true, strType, strValue}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinGetRegexFlags(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinGetRegexFlags(ArityType arity) { invariant(arity == 1); auto [regexOwned, regexType, regexValue] = getFromStack(0); @@ -4147,7 +4092,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinGetRegexFlags(A return {true, strType, strValue}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinGenerateSortKey(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinGenerateSortKey(ArityType arity) { invariant(arity == 2 || arity == 3); auto [ssOwned, ssTag, ssVal] = getFromStack(0); @@ -4185,7 +4130,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinGenerateSortKey new KeyString::Value(ss->generateSortKey(obj, collator)))}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinMakeBsonObj(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinMakeBsonObj(ArityType arity) { invariant(arity >= 2); tassert(6897002, str::stream() << "Unsupported number of arguments passed to makeBsonObj(): " << arity, @@ -4211,7 +4156,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinMakeBsonObj(Ari return {true, tag, val}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinReverseArray(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinReverseArray(ArityType arity) { invariant(arity == 1); auto [inputOwned, inputType, inputVal] = getFromStack(0); @@ -4275,7 +4220,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinReverseArray(Ar } } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinSortArray(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinSortArray(ArityType arity) { invariant(arity == 2 || arity == 3); auto [inputOwned, inputType, inputVal] = getFromStack(0); @@ -4366,7 +4311,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinSortArray(Arity } } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinDateAdd(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinDateAdd(ArityType arity) { invariant(arity == 5); auto [timezoneDBOwn, timezoneDBTag, timezoneDBVal] = getFromStack(0); @@ -4408,7 +4353,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinDateAdd(ArityTy false, value::TypeTags::Date, value::bitcastFrom<int64_t>(resDate.toMillisSinceEpoch())}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinFtsMatch(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinFtsMatch(ArityType arity) { invariant(arity == 2); auto [matcherOwn, matcherTag, matcherVal] = getFromStack(0); @@ -4433,7 +4378,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinFtsMatch(ArityT return {false, value::TypeTags::Boolean, value::bitcastFrom<bool>(matches)}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinTsSecond(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinTsSecond(ArityType arity) { invariant(arity == 1); auto [inputValueOwn, inputTypeTag, inputValue] = getFromStack(0); @@ -4446,7 +4391,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinTsSecond(ArityT return {false, value::TypeTags::NumberInt64, value::bitcastFrom<uint64_t>(timestamp.getSecs())}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinTsIncrement(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinTsIncrement(ArityType arity) { invariant(arity == 1); auto [inputValueOwn, inputTypeTag, inputValue] = getFromStack(0); @@ -4459,7 +4404,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinTsIncrement(Ari return {false, value::TypeTags::NumberInt64, value::bitcastFrom<uint64_t>(timestamp.getInc())}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinHash(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinHash(ArityType arity) { auto hashVal = value::hashInit(); for (ArityType idx = 0; idx < arity; ++idx) { auto [owned, tag, val] = getFromStack(idx); @@ -4469,7 +4414,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinHash(ArityType return {false, value::TypeTags::NumberInt64, value::bitcastFrom<decltype(hashVal)>(hashVal)}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinTypeMatch(ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::builtinTypeMatch(ArityType arity) { invariant(arity == 2); auto [inputOwn, inputTag, inputVal] = getFromStack(0); @@ -4484,8 +4429,8 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::builtinTypeMatch(Arity return {false, value::TypeTags::Nothing, 0}; } -std::tuple<bool, value::TypeTags, value::Value> ByteCode::dispatchBuiltin(Builtin f, - ArityType arity) { +FastTuple<bool, value::TypeTags, value::Value> ByteCode::dispatchBuiltin(Builtin f, + ArityType arity) { switch (f) { case Builtin::dateDiff: return builtinDateDiff(arity); @@ -4697,20 +4642,16 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::dispatchBuiltin(Builti MONGO_COMPILER_NORETURN void reportSwapFailure(); void ByteCode::swapStack() { - auto backOffset = _argStack.size() - 1; - auto& rhsOwnedTag = _argStack.ownedAndTag(backOffset); - auto& rhsValue = _argStack.value(backOffset); - --backOffset; - auto& lhsOwnedTag = _argStack.ownedAndTag(backOffset); - auto& lhsValue = _argStack.value(backOffset); - - if (rhsValue == lhsValue && rhsOwnedTag.tag == lhsOwnedTag.tag) { - if (rhsOwnedTag.owned && !isShallowType(rhsOwnedTag.tag)) { + auto [rhsOwned, rhsTag, rhsValue] = getFromStack(0); + auto [lhsOwned, lhsTag, lhsValue] = getFromStack(1); + + if (rhsValue == lhsValue && rhsTag == lhsTag) { + if (rhsOwned && !isShallowType(rhsTag)) { reportSwapFailure(); } } else { - std::swap(rhsOwnedTag, lhsOwnedTag); - std::swap(rhsValue, lhsValue); + setStack(0, lhsOwned, lhsTag, lhsValue); + setStack(1, rhsOwned, rhsTag, rhsValue); } } @@ -4718,6 +4659,57 @@ MONGO_COMPILER_NORETURN void reportSwapFailure() { tasserted(56123, "Attempting to swap two identical values when top of stack is owned"); } +MONGO_COMPILER_NORETURN void ByteCode::runFailInstruction() { + auto [ownedCode, tagCode, valCode] = getFromStack(1); + invariant(tagCode == value::TypeTags::NumberInt64); + + auto [ownedMsg, tagMsg, valMsg] = getFromStack(0); + invariant(value::isString(tagMsg)); + + ErrorCodes::Error code{static_cast<ErrorCodes::Error>(value::bitcastTo<int64_t>(valCode))}; + std::string message{value::getStringView(tagMsg, valMsg)}; + + uasserted(code, message); +} + +void ByteCode::runClassicMatcher(const MatchExpression* matcher) { + auto [ownedObj, tagObj, valObj] = getFromStack(0); + + BSONObj bsonObjForMatching; + if (tagObj == value::TypeTags::Object) { + BSONObjBuilder builder; + sbe::bson::convertToBsonObj(builder, sbe::value::getObjectView(valObj)); + bsonObjForMatching = builder.obj(); + } else if (tagObj == value::TypeTags::bsonObject) { + auto bson = value::getRawPointerView(valObj); + bsonObjForMatching = BSONObj(bson); + } else { + MONGO_UNREACHABLE_TASSERT(6681402); + } + + bool res = matcher->matchesBSON(bsonObjForMatching); + if (ownedObj) { + value::releaseValue(tagObj, valObj); + } + topStack(false, value::TypeTags::Boolean, value::bitcastFrom<bool>(res)); +} +template <typename T> +void ByteCode::runTagCheck(T&& predicate) { + auto [owned, tag, val] = getFromStack(0); + + if (tag != value::TypeTags::Nothing) { + topStack(false, value::TypeTags::Boolean, value::bitcastFrom<bool>(predicate(tag))); + } + + if (owned) { + value::releaseValue(tag, val); + } +} + +void ByteCode::runTagCheck(value::TypeTags tagRhs) { + runTagCheck([tagRhs](value::TypeTags tagLhs) { return tagLhs == tagRhs; }); +} + void ByteCode::runLambdaInternal(const CodeFragment* code, int64_t position) { runInternal(code, position); swapStack(); @@ -5599,101 +5591,31 @@ void ByteCode::runInternal(const CodeFragment* code, int64_t position) { break; } case Instruction::isNull: { - auto [owned, tag, val] = getFromStack(0); - - if (tag != value::TypeTags::Nothing) { - topStack(false, - value::TypeTags::Boolean, - value::bitcastFrom<bool>(tag == value::TypeTags::Null)); - } - - if (owned) { - value::releaseValue(tag, val); - } + runTagCheck(value::TypeTags::Null); break; } case Instruction::isObject: { - auto [owned, tag, val] = getFromStack(0); - - if (tag != value::TypeTags::Nothing) { - topStack(false, - value::TypeTags::Boolean, - value::bitcastFrom<bool>(value::isObject(tag))); - } - - if (owned) { - value::releaseValue(tag, val); - } + runTagCheck(value::isObject); break; } case Instruction::isArray: { - auto [owned, tag, val] = getFromStack(0); - - if (tag != value::TypeTags::Nothing) { - topStack(false, - value::TypeTags::Boolean, - value::bitcastFrom<bool>(value::isArray(tag))); - } - - if (owned) { - value::releaseValue(tag, val); - } + runTagCheck(value::isArray); break; } case Instruction::isString: { - auto [owned, tag, val] = getFromStack(0); - - if (tag != value::TypeTags::Nothing) { - topStack(false, - value::TypeTags::Boolean, - value::bitcastFrom<bool>(value::isString(tag))); - } - - if (owned) { - value::releaseValue(tag, val); - } + runTagCheck(value::isString); break; } case Instruction::isNumber: { - auto [owned, tag, val] = getFromStack(0); - - if (tag != value::TypeTags::Nothing) { - topStack(false, - value::TypeTags::Boolean, - value::bitcastFrom<bool>(value::isNumber(tag))); - } - - if (owned) { - value::releaseValue(tag, val); - } + runTagCheck(value::isNumber); break; } case Instruction::isBinData: { - auto [owned, tag, val] = getFromStack(0); - - if (tag != value::TypeTags::Nothing) { - topStack(false, - value::TypeTags::Boolean, - value::bitcastFrom<bool>(value::isBinData(tag))); - } - - if (owned) { - value::releaseValue(tag, val); - } + runTagCheck(value::isBinData); break; } case Instruction::isDate: { - auto [owned, tag, val] = getFromStack(0); - - if (tag != value::TypeTags::Nothing) { - topStack(false, - value::TypeTags::Boolean, - value::bitcastFrom<bool>(tag == value::TypeTags::Date)); - } - - if (owned) { - value::releaseValue(tag, val); - } + runTagCheck(value::TypeTags::Date); break; } case Instruction::isNaN: { @@ -5723,59 +5645,19 @@ void ByteCode::runInternal(const CodeFragment* code, int64_t position) { break; } case Instruction::isRecordId: { - auto [owned, tag, val] = getFromStack(0); - - if (tag != value::TypeTags::Nothing) { - topStack(false, - value::TypeTags::Boolean, - value::bitcastFrom<bool>(value::isRecordId(tag))); - } - - if (owned) { - value::releaseValue(tag, val); - } + runTagCheck(value::isRecordId); break; } case Instruction::isMinKey: { - auto [owned, tag, val] = getFromStack(0); - - if (tag != value::TypeTags::Nothing) { - topStack(false, - value::TypeTags::Boolean, - value::bitcastFrom<bool>(tag == value::TypeTags::MinKey)); - } - - if (owned) { - value::releaseValue(tag, val); - } + runTagCheck(value::TypeTags::MinKey); break; } case Instruction::isMaxKey: { - auto [owned, tag, val] = getFromStack(0); - - if (tag != value::TypeTags::Nothing) { - topStack(false, - value::TypeTags::Boolean, - value::bitcastFrom<bool>(tag == value::TypeTags::MaxKey)); - } - - if (owned) { - value::releaseValue(tag, val); - } + runTagCheck(value::TypeTags::MaxKey); break; } case Instruction::isTimestamp: { - auto [owned, tag, val] = getFromStack(0); - - if (tag != value::TypeTags::Nothing) { - topStack(false, - value::TypeTags::Boolean, - value::bitcastFrom<bool>(tag == value::TypeTags::Timestamp)); - } - - if (owned) { - value::releaseValue(tag, val); - } + runTagCheck(value::TypeTags::Timestamp); break; } case Instruction::function: @@ -5839,43 +5721,14 @@ void ByteCode::runInternal(const CodeFragment* code, int64_t position) { break; } case Instruction::fail: { - auto [ownedCode, tagCode, valCode] = getFromStack(1); - invariant(tagCode == value::TypeTags::NumberInt64); - - auto [ownedMsg, tagMsg, valMsg] = getFromStack(0); - invariant(value::isString(tagMsg)); - - ErrorCodes::Error code{ - static_cast<ErrorCodes::Error>(value::bitcastTo<int64_t>(valCode))}; - std::string message{value::getStringView(tagMsg, valMsg)}; - - uasserted(code, message); - + runFailInstruction(); break; } case Instruction::applyClassicMatcher: { const auto* matcher = readFromMemory<const MatchExpression*>(pcPointer); pcPointer += sizeof(matcher); - auto [ownedObj, tagObj, valObj] = getFromStack(0); - - BSONObj bsonObjForMatching; - if (tagObj == value::TypeTags::Object) { - BSONObjBuilder builder; - sbe::bson::convertToBsonObj(builder, sbe::value::getObjectView(valObj)); - bsonObjForMatching = builder.obj(); - } else if (tagObj == value::TypeTags::bsonObject) { - auto bson = value::getRawPointerView(valObj); - bsonObjForMatching = BSONObj(bson); - } else { - MONGO_UNREACHABLE_TASSERT(6681402); - } - - bool res = matcher->matchesBSON(bsonObjForMatching); - if (ownedObj) { - value::releaseValue(tagObj, valObj); - } - topStack(false, value::TypeTags::Boolean, value::bitcastFrom<bool>(res)); + runClassicMatcher(matcher); break; } default: @@ -5885,32 +5738,29 @@ void ByteCode::runInternal(const CodeFragment* code, int64_t position) { } } -std::tuple<uint8_t, value::TypeTags, value::Value> ByteCode::run(const CodeFragment* code) { - uassert(6040900, "The evaluation stack must be empty", _argStack.size() == 0); - - ON_BLOCK_EXIT([&] { - auto size = _argStack.size(); - for (size_t i = 0; i < size; ++i) { - auto [owned, tag] = _argStack.ownedAndTag(i); - if (owned) { - value::releaseValue(tag, _argStack.value(i)); - } - } - - _argStack.resize(0); - }); - - runInternal(code, 0); +FastTuple<bool, value::TypeTags, value::Value> ByteCode::run(const CodeFragment* code) { + try { + uassert(6040900, + "The evaluation stack must be empty", + _argStackTop + sizeOfElement == _argStack); - uassert(4822801, "The evaluation stack must hold only a single value", _argStack.size() == 1); + runInternal(code, 0); - auto [owned, tag] = _argStack.ownedAndTag(0); - auto val = _argStack.value(0); + uassert(4822801, + "The evaluation stack must hold only a single value", + _argStackTop == _argStack); - // Transfer ownership of tag/val to the caller - _argStack.resize(0); + // Transfer ownership of tag/val to the caller + stackReset(); - return {owned, tag, val}; + return readTuple(_argStack); + } catch (...) { + auto sentinel = _argStack - sizeOfElement; + while (_argStackTop != sentinel) { + popAndReleaseStack(); + } + throw; + } } bool ByteCode::runPredicate(const CodeFragment* code) { diff --git a/src/mongo/db/exec/sbe/vm/vm.h b/src/mongo/db/exec/sbe/vm/vm.h index 02f7146c42f..179d3aba3f2 100644 --- a/src/mongo/db/exec/sbe/vm/vm.h +++ b/src/mongo/db/exec/sbe/vm/vm.h @@ -830,129 +830,84 @@ private: size_t _stackSize{0}; }; -class ByteCode { -public: - std::tuple<uint8_t, value::TypeTags, value::Value> run(const CodeFragment* code); - bool runPredicate(const CodeFragment* code); - -private: - // The VM stack is used to pass inputs to instructions and hold the outputs produced by - // instructions. Each element of the VM stack is 3-tuple comprised of a boolean ('owned'), - // a value::TypeTags ('tag'), and a value::Value ('value'). - // - // In order to make the VM stack cache-friendly, for each element we want 'owned', 'tag', - // and 'value' to be located relatively close together, and we also want to avoid wasting - // any bytes due to padding. - // - // To achieve these goals, the VM stack is organized as a vector of "stack segments". Each - // "segment" is large enough to hold 4 elements. The first 8 bytes of a segment holds the - // 'owned' and 'tag' components, and the remaining 32 bytes hold the 'value' components. - static constexpr size_t ElementsPerSegment = 4; - - struct OwnedAndTag { - uint8_t owned; - value::TypeTags tag; - }; - - struct StackSegment { - OwnedAndTag ownedAndTags[ElementsPerSegment]; - value::Value values[ElementsPerSegment]; - }; - - class Stack { - public: - static constexpr size_t kMaxCapacity = - ((std::numeric_limits<size_t>::max() / 2) / sizeof(StackSegment)) * ElementsPerSegment; - - const auto& ownedAndTag(size_t index) const { - return _segments[index / ElementsPerSegment].ownedAndTags[index % ElementsPerSegment]; - } - auto& ownedAndTag(size_t index) { - return _segments[index / ElementsPerSegment].ownedAndTags[index % ElementsPerSegment]; - } - - const auto& owned(size_t index) const { - return ownedAndTag(index).owned; - } - auto& owned(size_t index) { - return ownedAndTag(index).owned; - } - - const auto& tag(size_t index) const { - return ownedAndTag(index).tag; - } - auto& tag(size_t index) { - return ownedAndTag(index).tag; - } - - const auto& value(size_t index) const { - return _segments[index / ElementsPerSegment].values[index % ElementsPerSegment]; - } - auto& value(size_t index) { - return _segments[index / ElementsPerSegment].values[index % ElementsPerSegment]; - } - - auto size() const { - return _size; - } +namespace { +template <typename T> +T readFromMemory(const uint8_t* ptr) noexcept { + static_assert(!IsEndian<T>::value); + + T val; + memcpy(&val, ptr, sizeof(T)); + return val; +} - auto capacity() const { - return _capacity; - } +template <typename T> +size_t writeToMemory(uint8_t* ptr, const T val) noexcept { + static_assert(!IsEndian<T>::value); - void resize(size_t newSize) { - if (MONGO_likely(newSize <= capacity())) { - _size = newSize; - return; - } - growAndResize(newSize); - } + memcpy(ptr, &val, sizeof(T)); + return sizeof(T); +} +} // namespace - void resizeDown() { - --_size; - } +class ByteCode { + static constexpr size_t sizeOfElement = + sizeof(bool) + sizeof(value::TypeTags) + sizeof(value::Value); + static_assert(sizeOfElement == 10); + static_assert(std::is_trivially_copyable_v<FastTuple<bool, value::TypeTags, value::Value>>); - private: - MONGO_COMPILER_NOINLINE void growAndResize(size_t newSize); +public: + ByteCode() { + _argStack = reinterpret_cast<uint8_t*>(mongoMalloc(sizeOfElement * 4)); + _argStackEnd = _argStack + sizeOfElement * 4; + _argStackTop = _argStack - sizeOfElement; + } - std::unique_ptr<StackSegment[]> _segments; - size_t _size = 0; - size_t _capacity = 0; - }; + ~ByteCode() { + std::free(_argStack); + } - Stack _argStack; + FastTuple<bool, value::TypeTags, value::Value> run(const CodeFragment* code); + bool runPredicate(const CodeFragment* code); +private: void runInternal(const CodeFragment* code, int64_t position); void runLambdaInternal(const CodeFragment* code, int64_t position); - std::tuple<bool, value::TypeTags, value::Value> genericDiv(value::TypeTags lhsTag, - value::Value lhsValue, - value::TypeTags rhsTag, - value::Value rhsValue); - std::tuple<bool, value::TypeTags, value::Value> genericIDiv(value::TypeTags lhsTag, - value::Value lhsValue, - value::TypeTags rhsTag, - value::Value rhsValue); - std::tuple<bool, value::TypeTags, value::Value> genericMod(value::TypeTags lhsTag, + MONGO_COMPILER_NORETURN void runFailInstruction(); + void runClassicMatcher(const MatchExpression* matcher); + + template <typename T> + void runTagCheck(T&& predicate); + void runTagCheck(value::TypeTags tagRhs); + + FastTuple<bool, value::TypeTags, value::Value> genericDiv(value::TypeTags lhsTag, + value::Value lhsValue, + value::TypeTags rhsTag, + value::Value rhsValue); + FastTuple<bool, value::TypeTags, value::Value> genericIDiv(value::TypeTags lhsTag, value::Value lhsValue, value::TypeTags rhsTag, value::Value rhsValue); - std::tuple<bool, value::TypeTags, value::Value> genericAbs(value::TypeTags operandTag, + FastTuple<bool, value::TypeTags, value::Value> genericMod(value::TypeTags lhsTag, + value::Value lhsValue, + value::TypeTags rhsTag, + value::Value rhsValue); + FastTuple<bool, value::TypeTags, value::Value> genericAbs(value::TypeTags operandTag, + value::Value operandValue); + FastTuple<bool, value::TypeTags, value::Value> genericCeil(value::TypeTags operandTag, value::Value operandValue); - std::tuple<bool, value::TypeTags, value::Value> genericCeil(value::TypeTags operandTag, + FastTuple<bool, value::TypeTags, value::Value> genericFloor(value::TypeTags operandTag, value::Value operandValue); - std::tuple<bool, value::TypeTags, value::Value> genericFloor(value::TypeTags operandTag, - value::Value operandValue); - std::tuple<bool, value::TypeTags, value::Value> genericTrunc(value::TypeTags operandTag, - value::Value operandValue); - std::tuple<bool, value::TypeTags, value::Value> genericExp(value::TypeTags operandTag, - value::Value operandValue); - std::tuple<bool, value::TypeTags, value::Value> genericLn(value::TypeTags operandTag, + FastTuple<bool, value::TypeTags, value::Value> genericTrunc(value::TypeTags operandTag, + value::Value operandValue); + FastTuple<bool, value::TypeTags, value::Value> genericExp(value::TypeTags operandTag, value::Value operandValue); - std::tuple<bool, value::TypeTags, value::Value> genericLog10(value::TypeTags operandTag, - value::Value operandValue); - std::tuple<bool, value::TypeTags, value::Value> genericSqrt(value::TypeTags operandTag, + FastTuple<bool, value::TypeTags, value::Value> genericLn(value::TypeTags operandTag, + value::Value operandValue); + FastTuple<bool, value::TypeTags, value::Value> genericLog10(value::TypeTags operandTag, value::Value operandValue); + FastTuple<bool, value::TypeTags, value::Value> genericSqrt(value::TypeTags operandTag, + value::Value operandValue); std::pair<value::TypeTags, value::Value> genericNot(value::TypeTags tag, value::Value value); std::pair<value::TypeTags, value::Value> genericIsMember(value::TypeTags lhsTag, value::Value lhsVal, @@ -965,9 +920,9 @@ private: value::Value rhsVal, value::TypeTags collTag, value::Value collVal); - std::tuple<bool, value::TypeTags, value::Value> genericNumConvert(value::TypeTags lhsTag, - value::Value lhsValue, - value::TypeTags rhsTag); + FastTuple<bool, value::TypeTags, value::Value> genericNumConvert(value::TypeTags lhsTag, + value::Value lhsValue, + value::TypeTags rhsTag); std::pair<value::TypeTags, value::Value> compare3way( value::TypeTags lhsTag, @@ -983,23 +938,23 @@ private: value::TypeTags collTag, value::Value collValue); - std::tuple<bool, value::TypeTags, value::Value> getField(value::TypeTags objTag, - value::Value objValue, - value::TypeTags fieldTag, - value::Value fieldValue); + FastTuple<bool, value::TypeTags, value::Value> getField(value::TypeTags objTag, + value::Value objValue, + value::TypeTags fieldTag, + value::Value fieldValue); - std::tuple<bool, value::TypeTags, value::Value> getField(value::TypeTags objTag, - value::Value objValue, - StringData fieldStr); + FastTuple<bool, value::TypeTags, value::Value> getField(value::TypeTags objTag, + value::Value objValue, + StringData fieldStr); - std::tuple<bool, value::TypeTags, value::Value> getElement(value::TypeTags objTag, - value::Value objValue, - value::TypeTags fieldTag, - value::Value fieldValue); - std::tuple<bool, value::TypeTags, value::Value> getFieldOrElement(value::TypeTags objTag, - value::Value objValue, - value::TypeTags fieldTag, - value::Value fieldValue); + FastTuple<bool, value::TypeTags, value::Value> getElement(value::TypeTags objTag, + value::Value objValue, + value::TypeTags fieldTag, + value::Value fieldValue); + FastTuple<bool, value::TypeTags, value::Value> getFieldOrElement(value::TypeTags objTag, + value::Value objValue, + value::TypeTags fieldTag, + value::Value fieldValue); void traverseP(const CodeFragment* code); void traverseP(const CodeFragment* code, int64_t position, int64_t maxDepth); @@ -1012,15 +967,15 @@ private: void traverseF(const CodeFragment* code); void traverseF(const CodeFragment* code, int64_t position, bool compareArray); void traverseFInArray(const CodeFragment* code, int64_t position, bool compareArray); - std::tuple<bool, value::TypeTags, value::Value> setField(); + FastTuple<bool, value::TypeTags, value::Value> setField(); - std::tuple<bool, value::TypeTags, value::Value> getArraySize(value::TypeTags tag, - value::Value val); + FastTuple<bool, value::TypeTags, value::Value> getArraySize(value::TypeTags tag, + value::Value val); - std::tuple<bool, value::TypeTags, value::Value> aggSum(value::TypeTags accTag, - value::Value accValue, - value::TypeTags fieldTag, - value::Value fieldValue); + FastTuple<bool, value::TypeTags, value::Value> aggSum(value::TypeTags accTag, + value::Value accValue, + value::TypeTags fieldTag, + value::Value fieldValue); void aggDoubleDoubleSumImpl(value::Array* arr, value::TypeTags rhsTag, value::Value rhsValue); @@ -1028,214 +983,239 @@ private: // https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Welford's_online_algorithm void aggStdDevImpl(value::Array* arr, value::TypeTags rhsTag, value::Value rhsValue); - std::tuple<bool, value::TypeTags, value::Value> aggStdDevFinalizeImpl(value::Value fieldValue, - bool isSamp); + FastTuple<bool, value::TypeTags, value::Value> aggStdDevFinalizeImpl(value::Value fieldValue, + bool isSamp); - std::tuple<bool, value::TypeTags, value::Value> aggMin(value::TypeTags accTag, - value::Value accValue, - value::TypeTags fieldTag, - value::Value fieldValue, - CollatorInterface* collator = nullptr); - - std::tuple<bool, value::TypeTags, value::Value> aggMax(value::TypeTags accTag, - value::Value accValue, - value::TypeTags fieldTag, - value::Value fieldValue, - CollatorInterface* collator = nullptr); + FastTuple<bool, value::TypeTags, value::Value> aggMin(value::TypeTags accTag, + value::Value accValue, + value::TypeTags fieldTag, + value::Value fieldValue, + CollatorInterface* collator = nullptr); - std::tuple<bool, value::TypeTags, value::Value> aggFirst(value::TypeTags accTag, - value::Value accValue, - value::TypeTags fieldTag, - value::Value fieldValue); + FastTuple<bool, value::TypeTags, value::Value> aggMax(value::TypeTags accTag, + value::Value accValue, + value::TypeTags fieldTag, + value::Value fieldValue, + CollatorInterface* collator = nullptr); - std::tuple<bool, value::TypeTags, value::Value> aggLast(value::TypeTags accTag, + FastTuple<bool, value::TypeTags, value::Value> aggFirst(value::TypeTags accTag, value::Value accValue, value::TypeTags fieldTag, value::Value fieldValue); - std::tuple<bool, value::TypeTags, value::Value> genericAcos(value::TypeTags operandTag, - value::Value operandValue); - std::tuple<bool, value::TypeTags, value::Value> genericAcosh(value::TypeTags operandTag, - value::Value operandValue); - std::tuple<bool, value::TypeTags, value::Value> genericAsin(value::TypeTags operandTag, + FastTuple<bool, value::TypeTags, value::Value> aggLast(value::TypeTags accTag, + value::Value accValue, + value::TypeTags fieldTag, + value::Value fieldValue); + + FastTuple<bool, value::TypeTags, value::Value> genericAcos(value::TypeTags operandTag, + value::Value operandValue); + FastTuple<bool, value::TypeTags, value::Value> genericAcosh(value::TypeTags operandTag, value::Value operandValue); - std::tuple<bool, value::TypeTags, value::Value> genericAsinh(value::TypeTags operandTag, - value::Value operandValue); - std::tuple<bool, value::TypeTags, value::Value> genericAtan(value::TypeTags operandTag, + FastTuple<bool, value::TypeTags, value::Value> genericAsin(value::TypeTags operandTag, + value::Value operandValue); + FastTuple<bool, value::TypeTags, value::Value> genericAsinh(value::TypeTags operandTag, value::Value operandValue); - std::tuple<bool, value::TypeTags, value::Value> genericAtanh(value::TypeTags operandTag, - value::Value operandValue); - std::tuple<bool, value::TypeTags, value::Value> genericAtan2(value::TypeTags operandTag1, - value::Value operandValue1, - value::TypeTags operandTag2, - value::Value operandValue2); - std::tuple<bool, value::TypeTags, value::Value> genericCos(value::TypeTags operandTag, + FastTuple<bool, value::TypeTags, value::Value> genericAtan(value::TypeTags operandTag, value::Value operandValue); - std::tuple<bool, value::TypeTags, value::Value> genericCosh(value::TypeTags operandTag, + FastTuple<bool, value::TypeTags, value::Value> genericAtanh(value::TypeTags operandTag, value::Value operandValue); - std::tuple<bool, value::TypeTags, value::Value> genericDegreesToRadians( + FastTuple<bool, value::TypeTags, value::Value> genericAtan2(value::TypeTags operandTag1, + value::Value operandValue1, + value::TypeTags operandTag2, + value::Value operandValue2); + FastTuple<bool, value::TypeTags, value::Value> genericCos(value::TypeTags operandTag, + value::Value operandValue); + FastTuple<bool, value::TypeTags, value::Value> genericCosh(value::TypeTags operandTag, + value::Value operandValue); + FastTuple<bool, value::TypeTags, value::Value> genericDegreesToRadians( value::TypeTags operandTag, value::Value operandValue); - std::tuple<bool, value::TypeTags, value::Value> genericRadiansToDegrees( + FastTuple<bool, value::TypeTags, value::Value> genericRadiansToDegrees( value::TypeTags operandTag, value::Value operandValue); - std::tuple<bool, value::TypeTags, value::Value> genericSin(value::TypeTags operandTag, + FastTuple<bool, value::TypeTags, value::Value> genericSin(value::TypeTags operandTag, + value::Value operandValue); + FastTuple<bool, value::TypeTags, value::Value> genericSinh(value::TypeTags operandTag, value::Value operandValue); - std::tuple<bool, value::TypeTags, value::Value> genericSinh(value::TypeTags operandTag, - value::Value operandValue); - std::tuple<bool, value::TypeTags, value::Value> genericTan(value::TypeTags operandTag, + FastTuple<bool, value::TypeTags, value::Value> genericTan(value::TypeTags operandTag, + value::Value operandValue); + FastTuple<bool, value::TypeTags, value::Value> genericTanh(value::TypeTags operandTag, value::Value operandValue); - std::tuple<bool, value::TypeTags, value::Value> genericTanh(value::TypeTags operandTag, - value::Value operandValue); - std::tuple<bool, value::TypeTags, value::Value> genericDayOfYear(value::TypeTags timezoneDBTag, + FastTuple<bool, value::TypeTags, value::Value> genericDayOfYear(value::TypeTags timezoneDBTag, + value::Value timezoneDBValue, + value::TypeTags dateTag, + value::Value dateValue, + value::TypeTags timezoneTag, + value::Value timezoneValue); + FastTuple<bool, value::TypeTags, value::Value> genericDayOfMonth(value::TypeTags timezoneDBTag, value::Value timezoneDBValue, value::TypeTags dateTag, value::Value dateValue, value::TypeTags timezoneTag, value::Value timezoneValue); - std::tuple<bool, value::TypeTags, value::Value> genericDayOfMonth(value::TypeTags timezoneDBTag, - value::Value timezoneDBValue, - value::TypeTags dateTag, - value::Value dateValue, - value::TypeTags timezoneTag, - value::Value timezoneValue); - std::tuple<bool, value::TypeTags, value::Value> genericDayOfWeek(value::TypeTags timezoneDBTag, - value::Value timezoneDBValue, - value::TypeTags dateTag, - value::Value dateValue, - value::TypeTags timezoneTag, - value::Value timezoneValue); - std::tuple<bool, value::TypeTags, value::Value> genericNewKeyString( + FastTuple<bool, value::TypeTags, value::Value> genericDayOfWeek(value::TypeTags timezoneDBTag, + value::Value timezoneDBValue, + value::TypeTags dateTag, + value::Value dateValue, + value::TypeTags timezoneTag, + value::Value timezoneValue); + FastTuple<bool, value::TypeTags, value::Value> genericNewKeyString( ArityType arity, CollatorInterface* collator = nullptr); - std::tuple<bool, value::TypeTags, value::Value> builtinSplit(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinDate(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinDateWeekYear(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinDateDiff(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinDateToParts(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinIsoDateToParts(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinDayOfYear(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinDayOfMonth(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinDayOfWeek(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinRegexMatch(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinKeepFields(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinReplaceOne(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinDropFields(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinNewArray(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinNewArrayFromRange(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinNewObj(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinKeyStringToString(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinNewKeyString(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinCollNewKeyString(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinAbs(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinCeil(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinFloor(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinTrunc(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinExp(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinLn(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinLog10(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinSqrt(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinAddToArray(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinAddToArrayCapped(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinMergeObjects(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinAddToSet(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinCollAddToSet(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> addToSetCappedImpl(value::TypeTags tagNewElem, - value::Value valNewElem, - int32_t sizeCap, - CollatorInterface* collator); - std::tuple<bool, value::TypeTags, value::Value> builtinAddToSetCapped(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinCollAddToSetCapped(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinDoubleDoubleSum(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinAggDoubleDoubleSum(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinDoubleDoubleSumFinalize(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinDoubleDoublePartialSumFinalize( + FastTuple<bool, value::TypeTags, value::Value> builtinSplit(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinDate(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinDateWeekYear(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinDateDiff(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinDateToParts(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinIsoDateToParts(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinDayOfYear(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinDayOfMonth(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinDayOfWeek(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinRegexMatch(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinKeepFields(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinReplaceOne(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinDropFields(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinNewArray(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinNewArrayFromRange(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinNewObj(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinKeyStringToString(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinNewKeyString(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinCollNewKeyString(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinAbs(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinCeil(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinFloor(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinTrunc(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinExp(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinLn(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinLog10(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinSqrt(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinAddToArray(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinAddToArrayCapped(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinMergeObjects(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinAddToSet(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinCollAddToSet(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> addToSetCappedImpl(value::TypeTags tagNewElem, + value::Value valNewElem, + int32_t sizeCap, + CollatorInterface* collator); + FastTuple<bool, value::TypeTags, value::Value> builtinAddToSetCapped(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinCollAddToSetCapped(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinDoubleDoubleSum(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinAggDoubleDoubleSum(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinDoubleDoubleSumFinalize(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinDoubleDoublePartialSumFinalize( ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinAggStdDev(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinStdDevPopFinalize(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinStdDevSampFinalize(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinBitTestZero(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinBitTestMask(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinBitTestPosition(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinBsonSize(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinToUpper(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinToLower(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinCoerceToString(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinAcos(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinAcosh(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinAsin(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinAsinh(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinAtan(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinAtanh(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinAtan2(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinCos(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinCosh(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinDegreesToRadians(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinRadiansToDegrees(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinSin(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinSinh(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinTan(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinTanh(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinRound(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinConcat(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinIsMember(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinCollIsMember(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinIndexOfBytes(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinIndexOfCP(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinIsDayOfWeek(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinIsTimeUnit(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinIsTimezone(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinSetUnion(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinSetIntersection(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinSetDifference(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinSetEquals(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinCollSetUnion(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinCollSetIntersection(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinCollSetDifference(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinCollSetEquals(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinRunJsPredicate(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinRegexCompile(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinRegexFind(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinRegexFindAll(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinShardFilter(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinShardHash(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinExtractSubArray(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinIsArrayEmpty(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinReverseArray(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinSortArray(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinDateAdd(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinHasNullBytes(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinGetRegexPattern(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinGetRegexFlags(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinHash(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinFtsMatch(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinGenerateSortKey(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinMakeBsonObj(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinTsSecond(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinTsIncrement(ArityType arity); - std::tuple<bool, value::TypeTags, value::Value> builtinTypeMatch(ArityType arity); - - std::tuple<bool, value::TypeTags, value::Value> dispatchBuiltin(Builtin f, ArityType arity); - - std::tuple<bool, value::TypeTags, value::Value> getFromStack(size_t offset) { - auto backOffset = _argStack.size() - 1 - offset; - - auto [owned, tag] = _argStack.ownedAndTag(backOffset); - auto val = _argStack.value(backOffset); - + FastTuple<bool, value::TypeTags, value::Value> builtinAggStdDev(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinStdDevPopFinalize(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinStdDevSampFinalize(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinBitTestZero(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinBitTestMask(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinBitTestPosition(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinBsonSize(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinToUpper(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinToLower(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinCoerceToString(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinAcos(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinAcosh(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinAsin(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinAsinh(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinAtan(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinAtanh(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinAtan2(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinCos(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinCosh(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinDegreesToRadians(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinRadiansToDegrees(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinSin(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinSinh(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinTan(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinTanh(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinRound(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinConcat(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinIsMember(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinCollIsMember(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinIndexOfBytes(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinIndexOfCP(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinIsDayOfWeek(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinIsTimeUnit(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinIsTimezone(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinSetUnion(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinSetIntersection(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinSetDifference(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinSetEquals(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinCollSetUnion(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinCollSetIntersection(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinCollSetDifference(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinCollSetEquals(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinRunJsPredicate(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinRegexCompile(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinRegexFind(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinRegexFindAll(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinShardFilter(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinShardHash(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinExtractSubArray(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinIsArrayEmpty(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinReverseArray(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinSortArray(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinDateAdd(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinHasNullBytes(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinGetRegexPattern(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinGetRegexFlags(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinHash(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinFtsMatch(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinGenerateSortKey(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinMakeBsonObj(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinTsSecond(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinTsIncrement(ArityType arity); + FastTuple<bool, value::TypeTags, value::Value> builtinTypeMatch(ArityType arity); + + FastTuple<bool, value::TypeTags, value::Value> dispatchBuiltin(Builtin f, ArityType arity); + + static constexpr size_t offsetOwned = 0; + static constexpr size_t offsetTag = 1; + static constexpr size_t offsetVal = 2; + + MONGO_COMPILER_ALWAYS_INLINE FastTuple<bool, value::TypeTags, value::Value> readTuple( + uint8_t* ptr) noexcept { + auto owned = readFromMemory<bool>(ptr + offsetOwned); + auto tag = readFromMemory<value::TypeTags>(ptr + offsetTag); + auto val = readFromMemory<value::Value>(ptr + offsetVal); return {owned, tag, val}; } - std::tuple<bool, value::TypeTags, value::Value> moveFromStack(size_t offset) { - auto backOffset = _argStack.size() - 1 - offset; - - auto [owned, tag] = _argStack.ownedAndTag(backOffset); - auto val = _argStack.value(backOffset); - _argStack.owned(backOffset) = false; + MONGO_COMPILER_ALWAYS_INLINE void writeTuple(uint8_t* ptr, + bool owned, + value::TypeTags tag, + value::Value val) noexcept { + writeToMemory(ptr + offsetOwned, owned); + writeToMemory(ptr + offsetTag, tag); + writeToMemory(ptr + offsetVal, val); + } + MONGO_COMPILER_ALWAYS_INLINE FastTuple<bool, value::TypeTags, value::Value> getFromStack( + size_t offset) noexcept { + if (MONGO_likely(offset == 0)) { + return readTuple(_argStackTop); + } else { + return readTuple(_argStackTop - offset * sizeOfElement); + } + } - return {owned, tag, val}; + MONGO_COMPILER_ALWAYS_INLINE FastTuple<bool, value::TypeTags, value::Value> moveFromStack( + size_t offset) noexcept { + if (MONGO_likely(offset == 0)) { + auto [owned, tag, val] = readTuple(_argStackTop); + writeToMemory(_argStackTop + offsetOwned, false); + return {owned, tag, val}; + } else { + auto ptr = _argStackTop - offset * sizeOfElement; + auto [owned, tag, val] = readTuple(ptr); + writeToMemory(ptr + offsetOwned, false); + return {owned, tag, val}; + } } - std::pair<value::TypeTags, value::Value> moveOwnedFromStack(size_t offset) { + MONGO_COMPILER_ALWAYS_INLINE std::pair<value::TypeTags, value::Value> moveOwnedFromStack( + size_t offset) { auto [owned, tag, val] = moveFromStack(offset); if (!owned) { std::tie(tag, val) = value::copyValue(tag, val); @@ -1244,39 +1224,57 @@ private: return {tag, val}; } - void setStack(size_t offset, bool owned, value::TypeTags tag, value::Value val) { - auto backOffset = _argStack.size() - 1 - offset; - _argStack.ownedAndTag(backOffset) = {owned, tag}; - _argStack.value(backOffset) = val; + MONGO_COMPILER_ALWAYS_INLINE void setStack(size_t offset, + bool owned, + value::TypeTags tag, + value::Value val) noexcept { + if (MONGO_likely(offset == 0)) { + topStack(owned, tag, val); + } else { + writeTuple(_argStackTop - offset * sizeOfElement, owned, tag, val); + } } - void pushStack(bool owned, value::TypeTags tag, value::Value val) { - _argStack.resize(_argStack.size() + 1); + MONGO_COMPILER_ALWAYS_INLINE void pushStack(bool owned, + value::TypeTags tag, + value::Value val) noexcept { + _argStackTop += sizeOfElement; + if (MONGO_unlikely(_argStackTop == _argStackEnd)) { + growAndResize(); + } topStack(owned, tag, val); } - void topStack(bool owned, value::TypeTags tag, value::Value val) { - size_t index = _argStack.size() - 1; - _argStack.ownedAndTag(index) = {owned, tag}; - _argStack.value(index) = val; + MONGO_COMPILER_ALWAYS_INLINE void topStack(bool owned, + value::TypeTags tag, + value::Value val) noexcept { + writeTuple(_argStackTop, owned, tag, val); } - void popStack() { - _argStack.resizeDown(); + MONGO_COMPILER_ALWAYS_INLINE void popStack() noexcept { + _argStackTop -= sizeOfElement; } - void popAndReleaseStack() { - size_t index = _argStack.size() - 1; - auto [owned, tag] = _argStack.ownedAndTag(index); - + MONGO_COMPILER_ALWAYS_INLINE void popAndReleaseStack() noexcept { + auto [owned, tag, val] = getFromStack(0); if (owned) { - value::releaseValue(tag, _argStack.value(index)); + value::releaseValue(tag, val); } popStack(); } + void stackReset() noexcept { + _argStackTop = _argStack - sizeOfElement; + } + + void growAndResize() noexcept; + void swapStack(); + + uint8_t* _argStackTop{nullptr}; + uint8_t* _argStack{nullptr}; + uint8_t* _argStackEnd{nullptr}; }; } // namespace vm } // namespace sbe |