diff options
author | Drew Paroski <drew.paroski@mongodb.com> | 2020-07-10 13:46:04 -0400 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2020-08-13 19:59:53 +0000 |
commit | 817ae16840a02b66ce2a50aca44754450bbce6b0 (patch) | |
tree | 86bdf1aad2c52da76b95a06ee6c4a62d1624f6c4 | |
parent | 1fde9c98c245050117f9f4891c3b5ddc8f8cd271 (diff) | |
download | mongo-817ae16840a02b66ce2a50aca44754450bbce6b0.tar.gz |
SERVER-49226 Create unittest fixture for SBE PlanStages
-rw-r--r-- | src/mongo/db/exec/sbe/SConscript | 9 | ||||
-rw-r--r-- | src/mongo/db/exec/sbe/expressions/expression.cpp | 2 | ||||
-rw-r--r-- | src/mongo/db/exec/sbe/sbe_filter_test.cpp | 178 | ||||
-rw-r--r-- | src/mongo/db/exec/sbe/sbe_limit_skip_test.cpp | 87 | ||||
-rw-r--r-- | src/mongo/db/exec/sbe/sbe_plan_stage_test.cpp | 239 | ||||
-rw-r--r-- | src/mongo/db/exec/sbe/sbe_plan_stage_test.h | 248 | ||||
-rw-r--r-- | src/mongo/db/exec/sbe/sbe_sort_test.cpp | 76 | ||||
-rw-r--r-- | src/mongo/db/exec/sbe/values/bson.cpp | 4 | ||||
-rw-r--r-- | src/mongo/db/exec/sbe/values/bson.h | 3 | ||||
-rw-r--r-- | src/mongo/db/exec/sbe/vm/arith.cpp | 21 | ||||
-rw-r--r-- | src/mongo/db/exec/sbe/vm/vm.cpp | 75 | ||||
-rw-r--r-- | src/mongo/db/exec/sbe/vm/vm.h | 10 |
12 files changed, 949 insertions, 3 deletions
diff --git a/src/mongo/db/exec/sbe/SConscript b/src/mongo/db/exec/sbe/SConscript index 368891f709a..514be74881a 100644 --- a/src/mongo/db/exec/sbe/SConscript +++ b/src/mongo/db/exec/sbe/SConscript @@ -87,13 +87,18 @@ env.Library( env.CppUnitTest( target='db_sbe_test', source=[ - 'sbe_test.cpp', + 'sbe_filter_test.cpp', 'sbe_key_string_test.cpp', + 'sbe_limit_skip_test.cpp', 'sbe_numeric_convert_test.cpp', + 'sbe_plan_stage_test.cpp', + 'sbe_sort_test.cpp', + 'sbe_test.cpp', ], LIBDEPS=[ '$BUILD_DIR/mongo/db/concurrency/lock_manager', '$BUILD_DIR/mongo/unittest/unittest', - 'query_sbe_parser' + '$BUILD_DIR/mongo/db/service_context_test_fixture', + 'query_sbe_parser', ], ) diff --git a/src/mongo/db/exec/sbe/expressions/expression.cpp b/src/mongo/db/exec/sbe/expressions/expression.cpp index 78c632c646b..0d4c7b9a709 100644 --- a/src/mongo/db/exec/sbe/expressions/expression.cpp +++ b/src/mongo/db/exec/sbe/expressions/expression.cpp @@ -382,6 +382,8 @@ struct InstrFn { static stdx::unordered_map<std::string, InstrFn> kInstrFunctions = { {"getField", InstrFn{[](size_t n) { return n == 2; }, &vm::CodeFragment::appendGetField, false}}, + {"getElement", + InstrFn{[](size_t n) { return n == 2; }, &vm::CodeFragment::appendGetElement, false}}, {"fillEmpty", InstrFn{[](size_t n) { return n == 2; }, &vm::CodeFragment::appendFillEmpty, false}}, {"exists", InstrFn{[](size_t n) { return n == 1; }, &vm::CodeFragment::appendExists, false}}, diff --git a/src/mongo/db/exec/sbe/sbe_filter_test.cpp b/src/mongo/db/exec/sbe/sbe_filter_test.cpp new file mode 100644 index 00000000000..3e55f041334 --- /dev/null +++ b/src/mongo/db/exec/sbe/sbe_filter_test.cpp @@ -0,0 +1,178 @@ +/** + * Copyright (C) 2018-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +/** + * This file contains tests for sbe::FilterStage. + */ + +#include "mongo/platform/basic.h" + +#include <string_view> + +#include "mongo/db/exec/sbe/sbe_plan_stage_test.h" +#include "mongo/db/exec/sbe/stages/filter.h" + +namespace mongo::sbe { + +using FilterStageTest = PlanStageTestFixture; + +TEST_F(FilterStageTest, ConstantFilterAlwaysTrueTest) { + auto [inputTag, inputVal] = + makeValue(BSON_ARRAY(12LL << "yar" << BSON_ARRAY(2.5) << 7.5 << BSON("foo" << 23))); + value::ValueGuard inputGuard{inputTag, inputVal}; + + auto [expectedTag, expectedVal] = value::copyValue(inputTag, inputVal); + value::ValueGuard expectedGuard{expectedTag, expectedVal}; + + auto makeStageFn = [](value::SlotId scanSlot, std::unique_ptr<PlanStage> scanStage) { + // Build a constant FilterStage whose filter expression is always boolean true. + auto filter = makeS<FilterStage<true>>(std::move(scanStage), + makeE<EConstant>(value::TypeTags::Boolean, 1)); + + return std::make_pair(scanSlot, std::move(filter)); + }; + + inputGuard.reset(); + expectedGuard.reset(); + runTest(inputTag, inputVal, expectedTag, expectedVal, makeStageFn); +} + +TEST_F(FilterStageTest, ConstantFilterAlwaysFalseTest) { + auto [inputTag, inputVal] = + makeValue(BSON_ARRAY(12LL << "yar" << BSON_ARRAY(2.5) << 7.5 << BSON("foo" << 23))); + value::ValueGuard inputGuard{inputTag, inputVal}; + + auto [expectedTag, expectedVal] = value::makeNewArray(); + value::ValueGuard expectedGuard{expectedTag, expectedVal}; + + auto makeStageFn = [](value::SlotId scanSlot, std::unique_ptr<PlanStage> scanStage) { + // Build a constant FilterStage whose filter expression is always boolean false. + auto filter = makeS<FilterStage<true>>(std::move(scanStage), + makeE<EConstant>(value::TypeTags::Boolean, 0)); + + return std::make_pair(scanSlot, std::move(filter)); + }; + + inputGuard.reset(); + expectedGuard.reset(); + runTest(inputTag, inputVal, expectedTag, expectedVal, makeStageFn); +} + +TEST_F(FilterStageTest, FilterAlwaysTrueTest) { + auto [inputTag, inputVal] = + makeValue(BSON_ARRAY(12LL << "yar" << BSON_ARRAY(2.5) << 7.5 << BSON("foo" << 23))); + value::ValueGuard inputGuard{inputTag, inputVal}; + + auto [expectedTag, expectedVal] = value::copyValue(inputTag, inputVal); + value::ValueGuard expectedGuard{expectedTag, expectedVal}; + + auto makeStageFn = [](value::SlotId scanSlot, std::unique_ptr<PlanStage> scanStage) { + // Build a non-constant FilterStage whose filter expression is always boolean true. + auto filter = makeS<FilterStage<false>>(std::move(scanStage), + makeE<EConstant>(value::TypeTags::Boolean, 1)); + + return std::make_pair(scanSlot, std::move(filter)); + }; + + inputGuard.reset(); + expectedGuard.reset(); + runTest(inputTag, inputVal, expectedTag, expectedVal, makeStageFn); +} + +TEST_F(FilterStageTest, FilterAlwaysFalseTest) { + auto [inputTag, inputVal] = + makeValue(BSON_ARRAY(12LL << "yar" << BSON_ARRAY(2.5) << 7.5 << BSON("foo" << 23))); + value::ValueGuard inputGuard{inputTag, inputVal}; + + auto [expectedTag, expectedVal] = value::makeNewArray(); + value::ValueGuard expectedGuard{expectedTag, expectedVal}; + + auto makeStageFn = [](value::SlotId scanSlot, std::unique_ptr<PlanStage> scanStage) { + // Build a non-constant FilterStage whose filter expression is always boolean false. + auto filter = makeS<FilterStage<false>>(std::move(scanStage), + makeE<EConstant>(value::TypeTags::Boolean, 0)); + + return std::make_pair(scanSlot, std::move(filter)); + }; + + inputGuard.reset(); + expectedGuard.reset(); + runTest(inputTag, inputVal, expectedTag, expectedVal, makeStageFn); +} + +TEST_F(FilterStageTest, FilterIsNumberTest) { + using namespace std::literals; + + auto [inputTag, inputVal] = + makeValue(BSON_ARRAY(12LL << "42" << BSON_ARRAY(2.5) << 7.5 << BSON("34" << 56))); + value::ValueGuard inputGuard{inputTag, inputVal}; + + auto [expectedTag, expectedVal] = makeValue(BSON_ARRAY(12LL << 7.5)); + value::ValueGuard expectedGuard{expectedTag, expectedVal}; + + auto makeStageFn = [](value::SlotId scanSlot, std::unique_ptr<PlanStage> scanStage) { + // Build a FilterStage whose filter expression is "isNumber(scanSlot)". + auto filter = makeS<FilterStage<false>>( + std::move(scanStage), + makeE<EFunction>("isNumber"sv, makeEs(makeE<EVariable>(scanSlot)))); + + return std::make_pair(scanSlot, std::move(filter)); + }; + + inputGuard.reset(); + expectedGuard.reset(); + runTest(inputTag, inputVal, expectedTag, expectedVal, makeStageFn); +} + +TEST_F(FilterStageTest, FilterLessThanTest) { + auto [inputTag, inputVal] = makeValue(BSON_ARRAY( + BSON_ARRAY(2.8 << 3) << BSON_ARRAY(7LL << 5.0) << BSON_ARRAY(4LL << 4.3) + << BSON_ARRAY(8 << 8) << BSON_ARRAY("1" << 2) << BSON_ARRAY(1 << "2") + << BSON_ARRAY(4.9 << 5) << BSON_ARRAY(6.0 << BSON_ARRAY(11.0)))); + value::ValueGuard inputGuard{inputTag, inputVal}; + + auto [expectedTag, expectedVal] = makeValue( + BSON_ARRAY(BSON_ARRAY(2.8 << 3) << BSON_ARRAY(4LL << 4.3) << BSON_ARRAY(4.9 << 5))); + value::ValueGuard expectedGuard{expectedTag, expectedVal}; + + auto makeStageFn = [](value::SlotVector scanSlots, std::unique_ptr<PlanStage> scanStage) { + // Build a FilterStage whose filter expression is "slot0 < slot1". + auto filter = makeS<FilterStage<false>>(std::move(scanStage), + makeE<EPrimBinary>(EPrimBinary::less, + makeE<EVariable>(scanSlots[0]), + makeE<EVariable>(scanSlots[1]))); + return std::make_pair(scanSlots, std::move(filter)); + }; + + inputGuard.reset(); + expectedGuard.reset(); + runTestMulti(2, inputTag, inputVal, expectedTag, expectedVal, makeStageFn); +} + +} // namespace mongo::sbe diff --git a/src/mongo/db/exec/sbe/sbe_limit_skip_test.cpp b/src/mongo/db/exec/sbe/sbe_limit_skip_test.cpp new file mode 100644 index 00000000000..b648cad5508 --- /dev/null +++ b/src/mongo/db/exec/sbe/sbe_limit_skip_test.cpp @@ -0,0 +1,87 @@ +/** + * Copyright (C) 2018-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +/** + * This file contains tests for sbe::LimitSkipStage. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/db/exec/sbe/sbe_plan_stage_test.h" +#include "mongo/db/exec/sbe/stages/limit_skip.h" + +namespace mongo::sbe { + +using LimitSkipStageTest = PlanStageTestFixture; + +TEST_F(LimitSkipStageTest, LimitSimpleTest) { + // Make a "limit 1000" stage. + auto limit = makeS<LimitSkipStage>(makeS<CoScanStage>(), 1000, boost::none); + + prepareTree(limit.get()); + + // Verify that `limit` produces at least 1000 values. + for (int i = 0; i < 1000; ++i) { + ASSERT_TRUE(limit->getNext() == PlanState::ADVANCED); + } + + // Verify that `limit` does not produce more than 1000 values. + ASSERT_TRUE(limit->getNext() == PlanState::IS_EOF); +} + +TEST_F(LimitSkipStageTest, LimitSkipSimpleTest) { + // Make an input array containing 64-integers 0 thru 999, inclusive. + auto [inputTag, inputVal] = value::makeNewArray(); + value::ValueGuard inputGuard{inputTag, inputVal}; + auto inputView = value::getArrayView(inputVal); + int i; + for (i = 0; i < 1000; ++i) { + inputView->push_back(value::TypeTags::NumberInt64, i); + } + + // Make a "limit 200 skip 300" stage. + inputGuard.reset(); + auto [scanSlot, scanStage] = generateMockScan(inputTag, inputVal); + auto limit = makeS<LimitSkipStage>(std::move(scanStage), 200, 300); + + auto resultAccessor = prepareTree(limit.get(), scanSlot); + + // Verify that `limit` produces exactly 300 thru 499, inclusive. + for (i = 0; i < 200; ++i) { + ASSERT_TRUE(limit->getNext() == PlanState::ADVANCED); + + auto [tag, val] = resultAccessor->getViewOfValue(); + ASSERT_TRUE(tag == value::TypeTags::NumberInt64); + ASSERT_TRUE(value::bitcastTo<int64_t>(val) == i + 300); + } + + ASSERT_TRUE(limit->getNext() == PlanState::IS_EOF); +} + +} // namespace mongo::sbe diff --git a/src/mongo/db/exec/sbe/sbe_plan_stage_test.cpp b/src/mongo/db/exec/sbe/sbe_plan_stage_test.cpp new file mode 100644 index 00000000000..f9a70523ee0 --- /dev/null +++ b/src/mongo/db/exec/sbe/sbe_plan_stage_test.cpp @@ -0,0 +1,239 @@ +/** + * Copyright (C) 2018-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +/** + * This file contains a test framework for testing sbe::PlanStages. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/db/exec/sbe/sbe_plan_stage_test.h" + +#include <string_view> + +namespace mongo::sbe { + +std::pair<value::TypeTags, value::Value> PlanStageTestFixture::makeValue(const BSONArray& ba) { + int numBytes = ba.objsize(); + uint8_t* data = new uint8_t[numBytes]; + memcpy(data, reinterpret_cast<const uint8_t*>(ba.objdata()), numBytes); + return {value::TypeTags::bsonArray, value::bitcastFrom(data)}; +} + +std::pair<value::TypeTags, value::Value> PlanStageTestFixture::makeValue(const BSONObj& bo) { + int numBytes = bo.objsize(); + uint8_t* data = new uint8_t[numBytes]; + memcpy(data, reinterpret_cast<const uint8_t*>(bo.objdata()), numBytes); + return {value::TypeTags::bsonObject, value::bitcastFrom(data)}; +} + +std::pair<value::SlotId, std::unique_ptr<PlanStage>> PlanStageTestFixture::generateMockScan( + value::TypeTags arrTag, value::Value arrVal) { + // The value passed in must be an array. + invariant(value::isArray(arrTag)); + + // Make an EConstant expression for the array. + auto arrayExpression = makeE<EConstant>(arrTag, arrVal); + + // Build the unwind/project/limit/coscan subtree. + auto projectSlot = generateSlotId(); + auto unwindSlot = generateSlotId(); + auto unwind = makeS<UnwindStage>( + makeProjectStage(makeS<LimitSkipStage>(makeS<CoScanStage>(), 1, boost::none), + projectSlot, + std::move(arrayExpression)), + projectSlot, + unwindSlot, + generateSlotId(), // We don't need an index slot but must to provide it. + false); // Don't preserve null and empty arrays. + + // Return the UnwindStage and its output slot. The UnwindStage can be used as an input + // to other PlanStages. + return {unwindSlot, std::move(unwind)}; +} + +std::pair<value::SlotVector, std::unique_ptr<PlanStage>> +PlanStageTestFixture::generateMockScanMulti(int64_t numSlots, + value::TypeTags arrTag, + value::Value arrVal) { + using namespace std::literals; + + invariant(numSlots >= 1); + + // Generate a mock scan with a single output slot. + auto [scanSlot, scanStage] = generateMockScan(arrTag, arrVal); + + // Create a ProjectStage that will read the data from `scanStage` and split it up + // across multiple output slots. + value::SlotVector projectSlots; + value::SlotMap<std::unique_ptr<EExpression>> projections; + for (int64_t i = 0; i < numSlots; ++i) { + projectSlots.emplace_back(generateSlotId()); + projections.emplace( + projectSlots.back(), + makeE<EFunction>("getElement"sv, + makeEs(makeE<EVariable>(scanSlot), + makeE<EConstant>(value::TypeTags::NumberInt64, i)))); + } + + return {std::move(projectSlots), + makeS<ProjectStage>(std::move(scanStage), std::move(projections))}; +} + +std::pair<value::SlotId, std::unique_ptr<PlanStage>> PlanStageTestFixture::generateMockScan( + const BSONArray& array) { + auto [arrTag, arrVal] = makeValue(array); + return generateMockScan(arrTag, arrVal); +} + +std::pair<value::SlotVector, std::unique_ptr<PlanStage>> +PlanStageTestFixture::generateMockScanMulti(int64_t numSlots, const BSONArray& array) { + auto [arrTag, arrVal] = makeValue(array); + return generateMockScanMulti(numSlots, arrTag, arrVal); +} + +void PlanStageTestFixture::prepareTree(PlanStage* root) { + root->prepare(*_compileCtx); + root->attachFromOperationContext(opCtx()); + root->open(false); +} + +value::SlotAccessor* PlanStageTestFixture::prepareTree(PlanStage* root, value::SlotId slot) { + prepareTree(root); + return root->getAccessor(*_compileCtx, slot); +} + +std::vector<value::SlotAccessor*> PlanStageTestFixture::prepareTree(PlanStage* root, + value::SlotVector slots) { + std::vector<value::SlotAccessor*> slotAccessors; + + prepareTree(root); + for (auto slot : slots) { + slotAccessors.emplace_back(root->getAccessor(*_compileCtx, slot)); + } + return slotAccessors; +} + +std::pair<value::TypeTags, value::Value> PlanStageTestFixture::getAllResults( + PlanStage* stage, value::SlotAccessor* accessor) { + // Allocate an array to hold the results. + auto [resultsTag, resultsVal] = value::makeNewArray(); + value::ValueGuard guard{resultsTag, resultsVal}; + auto resultsView = value::getArrayView(resultsVal); + // Loop and repeatedly call getNext() until we reach the end, storing the values produced + // into the array. + for (auto st = stage->getNext(); st == PlanState::ADVANCED; st = stage->getNext()) { + auto [tag, val] = accessor->copyOrMoveValue(); + resultsView->push_back(tag, val); + } + + guard.reset(); + return {resultsTag, resultsVal}; +} + +std::pair<value::TypeTags, value::Value> PlanStageTestFixture::getAllResultsMulti( + PlanStage* stage, std::vector<value::SlotAccessor*> accessors) { + // Allocate an SBE array to hold the results. + auto [resultsTag, resultsVal] = value::makeNewArray(); + value::ValueGuard resultsGuard{resultsTag, resultsVal}; + auto resultsView = value::getArrayView(resultsVal); + + // Loop and repeatedly call getNext() until we reach the end. + for (auto st = stage->getNext(); st == PlanState::ADVANCED; st = stage->getNext()) { + // Create a new SBE array (`arr`) containing the values produced by each SlotAccessor + // and insert `arr` into the array of results. + auto [arrTag, arrVal] = value::makeNewArray(); + value::ValueGuard guard{arrTag, arrVal}; + auto arrView = value::getArrayView(arrVal); + for (size_t i = 0; i < accessors.size(); ++i) { + auto [tag, val] = accessors[i]->copyOrMoveValue(); + arrView->push_back(tag, val); + } + guard.reset(); + resultsView->push_back(arrTag, arrVal); + } + + resultsGuard.reset(); + return {resultsTag, resultsVal}; +} + +void PlanStageTestFixture::runTest(value::TypeTags inputTag, + value::Value inputVal, + value::TypeTags expectedTag, + value::Value expectedVal, + const MakeStageFn<value::SlotId>& makeStage) { + // Set up a ValueGuard to ensure `expected` gets released. + value::ValueGuard expectedGuard{expectedTag, expectedVal}; + + // Generate a mock scan from `input` with a single output slot. + auto [scanSlot, scanStage] = generateMockScan(inputTag, inputVal); + + // Call the `makeStage` callback to create the PlanStage that we want to test, passing in + // the mock scan subtree and its output slot. + auto [outputSlot, stage] = makeStage(scanSlot, std::move(scanStage)); + + // Prepare the tree and get the SlotAccessor for the output slot. + auto resultAccessor = prepareTree(stage.get(), outputSlot); + + // Get all the results produced by the PlanStage we want to test. + auto [resultsTag, resultsVal] = getAllResults(stage.get(), resultAccessor); + value::ValueGuard resultGuard{resultsTag, resultsVal}; + + // Compare the results produced with the expected output and assert that they match. + ASSERT_TRUE(valueEquals(resultsTag, resultsVal, expectedTag, expectedVal)); +} + +void PlanStageTestFixture::runTestMulti(int64_t numInputSlots, + value::TypeTags inputTag, + value::Value inputVal, + value::TypeTags expectedTag, + value::Value expectedVal, + const MakeStageFn<value::SlotVector>& makeStageMulti) { + // Set up a ValueGuard to ensure `expected` gets released. + value::ValueGuard expectedGuard{expectedTag, expectedVal}; + + // Generate a mock scan from `input` with multiple output slots. + auto [scanSlots, scanStage] = generateMockScanMulti(numInputSlots, inputTag, inputVal); + + // Call the `makeStageMulti` callback to create the PlanStage that we want to test, passing + // in the mock scan subtree and its output slots. + auto [outputSlots, stage] = makeStageMulti(scanSlots, std::move(scanStage)); + + // Prepare the tree and get the SlotAccessors for the output slots. + auto resultAccessors = prepareTree(stage.get(), outputSlots); + + // Get all the results produced by the PlanStage we want to test. + auto [resultsTag, resultsVal] = getAllResultsMulti(stage.get(), resultAccessors); + value::ValueGuard resultGuard{resultsTag, resultsVal}; + + // Compare the results produced with the expected output and assert that they match. + ASSERT_TRUE(valueEquals(resultsTag, resultsVal, expectedTag, expectedVal)); +} + +} // namespace mongo::sbe diff --git a/src/mongo/db/exec/sbe/sbe_plan_stage_test.h b/src/mongo/db/exec/sbe/sbe_plan_stage_test.h new file mode 100644 index 00000000000..6fede2f7672 --- /dev/null +++ b/src/mongo/db/exec/sbe/sbe_plan_stage_test.h @@ -0,0 +1,248 @@ +/** + * Copyright (C) 2018-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +/** + * This file contains a unittest framework for testing sbe::PlanStages. + */ + +#pragma once + +#include "mongo/db/exec/sbe/stages/co_scan.h" +#include "mongo/db/exec/sbe/stages/limit_skip.h" +#include "mongo/db/exec/sbe/stages/project.h" +#include "mongo/db/exec/sbe/stages/unwind.h" +#include "mongo/db/exec/sbe/values/bson.h" +#include "mongo/db/exec/sbe/values/id_generators.h" +#include "mongo/db/exec/sbe/values/value.h" +#include "mongo/db/service_context_test_fixture.h" +#include "mongo/unittest/unittest.h" + +namespace mongo::sbe { + +template <typename T> +using MakeStageFn = std::function<std::pair<T, std::unique_ptr<PlanStage>>( + T scanSlots, std::unique_ptr<PlanStage> scanStage)>; + +/** + * PlanStageTestFixture is a unittest framework for testing sbe::PlanStages. + * + * To facilitate writing unittests for PlanStages, PlanStageTestFixture sets up an OperationContext + * and a CompileCtx and offers a number of methods to help unittest writers. From the perspective a + * unittest writer, the most important methods in the PlanStageTestFixture class are prepareTree(), + * runTest(), and runTestMulti(). Each unittest should directly call only one of these methods once. + * + * For unittests where you need more control and flexibility, calling prepareTree() directly is + * the way to go. prepareTree() takes the root stage of a PlanStage tree and 0 or more SlotIds as + * parameters. When invoked, prepareTree() calls prepare() on the root stage (passing in the + * CompileCtx), attaches the OperationContext to the root stage, calls open() on the root stage, + * and then returns the SlotAccessors corresponding to the specified SlotIds. For a given unittest + * that calls prepareTree() directly, you can think of the unittest as having two parts: (1) the + * part before prepareTree(); and (2) the part after prepareTree(). The first part of the test + * (before prepareTree()) should do whatever is needed to construct the desired PlanStage tree. + * The second part of the test (after prepareTree()) should drive the execution of the PlanStage + * tree (by calling getNext() on the root stage one or more times) and verify that the PlanStage + * tree behaves as expected. During the first part before prepareTree(), it's common to use + * generateMockScan() or generateMockScanMulti() which provide an easy way to build a PlanStage + * subtree that streams out the contents of an SBE array (mimicking a real collection scan). + * + * For unittests where you just need to stream the contents of an input array to a PlanStage and + * compare the values produced against an "expected output" array, runTest() or runTestMulti() are + * the way to go. For tests where the PlanStage only has 1 input slot and the test only needs to + * observe 1 output slot, use runTest(). For unittests where the PlanStage has multiple input slots + * and/or where the test needs to observe multiple output slots, use runTestMulti(). + */ +class PlanStageTestFixture : public ServiceContextTest { +public: + PlanStageTestFixture() = default; + + void setUp() override { + ServiceContextTest::setUp(); + _opCtx = makeOperationContext(); + _slotIdGenerator.reset(new value::SlotIdGenerator()); + _compileCtx.reset(new CompileCtx(std::make_unique<RuntimeEnvironment>())); + } + + void tearDown() override { + _compileCtx.reset(); + _slotIdGenerator.reset(); + _opCtx.reset(); + ServiceContextTest::tearDown(); + } + + OperationContext* opCtx() { + return _opCtx.get(); + } + + value::SlotId generateSlotId() { + return _slotIdGenerator->generate(); + } + + CompileCtx* compileCtx() { + return _compileCtx.get(); + } + + /** + * Compare two SBE values for equality. + */ + bool valueEquals(value::TypeTags lhsTag, + value::Value lhsVal, + value::TypeTags rhsTag, + value::Value rhsVal) { + auto [cmpTag, cmpVal] = value::compareValue(lhsTag, lhsVal, rhsTag, rhsVal); + return (cmpTag == value::TypeTags::NumberInt32 && value::bitcastTo<int32_t>(cmpVal) == 0); + } + + /** + * Converts a BSONArray to an SBE Array. Caller owns the SBE Array returned. This method + * does not assume ownership of the BSONArray. + */ + std::pair<value::TypeTags, value::Value> makeValue(const BSONArray& ba); + + /** + * Converts a BSONObj to an SBE Object. Caller owns the SBE Object returned. This method + * does not assume ownership of the BSONObj. + */ + std::pair<value::TypeTags, value::Value> makeValue(const BSONObj& bo); + + /** + * This method takes an SBE array and returns an output slot and a unwind/project/limit/coscan + * subtree that streams out the elements of the array one at a time via the output slot over a + * series of calls to getNext(), mimicking the output of a collection scan or an index scan. + * + * Note that this method assumes ownership of the SBE Array being passed in. + */ + std::pair<value::SlotId, std::unique_ptr<PlanStage>> generateMockScan(value::TypeTags arrTag, + value::Value arrVal); + + /** + * This method is similar to generateMockScan(), except that the subtree returned outputs to + * multiple slots instead of a single slot. `numSlots` specifies the number of output slots. + * `array` is expected to be an array of subarrays. Each subarray is expected to have exactly + * `numSlots` elements, where the value at index 0 corresponds to output slot 0, the value at + * index 1 corresponds to output slot 1, and so on. The first subarray supplies the values for + * the output slots for the first call to getNext(), the second subarray applies the values for + * the output slots for the second call to getNext(), and so on. + * + * Note that this method assumes ownership of the SBE Array being passed in. + */ + std::pair<value::SlotVector, std::unique_ptr<PlanStage>> generateMockScanMulti( + int64_t numSlots, value::TypeTags arrTag, value::Value arrVal); + + /** + * Make a mock scan from an BSON array. This method does NOT assume ownership of the BSONArray + * passed in. + */ + std::pair<value::SlotId, std::unique_ptr<PlanStage>> generateMockScan(const BSONArray& array); + + /** + * Make a mock scan with multiple output slots from an BSON array. This method does NOT assume + * ownership of the BSONArray passed in. + */ + std::pair<value::SlotVector, std::unique_ptr<PlanStage>> generateMockScanMulti( + int64_t numSlots, const BSONArray& array); + + /** + * Prepares the tree of PlanStages given by `root` and returns the SlotAccessor* for `slot`. + */ + void prepareTree(PlanStage* root); + + /** + * Prepares the tree of PlanStages given by `root` and returns the SlotAccessor* for `slot`. + */ + value::SlotAccessor* prepareTree(PlanStage* root, value::SlotId slot); + + /** + * Prepares the tree of PlanStages given by `root` and returns the SlotAccessor*'s for + * the specified slots. + */ + std::vector<value::SlotAccessor*> prepareTree(PlanStage* root, value::SlotVector slots); + + /** + * This method repeatedly calls getNext() on the specified PlanStage, stores all the values + * produced by the specified SlotAccessor into an SBE array, and returns the array. + * + * Note that the caller assumes ownership of the SBE array returned. + */ + std::pair<value::TypeTags, value::Value> getAllResults(PlanStage* stage, + value::SlotAccessor* accessor); + + /** + * This method is similar to getAllResults(), except that it supports multiple SlotAccessors. + * This method returns an array of subarrays. Each subarray contains exactly N elements (where + * N is the number of output slots) with the value at index 0 corresponding to output slot 0, + * the value at index 1 corresponding to output slot 1, and so on. The first subarray holds the + * first values produced by each slot, the second subarray holds the second values produced by + * each slot, and so on. + * + * Note that the caller assumes ownership of the SBE array returned. + */ + std::pair<value::TypeTags, value::Value> getAllResultsMulti( + PlanStage* stage, std::vector<value::SlotAccessor*> accessors); + + /** + * This method is intended to make it easy to write basic tests. The caller passes in an input + * array, an array containing the expected output, and a lambda for constructing the PlanStage + * to be tested. The `makeStage` lambda is passed the input stage and the input slot, and is + * expected to return a PlanStage and its output slot. + * + * This method assumes that the input array should be streamed to the PlanStage via a single + * slot. Also, for comparing the PlanStage's output to expected output, this method assumes + * there is only one relevant output slot. For writing basic tests that involve multiple input + * slots or that involve testing multiple output slots, runTestMulti() should be used instead. + */ + void runTest(value::TypeTags inputTag, + value::Value inputVal, + value::TypeTags expectedTag, + value::Value expectedVal, + const MakeStageFn<value::SlotId>& makeStage); + + /** + * This method is similar to runTest(), but it allows for streaming input via multiple slots as + * well as testing against multiple output slots. The caller passes in an integer indicating the + * number of input slots, an input array, an array containing the expected output, and a lambda + * for constructing the PlanStage to be tested. The `makeStage` lambda is passed the input stage + * and input slots, and is expected to return a PlanStage and its output slots. `input` should + * be an array of subarrays with each subarray having N elements, where N is the number of input + * slots. `output` should be an array of subarrays with each subarray having M elements, where M + * is the number of output slots. + */ + void runTestMulti(int64_t numInputSlots, + value::TypeTags inputTag, + value::Value inputVal, + value::TypeTags expectedTag, + value::Value expectedVal, + const MakeStageFn<value::SlotVector>& makeStageMulti); + +private: + ServiceContext::UniqueOperationContext _opCtx; + std::unique_ptr<value::SlotIdGenerator> _slotIdGenerator; + std::unique_ptr<CompileCtx> _compileCtx; +}; + +} // namespace mongo::sbe diff --git a/src/mongo/db/exec/sbe/sbe_sort_test.cpp b/src/mongo/db/exec/sbe/sbe_sort_test.cpp new file mode 100644 index 00000000000..6189f3c1e88 --- /dev/null +++ b/src/mongo/db/exec/sbe/sbe_sort_test.cpp @@ -0,0 +1,76 @@ +/** + * Copyright (C) 2018-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +/** + * This file contains tests for sbe::SortStage. + */ + +#include "mongo/platform/basic.h" + +#include <string_view> + +#include "mongo/db/exec/sbe/sbe_plan_stage_test.h" +#include "mongo/db/exec/sbe/stages/sort.h" + +namespace mongo::sbe { + +using SortStageTest = PlanStageTestFixture; + +TEST_F(SortStageTest, SortNumbersTest) { + auto [inputTag, inputVal] = makeValue( + BSON_ARRAY(BSON_ARRAY(12LL << "A") << BSON_ARRAY(2.5 << "B") << BSON_ARRAY(7 << "C") + << BSON_ARRAY(Decimal128(4) << "D"))); + value::ValueGuard inputGuard{inputTag, inputVal}; + + auto [expectedTag, expectedVal] = makeValue( + BSON_ARRAY(BSON_ARRAY(2.5 << "B") << BSON_ARRAY(Decimal128(4) << "D") + << BSON_ARRAY(7 << "C") << BSON_ARRAY(12LL << "A"))); + value::ValueGuard expectedGuard{expectedTag, expectedVal}; + + auto makeStageFn = [](value::SlotVector scanSlots, std::unique_ptr<PlanStage> scanStage) { + // Create a SortStage that sorts by slot0 in ascending order. + auto sortStage = + makeS<SortStage>(std::move(scanStage), + makeSV(scanSlots[0]), + std::vector<value::SortDirection>{value::SortDirection::Ascending}, + makeSV(scanSlots[1]), + std::numeric_limits<std::size_t>::max(), + 204857600, + false, + nullptr); + + return std::make_pair(scanSlots, std::move(sortStage)); + }; + + inputGuard.reset(); + expectedGuard.reset(); + runTestMulti(2, inputTag, inputVal, expectedTag, expectedVal, makeStageFn); +} + +} // namespace mongo::sbe diff --git a/src/mongo/db/exec/sbe/values/bson.cpp b/src/mongo/db/exec/sbe/values/bson.cpp index a9be90ece32..f45b666db3e 100644 --- a/src/mongo/db/exec/sbe/values/bson.cpp +++ b/src/mongo/db/exec/sbe/values/bson.cpp @@ -273,6 +273,10 @@ void convertToBsonObj(BSONArrayBuilder& builder, value::ArrayEnumerator arr) { } } } +void convertToBsonObj(BSONArrayBuilder& builder, value::Array* arr) { + return convertToBsonObj( + builder, value::ArrayEnumerator{value::TypeTags::Array, value::bitcastFrom(arr)}); +} void convertToBsonObj(BSONObjBuilder& builder, value::Object* obj) { for (size_t idx = 0; idx < obj->size(); ++idx) { auto [tag, val] = obj->getAt(idx); diff --git a/src/mongo/db/exec/sbe/values/bson.h b/src/mongo/db/exec/sbe/values/bson.h index 70f87ec204c..aa9c58a6557 100644 --- a/src/mongo/db/exec/sbe/values/bson.h +++ b/src/mongo/db/exec/sbe/values/bson.h @@ -45,7 +45,8 @@ inline auto fieldNameView(const char* be) noexcept { return std::string_view{be + 1}; } +void convertToBsonObj(BSONArrayBuilder& builder, value::Array* arr); void convertToBsonObj(BSONObjBuilder& builder, value::Object* obj); } // namespace bson } // namespace sbe -} // namespace mongo
\ No newline at end of file +} // namespace mongo diff --git a/src/mongo/db/exec/sbe/vm/arith.cpp b/src/mongo/db/exec/sbe/vm/arith.cpp index 41d55bbe8a7..afb1f3af6a4 100644 --- a/src/mongo/db/exec/sbe/vm/arith.cpp +++ b/src/mongo/db/exec/sbe/vm/arith.cpp @@ -366,6 +366,27 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::genericNumConvert( return {false, value::TypeTags::Nothing, 0}; } +static const double kDoubleLargestConsecutiveInteger = + pow(std::numeric_limits<double>::radix, std::numeric_limits<double>::digits); + +std::pair<value::TypeTags, value::Value> ByteCode::genericNumConvertToPreciseInt64( + value::TypeTags lhsTag, value::Value lhsValue) { + // If lhs is a double, we need to perform an extra check to ensure that lhs is within the range + // where double can represent consecutive integers precisely. This check isn't necessary for + // Decimal128, because Decimal128 can precisely represent every possible numeric value that can + // fit in an int64_t. + if (lhsTag == value::TypeTags::NumberDouble) { + auto d = value::bitcastTo<double>(lhsValue); + if (d > kDoubleLargestConsecutiveInteger || d < -kDoubleLargestConsecutiveInteger) { + return {value::TypeTags::Nothing, 0}; + } + } + + auto [owned, tag, val] = genericNumConvert(lhsTag, lhsValue, value::TypeTags::NumberInt64); + invariant(!owned); + return {tag, val}; +} + std::tuple<bool, value::TypeTags, value::Value> ByteCode::genericAbs(value::TypeTags operandTag, value::Value operandValue) { switch (operandTag) { diff --git a/src/mongo/db/exec/sbe/vm/vm.cpp b/src/mongo/db/exec/sbe/vm/vm.cpp index f304d186be0..475f1a2b875 100644 --- a/src/mongo/db/exec/sbe/vm/vm.cpp +++ b/src/mongo/db/exec/sbe/vm/vm.cpp @@ -80,6 +80,7 @@ int Instruction::stackOffset[Instruction::Tags::lastInstruction] = { -1, // fillEmpty -1, // getField + -1, // getElement -1, // sum -1, // min @@ -259,6 +260,10 @@ void CodeFragment::appendGetField() { appendSimpleInstruction(Instruction::getField); } +void CodeFragment::appendGetElement() { + appendSimpleInstruction(Instruction::getElement); +} + void CodeFragment::appendSum() { appendSimpleInstruction(Instruction::aggSum); } @@ -411,6 +416,59 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::getField(value::TypeTa return {false, value::TypeTags::Nothing, 0}; } +std::tuple<bool, value::TypeTags, value::Value> ByteCode::getElement(value::TypeTags arrTag, + value::Value arrValue, + value::TypeTags idxTag, + value::Value idxValue) { + if (arrTag != value::TypeTags::Array && arrTag != value::TypeTags::bsonArray) { + return {false, value::TypeTags::Nothing, 0}; + } + + // Bail out if the `idx` parameter isn't a number, or if it can't be converted to a 64-bit + // integer, or if it's outside of the range where the `lhsTag` type can represent consecutive + // integers precisely. + auto [numTag, numVal] = genericNumConvertToPreciseInt64(idxTag, idxValue); + if (numTag != value::TypeTags::NumberInt64) { + return {false, value::TypeTags::Nothing, 0}; + } + int64_t numInt64 = value::bitcastTo<int64_t>(numVal); + // Cast the `idx` parameter to size_t. Bail out if its negative or if it's too big for size_t. + if (numInt64 < 0 || + (sizeof(size_t) < sizeof(int64_t) && + numInt64 > static_cast<int64_t>(std::numeric_limits<size_t>::max()))) { + return {false, value::TypeTags::Nothing, 0}; + } + size_t idx = static_cast<size_t>(numInt64); + + if (arrTag == value::TypeTags::Array) { + // If `arr` is an SBE array, use Array::getAt() to retrieve the element at index `idx`. + auto [tag, val] = value::getArrayView(arrValue)->getAt(idx); + return {false, tag, val}; + } else if (arrTag == value::TypeTags::bsonArray) { + // If `arr` is a BSON array, loop over the elements until we reach the idx-th element. + auto be = value::bitcastTo<const char*>(arrValue); + auto end = be + ConstDataView(be).read<LittleEndian<uint32_t>>(); + // Skip document length. + be += 4; + // The field names of an array are always be 0 thru N-1 in order. Therefore we don't need to + // inspect the field names (aside from determining their length so we can skip over them). + for (size_t currentIdx = 0; *be != 0; ++currentIdx) { + size_t fieldNameLength = strlen(be + 1); + if (currentIdx == idx) { + auto [tag, val] = bson::convertFrom(true, be, end, fieldNameLength); + return {false, tag, val}; + } + be = bson::advance(be, fieldNameLength); + } + // If the array didn't have an element at index `idx`, return Nothing. + return {false, value::TypeTags::Nothing, 0}; + } else { + // Earlier in this function we bailed out if the `arrTag` wasn't Array or bsonArray, so it + // should be impossible to reach this point. + MONGO_UNREACHABLE + } +} + std::tuple<bool, value::TypeTags, value::Value> ByteCode::aggSum(value::TypeTags accTag, value::Value accValue, value::TypeTags fieldTag, @@ -1429,6 +1487,23 @@ std::tuple<uint8_t, value::TypeTags, value::Value> ByteCode::run(CodeFragment* c } break; } + case Instruction::getElement: { + auto [rhsOwned, rhsTag, rhsVal] = getFromStack(0); + popStack(); + auto [lhsOwned, lhsTag, lhsVal] = getFromStack(0); + + auto [owned, tag, val] = getElement(lhsTag, lhsVal, rhsTag, rhsVal); + + topStack(owned, tag, val); + + if (rhsOwned) { + value::releaseValue(rhsTag, rhsVal); + } + if (lhsOwned) { + value::releaseValue(lhsTag, lhsVal); + } + break; + } case Instruction::aggSum: { auto [rhsOwned, rhsTag, rhsVal] = getFromStack(0); popStack(); diff --git a/src/mongo/db/exec/sbe/vm/vm.h b/src/mongo/db/exec/sbe/vm/vm.h index 9c70fe30d9a..d7b5bce3cfb 100644 --- a/src/mongo/db/exec/sbe/vm/vm.h +++ b/src/mongo/db/exec/sbe/vm/vm.h @@ -134,6 +134,7 @@ struct Instruction { fillEmpty, getField, + getElement, aggSum, aggMin, @@ -237,6 +238,7 @@ public: appendSimpleInstruction(Instruction::fillEmpty); } void appendGetField(); + void appendGetElement(); void appendSum(); void appendMin(); void appendMax(); @@ -331,6 +333,9 @@ private: std::tuple<bool, value::TypeTags, value::Value> genericNumConvert(value::TypeTags lhsTag, value::Value lhsValue, value::TypeTags rhsTag); + std::pair<value::TypeTags, value::Value> genericNumConvertToPreciseInt64(value::TypeTags lhsTag, + value::Value lhsValue); + template <typename Op> std::pair<value::TypeTags, value::Value> genericCompare(value::TypeTags lhsTag, value::Value lhsValue, @@ -360,6 +365,11 @@ private: value::TypeTags fieldTag, value::Value fieldValue); + std::tuple<bool, value::TypeTags, value::Value> getElement(value::TypeTags objTag, + value::Value objValue, + value::TypeTags fieldTag, + value::Value fieldValue); + std::tuple<bool, value::TypeTags, value::Value> aggSum(value::TypeTags accTag, value::Value accValue, value::TypeTags fieldTag, |