diff options
author | Spencer Jackson <spencer.jackson@mongodb.com> | 2018-08-20 14:18:30 -0400 |
---|---|---|
committer | Daniel Gottlieb <daniel.gottlieb@mongodb.com> | 2019-02-14 22:59:29 -0500 |
commit | bbb83fa252895f36db69d69f249af2a98ba20890 (patch) | |
tree | c0915a024aad012bf9dc80c1918c8bf30d5d6be5 | |
parent | 1a7440011639302ca72df4d7c9395fceeda042bd (diff) | |
download | mongo-bbb83fa252895f36db69d69f249af2a98ba20890.tar.gz |
SERVER-36606: Allow construction of large BSON objects
(cherry picked from commit f99914d14b76718f1fef879cfaabe23c0c8f0857)
-rw-r--r-- | src/mongo/bson/bson_obj_test.cpp | 42 | ||||
-rw-r--r-- | src/mongo/bson/bsonelement.cpp | 4 | ||||
-rw-r--r-- | src/mongo/bson/bsonelement_test.cpp | 18 | ||||
-rw-r--r-- | src/mongo/bson/bsonobj.cpp | 4 | ||||
-rw-r--r-- | src/mongo/bson/bsonobj.h | 24 | ||||
-rw-r--r-- | src/mongo/bson/bsonobjbuilder.h | 8 | ||||
-rw-r--r-- | src/mongo/bson/bsonobjbuilder_test.cpp | 63 |
7 files changed, 150 insertions, 13 deletions
diff --git a/src/mongo/bson/bson_obj_test.cpp b/src/mongo/bson/bson_obj_test.cpp index f43680191c7..1c44edd01c6 100644 --- a/src/mongo/bson/bson_obj_test.cpp +++ b/src/mongo/bson/bson_obj_test.cpp @@ -678,4 +678,46 @@ TEST(BSONObj, addField) { ASSERT_BSONOBJ_EQ(obj, BSON("a" << 1 << "b" << 2)); } +TEST(BSONObj, sizeChecks) { + auto generateBuffer = [](std::int32_t size) { + std::vector<char> buffer(size); + DataRange bufferRange(&buffer.front(), &buffer.back()); + ASSERT_OK(bufferRange.write(LittleEndian<int32_t>(size))); + + return buffer; + }; + + { + // Implicitly assert that BSONObj constructor does not throw + // with standard size buffers. + auto normalBuffer = generateBuffer(15 * 1024 * 1024); + BSONObj obj(normalBuffer.data()); + } + + // Large buffers cause an exception to be thrown. + ASSERT_THROWS_CODE( + [&] { + auto largeBuffer = generateBuffer(17 * 1024 * 1024); + BSONObj obj(largeBuffer.data()); + }(), + DBException, + 10334); + + + // Assert that the max size can be increased by passing BSONObj a tag type. + { + auto largeBuffer = generateBuffer(17 * 1024 * 1024); + BSONObj obj(largeBuffer.data(), BSONObj::LargeSizeTrait{}); + } + + // But a size is in fact being enforced. + ASSERT_THROWS_CODE( + [&]() { + auto hugeBuffer = generateBuffer(70 * 1024 * 1024); + BSONObj obj(hugeBuffer.data(), BSONObj::LargeSizeTrait{}); + }(), + DBException, + 10334); +} + } // unnamed namespace diff --git a/src/mongo/bson/bsonelement.cpp b/src/mongo/bson/bsonelement.cpp index b0f39d5c874..ebc9fac6981 100644 --- a/src/mongo/bson/bsonelement.cpp +++ b/src/mongo/bson/bsonelement.cpp @@ -547,7 +547,7 @@ bool BSONElement::binaryEqualValues(const BSONElement& rhs) const { BSONObj BSONElement::embeddedObjectUserCheck() const { if (MONGO_likely(isABSONObj())) - return BSONObj(value()); + return BSONObj(value(), BSONObj::LargeSizeTrait{}); std::stringstream ss; ss << "invalid parameter: expected an object (" << fieldName() << ")"; uasserted(10065, ss.str()); @@ -556,7 +556,7 @@ BSONObj BSONElement::embeddedObjectUserCheck() const { BSONObj BSONElement::embeddedObject() const { verify(isABSONObj()); - return BSONObj(value()); + return BSONObj(value(), BSONObj::LargeSizeTrait{}); } BSONObj BSONElement::codeWScopeObject() const { diff --git a/src/mongo/bson/bsonelement_test.cpp b/src/mongo/bson/bsonelement_test.cpp index 6cdb8ac9ce7..a25329b0907 100644 --- a/src/mongo/bson/bsonelement_test.cpp +++ b/src/mongo/bson/bsonelement_test.cpp @@ -110,5 +110,23 @@ TEST(BSONElement, TimestampToString) { ASSERT_EQ(obj["ts3"].toString(false, false), "Timestamp(4294967295, 4294967295)"); } +TEST(BSONElement, ExtractLargeSubObject) { + std::int32_t size = 17 * 1024 * 1024; + std::vector<char> buffer(size); + DataRange bufferRange(&buffer.front(), &buffer.back()); + ASSERT_OK(bufferRange.write(LittleEndian<int32_t>(size))); + + BSONObj obj(buffer.data(), BSONObj::LargeSizeTrait{}); + + BSONObjBuilder bigObjectBuilder; + bigObjectBuilder.append("a", obj); + BSONObj bigObj = bigObjectBuilder.obj<BSONObj::LargeSizeTrait>(); + + BSONElement element = bigObj["a"]; + ASSERT_EQ(BSONType::Object, element.type()); + + BSONObj subObj = element.Obj(); +} + } // namespace } // namespace mongo diff --git a/src/mongo/bson/bsonobj.cpp b/src/mongo/bson/bsonobj.cpp index e490bf09b19..fba85418176 100644 --- a/src/mongo/bson/bsonobj.cpp +++ b/src/mongo/bson/bsonobj.cpp @@ -88,12 +88,12 @@ using namespace std; /* BSONObj ------------------------------------------------------------*/ -void BSONObj::_assertInvalid() const { +void BSONObj::_assertInvalid(int maxSize) const { StringBuilder ss; int os = objsize(); ss << "BSONObj size: " << os << " (0x" << integerToHex(os) << ") is invalid. " << "Size must be between 0 and " << BSONObjMaxInternalSize << "(" - << (BSONObjMaxInternalSize / (1024 * 1024)) << "MB)"; + << (maxSize / (1024 * 1024)) << "MB)"; try { BSONElement e = firstElement(); ss << " First element: " << e.toString(); diff --git a/src/mongo/bson/bsonobj.h b/src/mongo/bson/bsonobj.h index 97abcd62a35..476b51df03e 100644 --- a/src/mongo/bson/bsonobj.h +++ b/src/mongo/bson/bsonobj.h @@ -98,6 +98,13 @@ class BSONObjStlIterator; */ class BSONObj { public: + struct DefaultSizeTrait { + constexpr static int MaxSize = BSONObjMaxInternalSize; + }; + struct LargeSizeTrait { + constexpr static int MaxSize = BufferMaxSize; + }; + // Declared in bsonobj_comparator_interface.h. class ComparatorInterface; @@ -128,8 +135,9 @@ public: /** Construct a BSONObj from data in the proper format. * Use this constructor when something else owns bsonData's buffer */ - explicit BSONObj(const char* bsonData) { - init(bsonData); + template <typename Traits = DefaultSizeTrait> + explicit BSONObj(const char* bsonData, Traits t = Traits{}) { + init<Traits>(bsonData); } explicit BSONObj(ConstSharedBuffer ownedBuffer) @@ -374,9 +382,12 @@ public: } /** performs a cursory check on the object's size only. */ + template <typename Traits = DefaultSizeTrait> bool isValid() const { + static_assert(Traits::MaxSize > 0 && Traits::MaxSize <= std::numeric_limits<int>::max(), + "BSONObj maximum size must be within possible limits"); int x = objsize(); - return x > 0 && x <= BSONObjMaxInternalSize; + return x > 0 && x <= Traits::MaxSize; } /** @@ -559,12 +570,13 @@ public: } private: - void _assertInvalid() const; + void _assertInvalid(int maxSize) const; + template <typename Traits = DefaultSizeTrait> void init(const char* data) { _objdata = data; - if (!isValid()) - _assertInvalid(); + if (!isValid<Traits>()) + _assertInvalid(Traits::MaxSize); } const char* _objdata; diff --git a/src/mongo/bson/bsonobjbuilder.h b/src/mongo/bson/bsonobjbuilder.h index 978428eeb73..a27a742e719 100644 --- a/src/mongo/bson/bsonobjbuilder.h +++ b/src/mongo/bson/bsonobjbuilder.h @@ -660,9 +660,10 @@ public: * The returned BSONObj will free the buffer when it is finished. * @return owned BSONObj */ + template <typename BSONTraits = BSONObj::DefaultSizeTrait> BSONObj obj() { massert(10335, "builder does not own memory", owned()); - auto out = done(); + auto out = done<BSONTraits>(); out.shareOwnershipWith(_b.release()); return out; } @@ -672,8 +673,9 @@ public: scope -- very important to keep in mind. Use obj() if you would like the BSONObj to last longer than the builder. */ + template <typename BSONTraits = BSONObj::DefaultSizeTrait> BSONObj done() { - return BSONObj(_done()); + return BSONObj(_done(), BSONTraits{}); } // Like 'done' above, but does not construct a BSONObj to return to the caller. @@ -686,7 +688,7 @@ public: Intended use case: append a field if not already there. */ BSONObj asTempObj() { - BSONObj temp(_done()); + BSONObj temp(_done(), BSONObj::LargeSizeTrait{}); _b.setlen(_b.len() - 1); // next append should overwrite the EOO _b.reserveBytes(1); // Rereserve room for the real EOO _doneCalled = false; diff --git a/src/mongo/bson/bsonobjbuilder_test.cpp b/src/mongo/bson/bsonobjbuilder_test.cpp index 523d9914faa..2a8348b3a9c 100644 --- a/src/mongo/bson/bsonobjbuilder_test.cpp +++ b/src/mongo/bson/bsonobjbuilder_test.cpp @@ -448,5 +448,68 @@ TEST(BSONObjBuilderTest, SeedingBSONObjBuilderWithNonrootedUnownedBsonWorks) { ASSERT_NE(static_cast<const void*>(obj.objdata()), static_cast<const void*>(origObjPtr)); } +TEST(BSONObjBuilderTest, SizeChecks) { + auto generateBuffer = [](std::int32_t size) { + std::vector<char> buffer(size); + DataRange bufferRange(&buffer.front(), &buffer.back()); + ASSERT_OK(bufferRange.write(LittleEndian<int32_t>(size))); + + return buffer; + }; + + { + // Implicitly assert that BSONObjBuilder does not throw + // with standard size buffers. + auto normalBuffer = generateBuffer(15 * 1024 * 1024); + BSONObj obj(normalBuffer.data()); + + BSONObjBuilder builder; + builder.append("a", obj); + BSONObj finalObj = builder.obj(); + } + + // Large buffers cause an exception to be thrown. + ASSERT_THROWS_CODE( + [&] { + auto largeBuffer = generateBuffer(17 * 1024 * 1024); + BSONObj obj(largeBuffer.data(), BSONObj::LargeSizeTrait{}); + + BSONObjBuilder builder; + builder.append("a", obj); + BSONObj finalObj = builder.obj(); + }(), + DBException, + 10334); + + + // Assert that the max size can be increased by passing BSONObj a tag type. + { + auto largeBuffer = generateBuffer(17 * 1024 * 1024); + BSONObj obj(largeBuffer.data(), BSONObj::LargeSizeTrait{}); + + BSONObjBuilder builder; + builder.append("a", obj); + BSONObj finalObj = builder.obj<BSONObj::LargeSizeTrait>(); + } + + // But a size is in fact being enforced. + { + auto largeBuffer = generateBuffer(40 * 1024 * 1024); + BSONObj obj(largeBuffer.data(), BSONObj::LargeSizeTrait{}); + BSONObjBuilder builder; + ASSERT_THROWS( + [&]() { + + for (StringData character : {"a", "b", "c"}) { + builder.append(character, obj); + } + BSONObj finalObj = builder.obj<BSONObj::LargeSizeTrait>(); + + }(), + DBException); + } +} + + } // namespace } // namespace mongo |