diff options
author | Jonathan Reams <jbreams@mongodb.com> | 2017-10-10 16:49:18 -0400 |
---|---|---|
committer | Jonathan Reams <jbreams@mongodb.com> | 2017-10-27 12:22:00 -0400 |
commit | de34ba09d65fca6c2b331015dc9edebea358d96a (patch) | |
tree | 2d0a761e19569bef0bd28d5916dc19c7a3f1c002 | |
parent | 779bccf9dbd79aade8160a2d0e33c249ce38e286 (diff) | |
download | mongo-de34ba09d65fca6c2b331015dc9edebea358d96a.tar.gz |
SERVER-31505 Simplify Snappy compressor
(cherry picked from commit 835d351338fa945dfb4f6ba7696ebf4bdb1555c1)
-rw-r--r-- | src/mongo/transport/message_compressor_manager_test.cpp | 9 | ||||
-rw-r--r-- | src/mongo/transport/message_compressor_snappy.cpp | 139 |
2 files changed, 17 insertions, 131 deletions
diff --git a/src/mongo/transport/message_compressor_manager_test.cpp b/src/mongo/transport/message_compressor_manager_test.cpp index 868132171e9..5c533e45e75 100644 --- a/src/mongo/transport/message_compressor_manager_test.cpp +++ b/src/mongo/transport/message_compressor_manager_test.cpp @@ -127,10 +127,10 @@ void checkFidelity(const Message& msg, std::unique_ptr<MessageCompressorBase> co void checkOverflow(std::unique_ptr<MessageCompressorBase> compressor) { // This is our test data that we're going to try to compress/decompress into a buffer that's // way too small. - const auto data = std::string{ + const std::string data = "We embrace reality. We apply high-quality thinking and rigor." "We have courage in our convictions but work hard to ensure biases " - "or personal beliefs do not get in the way of finding the best solution."}; + "or personal beliefs do not get in the way of finding the best solution."; ConstDataRange input(data.data(), data.size()); // This is our tiny buffer that should cause an error. @@ -140,8 +140,9 @@ void checkOverflow(std::unique_ptr<MessageCompressorBase> compressor) { // This is a normal sized buffer that we can store a compressed version of our test data safely std::vector<char> normalBuffer; normalBuffer.resize(compressor->getMaxCompressedSize(data.size())); - DataRange normalRange(normalBuffer.data(), normalBuffer.size()); - ASSERT_OK(compressor->compressData(input, normalRange)); + auto sws = compressor->compressData(input, DataRange(normalBuffer.data(), normalBuffer.size())); + ASSERT_OK(sws); + DataRange normalRange = DataRange(normalBuffer.data(), sws.getValue()); // Check that compressing the test data into a small buffer fails ASSERT_NOT_OK(compressor->compressData(input, smallOutput)); diff --git a/src/mongo/transport/message_compressor_snappy.cpp b/src/mongo/transport/message_compressor_snappy.cpp index e58dd2f0747..9d523fca661 100644 --- a/src/mongo/transport/message_compressor_snappy.cpp +++ b/src/mongo/transport/message_compressor_snappy.cpp @@ -36,130 +36,24 @@ #include "mongo/transport/message_compressor_registry.h" #include "mongo/transport/message_compressor_snappy.h" -#include <snappy-sinksource.h> #include <snappy.h> namespace mongo { -namespace { -class SnappySourceSinkException : public DBException { -public: - SnappySourceSinkException(Status status) : DBException(status.reason(), status.code()) {} -}; - -// This is a bounds-checking version of snappy::UncheckedByteArraySink. -// -// If the amount of scratch buffer space requested by snappy is larger than the sink -// buffer, than it will allocate a new temporary buffer so that snappy can finish. -// If the amount of scratch buffer space requested by snappy is less than or equal to -// the size of the sink buffer, than it will just return the sink buffer. -// -// If the scratch buffer is the sink buffer, than Append will just advance the buffer -// cursor and do bounds checking without any copying. -// -// Appending data past the end of the sink buffer will throw a SnappySourcesinkexception. -class DataRangeSink final : public snappy::Sink { -public: - DataRangeSink(DataRange buffer) : _cursor(buffer) {} - - char* GetAppendBuffer(size_t length, char* scratch) final { - if (length > _cursor.length()) { - _scratch.resize(length); - return _scratch.data(); - } - - return const_cast<char*>(_cursor.data()); - } - - void AppendAndTakeOwnership(char* data, - size_t n, - void (*deleter)(void*, const char*, size_t), - void* deleterArg) final { - Append(data, n); - if (data != _cursor.data()) { - (*deleter)(deleterArg, data, n); - } - } - - void Append(const char* bytes, size_t n) final { - Status status = Status::OK(); - if (bytes == _cursor.data()) { - status = _cursor.advance(n); - } else { - ConstDataRange toWrite(bytes, n); - status = _cursor.writeAndAdvance(toWrite); - } - if (!status.isOK()) { - throw SnappySourceSinkException(std::move(status)); - } - } - - char* GetAppendBufferVariable(size_t minSize, - size_t desiredSizeHint, - char* scratch, - size_t scratchSize, - size_t* allocatedSize) { - if (desiredSizeHint > _cursor.length() || minSize > _cursor.length()) { - _scratch.resize(desiredSizeHint); - *allocatedSize = _scratch.size(); - return _scratch.data(); - } - - *allocatedSize = _cursor.length(); - return const_cast<char*>(_cursor.data()); - } - -private: - DataRangeCursor _cursor; - std::vector<char> _scratch; -}; - -class ConstDataRangeSource final : public snappy::Source { -public: - ConstDataRangeSource(ConstDataRange buffer) : _cursor(buffer) {} - - size_t Available() const final { - return _cursor.length(); - } - - const char* Peek(size_t* len) final { - *len = _cursor.length(); - return _cursor.data(); - } - - void Skip(size_t n) final { - auto status = _cursor.advance(n); - if (!status.isOK()) { - throw SnappySourceSinkException(std::move(status)); - } - } - -private: - ConstDataRangeCursor _cursor; -}; - -} // namespace SnappyMessageCompressor::SnappyMessageCompressor() : MessageCompressorBase(MessageCompressor::kSnappy) {} std::size_t SnappyMessageCompressor::getMaxCompressedSize(size_t inputSize) { - // Testing has shown that snappy typically requests two additional bytes of buffer space when - // compressing beyond what snappy::MaxCompressedLength returns. So by padding this by 2 more - // bytes, we can avoid additional allocations/copies during compression. - return snappy::MaxCompressedLength(inputSize) + 2; + return snappy::MaxCompressedLength(inputSize); } StatusWith<std::size_t> SnappyMessageCompressor::compressData(ConstDataRange input, DataRange output) { - size_t outLength; - ConstDataRangeSource source(input); - DataRangeSink sink(output); - - try { - outLength = snappy::Compress(&source, &sink); - } catch (const SnappySourceSinkException& e) { - return e.toStatus(); + size_t outLength = output.length(); + if (output.length() < getMaxCompressedSize(input.length())) { + return {ErrorCodes::BadValue, "Output too small for max size of compressed input"}; } + snappy::RawCompress(input.data(), input.length(), const_cast<char*>(output.data()), &outLength); counterHitCompress(input.length(), outLength); return {outLength}; @@ -167,23 +61,14 @@ StatusWith<std::size_t> SnappyMessageCompressor::compressData(ConstDataRange inp StatusWith<std::size_t> SnappyMessageCompressor::decompressData(ConstDataRange input, DataRange output) { - try { - uint32_t expectedLength = 0; - ConstDataRangeSource lengthCheckSource(input); - if (!snappy::GetUncompressedLength(&lengthCheckSource, &expectedLength) || - expectedLength > output.length()) { - return {ErrorCodes::BadValue, "Compressed message was invalid or corrupted"}; - } - - ConstDataRangeSource source(input); - DataRangeSink sink(output); + size_t expectedLength = 0; + if (!snappy::GetUncompressedLength(input.data(), input.length(), &expectedLength) || + expectedLength != output.length()) { + return {ErrorCodes::BadValue, "Compressed message was invalid or corrupted"}; + } - bool ret = snappy::Uncompress(&source, &sink); - if (!ret) { - return Status{ErrorCodes::BadValue, "Compressed message was invalid or corrupted"}; - } - } catch (const SnappySourceSinkException& e) { - return e.toStatus(); + if (!snappy::RawUncompress(input.data(), input.length(), const_cast<char*>(output.data()))) { + return Status{ErrorCodes::BadValue, "Compressed message was invalid or corrupted"}; } counterHitDecompress(input.length(), output.length()); |