summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJonathan Reams <jbreams@mongodb.com>2017-10-10 16:49:18 -0400
committerJonathan Reams <jbreams@mongodb.com>2017-10-27 12:22:00 -0400
commitde34ba09d65fca6c2b331015dc9edebea358d96a (patch)
tree2d0a761e19569bef0bd28d5916dc19c7a3f1c002
parent779bccf9dbd79aade8160a2d0e33c249ce38e286 (diff)
downloadmongo-de34ba09d65fca6c2b331015dc9edebea358d96a.tar.gz
SERVER-31505 Simplify Snappy compressor
(cherry picked from commit 835d351338fa945dfb4f6ba7696ebf4bdb1555c1)
-rw-r--r--src/mongo/transport/message_compressor_manager_test.cpp9
-rw-r--r--src/mongo/transport/message_compressor_snappy.cpp139
2 files changed, 17 insertions, 131 deletions
diff --git a/src/mongo/transport/message_compressor_manager_test.cpp b/src/mongo/transport/message_compressor_manager_test.cpp
index 868132171e9..5c533e45e75 100644
--- a/src/mongo/transport/message_compressor_manager_test.cpp
+++ b/src/mongo/transport/message_compressor_manager_test.cpp
@@ -127,10 +127,10 @@ void checkFidelity(const Message& msg, std::unique_ptr<MessageCompressorBase> co
void checkOverflow(std::unique_ptr<MessageCompressorBase> compressor) {
// This is our test data that we're going to try to compress/decompress into a buffer that's
// way too small.
- const auto data = std::string{
+ const std::string data =
"We embrace reality. We apply high-quality thinking and rigor."
"We have courage in our convictions but work hard to ensure biases "
- "or personal beliefs do not get in the way of finding the best solution."};
+ "or personal beliefs do not get in the way of finding the best solution.";
ConstDataRange input(data.data(), data.size());
// This is our tiny buffer that should cause an error.
@@ -140,8 +140,9 @@ void checkOverflow(std::unique_ptr<MessageCompressorBase> compressor) {
// This is a normal sized buffer that we can store a compressed version of our test data safely
std::vector<char> normalBuffer;
normalBuffer.resize(compressor->getMaxCompressedSize(data.size()));
- DataRange normalRange(normalBuffer.data(), normalBuffer.size());
- ASSERT_OK(compressor->compressData(input, normalRange));
+ auto sws = compressor->compressData(input, DataRange(normalBuffer.data(), normalBuffer.size()));
+ ASSERT_OK(sws);
+ DataRange normalRange = DataRange(normalBuffer.data(), sws.getValue());
// Check that compressing the test data into a small buffer fails
ASSERT_NOT_OK(compressor->compressData(input, smallOutput));
diff --git a/src/mongo/transport/message_compressor_snappy.cpp b/src/mongo/transport/message_compressor_snappy.cpp
index e58dd2f0747..9d523fca661 100644
--- a/src/mongo/transport/message_compressor_snappy.cpp
+++ b/src/mongo/transport/message_compressor_snappy.cpp
@@ -36,130 +36,24 @@
#include "mongo/transport/message_compressor_registry.h"
#include "mongo/transport/message_compressor_snappy.h"
-#include <snappy-sinksource.h>
#include <snappy.h>
namespace mongo {
-namespace {
-class SnappySourceSinkException : public DBException {
-public:
- SnappySourceSinkException(Status status) : DBException(status.reason(), status.code()) {}
-};
-
-// This is a bounds-checking version of snappy::UncheckedByteArraySink.
-//
-// If the amount of scratch buffer space requested by snappy is larger than the sink
-// buffer, than it will allocate a new temporary buffer so that snappy can finish.
-// If the amount of scratch buffer space requested by snappy is less than or equal to
-// the size of the sink buffer, than it will just return the sink buffer.
-//
-// If the scratch buffer is the sink buffer, than Append will just advance the buffer
-// cursor and do bounds checking without any copying.
-//
-// Appending data past the end of the sink buffer will throw a SnappySourcesinkexception.
-class DataRangeSink final : public snappy::Sink {
-public:
- DataRangeSink(DataRange buffer) : _cursor(buffer) {}
-
- char* GetAppendBuffer(size_t length, char* scratch) final {
- if (length > _cursor.length()) {
- _scratch.resize(length);
- return _scratch.data();
- }
-
- return const_cast<char*>(_cursor.data());
- }
-
- void AppendAndTakeOwnership(char* data,
- size_t n,
- void (*deleter)(void*, const char*, size_t),
- void* deleterArg) final {
- Append(data, n);
- if (data != _cursor.data()) {
- (*deleter)(deleterArg, data, n);
- }
- }
-
- void Append(const char* bytes, size_t n) final {
- Status status = Status::OK();
- if (bytes == _cursor.data()) {
- status = _cursor.advance(n);
- } else {
- ConstDataRange toWrite(bytes, n);
- status = _cursor.writeAndAdvance(toWrite);
- }
- if (!status.isOK()) {
- throw SnappySourceSinkException(std::move(status));
- }
- }
-
- char* GetAppendBufferVariable(size_t minSize,
- size_t desiredSizeHint,
- char* scratch,
- size_t scratchSize,
- size_t* allocatedSize) {
- if (desiredSizeHint > _cursor.length() || minSize > _cursor.length()) {
- _scratch.resize(desiredSizeHint);
- *allocatedSize = _scratch.size();
- return _scratch.data();
- }
-
- *allocatedSize = _cursor.length();
- return const_cast<char*>(_cursor.data());
- }
-
-private:
- DataRangeCursor _cursor;
- std::vector<char> _scratch;
-};
-
-class ConstDataRangeSource final : public snappy::Source {
-public:
- ConstDataRangeSource(ConstDataRange buffer) : _cursor(buffer) {}
-
- size_t Available() const final {
- return _cursor.length();
- }
-
- const char* Peek(size_t* len) final {
- *len = _cursor.length();
- return _cursor.data();
- }
-
- void Skip(size_t n) final {
- auto status = _cursor.advance(n);
- if (!status.isOK()) {
- throw SnappySourceSinkException(std::move(status));
- }
- }
-
-private:
- ConstDataRangeCursor _cursor;
-};
-
-} // namespace
SnappyMessageCompressor::SnappyMessageCompressor()
: MessageCompressorBase(MessageCompressor::kSnappy) {}
std::size_t SnappyMessageCompressor::getMaxCompressedSize(size_t inputSize) {
- // Testing has shown that snappy typically requests two additional bytes of buffer space when
- // compressing beyond what snappy::MaxCompressedLength returns. So by padding this by 2 more
- // bytes, we can avoid additional allocations/copies during compression.
- return snappy::MaxCompressedLength(inputSize) + 2;
+ return snappy::MaxCompressedLength(inputSize);
}
StatusWith<std::size_t> SnappyMessageCompressor::compressData(ConstDataRange input,
DataRange output) {
- size_t outLength;
- ConstDataRangeSource source(input);
- DataRangeSink sink(output);
-
- try {
- outLength = snappy::Compress(&source, &sink);
- } catch (const SnappySourceSinkException& e) {
- return e.toStatus();
+ size_t outLength = output.length();
+ if (output.length() < getMaxCompressedSize(input.length())) {
+ return {ErrorCodes::BadValue, "Output too small for max size of compressed input"};
}
+ snappy::RawCompress(input.data(), input.length(), const_cast<char*>(output.data()), &outLength);
counterHitCompress(input.length(), outLength);
return {outLength};
@@ -167,23 +61,14 @@ StatusWith<std::size_t> SnappyMessageCompressor::compressData(ConstDataRange inp
StatusWith<std::size_t> SnappyMessageCompressor::decompressData(ConstDataRange input,
DataRange output) {
- try {
- uint32_t expectedLength = 0;
- ConstDataRangeSource lengthCheckSource(input);
- if (!snappy::GetUncompressedLength(&lengthCheckSource, &expectedLength) ||
- expectedLength > output.length()) {
- return {ErrorCodes::BadValue, "Compressed message was invalid or corrupted"};
- }
-
- ConstDataRangeSource source(input);
- DataRangeSink sink(output);
+ size_t expectedLength = 0;
+ if (!snappy::GetUncompressedLength(input.data(), input.length(), &expectedLength) ||
+ expectedLength != output.length()) {
+ return {ErrorCodes::BadValue, "Compressed message was invalid or corrupted"};
+ }
- bool ret = snappy::Uncompress(&source, &sink);
- if (!ret) {
- return Status{ErrorCodes::BadValue, "Compressed message was invalid or corrupted"};
- }
- } catch (const SnappySourceSinkException& e) {
- return e.toStatus();
+ if (!snappy::RawUncompress(input.data(), input.length(), const_cast<char*>(output.data()))) {
+ return Status{ErrorCodes::BadValue, "Compressed message was invalid or corrupted"};
}
counterHitDecompress(input.length(), output.length());