SERVER-19583: Implement full-time diagnostic data capture compressor and decompressor

1 files changed, 429 insertions, 0 deletions

diff --git a/src/mongo/db/ftdc/compressor_test.cpp b/src/mongo/db/ftdc/compressor_test.cpp
new file mode 100644
index 00000000000..7b423679855
--- /dev/null
+++ b/src/mongo/db/ftdc/compressor_test.cpp
@@ -0,0 +1,429 @@
+/**
+ * Copyright (C) 2015 MongoDB Inc.
+ *
+ * This program is free software: you can redistribute it and/or  modify
+ * it under the terms of the GNU Affero General Public License, version 3,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * As a special exception, the copyright holders give permission to link the
+ * code of portions of this program with the OpenSSL library under certain
+ * conditions as described in each individual source file and distribute
+ * linked combinations including the program with the OpenSSL library. You
+ * must comply with the GNU Affero General Public License in all respects
+ * for all of the code used other than as permitted herein. If you modify
+ * file(s) with this exception, you may extend this exception to your
+ * version of the file(s), but you are not obligated to do so. If you do not
+ * wish to do so, delete this exception statement from your version. If you
+ * delete this exception statement from all source files in the program,
+ * then also delete it in the license file.
+ */
+
+#include "mongo/platform/basic.h"
+
+#include <limits>
+#include <random>
+
+#include "mongo/base/status_with.h"
+#include "mongo/bson/bsonmisc.h"
+#include "mongo/bson/bsonobjbuilder.h"
+#include "mongo/db/ftdc/compressor.h"
+#include "mongo/db/ftdc/config.h"
+#include "mongo/db/ftdc/decompressor.h"
+#include "mongo/db/ftdc/ftdc_test.h"
+#include "mongo/db/jsobj.h"
+#include "mongo/unittest/unittest.h"
+#include "mongo/util/assert_util.h"
+
+namespace mongo {
+
+#define ASSERT_HAS_SPACE(st) \
+    ASSERT_TRUE(st.isOK());  \
+    ASSERT_FALSE(st.getValue().is_initialized());
+
+#define ASSERT_SCHEMA_CHANGED(st)                                 \
+    ASSERT_TRUE(st.isOK());                                       \
+    ASSERT_TRUE(std::get<1>(st.getValue().get()) ==               \
+                FTDCCompressor::CompressorState::kSchemaChanged); \
+    ASSERT_TRUE(st.getValue().is_initialized());
+
+#define ASSERT_FULL(st)                                            \
+    ASSERT_TRUE(st.isOK());                                        \
+    ASSERT_TRUE(std::get<1>(st.getValue().get()) ==                \
+                FTDCCompressor::CompressorState::kCompressorFull); \
+    ASSERT_TRUE(st.getValue().is_initialized());
+
+// Sanity check
+TEST(FTDCCompressor, TestBasic) {
+    FTDCConfig config;
+    FTDCCompressor c(&config);
+
+    auto st = c.addSample(BSON("name"
+                               << "joe"
+                               << "key1" << 33 << "key2" << 42));
+    ASSERT_HAS_SPACE(st);
+    st = c.addSample(BSON("name"
+                          << "joe"
+                          << "key1" << 34 << "key2" << 45));
+    ASSERT_HAS_SPACE(st);
+
+
+    StatusWith<ConstDataRange> swBuf = c.getCompressedSamples();
+
+    ASSERT_TRUE(swBuf.isOK());
+    ASSERT_TRUE(swBuf.getValue().length() > 0);
+    ASSERT_TRUE(swBuf.getValue().data() != nullptr);
+}
+
+// Test strings only
+TEST(FTDCCompressor, TestStrings) {
+    FTDCConfig config;
+    FTDCCompressor c(&config);
+
+    auto st = c.addSample(BSON("name"
+                               << "joe"
+                               << "key1"
+                               << "value1"
+                               << "key2"
+                               << "value2"));
+    ASSERT_HAS_SPACE(st);
+    st = c.addSample(BSON("name"
+                          << "joe"
+                          << "key1"
+                          << "value3"
+                          << "key2"
+                          << "value6"));
+    ASSERT_HAS_SPACE(st);
+
+    StatusWith<ConstDataRange> swBuf = c.getCompressedSamples();
+
+    ASSERT_TRUE(swBuf.isOK());
+    ASSERT_TRUE(swBuf.getValue().length() > 0);
+    ASSERT_TRUE(swBuf.getValue().data() != nullptr);
+}
+
+/**
+ * Test class that records a series of samples and ensures that compress + decompress round trips
+ * them correctly.
+ */
+class TestTie {
+public:
+    TestTie() : _compressor(&_config) {}
+
+    ~TestTie() {
+        validate(boost::none_t());
+    }
+
+    StatusWith<boost::optional<std::tuple<ConstDataRange, FTDCCompressor::CompressorState>>>
+    addSample(const BSONObj& sample) {
+        auto st = _compressor.addSample(sample);
+
+        if (!st.getValue().is_initialized()) {
+            _docs.emplace_back(sample);
+        } else if (std::get<1>(st.getValue().get()) ==
+                   FTDCCompressor::CompressorState::kSchemaChanged) {
+            validate(std::get<0>(st.getValue().get()));
+            _docs.clear();
+            _docs.emplace_back(sample);
+        } else if (std::get<1>(st.getValue().get()) ==
+                   FTDCCompressor::CompressorState::kCompressorFull) {
+            _docs.emplace_back(sample);
+            validate(std::get<0>(st.getValue().get()));
+            _docs.clear();
+        } else {
+            MONGO_UNREACHABLE;
+        }
+
+        return st;
+    }
+
+    void validate(boost::optional<ConstDataRange> cdr) {
+        std::vector<BSONObj> list;
+        if (cdr.is_initialized()) {
+            auto sw = _decompressor.uncompress(cdr.get());
+            ASSERT_TRUE(sw.isOK());
+            list = sw.getValue();
+        } else {
+            auto swBuf = _compressor.getCompressedSamples();
+            ASSERT_TRUE(swBuf.isOK());
+            auto sw = _decompressor.uncompress(swBuf.getValue());
+            ASSERT_TRUE(sw.isOK());
+
+            list = sw.getValue();
+        }
+
+        ValidateDocumentList(list, _docs);
+    }
+
+private:
+    std::vector<BSONObj> _docs;
+    FTDCConfig _config;
+    FTDCCompressor _compressor;
+    FTDCDecompressor _decompressor;
+};
+
+// Test various schema changes
+TEST(FTDCCompressor, TestSchemaChanges) {
+    TestTie c;
+
+    auto st = c.addSample(BSON("name"
+                               << "joe"
+                               << "key1" << 33 << "key2" << 42));
+    ASSERT_HAS_SPACE(st);
+    st = c.addSample(BSON("name"
+                          << "joe"
+                          << "key1" << 34 << "key2" << 45));
+    ASSERT_HAS_SPACE(st);
+    st = c.addSample(BSON("name"
+                          << "joe"
+                          << "key1" << 34 << "key2" << 45));
+    ASSERT_HAS_SPACE(st);
+
+    // Add Field
+    st = c.addSample(BSON("name"
+                          << "joe"
+                          << "key1" << 34 << "key2" << 45 << "key3" << 47));
+    ASSERT_SCHEMA_CHANGED(st);
+
+    st = c.addSample(BSON("name"
+                          << "joe"
+                          << "key1" << 34 << "key2" << 45 << "key3" << 47));
+    ASSERT_HAS_SPACE(st);
+
+    // Rename field
+    st = c.addSample(BSON("name"
+                          << "joe"
+                          << "key1" << 34 << "key5" << 45 << "key3" << 47));
+    ASSERT_SCHEMA_CHANGED(st);
+
+    // Change type
+    st = c.addSample(BSON("name"
+                          << "joe"
+                          << "key1" << 34 << "key5"
+                          << "45"
+                          << "key3" << 47));
+    ASSERT_SCHEMA_CHANGED(st);
+
+    // Add Field
+    st = c.addSample(BSON("name"
+                          << "joe"
+                          << "key1" << 34 << "key2" << 45 << "key3" << 47 << "key7" << 34 << "key9"
+                          << 45 << "key13" << 47));
+    ASSERT_SCHEMA_CHANGED(st);
+
+    // Remove Field
+    st = c.addSample(BSON("name"
+                          << "joe"
+                          << "key7" << 34 << "key9" << 45 << "key13" << 47));
+    ASSERT_SCHEMA_CHANGED(st);
+
+    st = c.addSample(BSON("name"
+                          << "joe"
+                          << "key7" << 34 << "key9" << 45 << "key13" << 47));
+    ASSERT_HAS_SPACE(st);
+
+    // Start new batch
+    st = c.addSample(BSON("name"
+                          << "joe"
+                          << "key7" << 5));
+    ASSERT_SCHEMA_CHANGED(st);
+
+    // Change field to object
+    st = c.addSample(BSON("name"
+                          << "joe"
+                          << "key7" << BSON(  // nested object
+                                           "a" << 1)));
+    ASSERT_SCHEMA_CHANGED(st);
+
+    // Change field from object to number
+    st = c.addSample(BSON("name"
+                          << "joe"
+                          << "key7" << 7));
+    ASSERT_SCHEMA_CHANGED(st);
+
+    // Change field from number to array
+    st = c.addSample(BSON("name"
+                          << "joe"
+                          << "key7" << BSON_ARRAY(13 << 17)));
+    ASSERT_SCHEMA_CHANGED(st);
+
+    // Change field from array to number
+    st = c.addSample(BSON("name"
+                          << "joe"
+                          << "key7" << 19));
+    ASSERT_SCHEMA_CHANGED(st);
+
+
+    // New Schema
+    st = c.addSample(BSON("_id" << 1));
+    ASSERT_SCHEMA_CHANGED(st);
+
+    // Change field to oid
+    st = c.addSample(BSON(GENOID));
+    ASSERT_SCHEMA_CHANGED(st);
+
+    // Change field from oid to object
+    st = c.addSample(BSON("_id" << BSON("sub1" << 1)));
+    ASSERT_SCHEMA_CHANGED(st);
+
+    // Change field from object to oid
+    st = c.addSample(BSON(GENOID));
+    ASSERT_SCHEMA_CHANGED(st);
+}
+
+// Ensure changing between the various number formats is considered compatible
+TEST(FTDCCompressor, TestNumbersCompat) {
+    TestTie c;
+
+    auto st = c.addSample(BSON("name"
+                               << "joe"
+                               << "key1" << 33 << "key2" << 42LL));
+    ASSERT_HAS_SPACE(st);
+    st = c.addSample(BSON("name"
+                          << "joe"
+                          << "key1" << 34LL << "key2" << 45.0f));
+    ASSERT_HAS_SPACE(st);
+    st = c.addSample(BSON("name"
+                          << "joe"
+                          << "key1" << static_cast<char>(32) << "key2" << 45.0F));
+    ASSERT_HAS_SPACE(st);
+}
+
+// Test timestamp
+TEST(FTDCCompressor, TestTimeStamp) {
+    TestTie c;
+
+    BSONObjBuilder builder1;
+    BSONObj o = builder1.append("ts", Timestamp(0x55667788LL, 0x11223344LL)).obj();
+    auto st = c.addSample(o);
+    ASSERT_HAS_SPACE(st);
+}
+
+// Test all types
+TEST(FTDCCompressor, Types) {
+    TestTie c;
+
+    auto st = c.addSample(BSON("name"
+                               << "joe"
+                               << "key1" << 33 << "key2" << 42LL));
+    ASSERT_HAS_SPACE(st);
+
+    const char bytes[] = {0x1, 0x2, 0x3};
+    BSONObj o = BSON("created" << DATENOW                       // date_t
+                               << "null" << BSONNULL            // { a : null }
+                               << "undefined" << BSONUndefined  // { a : undefined }
+                               << "obj" << BSON(                // nested object
+                                               "a"
+                                               << "abc"
+                                               << "b" << 123LL) << "foo"
+                               << BSON_ARRAY("bar"
+                                             << "baz"
+                                             << "qux")               // array of strings
+                               << "foo2" << BSON_ARRAY(5 << 6 << 7)  // array of ints
+                               << "bindata" << BSONBinData(&bytes[0], 3, bdtCustom)  // bindata
+                               << "oid" << OID("010203040506070809101112")           // oid
+                               << "bool" << true                                     // bool
+                               << "regex" << BSONRegEx("mongodb")                    // regex
+                               << "ref" << BSONDBRef("c", OID("010203040506070809101112"))  // ref
+                               << "code" << BSONCode("func f() { return 1; }")              // code
+                               << "codewscope" << BSONCodeWScope("func f() { return 1; }",
+                                                                 BSON("c" << true))  // codew
+                               << "minkey" << MINKEY                                 // minkey
+                               << "maxkey" << MAXKEY                                 // maxkey
+                     );
+
+    st = c.addSample(o);
+    ASSERT_SCHEMA_CHANGED(st);
+
+    st = c.addSample(o);
+    ASSERT_HAS_SPACE(st);
+
+    st = c.addSample(BSON("name"
+                          << "joe"
+                          << "key1" << 34LL << "key2" << 45.0f));
+    ASSERT_SCHEMA_CHANGED(st);
+    st = c.addSample(BSON("name"
+                          << "joe"
+                          << "key1" << static_cast<char>(32) << "key2" << 45.0F));
+    ASSERT_HAS_SPACE(st);
+}
+
+// Test a full buffer
+TEST(FTDCCompressor, TestFull) {
+    // Test a large numbers of zeros, and incremental numbers in a full buffer
+    for (int j = 0; j < 2; j++) {
+        TestTie c;
+
+        auto st = c.addSample(BSON("name"
+                                   << "joe"
+                                   << "key1" << 33 << "key2" << 42));
+        ASSERT_HAS_SPACE(st);
+
+        for (size_t i = 0; i <= FTDCConfig::kMaxSamplesPerArchiveMetricChunkDefault - 2; i++) {
+            st = c.addSample(BSON("name"
+                                  << "joe"
+                                  << "key1" << static_cast<long long int>(i * j) << "key2" << 45));
+            ASSERT_HAS_SPACE(st);
+        }
+
+        st = c.addSample(BSON("name"
+                              << "joe"
+                              << "key1" << 34 << "key2" << 45));
+        ASSERT_FULL(st);
+
+        // Add Value
+        st = c.addSample(BSON("name"
+                              << "joe"
+                              << "key1" << 34 << "key2" << 45));
+        ASSERT_HAS_SPACE(st);
+    }
+}
+
+template <typename T>
+BSONObj generateSample(std::random_device& rd, T generator, size_t count) {
+    BSONObjBuilder builder;
+
+    for (size_t i = 0; i < count; ++i) {
+        builder.append("key", generator(rd));
+    }
+
+    return builder.obj();
+}
+
+// Test many metrics
+TEST(ZFTDCCompressor, TestManyMetrics) {
+    std::random_device rd;
+    std::mt19937 gen(rd());
+
+    std::uniform_int_distribution<long long> genValues(1, std::numeric_limits<long long>::max());
+    const size_t metrics = 1000;
+
+    // Test a large numbers of zeros, and incremental numbers in a full buffer
+    for (int j = 0; j < 2; j++) {
+        TestTie c;
+
+        auto st = c.addSample(generateSample(rd, genValues, metrics));
+        ASSERT_HAS_SPACE(st);
+
+        for (size_t i = 0; i <= FTDCConfig::kMaxSamplesPerArchiveMetricChunkDefault - 2; i++) {
+            st = c.addSample(generateSample(rd, genValues, metrics));
+            ASSERT_HAS_SPACE(st);
+        }
+
+        st = c.addSample(generateSample(rd, genValues, metrics));
+        ASSERT_FULL(st);
+
+        // Add Value
+        st = c.addSample(generateSample(rd, genValues, metrics));
+        ASSERT_HAS_SPACE(st);
+    }
+}
+
+}  // namespace mongo