From 9271c08121a0f511b3326a8a2eca449a6a523309 Mon Sep 17 00:00:00 2001 From: Mark Benvenuto Date: Mon, 17 Aug 2015 11:53:38 -0400 Subject: SERVER-19584: Implement full-time diagnostic data capture file reader and writer --- src/mongo/db/ftdc/file_manager_test.cpp | 320 ++++++++++++++++++++++++++++++++ 1 file changed, 320 insertions(+) create mode 100644 src/mongo/db/ftdc/file_manager_test.cpp (limited to 'src/mongo/db/ftdc/file_manager_test.cpp') diff --git a/src/mongo/db/ftdc/file_manager_test.cpp b/src/mongo/db/ftdc/file_manager_test.cpp new file mode 100644 index 00000000000..bd4d81a1b87 --- /dev/null +++ b/src/mongo/db/ftdc/file_manager_test.cpp @@ -0,0 +1,320 @@ +/** + * Copyright (C) 2015 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include +#include +#include +#include + +#include "mongo/base/init.h" +#include "mongo/bson/bson_validate.h" +#include "mongo/bson/bsonmisc.h" +#include "mongo/bson/bsonobjbuilder.h" +#include "mongo/db/client.h" +#include "mongo/db/ftdc/collector.h" +#include "mongo/db/ftdc/config.h" +#include "mongo/db/ftdc/file_manager.h" +#include "mongo/db/ftdc/file_writer.h" +#include "mongo/db/ftdc/ftdc_test.h" +#include "mongo/db/jsobj.h" +#include "mongo/stdx/memory.h" +#include "mongo/unittest/temp_dir.h" +#include "mongo/unittest/unittest.h" + +namespace mongo { + +// Test a full buffer +TEST(FTDCFileManagerTest, TestFull) { + Client* client = &cc(); + FTDCConfig c; + c.maxFileSizeBytes = 1000; + c.maxDirectorySizeBytes = 3000; + c.maxSamplesPerInterimMetricChunk = 1; + + unittest::TempDir tempdir("metrics_testpath"); + boost::filesystem::path dir(tempdir.path()); + createDirectoryClean(dir); + + FTDCCollectorCollection rotate; + auto swMgr = FTDCFileManager::create(&c, dir, &rotate, client); + ASSERT_OK(swMgr.getStatus()); + auto mgr = std::move(swMgr.getValue()); + + // Test a large numbers of zeros, and incremental numbers in a full buffer + for (int j = 0; j < 4; j++) { + ASSERT_OK( + mgr->writeSampleAndRotateIfNeeded(client, + BSON("name" + << "joe" + << "key1" << 3230792343LL << "key2" << 235135))); + + for (size_t i = 0; i <= FTDCConfig::kMaxSamplesPerArchiveMetricChunkDefault - 2; i++) { + ASSERT_OK(mgr->writeSampleAndRotateIfNeeded( + client, + BSON("name" + << "joe" + << "key1" << static_cast(i * j * 37) << "key2" + << static_cast(i * (645 << j))))); + } + + ASSERT_OK(mgr->writeSampleAndRotateIfNeeded(client, + BSON("name" + << "joe" + << "key1" << 34 << "key2" << 45))); + + // Add Value + ASSERT_OK(mgr->writeSampleAndRotateIfNeeded(client, + BSON("name" + << "joe" + << "key1" << 34 << "key2" << 45))); + } + + mgr->close(); + + auto files = scanDirectory(dir); + + int sum = 0; + for (auto& file : files) { + int fs = boost::filesystem::file_size(file); + ASSERT_TRUE(fs < c.maxFileSizeBytes * 1.10); + if (file.generic_string().find("interim") == std::string::npos) { + sum += fs; + } + } + + ASSERT_TRUE(sum < c.maxDirectorySizeBytes * 1.10); +} + +void ValidateInterimFileHasData(const boost::filesystem::path& dir, bool hasData) { + char buf[sizeof(std::int32_t)]; + + auto interimFile = FTDCUtil::getInterimFile(dir); + + std::fstream stream(interimFile.c_str()); + stream.read(&buf[0], sizeof(buf)); + + ASSERT_EQUALS(4, stream.gcount()); + std::uint32_t bsonLength = ConstDataView(buf).read>(); + + ASSERT_EQUALS(static_cast(bsonLength), hasData); +} + +// Test a normal restart +TEST(FTDCFileManagerTest, TestNormalRestart) { + Client* client = &cc(); + FTDCConfig c; + c.maxFileSizeBytes = 1000; + c.maxDirectorySizeBytes = 3000; + + unittest::TempDir tempdir("metrics_testpath"); + boost::filesystem::path dir(tempdir.path()); + + createDirectoryClean(dir); + + for (int i = 0; i < 3; i++) { + // Do a few cases of stop and start to ensure it works as expected + FTDCCollectorCollection rotate; + auto swMgr = FTDCFileManager::create(&c, dir, &rotate, client); + ASSERT_OK(swMgr.getStatus()); + auto mgr = std::move(swMgr.getValue()); + + // Test a large numbers of zeros, and incremental numbers in a full buffer + for (int j = 0; j < 4; j++) { + ASSERT_OK(mgr->writeSampleAndRotateIfNeeded(client, + BSON("name" + << "joe" + << "key1" << 3230792343LL << "key2" + << 235135))); + + for (size_t i = 0; i <= FTDCConfig::kMaxSamplesPerArchiveMetricChunkDefault - 2; i++) { + ASSERT_OK( + mgr->writeSampleAndRotateIfNeeded( + client, + BSON("name" + << "joe" + << "key1" << static_cast(i * j * 37) << "key2" + << static_cast(i * (645 << j))))); + } + + ASSERT_OK(mgr->writeSampleAndRotateIfNeeded(client, + BSON("name" + << "joe" + << "key1" << 34 << "key2" << 45))); + + // Add Value + ASSERT_OK(mgr->writeSampleAndRotateIfNeeded(client, + BSON("name" + << "joe" + << "key1" << 34 << "key2" << 45))); + } + + mgr->close(); + + // Validate the interim file does not have data + ValidateInterimFileHasData(dir, false); + } +} + +// Test a restart after a crash with a corrupt archive file +TEST(FTDCFileManagerTest, TestCorruptCrashRestart) { + Client* client = &cc(); + FTDCConfig c; + c.maxFileSizeBytes = 1000; + c.maxDirectorySizeBytes = 3000; + + unittest::TempDir tempdir("metrics_testpath"); + boost::filesystem::path dir(tempdir.path()); + + createDirectoryClean(dir); + + for (int i = 0; i < 2; i++) { + // Do a few cases of stop and start to ensure it works as expected + FTDCCollectorCollection rotate; + auto swMgr = FTDCFileManager::create(&c, dir, &rotate, client); + ASSERT_OK(swMgr.getStatus()); + auto mgr = std::move(swMgr.getValue()); + + // Test a large numbers of zeros, and incremental numbers in a full buffer + for (int j = 0; j < 4; j++) { + ASSERT_OK(mgr->writeSampleAndRotateIfNeeded(client, + BSON("name" + << "joe" + << "key1" << 3230792343LL << "key2" + << 235135))); + + for (size_t i = 0; i <= FTDCConfig::kMaxSamplesPerArchiveMetricChunkDefault - 2; i++) { + ASSERT_OK( + mgr->writeSampleAndRotateIfNeeded( + client, + BSON("name" + << "joe" + << "key1" << static_cast(i * j * 37) << "key2" + << static_cast(i * (645 << j))))); + } + + ASSERT_OK(mgr->writeSampleAndRotateIfNeeded(client, + BSON("name" + << "joe" + << "key1" << 34 << "key2" << 45))); + + // Add Value + ASSERT_OK(mgr->writeSampleAndRotateIfNeeded(client, + BSON("name" + << "joe" + << "key1" << 34 << "key2" << 45))); + } + + mgr->close(); + + auto swFile = FTDCFileManager::generateArchiveFileName(dir, "0test-crash"); + ASSERT_OK(swFile); + + std::ofstream stream(swFile.getValue().c_str()); + // This test case caused us to allocate more memory then the size of the file the first time + // I tried it + stream << "Hello World"; + + stream.close(); + } +} + +// Test a restart with a good interim file, and validate we have all the data +TEST(FTDCFileManagerTest, TestNormalCrashInterim) { + Client* client = &cc(); + FTDCConfig c; + c.maxSamplesPerInterimMetricChunk = 3; + c.maxFileSizeBytes = 10 * 1024 * 1024; + c.maxDirectorySizeBytes = 10 * 1024 * 1024; + + unittest::TempDir tempdir("metrics_testpath"); + boost::filesystem::path dir(tempdir.path()); + + createDirectoryClean(dir); + + BSONObj mdoc1 = BSON("name" + << "some_metadata" + << "key1" << 34 << "something" << 98); + + BSONObj sdoc1 = BSON("name" + << "joe" + << "key1" << 34 << "key2" << 45); + BSONObj sdoc2 = BSON("name" + << "joe" + << "key3" << 34 << "key5" << 45); + + auto swFile = FTDCFileManager::generateArchiveFileName(dir, "0test-crash"); + ASSERT_OK(swFile); + + { + FTDCFileWriter writer(&c); + + ASSERT_OK(writer.open(swFile.getValue())); + + ASSERT_OK(writer.writeMetadata(mdoc1)); + + ASSERT_OK(writer.writeSample(sdoc1)); + ASSERT_OK(writer.writeSample(sdoc1)); + ASSERT_OK(writer.writeSample(sdoc2)); + ASSERT_OK(writer.writeSample(sdoc2)); + ASSERT_OK(writer.writeSample(sdoc2)); + ASSERT_OK(writer.writeSample(sdoc2)); + + // leave some data in the interim file + writer.closeWithoutFlushForTest(); + } + + // Validate the interim file has data + ValidateInterimFileHasData(dir, true); + + // Let the manager run the recovery over the interim file + { + FTDCCollectorCollection rotate; + auto swMgr = FTDCFileManager::create(&c, dir, &rotate, client); + ASSERT_OK(swMgr.getStatus()); + auto mgr = std::move(swMgr.getValue()); + ASSERT_OK(mgr->close()); + } + + // Validate the file manager rolled over the changes to the current archive file + // and did not start a new archive file. + auto files = scanDirectory(dir); + + std::sort(files.begin(), files.end()); + + // Validate old file + std::vector docs1 = {mdoc1, sdoc1, sdoc1}; + ValidateDocumentList(files[0], docs1); + + // Validate new file + std::vector docs2 = {sdoc2, sdoc2, sdoc2, sdoc2}; + ValidateDocumentList(files[1], docs2); +} + +} // namespace mongo -- cgit v1.2.1