diff options
author | Yu Jin Kang Park <yujin.kang@mongodb.com> | 2022-11-29 14:46:06 +0000 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2022-11-29 18:03:30 +0000 |
commit | 8045c4a7772ab588679cdbf604a2875c27d97251 (patch) | |
tree | fb8b05a2627884648be86b3e49c623c57bd46584 | |
parent | 923b5fd65ad075c7da30af55022df936f4339128 (diff) | |
download | mongo-8045c4a7772ab588679cdbf604a2875c27d97251.tar.gz |
SERVER-68739 Do not reset WT session stats when fetching op stats
15 files changed, 429 insertions, 258 deletions
diff --git a/src/mongo/db/commands/find_cmd.cpp b/src/mongo/db/commands/find_cmd.cpp index 13cf5cff30c..6034f67f54a 100644 --- a/src/mongo/db/commands/find_cmd.cpp +++ b/src/mongo/db/commands/find_cmd.cpp @@ -712,7 +712,7 @@ public: // The stats collected here will not get overwritten, as the service entry // point layer will only set these stats when they're not empty. CurOp::get(opCtx)->debug().storageStats = - opCtx->recoveryUnit()->getOperationStatistics(); + opCtx->recoveryUnit()->computeOperationStatisticsSinceLastCall(); } } else { endQueryOp(opCtx, collection, *exec, numResults, cursorId); diff --git a/src/mongo/db/curop.cpp b/src/mongo/db/curop.cpp index f961ab83de4..b744e90ca3c 100644 --- a/src/mongo/db/curop.cpp +++ b/src/mongo/db/curop.cpp @@ -479,7 +479,8 @@ bool CurOp::completeAndLogOperation(OperationContext* opCtx, MODE_IS, Date_t::now() + Milliseconds(500), Lock::InterruptBehavior::kThrow); - _debug.storageStats = opCtx->recoveryUnit()->getOperationStatistics(); + _debug.storageStats = + opCtx->recoveryUnit()->computeOperationStatisticsSinceLastCall(); } catch (const DBException& ex) { LOGV2_WARNING_OPTIONS(20526, {component}, diff --git a/src/mongo/db/curop.h b/src/mongo/db/curop.h index 986a545e992..2d4f50872ed 100644 --- a/src/mongo/db/curop.h +++ b/src/mongo/db/curop.h @@ -326,7 +326,7 @@ public: AdditiveMetrics additiveMetrics; // Stores storage statistics. - std::shared_ptr<StorageStats> storageStats; + std::unique_ptr<StorageStats> storageStats; bool waitingForFlowControl{false}; diff --git a/src/mongo/db/storage/recovery_unit.h b/src/mongo/db/storage/recovery_unit.h index 9e173d632e7..dd0917bfaf2 100644 --- a/src/mongo/db/storage/recovery_unit.h +++ b/src/mongo/db/storage/recovery_unit.h @@ -391,9 +391,10 @@ public: virtual void allowUntimestampedWrite() {} /** - * Fetches the storage level statistics. + * Computes the storage level statistics accrued since the last call to this function, or + * since the recovery unit was instantiated. Should be called at the end of each operation. */ - virtual std::shared_ptr<StorageStats> getOperationStatistics() const { + virtual std::unique_ptr<StorageStats> computeOperationStatisticsSinceLastCall() { return (nullptr); } diff --git a/src/mongo/db/storage/storage_stats.h b/src/mongo/db/storage/storage_stats.h index b0afd2ba16d..0c7ea1ca179 100644 --- a/src/mongo/db/storage/storage_stats.h +++ b/src/mongo/db/storage/storage_stats.h @@ -39,19 +39,23 @@ namespace mongo { */ class StorageStats { public: + // This is a pure virtual class, so the constructors will never be called directly, and slicing + // should not be an issue. StorageStats() = default; + StorageStats(const StorageStats&) = default; + StorageStats(StorageStats&&) = default; - StorageStats(const StorageStats&) = delete; - StorageStats(StorageStats&&) = delete; StorageStats& operator=(const StorageStats&) = delete; + StorageStats& operator=(StorageStats&&) = delete; virtual ~StorageStats() = default; virtual BSONObj toBSON() const = 0; - virtual std::shared_ptr<StorageStats> clone() const = 0; + virtual std::unique_ptr<StorageStats> clone() const = 0; virtual StorageStats& operator+=(const StorageStats&) = 0; + virtual StorageStats& operator-=(const StorageStats&) = 0; }; } // namespace mongo diff --git a/src/mongo/db/storage/wiredtiger/SConscript b/src/mongo/db/storage/wiredtiger/SConscript index 638cf0b933c..7f6cee79dc4 100644 --- a/src/mongo/db/storage/wiredtiger/SConscript +++ b/src/mongo/db/storage/wiredtiger/SConscript @@ -37,7 +37,7 @@ wtEnv.Library( 'wiredtiger_index.cpp', 'wiredtiger_index_util.cpp', 'wiredtiger_kv_engine.cpp', - 'wiredtiger_operation_stats.cpp', + 'wiredtiger_stats.cpp', 'wiredtiger_oplog_manager.cpp', 'wiredtiger_parameters.cpp', 'wiredtiger_prepare_conflict.cpp', @@ -134,7 +134,7 @@ wtEnv.CppUnitTest( 'wiredtiger_init_test.cpp', 'wiredtiger_c_api_test.cpp', 'wiredtiger_kv_engine_test.cpp', - 'wiredtiger_operation_stats_test.cpp', + 'wiredtiger_stats_test.cpp', 'wiredtiger_recovery_unit_test.cpp', 'wiredtiger_session_cache_test.cpp', 'wiredtiger_util_test.cpp', diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_operation_stats_test.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_operation_stats_test.cpp deleted file mode 100644 index 5151f59cb48..00000000000 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_operation_stats_test.cpp +++ /dev/null @@ -1,215 +0,0 @@ -/** - * Copyright (C) 2022-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#include "mongo/db/storage/wiredtiger/wiredtiger_operation_stats.h" -#include "mongo/unittest/temp_dir.h" -#include "mongo/unittest/unittest.h" - -namespace mongo { -namespace { - -#define ASSERT_WT_OK(result) ASSERT_EQ(result, 0) << wiredtiger_strerror(result) - -class WiredTigerOperationStatsTest : public unittest::Test { -protected: - void setUp() override { - ASSERT_WT_OK( - wiredtiger_open(_path.path().c_str(), nullptr, "create,statistics=(fast),", &_conn)); - ASSERT_WT_OK(_conn->open_session(_conn, nullptr, "isolation=snapshot", &_session)); - ASSERT_WT_OK(_session->create( - _session, _uri.c_str(), "type=file,key_format=q,value_format=u,log=(enabled=false)")); - } - - void tearDown() override { - ASSERT_EQ(_conn->close(_conn, nullptr), 0); - } - - /** - * Writes the given data using WT. Causes the bytesWritten and timeWritingMicros stats to be - * incremented. - */ - void write(const std::string& data) { - ASSERT_WT_OK(_session->begin_transaction(_session, nullptr)); - - WT_CURSOR* cursor; - ASSERT_WT_OK(_session->open_cursor(_session, _uri.c_str(), nullptr, nullptr, &cursor)); - - cursor->set_key(cursor, _key++); - - WT_ITEM item{data.data(), data.size()}; - cursor->set_value(cursor, &item); - - ASSERT_WT_OK(cursor->insert(cursor)); - ASSERT_WT_OK(cursor->close(cursor)); - ASSERT_WT_OK(_session->commit_transaction(_session, nullptr)); - ASSERT_WT_OK(_session->checkpoint(_session, nullptr)); - } - - /** - * Reads all of the previously written data from WT. Causes the bytesRead and timeReadingMicros - * stats to be incremented. - */ - void read() { - tearDown(); - setUp(); - - ASSERT_WT_OK(_session->begin_transaction(_session, nullptr)); - - WT_CURSOR* cursor; - ASSERT_WT_OK(_session->open_cursor(_session, _uri.c_str(), nullptr, nullptr, &cursor)); - - for (int64_t i = 0; i < _key; ++i) { - cursor->set_key(cursor, i); - ASSERT_WT_OK(cursor->search(cursor)); - - WT_ITEM value; - ASSERT_WT_OK(cursor->get_value(cursor, &value)); - } - - ASSERT_WT_OK(cursor->close(cursor)); - ASSERT_WT_OK(_session->commit_transaction(_session, nullptr)); - } - - unittest::TempDir _path{"wiredtiger_operation_stats_test"}; - std::string _uri{"table:wiredtiger_operation_stats_test"}; - WT_CONNECTION* _conn; - WT_SESSION* _session; - int64_t _key = 0; -}; - -TEST_F(WiredTigerOperationStatsTest, Empty) { - ASSERT_BSONOBJ_EQ(WiredTigerOperationStats{_session}.toBSON(), BSONObj{}); -} - -TEST_F(WiredTigerOperationStatsTest, Write) { - write("a"); - - auto statsObj = WiredTigerOperationStats{_session}.toBSON(); - - auto dataSection = statsObj["data"]; - ASSERT_EQ(dataSection.type(), BSONType::Object) << statsObj; - - ASSERT(dataSection["bytesWritten"]) << statsObj; - for (auto&& [name, value] : dataSection.Obj()) { - ASSERT_EQ(value.type(), BSONType::NumberLong) << statsObj; - ASSERT_GT(value.numberLong(), 0) << statsObj; - } -} - -TEST_F(WiredTigerOperationStatsTest, Read) { - write("a"); - read(); - - auto statsObj = WiredTigerOperationStats{_session}.toBSON(); - - auto dataSection = statsObj["data"]; - ASSERT_EQ(dataSection.type(), BSONType::Object) << statsObj; - - ASSERT(dataSection["bytesRead"]) << statsObj; - for (auto&& [name, value] : dataSection.Obj()) { - ASSERT_EQ(value.type(), BSONType::NumberLong) << statsObj; - ASSERT_GT(value.numberLong(), 0) << statsObj; - } -} - -TEST_F(WiredTigerOperationStatsTest, Large) { - auto remaining = static_cast<int64_t>(std::numeric_limits<uint32_t>::max()) + 1; - while (remaining > 0) { - std::string data(1024 * 1024, 'a'); - remaining -= data.size(); - write(data); - } - - auto statsObj = WiredTigerOperationStats{_session}.toBSON(); - ASSERT_GT(statsObj["data"]["bytesWritten"].numberLong(), std::numeric_limits<uint32_t>::max()) - << statsObj; - - read(); - - statsObj = WiredTigerOperationStats{_session}.toBSON(); - ASSERT_GT(statsObj["data"]["bytesRead"].numberLong(), std::numeric_limits<uint32_t>::max()) - << statsObj; -} - -TEST_F(WiredTigerOperationStatsTest, Add) { - std::vector<std::unique_ptr<WiredTigerOperationStats>> stats; - - write("a"); - stats.push_back(std::make_unique<WiredTigerOperationStats>(_session)); - - read(); - stats.push_back(std::make_unique<WiredTigerOperationStats>(_session)); - - write("aa"); - stats.push_back(std::make_unique<WiredTigerOperationStats>(_session)); - - read(); - stats.push_back(std::make_unique<WiredTigerOperationStats>(_session)); - - long long bytesWritten = 0; - long long timeWritingMicros = 0; - long long bytesRead = 0; - long long timeReadingMicros = 0; - - WiredTigerOperationStats combined; - - for (auto&& op : stats) { - auto statsObj = op->toBSON(); - - bytesWritten += statsObj["data"]["bytesWritten"].numberLong(); - timeWritingMicros += statsObj["data"]["timeWritingMicros"].numberLong(); - bytesRead += statsObj["data"]["bytesRead"].numberLong(); - timeReadingMicros += statsObj["data"]["timeReadingMicros"].numberLong(); - - combined += *op; - } - - auto combinedObj = combined.toBSON(); - auto dataSection = combinedObj["data"]; - ASSERT_EQ(dataSection.type(), BSONType::Object) << combinedObj; - ASSERT_EQ(dataSection["bytesWritten"].numberLong(), bytesWritten) << combinedObj; - ASSERT_EQ(dataSection["timeWritingMicros"].numberLong(), timeWritingMicros) << combinedObj; - ASSERT_EQ(dataSection["bytesRead"].numberLong(), bytesRead) << combinedObj; - ASSERT_EQ(dataSection["timeReadingMicros"].numberLong(), timeReadingMicros) << combinedObj; -} - -TEST_F(WiredTigerOperationStatsTest, Clone) { - write("a"); - - WiredTigerOperationStats stats{_session}; - auto clone = stats.clone(); - - ASSERT_BSONOBJ_EQ(stats.toBSON(), clone->toBSON()); - - stats += *clone; - ASSERT_BSONOBJ_NE(stats.toBSON(), clone->toBSON()); -} - -} // namespace -} // namespace mongo diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.cpp index 26f91072bbd..aedd5d8e9cf 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.cpp @@ -37,9 +37,9 @@ #include "mongo/db/server_options.h" #include "mongo/db/storage/wiredtiger/wiredtiger_begin_transaction_block.h" #include "mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h" -#include "mongo/db/storage/wiredtiger/wiredtiger_operation_stats.h" #include "mongo/db/storage/wiredtiger/wiredtiger_prepare_conflict.h" #include "mongo/db/storage/wiredtiger/wiredtiger_session_cache.h" +#include "mongo/db/storage/wiredtiger/wiredtiger_stats.h" #include "mongo/db/storage/wiredtiger/wiredtiger_util.h" #include "mongo/logv2/log.h" #include "mongo/util/hex.h" @@ -48,6 +48,7 @@ #include <fmt/compile.h> #include <fmt/format.h> +#include <memory> #define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kStorage @@ -895,8 +896,21 @@ void WiredTigerRecoveryUnit::beginIdle() { } } -std::shared_ptr<StorageStats> WiredTigerRecoveryUnit::getOperationStatistics() const { - return _session ? std::make_shared<WiredTigerOperationStats>(_session->getSession()) : nullptr; +std::unique_ptr<StorageStats> WiredTigerRecoveryUnit::computeOperationStatisticsSinceLastCall() { + if (!_session) + return nullptr; + + // We compute operation statistics as the difference between the current session statistics and + // the session statistics of the last time the method was called, which should correspond to the + // end of one operation. + WiredTigerStats currentSessionStats{_session->getSession()}; + + auto operationStats = + std::make_unique<WiredTigerStats>(currentSessionStats - _sessionStatsAfterLastOperation); + + _sessionStatsAfterLastOperation = std::move(currentSessionStats); + + return operationStats; } void WiredTigerRecoveryUnit::setCatalogConflictingTimestamp(Timestamp timestamp) { diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.h b/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.h index 2d38888dc90..738bba18ff4 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.h +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.h @@ -45,8 +45,8 @@ #include "mongo/db/storage/recovery_unit.h" #include "mongo/db/storage/wiredtiger/wiredtiger_begin_transaction_block.h" #include "mongo/db/storage/wiredtiger/wiredtiger_session_cache.h" +#include "mongo/db/storage/wiredtiger/wiredtiger_stats.h" #include "mongo/util/timer.h" - namespace mongo { using RoundUpPreparedTimestamps = WiredTigerBeginTxnBlock::RoundUpPreparedTimestamps; @@ -144,7 +144,7 @@ public: return _readOnce; }; - std::shared_ptr<StorageStats> getOperationStatistics() const override; + std::unique_ptr<StorageStats> computeOperationStatisticsSinceLastCall() override; void ignoreAllMultiTimestampConstraints() { _multiTimestampConstraintTracker.ignoreAllMultiTimestampConstraints = true; @@ -287,6 +287,8 @@ private: boost::optional<int64_t> _oplogVisibleTs = boost::none; bool _gatherWriteContextForDebugging = false; std::vector<BSONObj> _writeContextForDebugging; + + WiredTigerStats _sessionStatsAfterLastOperation; }; } // namespace mongo diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_operation_stats.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_stats.cpp index ebf3842e7bc..da3d2f461e5 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_operation_stats.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_stats.cpp @@ -27,7 +27,7 @@ * it in the license file. */ -#include "mongo/db/storage/wiredtiger/wiredtiger_operation_stats.h" +#include "mongo/db/storage/wiredtiger/wiredtiger_stats.h" #include "mongo/bson/bsonobjbuilder.h" #include "mongo/db/storage/wiredtiger/wiredtiger_util.h" @@ -42,7 +42,7 @@ struct StatInfo { StatType type; }; -stdx::unordered_map<int, StatInfo> statInfo = { +const stdx::unordered_map<int, StatInfo> kWiredTigerStatCodeToStatInfo = { {WT_STAT_SESSION_BYTES_READ, {"bytesRead"_sd, StatType::kData}}, {WT_STAT_SESSION_BYTES_WRITE, {"bytesWritten"_sd, StatType::kData}}, {WT_STAT_SESSION_LOCK_DHANDLE_WAIT, {"handleLock"_sd, StatType::kWait}}, @@ -53,7 +53,7 @@ stdx::unordered_map<int, StatInfo> statInfo = { } // namespace -WiredTigerOperationStats::WiredTigerOperationStats(WT_SESSION* session) { +WiredTigerStats::WiredTigerStats(WT_SESSION* session) { invariant(session); WT_CURSOR* c; @@ -69,12 +69,9 @@ WiredTigerOperationStats::WiredTigerOperationStats(WT_SESSION* session) { fassert(51035, c->get_value(c, nullptr, nullptr, &value) == 0); _stats[key] = WiredTigerUtil::castStatisticsValue<long long>(value); } - - // Reset the statistics so that the next fetch gives the recent values. - invariantWTOK(c->reset(c), c->session); } -BSONObj WiredTigerOperationStats::toBSON() const { +BSONObj WiredTigerStats::toBSON() const { boost::optional<BSONObjBuilder> dataSection; boost::optional<BSONObjBuilder> waitSection; @@ -83,8 +80,8 @@ BSONObj WiredTigerOperationStats::toBSON() const { continue; } - auto it = statInfo.find(stat); - if (it == statInfo.end()) { + auto it = kWiredTigerStatCodeToStatInfo.find(stat); + if (it == kWiredTigerStatCodeToStatInfo.end()) { continue; } auto&& [name, type] = it->second; @@ -118,22 +115,36 @@ BSONObj WiredTigerOperationStats::toBSON() const { return builder.obj(); } -std::shared_ptr<StorageStats> WiredTigerOperationStats::clone() const { - auto copy = std::make_shared<WiredTigerOperationStats>(); - *copy += *this; - return copy; +std::unique_ptr<StorageStats> WiredTigerStats::clone() const { + return std::make_unique<WiredTigerStats>(*this); } -WiredTigerOperationStats& WiredTigerOperationStats::operator+=( - const WiredTigerOperationStats& other) { +WiredTigerStats& WiredTigerStats::operator=(WiredTigerStats&& other) { + _stats = std::move(other._stats); + return *this; +} + +WiredTigerStats& WiredTigerStats::operator+=(const WiredTigerStats& other) { for (auto&& [stat, value] : other._stats) { _stats[stat] += value; } return *this; } -StorageStats& WiredTigerOperationStats::operator+=(const StorageStats& other) { - return *this += checked_cast<const WiredTigerOperationStats&>(other); +StorageStats& WiredTigerStats::operator+=(const StorageStats& other) { + return *this += checked_cast<const WiredTigerStats&>(other); +} + +WiredTigerStats& WiredTigerStats::operator-=(const WiredTigerStats& other) { + for (auto const& otherStat : other._stats) { + _stats[otherStat.first] -= otherStat.second; + } + return (*this); +} + +StorageStats& WiredTigerStats::operator-=(const StorageStats& other) { + *this -= checked_cast<const WiredTigerStats&>(other); + return (*this); } } // namespace mongo diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_operation_stats.h b/src/mongo/db/storage/wiredtiger/wiredtiger_stats.h index 581303e30b8..d35a582cd34 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_operation_stats.h +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_stats.h @@ -35,21 +35,38 @@ namespace mongo { -class WiredTigerOperationStats final : public StorageStats { +class WiredTigerStats final : public StorageStats { public: - WiredTigerOperationStats() = default; - WiredTigerOperationStats(WT_SESSION* session); + /** + * Construct a new WiredTigerStats object with the statistics of the specified session. + */ + WiredTigerStats(WT_SESSION*); + + WiredTigerStats() = default; + WiredTigerStats(const WiredTigerStats&) = default; + WiredTigerStats(WiredTigerStats&&) = default; BSONObj toBSON() const final; - std::shared_ptr<StorageStats> clone() const final; + std::unique_ptr<StorageStats> clone() const final; + + WiredTigerStats& operator=(WiredTigerStats&&); StorageStats& operator+=(const StorageStats&) final; - WiredTigerOperationStats& operator+=(const WiredTigerOperationStats&); + WiredTigerStats& operator+=(const WiredTigerStats&); + + StorageStats& operator-=(const StorageStats&) final; -private: + WiredTigerStats& operator-=(const WiredTigerStats&); + +protected: std::map<int, long long> _stats; }; +inline WiredTigerStats operator-(WiredTigerStats lhs, const WiredTigerStats& rhs) { + lhs -= rhs; + return lhs; +} + } // namespace mongo diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_stats_test.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_stats_test.cpp new file mode 100644 index 00000000000..f29f360772c --- /dev/null +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_stats_test.cpp @@ -0,0 +1,328 @@ +/** + * Copyright (C) 2022-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/storage/wiredtiger/wiredtiger_stats.h" +#include "mongo/unittest/assert.h" +#include "mongo/unittest/temp_dir.h" +#include "mongo/unittest/unittest.h" +#include <memory> + +namespace mongo { +namespace { + +#define ASSERT_WT_OK(result) ASSERT_EQ(result, 0) << wiredtiger_strerror(result) + +class WiredTigerStatsTest : public unittest::Test { +protected: + void setUp() override { + openConnectionAndCreateSession(); + // Prepare data to be read by tests. Reading data written within the same transaction does + // not count towards bytes read into cache. + + // Tests in the fixture do up to _maxReads reads. + for (int64_t i = 0; i < _kMaxReads; ++i) { + // Make the write big enough to span different pages. + writeAtKey(std::string(20000, 'a'), i); + } + + // Closing the connection will ensure that tests actually have to read into cache. + closeConnection(); + openConnectionAndCreateSession(); + } + + void tearDown() override { + closeConnection(); + } + + void openConnectionAndCreateSession() { + ASSERT_WT_OK( + wiredtiger_open(_path.path().c_str(), nullptr, "create,statistics=(fast),", &_conn)); + ASSERT_WT_OK(_conn->open_session(_conn, nullptr, "isolation=snapshot", &_session)); + ASSERT_WT_OK(_session->create( + _session, _uri.c_str(), "type=file,key_format=q,value_format=u,log=(enabled=false)")); + } + + void closeConnection() { + ASSERT_EQ(_conn->close(_conn, nullptr), 0); + } + + /** + * Writes some data using WT. Causes the bytesWritten stat to be incremented, and may also + * increment timeWritingMicros. + */ + void write() { + writeAtKey(std::string(_writeKey + 1, 'a'), _writeKey); + ++_writeKey; + } + + /** + * Writes the specified data using WT. Causes the bytesWritten stat to be incremented, and may + * also increment timeWritingMicros. + */ + void write(const std::string& data) { + writeAtKey(data, _writeKey); + ++_writeKey; + } + + /** + * Writes at the specified key to WT. + */ + void writeAtKey(const std::string& data, int64_t key) { + ASSERT_WT_OK(_session->begin_transaction(_session, nullptr)); + + WT_CURSOR* cursor; + ASSERT_WT_OK(_session->open_cursor(_session, _uri.c_str(), nullptr, nullptr, &cursor)); + + cursor->set_key(cursor, key); + + WT_ITEM item{data.data(), data.size()}; + cursor->set_value(cursor, &item); + + ASSERT_WT_OK(cursor->insert(cursor)); + ASSERT_WT_OK(cursor->close(cursor)); + ASSERT_WT_OK(_session->commit_transaction(_session, nullptr)); + + // Without a checkpoint, an operation is not guaranteed to write to disk. + ASSERT_WT_OK(_session->checkpoint(_session, nullptr)); + } + + /** + * Reads at the specified key from WT. + */ + void readAtKey(int64_t key) { + ASSERT_WT_OK(_session->begin_transaction(_session, nullptr)); + + WT_CURSOR* cursor; + ASSERT_WT_OK(_session->open_cursor(_session, _uri.c_str(), nullptr, nullptr, &cursor)); + + cursor->set_key(cursor, key); + ASSERT_WT_OK(cursor->search(cursor)); + + WT_ITEM value; + ASSERT_WT_OK(cursor->get_value(cursor, &value)); + + ASSERT_WT_OK(cursor->close(cursor)); + ASSERT_WT_OK(_session->commit_transaction(_session, nullptr)); + } + + /** + * Reads fixture data from WT. Causes the bytesRead stat to be incremented. May also cause + * timeReadingMicros to be incremented, but not always. This function can only be called up to + * _kMaxReads times within a test. + */ + void read() { + ASSERT_LT(_readKey, _kMaxReads); + readAtKey(_readKey++); + } + + /** + * Reads data written by the test from WT. Causes the bytesRead stat to be incremented. May + * also cause timeReadingMicros to be incremented, but not always. + */ + void readTestWrites() { + for (int64_t i = _kMaxReads; i < _writeKey; i++) { + readAtKey(i); + } + } + + unittest::TempDir _path{"wiredtiger_operation_stats_test"}; + std::string _uri{"table:wiredtiger_operation_stats_test"}; + WT_CONNECTION* _conn; + WT_SESSION* _session; + /* Number of reads the fixture will prepare in setUp(), consequently max amount of times read() + * can be called in a test. */ + static constexpr int64_t _kMaxReads = 2; + /* Next key to be used by read(), must be initialized at 0. */ + int64_t _readKey = 0; + /* Next key to be used by write(), must be initialized >= _kMaxReads. */ + int64_t _writeKey = _kMaxReads; +}; + +TEST_F(WiredTigerStatsTest, EmptySession) { + // Read and write statistics should be empty. Check "data" field does not exist. "wait" fields + // such as the schemaLock might have some value. + auto statsBson = WiredTigerStats{_session}.toBSON(); + ASSERT_FALSE(statsBson.hasField("data")); +} + +TEST_F(WiredTigerStatsTest, SessionWithWrite) { + write(); + + auto statsObj = WiredTigerStats{_session}.toBSON(); + auto dataSection = statsObj["data"]; + ASSERT_EQ(dataSection.type(), BSONType::Object) << statsObj; + + ASSERT(dataSection["bytesWritten"]) << statsObj; + for (auto&& [name, value] : dataSection.Obj()) { + ASSERT_EQ(value.type(), BSONType::NumberLong) << statsObj; + ASSERT_GT(value.numberLong(), 0) << statsObj; + } +} + +TEST_F(WiredTigerStatsTest, SessionWithRead) { + read(); + + auto statsObj = WiredTigerStats{_session}.toBSON(); + + auto dataSection = statsObj["data"]; + ASSERT_EQ(dataSection.type(), BSONType::Object) << statsObj; + + ASSERT(dataSection["bytesRead"]) << statsObj; + for (auto&& [name, value] : dataSection.Obj()) { + ASSERT_EQ(value.type(), BSONType::NumberLong) << statsObj; + ASSERT_GT(value.numberLong(), 0) << statsObj; + } +} + +TEST_F(WiredTigerStatsTest, SessionWithLargeWriteAndLargeRead) { + auto remaining = static_cast<int64_t>(std::numeric_limits<uint32_t>::max()) + 1; + while (remaining > 0) { + std::string data(1024 * 1024, 'a'); + remaining -= data.size(); + write(data); + } + + auto statsObj = WiredTigerStats{_session}.toBSON(); + ASSERT_GT(statsObj["data"]["bytesWritten"].numberLong(), std::numeric_limits<uint32_t>::max()) + << statsObj; + + // Closing the connection will ensure that tests actually have to read into cache. + closeConnection(); + openConnectionAndCreateSession(); + + readTestWrites(); + + statsObj = WiredTigerStats{_session}.toBSON(); + ASSERT_GT(statsObj["data"]["bytesRead"].numberLong(), std::numeric_limits<uint32_t>::max()) + << statsObj; +} + +TEST_F(WiredTigerStatsTest, OperationsAddToSessionStats) { + std::vector<std::unique_ptr<WiredTigerStats>> operationStats; + + write(); + WiredTigerStats firstWrite(_session); + operationStats.push_back(std::make_unique<WiredTigerStats>(firstWrite - WiredTigerStats{})); + read(); + WiredTigerStats firstRead(_session); + operationStats.push_back(std::make_unique<WiredTigerStats>(firstRead - firstWrite)); + write(); + WiredTigerStats secondWrite(_session); + operationStats.push_back(std::make_unique<WiredTigerStats>(secondWrite - firstRead)); + read(); + WiredTigerStats secondRead(_session); + operationStats.push_back(std::make_unique<WiredTigerStats>(secondRead - secondWrite)); + + const WiredTigerStats& fetchedSessionStats = secondRead; + + long long bytesWritten = 0; + long long timeWritingMicros = 0; + long long bytesRead = 0; + long long timeReadingMicros = 0; + + WiredTigerStats addedSessionStats; + + for (auto&& op : operationStats) { + auto statsObj = op->toBSON(); + + bytesWritten += statsObj["data"]["bytesWritten"].numberLong(); + timeWritingMicros += statsObj["data"]["timeWritingMicros"].numberLong(); + bytesRead += statsObj["data"]["bytesRead"].numberLong(); + timeReadingMicros += statsObj["data"]["timeReadingMicros"].numberLong(); + + addedSessionStats += *op; + } + + auto addedObj = addedSessionStats.toBSON(); + auto dataSection = addedObj["data"]; + ASSERT_EQ(dataSection.type(), BSONType::Object) << addedObj; + ASSERT_EQ(dataSection["bytesWritten"].numberLong(), bytesWritten) << addedObj; + ASSERT_EQ(dataSection["timeWritingMicros"].numberLong(), timeWritingMicros) << addedObj; + ASSERT_EQ(dataSection["bytesRead"].numberLong(), bytesRead) << addedObj; + ASSERT_EQ(dataSection["timeReadingMicros"].numberLong(), timeReadingMicros) << addedObj; + + auto fetchedObj = fetchedSessionStats.toBSON(); + auto fetchedDataSection = fetchedObj["data"]; + ASSERT_EQ(fetchedDataSection.type(), BSONType::Object) << fetchedObj; + ASSERT_EQ(fetchedDataSection["bytesWritten"].numberLong(), bytesWritten) << fetchedObj; + ASSERT_EQ(fetchedDataSection["timeWritingMicros"].numberLong(), timeWritingMicros) + << fetchedObj; + ASSERT_EQ(fetchedDataSection["bytesRead"].numberLong(), bytesRead) << fetchedObj; + ASSERT_EQ(fetchedDataSection["timeReadingMicros"].numberLong(), timeReadingMicros) + << fetchedObj; +} + +TEST_F(WiredTigerStatsTest, OperationsSubtractToZero) { + std::vector<std::unique_ptr<WiredTigerStats>> operationStats; + + write(); + WiredTigerStats firstWrite(_session); + operationStats.push_back(std::make_unique<WiredTigerStats>(firstWrite - WiredTigerStats{})); + read(); + WiredTigerStats firstRead(_session); + operationStats.push_back(std::make_unique<WiredTigerStats>(firstRead - firstWrite)); + write(); + WiredTigerStats secondWrite(_session); + operationStats.push_back(std::make_unique<WiredTigerStats>(secondWrite - firstRead)); + read(); + WiredTigerStats secondRead(_session); + operationStats.push_back(std::make_unique<WiredTigerStats>(secondRead - secondWrite)); + + WiredTigerStats& fetchedSessionStats = secondRead; + + // Assert fetchedSessionStats was not zero before checking subtract results in it being zero. + // We ignore the time statistics as those might still be 0 from time to time. + auto preSubtractObj = fetchedSessionStats.toBSON(); + auto preSubtract = preSubtractObj["data"]; + ASSERT_EQ(preSubtract.type(), BSONType::Object) << preSubtractObj; + ASSERT_GT(preSubtract["bytesWritten"].numberLong(), 0) << preSubtractObj; + ASSERT_GT(preSubtract["bytesRead"].numberLong(), 0) << preSubtractObj; + + for (auto&& op : operationStats) { + fetchedSessionStats -= *op; + } + + auto subtractedObj = fetchedSessionStats.toBSON(); + ASSERT_BSONOBJ_EQ(subtractedObj, BSONObj{}); +} + +TEST_F(WiredTigerStatsTest, Clone) { + write(); + + WiredTigerStats stats{_session}; + auto clone = stats.clone(); + + ASSERT_BSONOBJ_EQ(stats.toBSON(), clone->toBSON()); + + stats += *clone; + ASSERT_BSONOBJ_NE(stats.toBSON(), clone->toBSON()); +} + +} // namespace +} // namespace mongo diff --git a/src/mongo/db/transaction/transaction_metrics_observer.cpp b/src/mongo/db/transaction/transaction_metrics_observer.cpp index 7a6544a1b3b..e1f703daca2 100644 --- a/src/mongo/db/transaction/transaction_metrics_observer.cpp +++ b/src/mongo/db/transaction/transaction_metrics_observer.cpp @@ -183,15 +183,15 @@ void TransactionMetricsObserver::onTransactionOperation(OperationContext* opCtx, // into an existing storageStats instance stored in SingleTransactionStats. // WiredTiger doesn't let storage statistics be collected when transaction is prepared. if (!isPrepared) { - std::shared_ptr<StorageStats> storageStats = - opCtx->recoveryUnit()->getOperationStatistics(); + std::unique_ptr<StorageStats> storageStats = + opCtx->recoveryUnit()->computeOperationStatisticsSinceLastCall(); if (storageStats) { - CurOp::get(opCtx)->debug().storageStats = storageStats; if (!_singleTransactionStats.getOpDebug()->storageStats) { _singleTransactionStats.getOpDebug()->storageStats = storageStats->clone(); } else { *_singleTransactionStats.getOpDebug()->storageStats += *storageStats; } + CurOp::get(opCtx)->debug().storageStats = std::move(storageStats); } } diff --git a/src/mongo/db/transaction/transaction_metrics_observer.h b/src/mongo/db/transaction/transaction_metrics_observer.h index 833c4e5cfb7..1c51f51fe02 100644 --- a/src/mongo/db/transaction/transaction_metrics_observer.h +++ b/src/mongo/db/transaction/transaction_metrics_observer.h @@ -107,6 +107,14 @@ public: } /** + * Returns a reference to the SingleTransactionStats object stored in this + * TransactionMetricsObserver instance. + */ + SingleTransactionStats& getSingleTransactionStats() { + return _singleTransactionStats; + } + + /** * Resets the SingleTransactionStats object stored in this TransactionMetricsObserver instance, * preparing it for the new transaction or retryable write with the given number. */ diff --git a/src/mongo/db/transaction/transaction_participant.h b/src/mongo/db/transaction/transaction_participant.h index ac32f9fb32d..4a9b63eeb5e 100644 --- a/src/mongo/db/transaction/transaction_participant.h +++ b/src/mongo/db/transaction/transaction_participant.h @@ -749,8 +749,8 @@ public: } - SingleTransactionStats getSingleTransactionStatsForTest() const { - return o().transactionMetricsObserver.getSingleTransactionStats(); + SingleTransactionStats& getSingleTransactionStatsForTest() { + return _tp->_o.transactionMetricsObserver.getSingleTransactionStats(); } std::vector<repl::ReplOperation> getTransactionOperationsForTest() const { |