summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorYu Jin Kang Park <yujin.kang@mongodb.com>2022-11-29 14:46:06 +0000
committerEvergreen Agent <no-reply@evergreen.mongodb.com>2022-11-29 18:03:30 +0000
commit8045c4a7772ab588679cdbf604a2875c27d97251 (patch)
treefb8b05a2627884648be86b3e49c623c57bd46584
parent923b5fd65ad075c7da30af55022df936f4339128 (diff)
downloadmongo-8045c4a7772ab588679cdbf604a2875c27d97251.tar.gz
SERVER-68739 Do not reset WT session stats when fetching op stats
-rw-r--r--src/mongo/db/commands/find_cmd.cpp2
-rw-r--r--src/mongo/db/curop.cpp3
-rw-r--r--src/mongo/db/curop.h2
-rw-r--r--src/mongo/db/storage/recovery_unit.h5
-rw-r--r--src/mongo/db/storage/storage_stats.h10
-rw-r--r--src/mongo/db/storage/wiredtiger/SConscript4
-rw-r--r--src/mongo/db/storage/wiredtiger/wiredtiger_operation_stats_test.cpp215
-rw-r--r--src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.cpp20
-rw-r--r--src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.h6
-rw-r--r--src/mongo/db/storage/wiredtiger/wiredtiger_stats.cpp (renamed from src/mongo/db/storage/wiredtiger/wiredtiger_operation_stats.cpp)45
-rw-r--r--src/mongo/db/storage/wiredtiger/wiredtiger_stats.h (renamed from src/mongo/db/storage/wiredtiger/wiredtiger_operation_stats.h)29
-rw-r--r--src/mongo/db/storage/wiredtiger/wiredtiger_stats_test.cpp328
-rw-r--r--src/mongo/db/transaction/transaction_metrics_observer.cpp6
-rw-r--r--src/mongo/db/transaction/transaction_metrics_observer.h8
-rw-r--r--src/mongo/db/transaction/transaction_participant.h4
15 files changed, 429 insertions, 258 deletions
diff --git a/src/mongo/db/commands/find_cmd.cpp b/src/mongo/db/commands/find_cmd.cpp
index 13cf5cff30c..6034f67f54a 100644
--- a/src/mongo/db/commands/find_cmd.cpp
+++ b/src/mongo/db/commands/find_cmd.cpp
@@ -712,7 +712,7 @@ public:
// The stats collected here will not get overwritten, as the service entry
// point layer will only set these stats when they're not empty.
CurOp::get(opCtx)->debug().storageStats =
- opCtx->recoveryUnit()->getOperationStatistics();
+ opCtx->recoveryUnit()->computeOperationStatisticsSinceLastCall();
}
} else {
endQueryOp(opCtx, collection, *exec, numResults, cursorId);
diff --git a/src/mongo/db/curop.cpp b/src/mongo/db/curop.cpp
index f961ab83de4..b744e90ca3c 100644
--- a/src/mongo/db/curop.cpp
+++ b/src/mongo/db/curop.cpp
@@ -479,7 +479,8 @@ bool CurOp::completeAndLogOperation(OperationContext* opCtx,
MODE_IS,
Date_t::now() + Milliseconds(500),
Lock::InterruptBehavior::kThrow);
- _debug.storageStats = opCtx->recoveryUnit()->getOperationStatistics();
+ _debug.storageStats =
+ opCtx->recoveryUnit()->computeOperationStatisticsSinceLastCall();
} catch (const DBException& ex) {
LOGV2_WARNING_OPTIONS(20526,
{component},
diff --git a/src/mongo/db/curop.h b/src/mongo/db/curop.h
index 986a545e992..2d4f50872ed 100644
--- a/src/mongo/db/curop.h
+++ b/src/mongo/db/curop.h
@@ -326,7 +326,7 @@ public:
AdditiveMetrics additiveMetrics;
// Stores storage statistics.
- std::shared_ptr<StorageStats> storageStats;
+ std::unique_ptr<StorageStats> storageStats;
bool waitingForFlowControl{false};
diff --git a/src/mongo/db/storage/recovery_unit.h b/src/mongo/db/storage/recovery_unit.h
index 9e173d632e7..dd0917bfaf2 100644
--- a/src/mongo/db/storage/recovery_unit.h
+++ b/src/mongo/db/storage/recovery_unit.h
@@ -391,9 +391,10 @@ public:
virtual void allowUntimestampedWrite() {}
/**
- * Fetches the storage level statistics.
+ * Computes the storage level statistics accrued since the last call to this function, or
+ * since the recovery unit was instantiated. Should be called at the end of each operation.
*/
- virtual std::shared_ptr<StorageStats> getOperationStatistics() const {
+ virtual std::unique_ptr<StorageStats> computeOperationStatisticsSinceLastCall() {
return (nullptr);
}
diff --git a/src/mongo/db/storage/storage_stats.h b/src/mongo/db/storage/storage_stats.h
index b0afd2ba16d..0c7ea1ca179 100644
--- a/src/mongo/db/storage/storage_stats.h
+++ b/src/mongo/db/storage/storage_stats.h
@@ -39,19 +39,23 @@ namespace mongo {
*/
class StorageStats {
public:
+ // This is a pure virtual class, so the constructors will never be called directly, and slicing
+ // should not be an issue.
StorageStats() = default;
+ StorageStats(const StorageStats&) = default;
+ StorageStats(StorageStats&&) = default;
- StorageStats(const StorageStats&) = delete;
- StorageStats(StorageStats&&) = delete;
StorageStats& operator=(const StorageStats&) = delete;
+ StorageStats& operator=(StorageStats&&) = delete;
virtual ~StorageStats() = default;
virtual BSONObj toBSON() const = 0;
- virtual std::shared_ptr<StorageStats> clone() const = 0;
+ virtual std::unique_ptr<StorageStats> clone() const = 0;
virtual StorageStats& operator+=(const StorageStats&) = 0;
+ virtual StorageStats& operator-=(const StorageStats&) = 0;
};
} // namespace mongo
diff --git a/src/mongo/db/storage/wiredtiger/SConscript b/src/mongo/db/storage/wiredtiger/SConscript
index 638cf0b933c..7f6cee79dc4 100644
--- a/src/mongo/db/storage/wiredtiger/SConscript
+++ b/src/mongo/db/storage/wiredtiger/SConscript
@@ -37,7 +37,7 @@ wtEnv.Library(
'wiredtiger_index.cpp',
'wiredtiger_index_util.cpp',
'wiredtiger_kv_engine.cpp',
- 'wiredtiger_operation_stats.cpp',
+ 'wiredtiger_stats.cpp',
'wiredtiger_oplog_manager.cpp',
'wiredtiger_parameters.cpp',
'wiredtiger_prepare_conflict.cpp',
@@ -134,7 +134,7 @@ wtEnv.CppUnitTest(
'wiredtiger_init_test.cpp',
'wiredtiger_c_api_test.cpp',
'wiredtiger_kv_engine_test.cpp',
- 'wiredtiger_operation_stats_test.cpp',
+ 'wiredtiger_stats_test.cpp',
'wiredtiger_recovery_unit_test.cpp',
'wiredtiger_session_cache_test.cpp',
'wiredtiger_util_test.cpp',
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_operation_stats_test.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_operation_stats_test.cpp
deleted file mode 100644
index 5151f59cb48..00000000000
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_operation_stats_test.cpp
+++ /dev/null
@@ -1,215 +0,0 @@
-/**
- * Copyright (C) 2022-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#include "mongo/db/storage/wiredtiger/wiredtiger_operation_stats.h"
-#include "mongo/unittest/temp_dir.h"
-#include "mongo/unittest/unittest.h"
-
-namespace mongo {
-namespace {
-
-#define ASSERT_WT_OK(result) ASSERT_EQ(result, 0) << wiredtiger_strerror(result)
-
-class WiredTigerOperationStatsTest : public unittest::Test {
-protected:
- void setUp() override {
- ASSERT_WT_OK(
- wiredtiger_open(_path.path().c_str(), nullptr, "create,statistics=(fast),", &_conn));
- ASSERT_WT_OK(_conn->open_session(_conn, nullptr, "isolation=snapshot", &_session));
- ASSERT_WT_OK(_session->create(
- _session, _uri.c_str(), "type=file,key_format=q,value_format=u,log=(enabled=false)"));
- }
-
- void tearDown() override {
- ASSERT_EQ(_conn->close(_conn, nullptr), 0);
- }
-
- /**
- * Writes the given data using WT. Causes the bytesWritten and timeWritingMicros stats to be
- * incremented.
- */
- void write(const std::string& data) {
- ASSERT_WT_OK(_session->begin_transaction(_session, nullptr));
-
- WT_CURSOR* cursor;
- ASSERT_WT_OK(_session->open_cursor(_session, _uri.c_str(), nullptr, nullptr, &cursor));
-
- cursor->set_key(cursor, _key++);
-
- WT_ITEM item{data.data(), data.size()};
- cursor->set_value(cursor, &item);
-
- ASSERT_WT_OK(cursor->insert(cursor));
- ASSERT_WT_OK(cursor->close(cursor));
- ASSERT_WT_OK(_session->commit_transaction(_session, nullptr));
- ASSERT_WT_OK(_session->checkpoint(_session, nullptr));
- }
-
- /**
- * Reads all of the previously written data from WT. Causes the bytesRead and timeReadingMicros
- * stats to be incremented.
- */
- void read() {
- tearDown();
- setUp();
-
- ASSERT_WT_OK(_session->begin_transaction(_session, nullptr));
-
- WT_CURSOR* cursor;
- ASSERT_WT_OK(_session->open_cursor(_session, _uri.c_str(), nullptr, nullptr, &cursor));
-
- for (int64_t i = 0; i < _key; ++i) {
- cursor->set_key(cursor, i);
- ASSERT_WT_OK(cursor->search(cursor));
-
- WT_ITEM value;
- ASSERT_WT_OK(cursor->get_value(cursor, &value));
- }
-
- ASSERT_WT_OK(cursor->close(cursor));
- ASSERT_WT_OK(_session->commit_transaction(_session, nullptr));
- }
-
- unittest::TempDir _path{"wiredtiger_operation_stats_test"};
- std::string _uri{"table:wiredtiger_operation_stats_test"};
- WT_CONNECTION* _conn;
- WT_SESSION* _session;
- int64_t _key = 0;
-};
-
-TEST_F(WiredTigerOperationStatsTest, Empty) {
- ASSERT_BSONOBJ_EQ(WiredTigerOperationStats{_session}.toBSON(), BSONObj{});
-}
-
-TEST_F(WiredTigerOperationStatsTest, Write) {
- write("a");
-
- auto statsObj = WiredTigerOperationStats{_session}.toBSON();
-
- auto dataSection = statsObj["data"];
- ASSERT_EQ(dataSection.type(), BSONType::Object) << statsObj;
-
- ASSERT(dataSection["bytesWritten"]) << statsObj;
- for (auto&& [name, value] : dataSection.Obj()) {
- ASSERT_EQ(value.type(), BSONType::NumberLong) << statsObj;
- ASSERT_GT(value.numberLong(), 0) << statsObj;
- }
-}
-
-TEST_F(WiredTigerOperationStatsTest, Read) {
- write("a");
- read();
-
- auto statsObj = WiredTigerOperationStats{_session}.toBSON();
-
- auto dataSection = statsObj["data"];
- ASSERT_EQ(dataSection.type(), BSONType::Object) << statsObj;
-
- ASSERT(dataSection["bytesRead"]) << statsObj;
- for (auto&& [name, value] : dataSection.Obj()) {
- ASSERT_EQ(value.type(), BSONType::NumberLong) << statsObj;
- ASSERT_GT(value.numberLong(), 0) << statsObj;
- }
-}
-
-TEST_F(WiredTigerOperationStatsTest, Large) {
- auto remaining = static_cast<int64_t>(std::numeric_limits<uint32_t>::max()) + 1;
- while (remaining > 0) {
- std::string data(1024 * 1024, 'a');
- remaining -= data.size();
- write(data);
- }
-
- auto statsObj = WiredTigerOperationStats{_session}.toBSON();
- ASSERT_GT(statsObj["data"]["bytesWritten"].numberLong(), std::numeric_limits<uint32_t>::max())
- << statsObj;
-
- read();
-
- statsObj = WiredTigerOperationStats{_session}.toBSON();
- ASSERT_GT(statsObj["data"]["bytesRead"].numberLong(), std::numeric_limits<uint32_t>::max())
- << statsObj;
-}
-
-TEST_F(WiredTigerOperationStatsTest, Add) {
- std::vector<std::unique_ptr<WiredTigerOperationStats>> stats;
-
- write("a");
- stats.push_back(std::make_unique<WiredTigerOperationStats>(_session));
-
- read();
- stats.push_back(std::make_unique<WiredTigerOperationStats>(_session));
-
- write("aa");
- stats.push_back(std::make_unique<WiredTigerOperationStats>(_session));
-
- read();
- stats.push_back(std::make_unique<WiredTigerOperationStats>(_session));
-
- long long bytesWritten = 0;
- long long timeWritingMicros = 0;
- long long bytesRead = 0;
- long long timeReadingMicros = 0;
-
- WiredTigerOperationStats combined;
-
- for (auto&& op : stats) {
- auto statsObj = op->toBSON();
-
- bytesWritten += statsObj["data"]["bytesWritten"].numberLong();
- timeWritingMicros += statsObj["data"]["timeWritingMicros"].numberLong();
- bytesRead += statsObj["data"]["bytesRead"].numberLong();
- timeReadingMicros += statsObj["data"]["timeReadingMicros"].numberLong();
-
- combined += *op;
- }
-
- auto combinedObj = combined.toBSON();
- auto dataSection = combinedObj["data"];
- ASSERT_EQ(dataSection.type(), BSONType::Object) << combinedObj;
- ASSERT_EQ(dataSection["bytesWritten"].numberLong(), bytesWritten) << combinedObj;
- ASSERT_EQ(dataSection["timeWritingMicros"].numberLong(), timeWritingMicros) << combinedObj;
- ASSERT_EQ(dataSection["bytesRead"].numberLong(), bytesRead) << combinedObj;
- ASSERT_EQ(dataSection["timeReadingMicros"].numberLong(), timeReadingMicros) << combinedObj;
-}
-
-TEST_F(WiredTigerOperationStatsTest, Clone) {
- write("a");
-
- WiredTigerOperationStats stats{_session};
- auto clone = stats.clone();
-
- ASSERT_BSONOBJ_EQ(stats.toBSON(), clone->toBSON());
-
- stats += *clone;
- ASSERT_BSONOBJ_NE(stats.toBSON(), clone->toBSON());
-}
-
-} // namespace
-} // namespace mongo
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.cpp
index 26f91072bbd..aedd5d8e9cf 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.cpp
@@ -37,9 +37,9 @@
#include "mongo/db/server_options.h"
#include "mongo/db/storage/wiredtiger/wiredtiger_begin_transaction_block.h"
#include "mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h"
-#include "mongo/db/storage/wiredtiger/wiredtiger_operation_stats.h"
#include "mongo/db/storage/wiredtiger/wiredtiger_prepare_conflict.h"
#include "mongo/db/storage/wiredtiger/wiredtiger_session_cache.h"
+#include "mongo/db/storage/wiredtiger/wiredtiger_stats.h"
#include "mongo/db/storage/wiredtiger/wiredtiger_util.h"
#include "mongo/logv2/log.h"
#include "mongo/util/hex.h"
@@ -48,6 +48,7 @@
#include <fmt/compile.h>
#include <fmt/format.h>
+#include <memory>
#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kStorage
@@ -895,8 +896,21 @@ void WiredTigerRecoveryUnit::beginIdle() {
}
}
-std::shared_ptr<StorageStats> WiredTigerRecoveryUnit::getOperationStatistics() const {
- return _session ? std::make_shared<WiredTigerOperationStats>(_session->getSession()) : nullptr;
+std::unique_ptr<StorageStats> WiredTigerRecoveryUnit::computeOperationStatisticsSinceLastCall() {
+ if (!_session)
+ return nullptr;
+
+ // We compute operation statistics as the difference between the current session statistics and
+ // the session statistics of the last time the method was called, which should correspond to the
+ // end of one operation.
+ WiredTigerStats currentSessionStats{_session->getSession()};
+
+ auto operationStats =
+ std::make_unique<WiredTigerStats>(currentSessionStats - _sessionStatsAfterLastOperation);
+
+ _sessionStatsAfterLastOperation = std::move(currentSessionStats);
+
+ return operationStats;
}
void WiredTigerRecoveryUnit::setCatalogConflictingTimestamp(Timestamp timestamp) {
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.h b/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.h
index 2d38888dc90..738bba18ff4 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.h
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.h
@@ -45,8 +45,8 @@
#include "mongo/db/storage/recovery_unit.h"
#include "mongo/db/storage/wiredtiger/wiredtiger_begin_transaction_block.h"
#include "mongo/db/storage/wiredtiger/wiredtiger_session_cache.h"
+#include "mongo/db/storage/wiredtiger/wiredtiger_stats.h"
#include "mongo/util/timer.h"
-
namespace mongo {
using RoundUpPreparedTimestamps = WiredTigerBeginTxnBlock::RoundUpPreparedTimestamps;
@@ -144,7 +144,7 @@ public:
return _readOnce;
};
- std::shared_ptr<StorageStats> getOperationStatistics() const override;
+ std::unique_ptr<StorageStats> computeOperationStatisticsSinceLastCall() override;
void ignoreAllMultiTimestampConstraints() {
_multiTimestampConstraintTracker.ignoreAllMultiTimestampConstraints = true;
@@ -287,6 +287,8 @@ private:
boost::optional<int64_t> _oplogVisibleTs = boost::none;
bool _gatherWriteContextForDebugging = false;
std::vector<BSONObj> _writeContextForDebugging;
+
+ WiredTigerStats _sessionStatsAfterLastOperation;
};
} // namespace mongo
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_operation_stats.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_stats.cpp
index ebf3842e7bc..da3d2f461e5 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_operation_stats.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_stats.cpp
@@ -27,7 +27,7 @@
* it in the license file.
*/
-#include "mongo/db/storage/wiredtiger/wiredtiger_operation_stats.h"
+#include "mongo/db/storage/wiredtiger/wiredtiger_stats.h"
#include "mongo/bson/bsonobjbuilder.h"
#include "mongo/db/storage/wiredtiger/wiredtiger_util.h"
@@ -42,7 +42,7 @@ struct StatInfo {
StatType type;
};
-stdx::unordered_map<int, StatInfo> statInfo = {
+const stdx::unordered_map<int, StatInfo> kWiredTigerStatCodeToStatInfo = {
{WT_STAT_SESSION_BYTES_READ, {"bytesRead"_sd, StatType::kData}},
{WT_STAT_SESSION_BYTES_WRITE, {"bytesWritten"_sd, StatType::kData}},
{WT_STAT_SESSION_LOCK_DHANDLE_WAIT, {"handleLock"_sd, StatType::kWait}},
@@ -53,7 +53,7 @@ stdx::unordered_map<int, StatInfo> statInfo = {
} // namespace
-WiredTigerOperationStats::WiredTigerOperationStats(WT_SESSION* session) {
+WiredTigerStats::WiredTigerStats(WT_SESSION* session) {
invariant(session);
WT_CURSOR* c;
@@ -69,12 +69,9 @@ WiredTigerOperationStats::WiredTigerOperationStats(WT_SESSION* session) {
fassert(51035, c->get_value(c, nullptr, nullptr, &value) == 0);
_stats[key] = WiredTigerUtil::castStatisticsValue<long long>(value);
}
-
- // Reset the statistics so that the next fetch gives the recent values.
- invariantWTOK(c->reset(c), c->session);
}
-BSONObj WiredTigerOperationStats::toBSON() const {
+BSONObj WiredTigerStats::toBSON() const {
boost::optional<BSONObjBuilder> dataSection;
boost::optional<BSONObjBuilder> waitSection;
@@ -83,8 +80,8 @@ BSONObj WiredTigerOperationStats::toBSON() const {
continue;
}
- auto it = statInfo.find(stat);
- if (it == statInfo.end()) {
+ auto it = kWiredTigerStatCodeToStatInfo.find(stat);
+ if (it == kWiredTigerStatCodeToStatInfo.end()) {
continue;
}
auto&& [name, type] = it->second;
@@ -118,22 +115,36 @@ BSONObj WiredTigerOperationStats::toBSON() const {
return builder.obj();
}
-std::shared_ptr<StorageStats> WiredTigerOperationStats::clone() const {
- auto copy = std::make_shared<WiredTigerOperationStats>();
- *copy += *this;
- return copy;
+std::unique_ptr<StorageStats> WiredTigerStats::clone() const {
+ return std::make_unique<WiredTigerStats>(*this);
}
-WiredTigerOperationStats& WiredTigerOperationStats::operator+=(
- const WiredTigerOperationStats& other) {
+WiredTigerStats& WiredTigerStats::operator=(WiredTigerStats&& other) {
+ _stats = std::move(other._stats);
+ return *this;
+}
+
+WiredTigerStats& WiredTigerStats::operator+=(const WiredTigerStats& other) {
for (auto&& [stat, value] : other._stats) {
_stats[stat] += value;
}
return *this;
}
-StorageStats& WiredTigerOperationStats::operator+=(const StorageStats& other) {
- return *this += checked_cast<const WiredTigerOperationStats&>(other);
+StorageStats& WiredTigerStats::operator+=(const StorageStats& other) {
+ return *this += checked_cast<const WiredTigerStats&>(other);
+}
+
+WiredTigerStats& WiredTigerStats::operator-=(const WiredTigerStats& other) {
+ for (auto const& otherStat : other._stats) {
+ _stats[otherStat.first] -= otherStat.second;
+ }
+ return (*this);
+}
+
+StorageStats& WiredTigerStats::operator-=(const StorageStats& other) {
+ *this -= checked_cast<const WiredTigerStats&>(other);
+ return (*this);
}
} // namespace mongo
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_operation_stats.h b/src/mongo/db/storage/wiredtiger/wiredtiger_stats.h
index 581303e30b8..d35a582cd34 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_operation_stats.h
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_stats.h
@@ -35,21 +35,38 @@
namespace mongo {
-class WiredTigerOperationStats final : public StorageStats {
+class WiredTigerStats final : public StorageStats {
public:
- WiredTigerOperationStats() = default;
- WiredTigerOperationStats(WT_SESSION* session);
+ /**
+ * Construct a new WiredTigerStats object with the statistics of the specified session.
+ */
+ WiredTigerStats(WT_SESSION*);
+
+ WiredTigerStats() = default;
+ WiredTigerStats(const WiredTigerStats&) = default;
+ WiredTigerStats(WiredTigerStats&&) = default;
BSONObj toBSON() const final;
- std::shared_ptr<StorageStats> clone() const final;
+ std::unique_ptr<StorageStats> clone() const final;
+
+ WiredTigerStats& operator=(WiredTigerStats&&);
StorageStats& operator+=(const StorageStats&) final;
- WiredTigerOperationStats& operator+=(const WiredTigerOperationStats&);
+ WiredTigerStats& operator+=(const WiredTigerStats&);
+
+ StorageStats& operator-=(const StorageStats&) final;
-private:
+ WiredTigerStats& operator-=(const WiredTigerStats&);
+
+protected:
std::map<int, long long> _stats;
};
+inline WiredTigerStats operator-(WiredTigerStats lhs, const WiredTigerStats& rhs) {
+ lhs -= rhs;
+ return lhs;
+}
+
} // namespace mongo
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_stats_test.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_stats_test.cpp
new file mode 100644
index 00000000000..f29f360772c
--- /dev/null
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_stats_test.cpp
@@ -0,0 +1,328 @@
+/**
+ * Copyright (C) 2022-present MongoDB, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ * As a special exception, the copyright holders give permission to link the
+ * code of portions of this program with the OpenSSL library under certain
+ * conditions as described in each individual source file and distribute
+ * linked combinations including the program with the OpenSSL library. You
+ * must comply with the Server Side Public License in all respects for
+ * all of the code used other than as permitted herein. If you modify file(s)
+ * with this exception, you may extend this exception to your version of the
+ * file(s), but you are not obligated to do so. If you do not wish to do so,
+ * delete this exception statement from your version. If you delete this
+ * exception statement from all source files in the program, then also delete
+ * it in the license file.
+ */
+
+#include "mongo/db/storage/wiredtiger/wiredtiger_stats.h"
+#include "mongo/unittest/assert.h"
+#include "mongo/unittest/temp_dir.h"
+#include "mongo/unittest/unittest.h"
+#include <memory>
+
+namespace mongo {
+namespace {
+
+#define ASSERT_WT_OK(result) ASSERT_EQ(result, 0) << wiredtiger_strerror(result)
+
+class WiredTigerStatsTest : public unittest::Test {
+protected:
+ void setUp() override {
+ openConnectionAndCreateSession();
+ // Prepare data to be read by tests. Reading data written within the same transaction does
+ // not count towards bytes read into cache.
+
+ // Tests in the fixture do up to _maxReads reads.
+ for (int64_t i = 0; i < _kMaxReads; ++i) {
+ // Make the write big enough to span different pages.
+ writeAtKey(std::string(20000, 'a'), i);
+ }
+
+ // Closing the connection will ensure that tests actually have to read into cache.
+ closeConnection();
+ openConnectionAndCreateSession();
+ }
+
+ void tearDown() override {
+ closeConnection();
+ }
+
+ void openConnectionAndCreateSession() {
+ ASSERT_WT_OK(
+ wiredtiger_open(_path.path().c_str(), nullptr, "create,statistics=(fast),", &_conn));
+ ASSERT_WT_OK(_conn->open_session(_conn, nullptr, "isolation=snapshot", &_session));
+ ASSERT_WT_OK(_session->create(
+ _session, _uri.c_str(), "type=file,key_format=q,value_format=u,log=(enabled=false)"));
+ }
+
+ void closeConnection() {
+ ASSERT_EQ(_conn->close(_conn, nullptr), 0);
+ }
+
+ /**
+ * Writes some data using WT. Causes the bytesWritten stat to be incremented, and may also
+ * increment timeWritingMicros.
+ */
+ void write() {
+ writeAtKey(std::string(_writeKey + 1, 'a'), _writeKey);
+ ++_writeKey;
+ }
+
+ /**
+ * Writes the specified data using WT. Causes the bytesWritten stat to be incremented, and may
+ * also increment timeWritingMicros.
+ */
+ void write(const std::string& data) {
+ writeAtKey(data, _writeKey);
+ ++_writeKey;
+ }
+
+ /**
+ * Writes at the specified key to WT.
+ */
+ void writeAtKey(const std::string& data, int64_t key) {
+ ASSERT_WT_OK(_session->begin_transaction(_session, nullptr));
+
+ WT_CURSOR* cursor;
+ ASSERT_WT_OK(_session->open_cursor(_session, _uri.c_str(), nullptr, nullptr, &cursor));
+
+ cursor->set_key(cursor, key);
+
+ WT_ITEM item{data.data(), data.size()};
+ cursor->set_value(cursor, &item);
+
+ ASSERT_WT_OK(cursor->insert(cursor));
+ ASSERT_WT_OK(cursor->close(cursor));
+ ASSERT_WT_OK(_session->commit_transaction(_session, nullptr));
+
+ // Without a checkpoint, an operation is not guaranteed to write to disk.
+ ASSERT_WT_OK(_session->checkpoint(_session, nullptr));
+ }
+
+ /**
+ * Reads at the specified key from WT.
+ */
+ void readAtKey(int64_t key) {
+ ASSERT_WT_OK(_session->begin_transaction(_session, nullptr));
+
+ WT_CURSOR* cursor;
+ ASSERT_WT_OK(_session->open_cursor(_session, _uri.c_str(), nullptr, nullptr, &cursor));
+
+ cursor->set_key(cursor, key);
+ ASSERT_WT_OK(cursor->search(cursor));
+
+ WT_ITEM value;
+ ASSERT_WT_OK(cursor->get_value(cursor, &value));
+
+ ASSERT_WT_OK(cursor->close(cursor));
+ ASSERT_WT_OK(_session->commit_transaction(_session, nullptr));
+ }
+
+ /**
+ * Reads fixture data from WT. Causes the bytesRead stat to be incremented. May also cause
+ * timeReadingMicros to be incremented, but not always. This function can only be called up to
+ * _kMaxReads times within a test.
+ */
+ void read() {
+ ASSERT_LT(_readKey, _kMaxReads);
+ readAtKey(_readKey++);
+ }
+
+ /**
+ * Reads data written by the test from WT. Causes the bytesRead stat to be incremented. May
+ * also cause timeReadingMicros to be incremented, but not always.
+ */
+ void readTestWrites() {
+ for (int64_t i = _kMaxReads; i < _writeKey; i++) {
+ readAtKey(i);
+ }
+ }
+
+ unittest::TempDir _path{"wiredtiger_operation_stats_test"};
+ std::string _uri{"table:wiredtiger_operation_stats_test"};
+ WT_CONNECTION* _conn;
+ WT_SESSION* _session;
+ /* Number of reads the fixture will prepare in setUp(), consequently max amount of times read()
+ * can be called in a test. */
+ static constexpr int64_t _kMaxReads = 2;
+ /* Next key to be used by read(), must be initialized at 0. */
+ int64_t _readKey = 0;
+ /* Next key to be used by write(), must be initialized >= _kMaxReads. */
+ int64_t _writeKey = _kMaxReads;
+};
+
+TEST_F(WiredTigerStatsTest, EmptySession) {
+ // Read and write statistics should be empty. Check "data" field does not exist. "wait" fields
+ // such as the schemaLock might have some value.
+ auto statsBson = WiredTigerStats{_session}.toBSON();
+ ASSERT_FALSE(statsBson.hasField("data"));
+}
+
+TEST_F(WiredTigerStatsTest, SessionWithWrite) {
+ write();
+
+ auto statsObj = WiredTigerStats{_session}.toBSON();
+ auto dataSection = statsObj["data"];
+ ASSERT_EQ(dataSection.type(), BSONType::Object) << statsObj;
+
+ ASSERT(dataSection["bytesWritten"]) << statsObj;
+ for (auto&& [name, value] : dataSection.Obj()) {
+ ASSERT_EQ(value.type(), BSONType::NumberLong) << statsObj;
+ ASSERT_GT(value.numberLong(), 0) << statsObj;
+ }
+}
+
+TEST_F(WiredTigerStatsTest, SessionWithRead) {
+ read();
+
+ auto statsObj = WiredTigerStats{_session}.toBSON();
+
+ auto dataSection = statsObj["data"];
+ ASSERT_EQ(dataSection.type(), BSONType::Object) << statsObj;
+
+ ASSERT(dataSection["bytesRead"]) << statsObj;
+ for (auto&& [name, value] : dataSection.Obj()) {
+ ASSERT_EQ(value.type(), BSONType::NumberLong) << statsObj;
+ ASSERT_GT(value.numberLong(), 0) << statsObj;
+ }
+}
+
+TEST_F(WiredTigerStatsTest, SessionWithLargeWriteAndLargeRead) {
+ auto remaining = static_cast<int64_t>(std::numeric_limits<uint32_t>::max()) + 1;
+ while (remaining > 0) {
+ std::string data(1024 * 1024, 'a');
+ remaining -= data.size();
+ write(data);
+ }
+
+ auto statsObj = WiredTigerStats{_session}.toBSON();
+ ASSERT_GT(statsObj["data"]["bytesWritten"].numberLong(), std::numeric_limits<uint32_t>::max())
+ << statsObj;
+
+ // Closing the connection will ensure that tests actually have to read into cache.
+ closeConnection();
+ openConnectionAndCreateSession();
+
+ readTestWrites();
+
+ statsObj = WiredTigerStats{_session}.toBSON();
+ ASSERT_GT(statsObj["data"]["bytesRead"].numberLong(), std::numeric_limits<uint32_t>::max())
+ << statsObj;
+}
+
+TEST_F(WiredTigerStatsTest, OperationsAddToSessionStats) {
+ std::vector<std::unique_ptr<WiredTigerStats>> operationStats;
+
+ write();
+ WiredTigerStats firstWrite(_session);
+ operationStats.push_back(std::make_unique<WiredTigerStats>(firstWrite - WiredTigerStats{}));
+ read();
+ WiredTigerStats firstRead(_session);
+ operationStats.push_back(std::make_unique<WiredTigerStats>(firstRead - firstWrite));
+ write();
+ WiredTigerStats secondWrite(_session);
+ operationStats.push_back(std::make_unique<WiredTigerStats>(secondWrite - firstRead));
+ read();
+ WiredTigerStats secondRead(_session);
+ operationStats.push_back(std::make_unique<WiredTigerStats>(secondRead - secondWrite));
+
+ const WiredTigerStats& fetchedSessionStats = secondRead;
+
+ long long bytesWritten = 0;
+ long long timeWritingMicros = 0;
+ long long bytesRead = 0;
+ long long timeReadingMicros = 0;
+
+ WiredTigerStats addedSessionStats;
+
+ for (auto&& op : operationStats) {
+ auto statsObj = op->toBSON();
+
+ bytesWritten += statsObj["data"]["bytesWritten"].numberLong();
+ timeWritingMicros += statsObj["data"]["timeWritingMicros"].numberLong();
+ bytesRead += statsObj["data"]["bytesRead"].numberLong();
+ timeReadingMicros += statsObj["data"]["timeReadingMicros"].numberLong();
+
+ addedSessionStats += *op;
+ }
+
+ auto addedObj = addedSessionStats.toBSON();
+ auto dataSection = addedObj["data"];
+ ASSERT_EQ(dataSection.type(), BSONType::Object) << addedObj;
+ ASSERT_EQ(dataSection["bytesWritten"].numberLong(), bytesWritten) << addedObj;
+ ASSERT_EQ(dataSection["timeWritingMicros"].numberLong(), timeWritingMicros) << addedObj;
+ ASSERT_EQ(dataSection["bytesRead"].numberLong(), bytesRead) << addedObj;
+ ASSERT_EQ(dataSection["timeReadingMicros"].numberLong(), timeReadingMicros) << addedObj;
+
+ auto fetchedObj = fetchedSessionStats.toBSON();
+ auto fetchedDataSection = fetchedObj["data"];
+ ASSERT_EQ(fetchedDataSection.type(), BSONType::Object) << fetchedObj;
+ ASSERT_EQ(fetchedDataSection["bytesWritten"].numberLong(), bytesWritten) << fetchedObj;
+ ASSERT_EQ(fetchedDataSection["timeWritingMicros"].numberLong(), timeWritingMicros)
+ << fetchedObj;
+ ASSERT_EQ(fetchedDataSection["bytesRead"].numberLong(), bytesRead) << fetchedObj;
+ ASSERT_EQ(fetchedDataSection["timeReadingMicros"].numberLong(), timeReadingMicros)
+ << fetchedObj;
+}
+
+TEST_F(WiredTigerStatsTest, OperationsSubtractToZero) {
+ std::vector<std::unique_ptr<WiredTigerStats>> operationStats;
+
+ write();
+ WiredTigerStats firstWrite(_session);
+ operationStats.push_back(std::make_unique<WiredTigerStats>(firstWrite - WiredTigerStats{}));
+ read();
+ WiredTigerStats firstRead(_session);
+ operationStats.push_back(std::make_unique<WiredTigerStats>(firstRead - firstWrite));
+ write();
+ WiredTigerStats secondWrite(_session);
+ operationStats.push_back(std::make_unique<WiredTigerStats>(secondWrite - firstRead));
+ read();
+ WiredTigerStats secondRead(_session);
+ operationStats.push_back(std::make_unique<WiredTigerStats>(secondRead - secondWrite));
+
+ WiredTigerStats& fetchedSessionStats = secondRead;
+
+ // Assert fetchedSessionStats was not zero before checking subtract results in it being zero.
+ // We ignore the time statistics as those might still be 0 from time to time.
+ auto preSubtractObj = fetchedSessionStats.toBSON();
+ auto preSubtract = preSubtractObj["data"];
+ ASSERT_EQ(preSubtract.type(), BSONType::Object) << preSubtractObj;
+ ASSERT_GT(preSubtract["bytesWritten"].numberLong(), 0) << preSubtractObj;
+ ASSERT_GT(preSubtract["bytesRead"].numberLong(), 0) << preSubtractObj;
+
+ for (auto&& op : operationStats) {
+ fetchedSessionStats -= *op;
+ }
+
+ auto subtractedObj = fetchedSessionStats.toBSON();
+ ASSERT_BSONOBJ_EQ(subtractedObj, BSONObj{});
+}
+
+TEST_F(WiredTigerStatsTest, Clone) {
+ write();
+
+ WiredTigerStats stats{_session};
+ auto clone = stats.clone();
+
+ ASSERT_BSONOBJ_EQ(stats.toBSON(), clone->toBSON());
+
+ stats += *clone;
+ ASSERT_BSONOBJ_NE(stats.toBSON(), clone->toBSON());
+}
+
+} // namespace
+} // namespace mongo
diff --git a/src/mongo/db/transaction/transaction_metrics_observer.cpp b/src/mongo/db/transaction/transaction_metrics_observer.cpp
index 7a6544a1b3b..e1f703daca2 100644
--- a/src/mongo/db/transaction/transaction_metrics_observer.cpp
+++ b/src/mongo/db/transaction/transaction_metrics_observer.cpp
@@ -183,15 +183,15 @@ void TransactionMetricsObserver::onTransactionOperation(OperationContext* opCtx,
// into an existing storageStats instance stored in SingleTransactionStats.
// WiredTiger doesn't let storage statistics be collected when transaction is prepared.
if (!isPrepared) {
- std::shared_ptr<StorageStats> storageStats =
- opCtx->recoveryUnit()->getOperationStatistics();
+ std::unique_ptr<StorageStats> storageStats =
+ opCtx->recoveryUnit()->computeOperationStatisticsSinceLastCall();
if (storageStats) {
- CurOp::get(opCtx)->debug().storageStats = storageStats;
if (!_singleTransactionStats.getOpDebug()->storageStats) {
_singleTransactionStats.getOpDebug()->storageStats = storageStats->clone();
} else {
*_singleTransactionStats.getOpDebug()->storageStats += *storageStats;
}
+ CurOp::get(opCtx)->debug().storageStats = std::move(storageStats);
}
}
diff --git a/src/mongo/db/transaction/transaction_metrics_observer.h b/src/mongo/db/transaction/transaction_metrics_observer.h
index 833c4e5cfb7..1c51f51fe02 100644
--- a/src/mongo/db/transaction/transaction_metrics_observer.h
+++ b/src/mongo/db/transaction/transaction_metrics_observer.h
@@ -107,6 +107,14 @@ public:
}
/**
+ * Returns a reference to the SingleTransactionStats object stored in this
+ * TransactionMetricsObserver instance.
+ */
+ SingleTransactionStats& getSingleTransactionStats() {
+ return _singleTransactionStats;
+ }
+
+ /**
* Resets the SingleTransactionStats object stored in this TransactionMetricsObserver instance,
* preparing it for the new transaction or retryable write with the given number.
*/
diff --git a/src/mongo/db/transaction/transaction_participant.h b/src/mongo/db/transaction/transaction_participant.h
index ac32f9fb32d..4a9b63eeb5e 100644
--- a/src/mongo/db/transaction/transaction_participant.h
+++ b/src/mongo/db/transaction/transaction_participant.h
@@ -749,8 +749,8 @@ public:
}
- SingleTransactionStats getSingleTransactionStatsForTest() const {
- return o().transactionMetricsObserver.getSingleTransactionStats();
+ SingleTransactionStats& getSingleTransactionStatsForTest() {
+ return _tp->_o.transactionMetricsObserver.getSingleTransactionStats();
}
std::vector<repl::ReplOperation> getTransactionOperationsForTest() const {