diff options
author | Moustafa Maher <m.maher@10gen.com> | 2021-06-02 02:20:35 +0000 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2021-06-03 03:51:35 +0000 |
commit | 25bf17229bdddea523ae94571cc22ae4939cfd3c (patch) | |
tree | a1bb40a36ccbcd687bd97c2ff867a644cd12c48c | |
parent | 4d28f81e867485d0bef470bdd7f9e2b0344e6597 (diff) | |
download | mongo-25bf17229bdddea523ae94571cc22ae4939cfd3c.tar.gz |
SERVER-56861 Update semantics for server status's opWriteConcernCounters
-rw-r--r-- | jstests/noPassthrough/server_write_concern_metrics.js | 263 | ||||
-rw-r--r-- | src/mongo/db/read_write_concern_provenance.h | 7 | ||||
-rw-r--r-- | src/mongo/db/stats/server_write_concern_metrics.cpp | 59 | ||||
-rw-r--r-- | src/mongo/db/stats/server_write_concern_metrics.h | 43 |
4 files changed, 263 insertions, 109 deletions
diff --git a/jstests/noPassthrough/server_write_concern_metrics.js b/jstests/noPassthrough/server_write_concern_metrics.js index 0bbfd88c683..82b2f150762 100644 --- a/jstests/noPassthrough/server_write_concern_metrics.js +++ b/jstests/noPassthrough/server_write_concern_metrics.js @@ -7,6 +7,8 @@ (function() { "use strict"; +load("jstests/libs/write_concern_util.js"); // For isDefaultWriteConcernMajorityFlagEnabled. + // Verifies that the server status response has the fields that we expect. function verifyServerStatusFields(serverStatusResponse) { assert(serverStatusResponse.hasOwnProperty("opWriteConcernCounters"), @@ -25,67 +27,79 @@ function verifyServerStatusFields(serverStatusResponse) { // Verifies that the given path of the server status response is incremented in the way we // expect, and no other changes occurred. This function modifies its inputs. -function verifyServerStatusChange(initialStats, newStats, path, expectedIncrement) { - // Traverse to the parent of the changed element. - let pathComponents = path.split("."); - let initialParent = initialStats; - let newParent = newStats; - for (let i = 0; i < pathComponents.length - 1; i++) { - assert(initialParent.hasOwnProperty(pathComponents[i]), - "initialStats did not contain component " + i + " of path " + path + - ", initialStats: " + tojson(initialStats)); - initialParent = initialParent[pathComponents[i]]; - - assert(newParent.hasOwnProperty(pathComponents[i]), - "newStats did not contain component " + i + " of path " + path + - ", newStats: " + tojson(newStats)); - newParent = newParent[pathComponents[i]]; - } +function verifyServerStatusChange(initialStats, newStats, paths, expectedIncrement) { + paths.forEach(path => { + // Traverse to the parent of the changed element. + let pathComponents = path.split("."); + let initialParent = initialStats; + let newParent = newStats; + for (let i = 0; i < pathComponents.length - 1; i++) { + assert(initialParent.hasOwnProperty(pathComponents[i]), + "initialStats did not contain component " + i + " of path " + path + + ", initialStats: " + tojson(initialStats)); + initialParent = initialParent[pathComponents[i]]; - // Test the expected increment of the changed element. The element may not exist in the - // initial stats, in which case it is treated as 0. - let lastPathComponent = pathComponents[pathComponents.length - 1]; - let initialValue = 0; - if (initialParent.hasOwnProperty(lastPathComponent)) { - initialValue = initialParent[lastPathComponent]; - } - assert(newParent.hasOwnProperty(lastPathComponent), - "newStats did not contain last component of path " + path + - ", newStats: " + tojson(newStats)); - assert.eq(initialValue + expectedIncrement, - newParent[lastPathComponent], - "expected " + path + " to increase by " + expectedIncrement + - ", initialStats: " + tojson(initialStats) + ", newStats: " + tojson(newStats)); + assert(newParent.hasOwnProperty(pathComponents[i]), + "newStats did not contain component " + i + " of path " + path + + ", newStats: " + tojson(newStats)); + newParent = newParent[pathComponents[i]]; + } + + // Test the expected increment of the changed element. The element may not exist in the + // initial stats, in which case it is treated as 0. + let lastPathComponent = pathComponents[pathComponents.length - 1]; + let initialValue = 0; + if (initialParent.hasOwnProperty(lastPathComponent)) { + initialValue = initialParent[lastPathComponent]; + } + assert(newParent.hasOwnProperty(lastPathComponent), + "newStats did not contain last component of path " + path + + ", newStats: " + tojson(newStats)); + assert.eq(initialValue + expectedIncrement, + newParent[lastPathComponent], + "expected " + path + " to increase by " + expectedIncrement + ", initialStats: " + + tojson(initialStats) + ", newStats: " + tojson(newStats)); - // Delete the changed element. - delete initialParent[lastPathComponent]; - delete newParent[lastPathComponent]; + // Delete the changed element. + delete initialParent[lastPathComponent]; + delete newParent[lastPathComponent]; + }); // The stats objects should be equal without the changed element. assert.eq(0, bsonWoCompare(initialStats, newStats), - "expected initialStats and newStats to be equal after removing " + path + + "expected initialStats and newStats to be equal after removing " + tojson(paths) + ", initialStats: " + tojson(initialStats) + ", newStats: " + tojson(newStats)); } -const rst = new ReplSetTest( - {nodes: 2, nodeOptions: {setParameter: 'reportOpWriteConcernCountersInServerStatus=true'}}); -rst.startSet(); -let config = rst.getReplSetConfig(); -config.members[1].priority = 0; -config.members[0].tags = { - dc_va: "rack1" -}; -config.settings = { - getLastErrorModes: {myTag: {dc_va: 1}} -}; -rst.initiate(config); -const primary = rst.getPrimary(); -const secondary = rst.getSecondary(); +let rst; +let primary; +let secondary; const dbName = "test"; const collName = "server_write_concern_metrics"; -const testDB = primary.getDB(dbName); -const testColl = testDB[collName]; +let testDB; +let testColl; + +function initializeReplicaSet(isPSASet) { + let replSetNodes = [{}, {}]; + if (isPSASet) { + replSetNodes.push({arbiter: true}); + } + rst = new ReplSetTest({ + nodes: replSetNodes, + nodeOptions: {setParameter: 'reportOpWriteConcernCountersInServerStatus=true'} + }); + rst.startSet(); + let config = rst.getReplSetConfig(); + config.members[1].priority = 0; + config.members[0].tags = {dc_va: "rack1"}; + config.settings = {getLastErrorModes: {myTag: {dc_va: 1}}}; + rst.initiate(config); + primary = rst.getPrimary(); + secondary = rst.getSecondary(); + testDB = primary.getDB(dbName); + testColl = testDB[collName]; +} function resetCollection(setupCommand) { testColl.drop(); @@ -95,16 +109,84 @@ function resetCollection(setupCommand) { } } -function testWriteConcernMetrics(cmd, opName, inc, setupCommand) { - // Run command with no writeConcern. +function testWriteConcernMetrics(cmd, opName, inc, isPSASet, setupCommand) { + initializeReplicaSet(isPSASet); + const isDefaultWCMajorityFlagEnabled = isDefaultWriteConcernMajorityFlagEnabled(primary); + + // Run command with no writeConcern and no CWWC set. resetCollection(setupCommand); let serverStatus = assert.commandWorked(testDB.adminCommand({serverStatus: 1})); verifyServerStatusFields(serverStatus); assert.commandWorked(testDB.runCommand(cmd)); let newStatus = assert.commandWorked(testDB.adminCommand({serverStatus: 1})); + verifyServerStatusChange( + serverStatus.opWriteConcernCounters, + newStatus.opWriteConcernCounters, + [ + opName + + (isDefaultWCMajorityFlagEnabled ? (isPSASet ? ".noneInfo.implicitDefault.wnum.1" + : ".noneInfo.implicitDefault.wmajority") + : ".noneInfo.implicitDefault.wnum.1"), + opName + ".none" + ], + inc); + + // Run command with no writeConcern with CWWC set to majority. + resetCollection(setupCommand); + assert.commandWorked(primary.adminCommand({ + setDefaultRWConcern: 1, + defaultWriteConcern: {w: "majority"}, + writeConcern: {w: "majority"} + })); + serverStatus = assert.commandWorked(testDB.adminCommand({serverStatus: 1})); + verifyServerStatusFields(serverStatus); + assert.commandWorked(testDB.runCommand(cmd)); + newStatus = assert.commandWorked(testDB.adminCommand({serverStatus: 1})); + verifyServerStatusChange(serverStatus.opWriteConcernCounters, + newStatus.opWriteConcernCounters, + [opName + ".noneInfo.CWWC.wmajority", opName + ".none"], + inc); + + // Run command with no writeConcern with CWWC set to w:1. + resetCollection(setupCommand); + assert.commandWorked(primary.adminCommand( + {setDefaultRWConcern: 1, defaultWriteConcern: {w: 1}, writeConcern: {w: "majority"}})); + serverStatus = assert.commandWorked(testDB.adminCommand({serverStatus: 1})); + verifyServerStatusFields(serverStatus); + assert.commandWorked(testDB.runCommand(cmd)); + newStatus = assert.commandWorked(testDB.adminCommand({serverStatus: 1})); + verifyServerStatusChange(serverStatus.opWriteConcernCounters, + newStatus.opWriteConcernCounters, + [opName + ".noneInfo.CWWC.wnum.1", opName + ".none"], + inc); + + // Run command with no writeConcern and with CWWC set to j:true. + resetCollection(setupCommand); + assert.commandWorked(primary.adminCommand( + {setDefaultRWConcern: 1, defaultWriteConcern: {j: true}, writeConcern: {w: "majority"}})); + serverStatus = assert.commandWorked(testDB.adminCommand({serverStatus: 1})); + verifyServerStatusFields(serverStatus); + assert.commandWorked(testDB.runCommand(cmd)); + newStatus = assert.commandWorked(testDB.adminCommand({serverStatus: 1})); + verifyServerStatusChange(serverStatus.opWriteConcernCounters, + newStatus.opWriteConcernCounters, + [opName + ".noneInfo.CWWC.wnum.1", opName + ".none"], + inc); + + // Run command with no writeConcern and with CWWC set with (w: "myTag"). + resetCollection(setupCommand); + assert.commandWorked(primary.adminCommand({ + setDefaultRWConcern: 1, + defaultWriteConcern: {w: "myTag"}, + writeConcern: {w: "majority"} + })); + serverStatus = assert.commandWorked(testDB.adminCommand({serverStatus: 1})); + verifyServerStatusFields(serverStatus); + assert.commandWorked(testDB.runCommand(cmd)); + newStatus = assert.commandWorked(testDB.adminCommand({serverStatus: 1})); verifyServerStatusChange(serverStatus.opWriteConcernCounters, newStatus.opWriteConcernCounters, - opName + ".none", + [opName + ".noneInfo.CWWC.wtag.myTag", opName + ".none"], inc); // Run command with writeConcern {j: true}. This should be counted as having no 'w' value. @@ -116,7 +198,7 @@ function testWriteConcernMetrics(cmd, opName, inc, setupCommand) { newStatus = assert.commandWorked(testDB.adminCommand({serverStatus: 1})); verifyServerStatusChange(serverStatus.opWriteConcernCounters, newStatus.opWriteConcernCounters, - opName + ".none", + [opName + ".noneInfo.implicitDefault.wnum.1", opName + ".none"], inc); // Run command with writeConcern {w: "majority"}. @@ -128,7 +210,7 @@ function testWriteConcernMetrics(cmd, opName, inc, setupCommand) { newStatus = assert.commandWorked(testDB.adminCommand({serverStatus: 1})); verifyServerStatusChange(serverStatus.opWriteConcernCounters, newStatus.opWriteConcernCounters, - opName + ".wmajority", + [opName + ".wmajority"], inc); // Run command with writeConcern {w: 0}. @@ -140,7 +222,7 @@ function testWriteConcernMetrics(cmd, opName, inc, setupCommand) { newStatus = assert.commandWorked(testDB.adminCommand({serverStatus: 1})); verifyServerStatusChange(serverStatus.opWriteConcernCounters, newStatus.opWriteConcernCounters, - opName + ".wnum.0", + [opName + ".wnum.0"], inc); // Run command with writeConcern {w: 1}. @@ -152,7 +234,7 @@ function testWriteConcernMetrics(cmd, opName, inc, setupCommand) { newStatus = assert.commandWorked(testDB.adminCommand({serverStatus: 1})); verifyServerStatusChange(serverStatus.opWriteConcernCounters, newStatus.opWriteConcernCounters, - opName + ".wnum.1", + [opName + ".wnum.1"], inc); // Run command with writeConcern {w: 2}. @@ -164,7 +246,7 @@ function testWriteConcernMetrics(cmd, opName, inc, setupCommand) { newStatus = assert.commandWorked(testDB.adminCommand({serverStatus: 1})); verifyServerStatusChange(serverStatus.opWriteConcernCounters, newStatus.opWriteConcernCounters, - opName + ".wnum.2", + [opName + ".wnum.2"], inc); // Run command with writeConcern {w: "myTag"}. @@ -176,7 +258,7 @@ function testWriteConcernMetrics(cmd, opName, inc, setupCommand) { newStatus = assert.commandWorked(testDB.adminCommand({serverStatus: 1})); verifyServerStatusChange(serverStatus.opWriteConcernCounters, newStatus.opWriteConcernCounters, - opName + ".wtag.myTag", + [opName + ".wtag.myTag"], inc); // writeConcern metrics are not tracked on the secondary. @@ -189,34 +271,41 @@ function testWriteConcernMetrics(cmd, opName, inc, setupCommand) { bsonWoCompare(serverStatus.opWriteConcernCounters, newStatus.opWriteConcernCounters), "expected no change in secondary writeConcern metrics, before: " + tojson(serverStatus) + ", after: " + tojson(newStatus)); + + rst.stopSet(); } -// Test single insert/update/delete. -testWriteConcernMetrics({insert: collName, documents: [{}]}, "insert", 1); -testWriteConcernMetrics({update: collName, updates: [{q: {}, u: {$set: {a: 1}}}]}, "update", 1); -testWriteConcernMetrics({delete: collName, deletes: [{q: {}, limit: 1}]}, "delete", 1); - -// Test batch writes. -testWriteConcernMetrics({insert: collName, documents: [{}, {}]}, "insert", 2); -testWriteConcernMetrics( - {update: collName, updates: [{q: {}, u: {$set: {a: 1}}}, {q: {}, u: {$set: {a: 1}}}]}, - "update", - 2); -testWriteConcernMetrics( - {delete: collName, deletes: [{q: {}, limit: 1}, {q: {}, limit: 1}]}, "delete", 2); - -// Test applyOps. All sequences of setup + command must be idempotent in steady-state oplog -// application, as testWriteConcernMetrics will run them multiple times. -testWriteConcernMetrics( - {applyOps: [{op: "i", ns: testColl.getFullName(), o: {_id: 0}}]}, "insert", 1); -testWriteConcernMetrics( - {applyOps: [{op: "u", ns: testColl.getFullName(), o2: {_id: 0}, o: {$set: {a: 1}}}]}, - "update", - 1); -testWriteConcernMetrics({applyOps: [{op: "d", ns: testColl.getFullName(), o: {_id: 0}}]}, - "delete", - 1, - {insert: collName, documents: [{_id: 0}]}); - -rst.stopSet(); +for (const isPSASet of [true, false]) { + // Test single insert/update/delete. + testWriteConcernMetrics({insert: collName, documents: [{}]}, "insert", 1, isPSASet); + testWriteConcernMetrics( + {update: collName, updates: [{q: {}, u: {$set: {a: 1}}}]}, "update", 1, isPSASet); + testWriteConcernMetrics( + {delete: collName, deletes: [{q: {}, limit: 1}]}, "delete", 1, isPSASet); + + // Test batch writes. + testWriteConcernMetrics({insert: collName, documents: [{}, {}]}, "insert", 2, isPSASet); + testWriteConcernMetrics( + {update: collName, updates: [{q: {}, u: {$set: {a: 1}}}, {q: {}, u: {$set: {a: 1}}}]}, + "update", + 2, + isPSASet); + testWriteConcernMetrics( + {delete: collName, deletes: [{q: {}, limit: 1}, {q: {}, limit: 1}]}, "delete", 2, isPSASet); + + // Test applyOps. All sequences of setup + command must be idempotent in steady-state oplog + // application, as testWriteConcernMetrics will run them multiple times. + testWriteConcernMetrics( + {applyOps: [{op: "i", ns: testColl.getFullName(), o: {_id: 0}}]}, "insert", 1, isPSASet); + testWriteConcernMetrics( + {applyOps: [{op: "u", ns: testColl.getFullName(), o2: {_id: 0}, o: {$set: {a: 1}}}]}, + "update", + 1, + isPSASet); + testWriteConcernMetrics({applyOps: [{op: "d", ns: testColl.getFullName(), o: {_id: 0}}]}, + "delete", + 1, + isPSASet, + {insert: collName, documents: [{_id: 0}]}); +} }()); diff --git a/src/mongo/db/read_write_concern_provenance.h b/src/mongo/db/read_write_concern_provenance.h index 51e49121c78..57b4f9f64f5 100644 --- a/src/mongo/db/read_write_concern_provenance.h +++ b/src/mongo/db/read_write_concern_provenance.h @@ -123,6 +123,13 @@ public: } /** + * Returns true if the RWC was a custom default. + */ + const bool isCustomDefault() const { + return hasSource() && *getSource() == Source::customDefault; + } + + /** * Sets the source of this provenance. In order to prevent accidental clobbering of provenance * with incorrect values, a source cannot change during the provenance's lifetime, except for * the initial transition from kUnset to some other Source value. diff --git a/src/mongo/db/stats/server_write_concern_metrics.cpp b/src/mongo/db/stats/server_write_concern_metrics.cpp index bfc14025d73..f9a72590d8a 100644 --- a/src/mongo/db/stats/server_write_concern_metrics.cpp +++ b/src/mongo/db/stats/server_write_concern_metrics.cpp @@ -106,13 +106,8 @@ BSONObj ServerWriteConcernMetrics::toBSON() const { return builder.obj(); } -void ServerWriteConcernMetrics::WriteConcernMetricsForOperationType::recordWriteConcern( +void ServerWriteConcernMetrics::WriteConcernCounters::recordWriteConcern( const WriteConcernOptions& writeConcernOptions, size_t numOps) { - if (writeConcernOptions.usedDefaultW) { - noWCount += numOps; - return; - } - if (!writeConcernOptions.wMode.empty()) { if (writeConcernOptions.wMode == WriteConcernOptions::kMajority) { wMajorityCount += numOps; @@ -126,8 +121,28 @@ void ServerWriteConcernMetrics::WriteConcernMetricsForOperationType::recordWrite wNumCounts[writeConcernOptions.wNumNodes] += numOps; } -void ServerWriteConcernMetrics::WriteConcernMetricsForOperationType::toBSON( - BSONObjBuilder* builder) const { +void ServerWriteConcernMetrics::WriteConcernMetricsForOperationType::recordWriteConcern( + const WriteConcernOptions& writeConcernOptions, size_t numOps) { + if (writeConcernOptions.usedDefaultW) { + if (writeConcernOptions.getProvenance().isCustomDefault()) { + cWWC.recordWriteConcern(writeConcernOptions, numOps); + } else { + // Provenance is either: + // - "implicitDefault" : implicit default WC (w:1 or w:"majority") is used. + // - "clientSupplied" : set without "w" value, so implicit default WC (w:1) is used. + // - "internalWriteDefault" : if internal command sets empty WC ({writeConcern: {}}), + // then default constructed WC (w:1) is used. + implicitDefaultWC.recordWriteConcern(writeConcernOptions, numOps); + } + + notExplicitWCount += numOps; + return; + } + + explicitWC.recordWriteConcern(writeConcernOptions, numOps); +} + +void ServerWriteConcernMetrics::WriteConcernCounters::toBSON(BSONObjBuilder* builder) const { builder->append("wmajority", static_cast<long long>(wMajorityCount)); BSONObjBuilder wNumBuilder(builder->subobjStart("wnum")); @@ -136,13 +151,31 @@ void ServerWriteConcernMetrics::WriteConcernMetricsForOperationType::toBSON( } wNumBuilder.done(); - BSONObjBuilder wTagBuilder(builder->subobjStart("wtag")); - for (auto const& pair : wTagCounts) { - wTagBuilder.append(pair.first, static_cast<long long>(pair.second)); + if (exportWTag) { + BSONObjBuilder wTagBuilder(builder->subobjStart("wtag")); + for (auto const& pair : wTagCounts) { + wTagBuilder.append(pair.first, static_cast<long long>(pair.second)); + } + wTagBuilder.done(); } - wTagBuilder.done(); +} + +void ServerWriteConcernMetrics::WriteConcernMetricsForOperationType::toBSON( + BSONObjBuilder* builder) const { + explicitWC.toBSON(builder); + + builder->append("none", static_cast<long long>(notExplicitWCount)); + BSONObjBuilder noneBuilder(builder->subobjStart("noneInfo")); + + BSONObjBuilder cWWCBuilder(noneBuilder.subobjStart("CWWC")); + cWWC.toBSON(&cWWCBuilder); + cWWCBuilder.done(); + + BSONObjBuilder implicitBuilder(noneBuilder.subobjStart("implicitDefault")); + implicitDefaultWC.toBSON(&implicitBuilder); + implicitBuilder.done(); - builder->append("none", static_cast<long long>(noWCount)); + noneBuilder.done(); } namespace { diff --git a/src/mongo/db/stats/server_write_concern_metrics.h b/src/mongo/db/stats/server_write_concern_metrics.h index b1e17f53e38..46a99bab364 100644 --- a/src/mongo/db/stats/server_write_concern_metrics.h +++ b/src/mongo/db/stats/server_write_concern_metrics.h @@ -76,25 +76,50 @@ public: BSONObj toBSON() const; private: - struct WriteConcernMetricsForOperationType { - /** - * Updates counter for the 'w' value of 'writeConcernOptions'. - */ - void recordWriteConcern(const WriteConcernOptions& writeConcernOptions, size_t numOps = 1); + struct WriteConcernCounters { + WriteConcernCounters() = default; - void toBSON(BSONObjBuilder* builder) const; + WriteConcernCounters(bool exportWTag) : exportWTag(exportWTag) {} // Count of operations with writeConcern w:"majority". std::uint64_t wMajorityCount = 0; - // Count of operations without a writeConcern "w" value. - std::uint64_t noWCount = 0; - // Counts of operations with writeConcern w:<num>. std::map<int, std::uint64_t> wNumCounts; + // Set to true to include "wTag" section when exporting to BSON. + bool exportWTag = true; + // Counts of operations with writeConcern w:"tag". StringMap<std::uint64_t> wTagCounts; + + /** + * Updates counters for the 'w' value of 'writeConcernOptions'. + */ + void recordWriteConcern(const WriteConcernOptions& writeConcernOptions, size_t numOps); + + void toBSON(BSONObjBuilder* builder) const; + }; + + struct WriteConcernMetricsForOperationType { + /** + * Updates the corresponding WC counters for the 'w' value of 'writeConcernOptions'. + */ + void recordWriteConcern(const WriteConcernOptions& writeConcernOptions, size_t numOps = 1); + + void toBSON(BSONObjBuilder* builder) const; + + // Counts of operations with writeConcern with 'w' value explicitly set by client. + WriteConcernCounters explicitWC; + + // Counts of operations used cluster-wide writeConcern. + WriteConcernCounters cWWC; + + // Counts of operations used implicit default writeConcern. + WriteConcernCounters implicitDefaultWC = WriteConcernCounters(false); + + // Count of operations without an explicit writeConcern with "w" value. + std::uint64_t notExplicitWCount = 0; }; mutable Mutex _mutex = MONGO_MAKE_LATCH("ServerWriteConcernMetrics::_mutex"); |