diff options
author | Ted Tuckman <ted.tuckman@mongodb.com> | 2018-09-04 16:27:31 -0400 |
---|---|---|
committer | Ted Tuckman <ted.tuckman@mongodb.com> | 2018-09-14 13:34:41 -0400 |
commit | 8f9cf06033d7b1e0942c76eecfb69b5eee044ed6 (patch) | |
tree | 7c1a4fa59521b125f7fe9283b5830943a4ebcc2b | |
parent | 846a6c19839601ce66f27877b348a4a5150a453d (diff) | |
download | mongo-8f9cf06033d7b1e0942c76eecfb69b5eee044ed6.tar.gz |
SERVER-37001 Add idleCursor to $currentOp
25 files changed, 329 insertions, 130 deletions
diff --git a/buildscripts/resmokeconfig/suites/sharded_causally_consistent_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/sharded_causally_consistent_jscore_passthrough.yml index 0ccb4112ef5..2720928e0bc 100644 --- a/buildscripts/resmokeconfig/suites/sharded_causally_consistent_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/sharded_causally_consistent_jscore_passthrough.yml @@ -130,6 +130,7 @@ selector: - jstests/core/dbstats.js exclude_with_any_tags: # Tests tagged with the following will fail because they assume collections are not sharded. + - assumes_against_mongod_not_mongos - assumes_no_implicit_collection_creation_after_drop - assumes_no_implicit_index_creation - assumes_unsharded_collection diff --git a/jstests/core/currentop_cursors.js b/jstests/core/currentop_cursors.js new file mode 100644 index 00000000000..4e8feffa9cc --- /dev/null +++ b/jstests/core/currentop_cursors.js @@ -0,0 +1,41 @@ +/** + * Tests that an idle cursor will appear in the $currentOp output if the idleCursors option is + * set to true. + * + * The work to make this feature available on mongos is deferred to SERVER-37004 + * and SERVER-37005. Those tickets will make the idleCursor fields available to curOp. + * @tags: [assumes_against_mongod_not_mongos, assumes_read_concern_unchanged] + * + */ + +(function() { + "use strict"; + const coll = db.jstests_currentop; + const adminDB = db.getSiblingDB("admin"); + coll.drop(); + for (let i = 0; i < 5; ++i) { + assert.commandWorked(coll.insert({"val": i})); + } + const findOut = + assert.commandWorked(db.runCommand({find: "jstests_currentop", batchSize: 2})).cursor.id; + const result = adminDB + .aggregate([ + {$currentOp: {allUsers: false, idleCursors: true}}, + {$match: {$and: [{type: "idleCursor"}, {"cursor.cursorId": findOut}]}} + ]) + .toArray(); + assert.eq(result.length, 1, tojson(result)); + assert.eq(result[0].cursor.nDocsReturned, 2, tojson(result)); + const noIdle = adminDB + .aggregate([ + {$currentOp: {allUsers: false, idleCursors: false}}, + {$match: {$and: [{type: "idleCursor"}, {"cursor.cursorId": findOut}]}} + ]) + .toArray(); + assert.eq(noIdle.length, 0, tojson(noIdle)); + const noFlag = + adminDB.aggregate([{$currentOp: {allUsers: false}}, {$match: {type: "idleCursor"}}]) + .toArray(); + + assert.eq(noIdle.length, 0, tojson(noFlag)); +})(); diff --git a/jstests/core/list_all_local_cursors.js b/jstests/core/list_all_local_cursors.js index 09397467b55..02a5e956ecd 100644 --- a/jstests/core/list_all_local_cursors.js +++ b/jstests/core/list_all_local_cursors.js @@ -8,11 +8,7 @@ const admin = db.getSisterDB("admin"); function listAllCursorsWithId(cursorId) { - return admin - .aggregate([ - {"$listLocalCursors": {}}, - {"$match": {"id": cursorId}}, - ]) + return admin.aggregate([{"$listLocalCursors": {}}, {"$match": {cursorId: cursorId}}]) .toArray(); } @@ -31,13 +27,13 @@ let foundCursors = listAllCursorsWithId(cursorIdWithoutSession); assert.eq(foundCursors.length, 1, tojson(foundCursors)); assert.eq(foundCursors[0].ns, "listAllLocalCursors.data", tojson(foundCursors)); - assert.eq(foundCursors[0].id, cursorIdWithoutSession, tojson(foundCursors)); + assert.eq(foundCursors[0].cursorId, cursorIdWithoutSession, tojson(foundCursors)); // Verify that we correctly list the cursor which is inside of a session. foundCursors = listAllCursorsWithId(cursorIdWithSession); assert.eq(foundCursors.length, 1, tojson(foundCursors)); assert.eq(foundCursors[0].ns, "listAllLocalCursors.data", tojson(foundCursors)); - assert.eq(foundCursors[0].id, cursorIdWithSession, tojson(foundCursors)); + assert.eq(foundCursors[0].cursorId, cursorIdWithSession, tojson(foundCursors)); assert(foundCursors[0].hasOwnProperty("lsid"), tojson(foundCursors)); assert.eq( foundCursors[0].lsid.id, session._serverSession.handle.getId().id, tojson(foundCursors)); diff --git a/jstests/libs/pin_getmore_cursor.js b/jstests/libs/pin_getmore_cursor.js new file mode 100644 index 00000000000..d362eb347fa --- /dev/null +++ b/jstests/libs/pin_getmore_cursor.js @@ -0,0 +1,61 @@ +/** + * Pins a cursor in a seperate shell and then runs the given function. + * 'conn': a connection to an instance of a mongod or mongos. + * 'assertFunction': a function containing the test to be run after a cursor is pinned and hanging. + * 'runGetMoreFunc': A function to generate a string that will be executed in the parallel shell. + * 'failPointName': The string name of the failpoint where the cursor will hang. The function turns + * the failpoint on, the assert function should turn it off whenever it is appropriate for the test. + */ + +function withPinnedCursor({conn, assertFunction, runGetMoreFunc, failPointName}) { + // This test runs manual getMores using different connections, which will not inherit the + // implicit session of the cursor establishing command. + TestData.disableImplicitSessions = true; + + const db = conn.getDB("test"); + const coll = db.jstest_with_pinned_cursor; + coll.drop(); + for (let i = 0; i < 100; ++i) { + assert.writeOK(coll.insert({value: i})); + } + let cleanup = null; + try { + // Enable the specified failpoint. + assert.commandWorked( + db.adminCommand({configureFailPoint: failPointName, mode: "alwaysOn"})); + // Issue an initial find in order to create a cursor and obtain its cursorID. + let cmdRes = db.runCommand({find: coll.getName(), batchSize: 2}); + assert.commandWorked(cmdRes); + let cursorId = cmdRes.cursor.id; + assert.neq(cursorId, NumberLong(0)); + // Let the cursor hang in a different shell + let code = "let cursorId = " + cursorId.toString() + ";"; + code += "let collName = '" + coll.getName() + "';"; + code += "(" + runGetMoreFunc.toString() + ")();"; + cleanup = startParallelShell(code, conn.port); + // Wait until we know the failpoint has been reached. + assert.soon(function() { + const arr = + db.getSiblingDB("admin") + .aggregate([{$currentOp: {localOps: true}}, {$match: {"msg": failPointName}}]) + .toArray(); + return arr.length > 0; + }); + assertFunction(cursorId, coll); + // Eventually the cursor should be cleaned up. + assert.commandWorked(db.adminCommand({configureFailPoint: failPointName, mode: "off"})); + assert.soon(() => db.serverStatus().metrics.cursor.open.pinned == 0); + + // Trying to kill the cursor again should result in the cursor not being found. + cmdRes = db.runCommand({killCursors: coll.getName(), cursors: [cursorId]}); + assert.commandWorked(cmdRes); + assert.eq(cmdRes.cursorsKilled, []); + assert.eq(cmdRes.cursorsAlive, []); + assert.eq(cmdRes.cursorsNotFound, [cursorId]); + assert.eq(cmdRes.cursorsUnknown, []); + } finally { + if (cleanup) { + cleanup(); + } + } +} diff --git a/jstests/noPassthrough/currentop_active_cursor.js b/jstests/noPassthrough/currentop_active_cursor.js new file mode 100644 index 00000000000..978616d0f27 --- /dev/null +++ b/jstests/noPassthrough/currentop_active_cursor.js @@ -0,0 +1,31 @@ +// Test whether a pinned cursor does not show up as an idle cursor in curOp. +(function() { + "use strict"; + load("jstests/libs/pin_getmore_cursor.js"); // for "withPinnedCursor" + + function runTest(cursorId, coll) { + const db = coll.getDB(); + const adminDB = db.getSiblingDB("admin"); + const result = adminDB + .aggregate([ + {"$currentOp": {"idleCursors": true, "allUsers": false}}, + {"$match": {"type": "idleCursor"}} + ]) + .toArray(); + assert.eq(result.length, 0, tojson(result)); + } + + const conn = MongoRunner.runMongod({}); + const failPointName = "waitAfterPinningCursorBeforeGetMoreBatch"; + withPinnedCursor({ + conn: conn, + assertFunction: runTest, + runGetMoreFunc: function() { + const response = + assert.commandWorked(db.runCommand({getMore: cursorId, collection: collName})); + }, + failPointName: failPointName + }); + MongoRunner.stopMongod(conn); + +})(); diff --git a/jstests/noPassthrough/kill_pinned_cursor.js b/jstests/noPassthrough/kill_pinned_cursor.js index e70f7f98dfb..8e7a953d110 100644 --- a/jstests/noPassthrough/kill_pinned_cursor.js +++ b/jstests/noPassthrough/kill_pinned_cursor.js @@ -17,79 +17,30 @@ // implicit session of the cursor establishing command. TestData.disableImplicitSessions = true; - load("jstests/libs/fixture_helpers.js"); // For "isMongos". - + load("jstests/libs/fixture_helpers.js"); // For "isMongos". + load("jstests/libs/pin_getmore_cursor.js"); // For "withPinnedCursor". const st = new ShardingTest({shards: 2}); // Enables the specified 'failPointName', executes 'runGetMoreFunc' function in a parallel // shell, waits for the the failpoint to be hit, then kills the cursor and confirms that the // kill was successful. function runPinnedCursorKillTest({conn, failPointName, runGetMoreFunc}) { - const db = conn.getDB("test"); - jsTestLog("Running test with failPoint: " + failPointName); - - const coll = db.jstest_kill_pinned_cursor; - coll.drop(); - - for (let i = 0; i < 10; i++) { - assert.writeOK(coll.insert({_id: i})); - } - - let cleanup = null; - let cursorId; - - try { - // Enable the specified failpoint. - assert.commandWorked( - db.adminCommand({configureFailPoint: failPointName, mode: "alwaysOn"})); - - // Issue an initial find in order to create a cursor and obtain its ID. - let cmdRes = db.runCommand({find: coll.getName(), batchSize: 2}); - assert.commandWorked(cmdRes); - cursorId = cmdRes.cursor.id; - assert.neq(cursorId, NumberLong(0)); - - // Serialize 'runGetMoreFunc' along with the cursor ID and collection name, then execute - // the function in a parallel shell. - let code = "let cursorId = " + cursorId.toString() + ";"; - code += "let collName = '" + coll.getName() + "';"; - code += "(" + runGetMoreFunc.toString() + ")();"; - cleanup = startParallelShell(code, conn.port); - - // Wait until we know the failpoint has been reached. - assert.soon(function() { - const arr = - db.getSiblingDB("admin") - .aggregate( - [{$currentOp: {localOps: true}}, {$match: {"msg": failPointName}}]) - .toArray(); - return arr.length > 0; - }); - + function assertFunction(cursorId, coll) { + const db = coll.getDB(); // Kill the cursor associated with the command and assert that the kill succeeded. - cmdRes = db.runCommand({killCursors: coll.getName(), cursors: [cursorId]}); + let cmdRes = db.runCommand({killCursors: coll.getName(), cursors: [cursorId]}); assert.commandWorked(cmdRes); assert.eq(cmdRes.cursorsKilled, [cursorId]); assert.eq(cmdRes.cursorsAlive, []); assert.eq(cmdRes.cursorsNotFound, []); assert.eq(cmdRes.cursorsUnknown, []); - } finally { - assert.commandWorked(db.adminCommand({configureFailPoint: failPointName, mode: "off"})); - if (cleanup) { - cleanup(); - } } - - // Eventually the cursor should be cleaned up. - assert.soon(() => db.serverStatus().metrics.cursor.open.pinned == 0); - - // Trying to kill the cursor again should result in the cursor not being found. - const cmdRes = db.runCommand({killCursors: coll.getName(), cursors: [cursorId]}); - assert.commandWorked(cmdRes); - assert.eq(cmdRes.cursorsKilled, []); - assert.eq(cmdRes.cursorsAlive, []); - assert.eq(cmdRes.cursorsNotFound, [cursorId]); - assert.eq(cmdRes.cursorsUnknown, []); + withPinnedCursor({ + conn: conn, + assertFunction: assertFunction, + runGetMoreFunc: runGetMoreFunc, + failPointName: failPointName + }); } // Test that killing the pinned cursor before it starts building the batch results in a diff --git a/src/mongo/db/clientcursor.h b/src/mongo/db/clientcursor.h index 59af3fc3929..cc698e12bfd 100644 --- a/src/mongo/db/clientcursor.h +++ b/src/mongo/db/clientcursor.h @@ -271,7 +271,7 @@ private: } // The ID of the ClientCursor. A value of 0 is used to mean that no cursor id has been assigned. - CursorId _cursorid = 0; + const CursorId _cursorid = 0; // Threads may read from this field even if they don't have the cursor pinned, as long as they // have the correct partition of the CursorManager locked (just like _authenticatedUsers). diff --git a/src/mongo/db/cursor_manager.cpp b/src/mongo/db/cursor_manager.cpp index b0537afcf95..fb7cd6058d4 100644 --- a/src/mongo/db/cursor_manager.cpp +++ b/src/mongo/db/cursor_manager.cpp @@ -340,9 +340,11 @@ void CursorManager::appendAllActiveSessions(OperationContext* opCtx, LogicalSess globalCursorIdCache->visitAllCursorManagers(opCtx, &visitor); } -std::vector<GenericCursor> CursorManager::getAllCursors(OperationContext* opCtx) { +std::vector<GenericCursor> CursorManager::getIdleCursors( + OperationContext* opCtx, MongoProcessInterface::CurrentOpUserMode userMode) { std::vector<GenericCursor> cursors; - auto visitor = [&](CursorManager& mgr) { mgr.appendActiveCursors(&cursors); }; + AuthorizationSession* ctxAuth = AuthorizationSession::get(opCtx->getClient()); + auto visitor = [&](CursorManager& mgr) { mgr.appendIdleCursors(ctxAuth, userMode, &cursors); }; globalCursorIdCache->visitAllCursorManagers(opCtx, &visitor); return cursors; @@ -620,16 +622,38 @@ void CursorManager::appendActiveSessions(LogicalSessionIdSet* lsids) const { } } -void CursorManager::appendActiveCursors(std::vector<GenericCursor>* cursors) const { +GenericCursor CursorManager::buildGenericCursor_inlock(const ClientCursor* cursor) const { + GenericCursor gc; + gc.setCursorId(cursor->_cursorid); + gc.setNs(cursor->nss()); + gc.setLsid(cursor->getSessionId()); + gc.setNDocsReturned(cursor->pos()); + gc.setTailable(cursor->isTailable()); + gc.setAwaitData(cursor->isAwaitData()); + gc.setOriginatingCommand(cursor->getOriginatingCommandObj()); + gc.setNoCursorTimeout(cursor->isNoTimeout()); + return gc; +} + +void CursorManager::appendIdleCursors(AuthorizationSession* ctxAuth, + MongoProcessInterface::CurrentOpUserMode userMode, + std::vector<GenericCursor>* cursors) const { auto allPartitions = _cursorMap->lockAllPartitions(); for (auto&& partition : allPartitions) { for (auto&& entry : partition) { auto cursor = entry.second; - cursors->emplace_back(); - auto& gc = cursors->back(); - gc.setId(cursor->_cursorid); - gc.setNs(cursor->nss()); - gc.setLsid(cursor->getSessionId()); + + // Exclude cursors that this user does not own if auth is enabled. + if (ctxAuth->getAuthorizationManager().isAuthEnabled() && + userMode == MongoProcessInterface::CurrentOpUserMode::kExcludeOthers && + !ctxAuth->isCoauthorizedWith(cursor->getAuthenticatedUsers())) { + continue; + } + // Exclude pinned cursors. + if (cursor->_operationUsingCursor) { + continue; + } + cursors->emplace_back(buildGenericCursor_inlock(cursor)); } } } diff --git a/src/mongo/db/cursor_manager.h b/src/mongo/db/cursor_manager.h index 5714816d450..4202afadc65 100644 --- a/src/mongo/db/cursor_manager.h +++ b/src/mongo/db/cursor_manager.h @@ -45,6 +45,7 @@ namespace mongo { +class AuthorizationSession; class OperationContext; class PseudoRandom; class PlanExecutor; @@ -87,10 +88,13 @@ public: static void appendAllActiveSessions(OperationContext* opCtx, LogicalSessionIdSet* lsids); /** - * Returns a list of GenericCursors for all cursors on the global cursor manager and across all - * collection-level cursor maangers. + * Returns a list of GenericCursors for all idle cursors on the global cursor manager and across + * all collection-level cursor managers. Does not include currently pinned cursors. + * 'userMode': If auth is on, calling with userMode as kExcludeOthers will cause this function + * to only return cursors owned by the caller. If auth is off, this argument does not matter. */ - static std::vector<GenericCursor> getAllCursors(OperationContext* opCtx); + static std::vector<GenericCursor> getIdleCursors( + OperationContext* opCtx, MongoProcessInterface::CurrentOpUserMode userMode); /** * Kills cursors with matching logical sessions. Returns a pair with the overall @@ -188,13 +192,17 @@ public: /** * Appends sessions that have open cursors in this cursor manager to the given set of lsids. + * 'userMode': If auth is on, calling with userMode as kExcludeOthers will cause this function + * to only return cursors owned by the caller. If auth is off, this argument does not matter. */ void appendActiveSessions(LogicalSessionIdSet* lsids) const; /** - * Appends all active cursors in this cursor manager to the output vector. + * Appends all idle (non-pinned) cursors in this cursor manager to the output vector. */ - void appendActiveCursors(std::vector<GenericCursor>* cursors) const; + void appendIdleCursors(AuthorizationSession* ctxAuth, + MongoProcessInterface::CurrentOpUserMode userMode, + std::vector<GenericCursor>* cursors) const; /* * Returns a list of all open cursors for the given session. @@ -251,6 +259,13 @@ private: CursorId allocateCursorId_inlock(); + /** + * Creates a generic cursor from a ClientCursor. Can only be called while holding the + * CursorManager partition lock. This is neccessary to protect concurrent access to the data + * members of 'cursor', as it prevents other threads from pinning this cursor. + */ + GenericCursor buildGenericCursor_inlock(const ClientCursor* cursor) const; + ClientCursorPin _registerCursor( OperationContext* opCtx, std::unique_ptr<ClientCursor, ClientCursor::Deleter> clientCursor); diff --git a/src/mongo/db/generic_cursor.idl b/src/mongo/db/generic_cursor.idl index f1359c8f2b8..09a261baac1 100644 --- a/src/mongo/db/generic_cursor.idl +++ b/src/mongo/db/generic_cursor.idl @@ -28,8 +28,38 @@ structs: GenericCursor: description: "A struct representing a cursor in either mongod or mongos" fields: - id: long - ns: namespacestring + cursorId: + description: The cursor id of the cursor. + type: long + optional: true + ns: + description: The namespace of the cursor. + type: namespacestring + optional: true + nDocsReturned: + description: The number of docs returned by the cursor. + type: long + optional: true + tailable: + description: Whether the cursor is tailable and remains open after exhausting all documents in the find. + type: bool + optional: true + awaitData: + description: Whether this is a tailable and awaitData cursor that will block waiting for new data to be inserted into a capped collection. + type: bool + optional: true + noCursorTimeout: + description: If true the cursor will not be timed out because of inactivity. + type: bool + optional: true + originatingCommand: + description: The bson object containing the command that created the cursor. + type: object + optional: true lsid: type: LogicalSessionId optional: true + operationUsingCursorId: + description: The op ID of the operation pinning the cursor. Will be empty for idle cursors. + type: long + optional: true diff --git a/src/mongo/db/pipeline/SConscript b/src/mongo/db/pipeline/SConscript index 5999ec5e3ac..40d1afff528 100644 --- a/src/mongo/db/pipeline/SConscript +++ b/src/mongo/db/pipeline/SConscript @@ -270,6 +270,7 @@ env.Library( ], LIBDEPS=[ '$BUILD_DIR/mongo/db/auth/authorization_manager_global', + '$BUILD_DIR/mongo/db/generic_cursor', ] ) diff --git a/src/mongo/db/pipeline/document_source_current_op.cpp b/src/mongo/db/pipeline/document_source_current_op.cpp index 3c982d53487..b0db06b4e83 100644 --- a/src/mongo/db/pipeline/document_source_current_op.cpp +++ b/src/mongo/db/pipeline/document_source_current_op.cpp @@ -40,6 +40,7 @@ const StringData kIdleConnectionsFieldName = "idleConnections"_sd; const StringData kIdleSessionsFieldName = "idleSessions"_sd; const StringData kLocalOpsFieldName = "localOps"_sd; const StringData kTruncateOpsFieldName = "truncateOps"_sd; +const StringData kIdleCursorsFieldName = "idleCursors"_sd; const StringData kOpIdFieldName = "opid"_sd; const StringData kClientFieldName = "client"_sd; @@ -109,11 +110,12 @@ DocumentSource::GetNextResult DocumentSourceCurrentOp::getNext() { pExpCtx->checkForInterrupt(); if (_ops.empty()) { - _ops = pExpCtx->mongoProcessInterface->getCurrentOps(pExpCtx->opCtx, + _ops = pExpCtx->mongoProcessInterface->getCurrentOps(pExpCtx, _includeIdleConnections, _includeIdleSessions, _includeOpsFromAllUsers, - _truncateOps); + _truncateOps, + _idleCursors); _opsIter = _ops.begin(); @@ -188,6 +190,7 @@ intrusive_ptr<DocumentSource> DocumentSourceCurrentOp::createFromBson( UserMode includeOpsFromAllUsers = UserMode::kExcludeOthers; LocalOpsMode showLocalOpsOnMongoS = LocalOpsMode::kRemoteShardOps; TruncationMode truncateOps = TruncationMode::kNoTruncation; + CursorMode idleCursors = CursorMode::kExcludeCursors; for (auto&& elem : spec.embeddedObject()) { const auto fieldName = elem.fieldNameStringData(); @@ -233,6 +236,14 @@ intrusive_ptr<DocumentSource> DocumentSourceCurrentOp::createFromBson( elem.type() == BSONType::Bool); truncateOps = (elem.boolean() ? TruncationMode::kTruncateOps : TruncationMode::kNoTruncation); + } else if (fieldName == kIdleCursorsFieldName) { + uassert(ErrorCodes::FailedToParse, + str::stream() << "The 'idleCursors' parameter of the $currentOp stage must be " + "a boolean value, but found: " + << typeName(elem.type()), + elem.type() == BSONType::Bool); + idleCursors = + (elem.boolean() ? CursorMode::kIncludeCursors : CursorMode::kExcludeCursors); } else { uasserted(ErrorCodes::FailedToParse, str::stream() << "Unrecognized option '" << fieldName @@ -245,7 +256,8 @@ intrusive_ptr<DocumentSource> DocumentSourceCurrentOp::createFromBson( includeIdleSessions, includeOpsFromAllUsers, showLocalOpsOnMongoS, - truncateOps); + truncateOps, + idleCursors); } intrusive_ptr<DocumentSourceCurrentOp> DocumentSourceCurrentOp::create( @@ -254,13 +266,15 @@ intrusive_ptr<DocumentSourceCurrentOp> DocumentSourceCurrentOp::create( SessionMode includeIdleSessions, UserMode includeOpsFromAllUsers, LocalOpsMode showLocalOpsOnMongoS, - TruncationMode truncateOps) { + TruncationMode truncateOps, + CursorMode idleCursors) { return new DocumentSourceCurrentOp(pExpCtx, includeIdleConnections, includeIdleSessions, includeOpsFromAllUsers, showLocalOpsOnMongoS, - truncateOps); + truncateOps, + idleCursors); } Value DocumentSourceCurrentOp::serialize(boost::optional<ExplainOptions::Verbosity> explain) const { @@ -275,6 +289,8 @@ Value DocumentSourceCurrentOp::serialize(boost::optional<ExplainOptions::Verbosi {kLocalOpsFieldName, _showLocalOpsOnMongoS == LocalOpsMode::kLocalMongosOps ? Value(true) : Value()}, {kTruncateOpsFieldName, - _truncateOps == TruncationMode::kTruncateOps ? Value(true) : Value()}}}}); + _truncateOps == TruncationMode::kTruncateOps ? Value(true) : Value()}, + {kIdleCursorsFieldName, + _idleCursors == CursorMode::kIncludeCursors ? Value(true) : Value()}}}}); } } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_current_op.h b/src/mongo/db/pipeline/document_source_current_op.h index cbd3e266c57..f24dd88baa7 100644 --- a/src/mongo/db/pipeline/document_source_current_op.h +++ b/src/mongo/db/pipeline/document_source_current_op.h @@ -39,6 +39,7 @@ public: using LocalOpsMode = MongoProcessInterface::CurrentOpLocalOpsMode; using SessionMode = MongoProcessInterface::CurrentOpSessionsMode; using UserMode = MongoProcessInterface::CurrentOpUserMode; + using CursorMode = MongoProcessInterface::CurrentOpCursorMode; static constexpr StringData kStageName = "$currentOp"_sd; @@ -100,7 +101,8 @@ public: SessionMode includeIdleSessions = SessionMode::kIncludeIdle, UserMode includeOpsFromAllUsers = UserMode::kExcludeOthers, LocalOpsMode showLocalOpsOnMongoS = LocalOpsMode::kRemoteShardOps, - TruncationMode truncateOps = TruncationMode::kNoTruncation); + TruncationMode truncateOps = TruncationMode::kNoTruncation, + CursorMode idleCursors = CursorMode::kExcludeCursors); GetNextResult getNext() final; @@ -132,19 +134,22 @@ private: SessionMode includeIdleSessions, UserMode includeOpsFromAllUsers, LocalOpsMode showLocalOpsOnMongoS, - TruncationMode truncateOps) + TruncationMode truncateOps, + CursorMode idleCursors) : DocumentSource(pExpCtx), _includeIdleConnections(includeIdleConnections), _includeIdleSessions(includeIdleSessions), _includeOpsFromAllUsers(includeOpsFromAllUsers), _showLocalOpsOnMongoS(showLocalOpsOnMongoS), - _truncateOps(truncateOps) {} + _truncateOps(truncateOps), + _idleCursors(idleCursors) {} ConnMode _includeIdleConnections = ConnMode::kExcludeIdle; SessionMode _includeIdleSessions = SessionMode::kIncludeIdle; UserMode _includeOpsFromAllUsers = UserMode::kExcludeOthers; LocalOpsMode _showLocalOpsOnMongoS = LocalOpsMode::kRemoteShardOps; TruncationMode _truncateOps = TruncationMode::kNoTruncation; + CursorMode _idleCursors = CursorMode::kExcludeCursors; std::string _shardName; diff --git a/src/mongo/db/pipeline/document_source_current_op_test.cpp b/src/mongo/db/pipeline/document_source_current_op_test.cpp index 4dce695980d..4e424be390f 100644 --- a/src/mongo/db/pipeline/document_source_current_op_test.cpp +++ b/src/mongo/db/pipeline/document_source_current_op_test.cpp @@ -65,11 +65,12 @@ public: MockMongoInterface(bool hasShardName = true) : _hasShardName(hasShardName) {} - std::vector<BSONObj> getCurrentOps(OperationContext* opCtx, + std::vector<BSONObj> getCurrentOps(const boost::intrusive_ptr<ExpressionContext>& expCtx, CurrentOpConnectionsMode connMode, CurrentOpSessionsMode sessionMode, CurrentOpUserMode userMode, - CurrentOpTruncateMode truncateMode) const { + CurrentOpTruncateMode truncateMode, + CurrentOpCursorMode cursorMode) const { return _ops; } diff --git a/src/mongo/db/pipeline/document_source_list_local_cursors.cpp b/src/mongo/db/pipeline/document_source_list_local_cursors.cpp index 7246bc1d749..4e63c792df8 100644 --- a/src/mongo/db/pipeline/document_source_list_local_cursors.cpp +++ b/src/mongo/db/pipeline/document_source_list_local_cursors.cpp @@ -67,5 +67,7 @@ boost::intrusive_ptr<DocumentSource> DocumentSourceListLocalCursors::createFromB DocumentSourceListLocalCursors::DocumentSourceListLocalCursors( const boost::intrusive_ptr<ExpressionContext>& pExpCtx) - : DocumentSource(pExpCtx), _cursors(pExpCtx->mongoProcessInterface->getCursors(pExpCtx)) {} + : DocumentSource(pExpCtx), + _cursors(pExpCtx->mongoProcessInterface->getIdleCursors( + pExpCtx, MongoProcessInterface::CurrentOpUserMode::kIncludeAll)) {} } diff --git a/src/mongo/db/pipeline/mongo_process_common.cpp b/src/mongo/db/pipeline/mongo_process_common.cpp index ff84540cd01..87b6bd75c75 100644 --- a/src/mongo/db/pipeline/mongo_process_common.cpp +++ b/src/mongo/db/pipeline/mongo_process_common.cpp @@ -30,19 +30,24 @@ #include "mongo/db/pipeline/mongo_process_common.h" +#include "mongo/bson/mutable/document.h" #include "mongo/db/auth/authorization_manager.h" #include "mongo/db/auth/authorization_session.h" #include "mongo/db/client.h" #include "mongo/db/operation_context.h" +#include "mongo/db/pipeline/expression_context.h" #include "mongo/db/service_context.h" namespace mongo { -std::vector<BSONObj> MongoProcessCommon::getCurrentOps(OperationContext* opCtx, - CurrentOpConnectionsMode connMode, - CurrentOpSessionsMode sessionMode, - CurrentOpUserMode userMode, - CurrentOpTruncateMode truncateMode) const { +std::vector<BSONObj> MongoProcessCommon::getCurrentOps( + const boost::intrusive_ptr<ExpressionContext>& expCtx, + CurrentOpConnectionsMode connMode, + CurrentOpSessionsMode sessionMode, + CurrentOpUserMode userMode, + CurrentOpTruncateMode truncateMode, + CurrentOpCursorMode cursorMode) const { + OperationContext* opCtx = expCtx->opCtx; AuthorizationSession* ctxAuth = AuthorizationSession::get(opCtx->getClient()); std::vector<BSONObj> ops; @@ -69,6 +74,17 @@ std::vector<BSONObj> MongoProcessCommon::getCurrentOps(OperationContext* opCtx, ops.emplace_back(_reportCurrentOpForClient(opCtx, client, truncateMode)); } + // If 'cursorMode' is set to include idle cursors, retrieve them and add them to ops. + if (cursorMode == CurrentOpCursorMode::kIncludeCursors) { + + for (auto&& cursor : getIdleCursors(expCtx, userMode)) { + ops.push_back(BSON("type" + << "idleCursor" + << "cursor" + << cursor.toBSON())); + } + } + // If we need to report on idle Sessions, defer to the mongoD or mongoS implementations. if (sessionMode == CurrentOpSessionsMode::kIncludeIdle) { _reportCurrentOpsForIdleSessions(opCtx, userMode, &ops); diff --git a/src/mongo/db/pipeline/mongo_process_common.h b/src/mongo/db/pipeline/mongo_process_common.h index 2fcdf1e54d4..e24b7c0c286 100644 --- a/src/mongo/db/pipeline/mongo_process_common.h +++ b/src/mongo/db/pipeline/mongo_process_common.h @@ -43,11 +43,12 @@ class MongoProcessCommon : public MongoProcessInterface { public: virtual ~MongoProcessCommon() = default; - std::vector<BSONObj> getCurrentOps(OperationContext* opCtx, + std::vector<BSONObj> getCurrentOps(const boost::intrusive_ptr<ExpressionContext>& expCtx, CurrentOpConnectionsMode connMode, CurrentOpSessionsMode sessionMode, CurrentOpUserMode userMode, - CurrentOpTruncateMode) const final; + CurrentOpTruncateMode truncateMode, + CurrentOpCursorMode cursorMode) const final; protected: /** diff --git a/src/mongo/db/pipeline/mongo_process_interface.h b/src/mongo/db/pipeline/mongo_process_interface.h index 1d7add701d7..f457c184706 100644 --- a/src/mongo/db/pipeline/mongo_process_interface.h +++ b/src/mongo/db/pipeline/mongo_process_interface.h @@ -66,6 +66,7 @@ public: enum class CurrentOpTruncateMode { kNoTruncation, kTruncateOps }; enum class CurrentOpLocalOpsMode { kLocalMongosOps, kRemoteShardOps }; enum class CurrentOpSessionsMode { kIncludeIdle, kExcludeIdle }; + enum class CurrentOpCursorMode { kIncludeCursors, kExcludeCursors }; struct MakePipelineOptions { MakePipelineOptions(){}; @@ -186,11 +187,13 @@ public: * operation or, optionally, an idle connection. If userMode is kIncludeAllUsers, report * operations for all authenticated users; otherwise, report only the current user's operations. */ - virtual std::vector<BSONObj> getCurrentOps(OperationContext* opCtx, - CurrentOpConnectionsMode connMode, - CurrentOpSessionsMode sessionMode, - CurrentOpUserMode userMode, - CurrentOpTruncateMode) const = 0; + virtual std::vector<BSONObj> getCurrentOps( + const boost::intrusive_ptr<ExpressionContext>& expCtx, + CurrentOpConnectionsMode connMode, + CurrentOpSessionsMode sessionMode, + CurrentOpUserMode userMode, + CurrentOpTruncateMode, + CurrentOpCursorMode) const = 0; /** * Returns the name of the local shard if sharding is enabled, or an empty string. @@ -221,10 +224,11 @@ public: boost::optional<BSONObj> readConcern) = 0; /** - * Returns a vector of all local cursors. + * Returns a vector of all idle (non-pinned) local cursors. */ - virtual std::vector<GenericCursor> getCursors( - const boost::intrusive_ptr<ExpressionContext>& expCtx) const = 0; + virtual std::vector<GenericCursor> getIdleCursors( + const boost::intrusive_ptr<ExpressionContext>& expCtx, + CurrentOpUserMode userMode) const = 0; /** * The following methods forward to the BackupCursorService decorating the ServiceContext. diff --git a/src/mongo/db/pipeline/mongod_process_interface.cpp b/src/mongo/db/pipeline/mongod_process_interface.cpp index cb6f297946e..14830712722 100644 --- a/src/mongo/db/pipeline/mongod_process_interface.cpp +++ b/src/mongo/db/pipeline/mongod_process_interface.cpp @@ -393,9 +393,9 @@ std::pair<std::vector<FieldPath>, bool> MongoDInterface::collectDocumentKeyField return {result, true}; } -std::vector<GenericCursor> MongoDInterface::getCursors( - const intrusive_ptr<ExpressionContext>& expCtx) const { - return CursorManager::getAllCursors(expCtx->opCtx); +std::vector<GenericCursor> MongoDInterface::getIdleCursors( + const intrusive_ptr<ExpressionContext>& expCtx, CurrentOpUserMode userMode) const { + return CursorManager::getIdleCursors(expCtx->opCtx, userMode); } boost::optional<Document> MongoDInterface::lookupSingleDocument( diff --git a/src/mongo/db/pipeline/mongod_process_interface.h b/src/mongo/db/pipeline/mongod_process_interface.h index 0b0ccb7e7e7..915a8946118 100644 --- a/src/mongo/db/pipeline/mongod_process_interface.h +++ b/src/mongo/db/pipeline/mongod_process_interface.h @@ -91,8 +91,8 @@ public: UUID collectionUUID, const Document& documentKey, boost::optional<BSONObj> readConcern) final; - std::vector<GenericCursor> getCursors( - const boost::intrusive_ptr<ExpressionContext>& expCtx) const final; + std::vector<GenericCursor> getIdleCursors(const boost::intrusive_ptr<ExpressionContext>& expCtx, + CurrentOpUserMode userMode) const final; void fsyncLock(OperationContext* opCtx) final; void fsyncUnlock(OperationContext* opCtx) final; BackupCursorState openBackupCursor(OperationContext* opCtx) final; diff --git a/src/mongo/db/pipeline/mongos_process_interface.cpp b/src/mongo/db/pipeline/mongos_process_interface.cpp index 0ec76eaf200..e5a2904a8e0 100644 --- a/src/mongo/db/pipeline/mongos_process_interface.cpp +++ b/src/mongo/db/pipeline/mongos_process_interface.cpp @@ -228,12 +228,12 @@ BSONObj MongoSInterface::_reportCurrentOpForClient(OperationContext* opCtx, return builder.obj(); } -std::vector<GenericCursor> MongoSInterface::getCursors( - const intrusive_ptr<ExpressionContext>& expCtx) const { +std::vector<GenericCursor> MongoSInterface::getIdleCursors( + const intrusive_ptr<ExpressionContext>& expCtx, CurrentOpUserMode userMode) const { invariant(hasGlobalServiceContext()); auto cursorManager = Grid::get(expCtx->opCtx->getServiceContext())->getCursorManager(); invariant(cursorManager); - return cursorManager->getAllCursors(); + return cursorManager->getIdleCursors(); } bool MongoSInterface::isSharded(OperationContext* opCtx, const NamespaceString& nss) { diff --git a/src/mongo/db/pipeline/mongos_process_interface.h b/src/mongo/db/pipeline/mongos_process_interface.h index df96e9f1043..05182b4116a 100644 --- a/src/mongo/db/pipeline/mongos_process_interface.h +++ b/src/mongo/db/pipeline/mongos_process_interface.h @@ -52,8 +52,10 @@ public: const Document& documentKey, boost::optional<BSONObj> readConcern) final; - std::vector<GenericCursor> getCursors( - const boost::intrusive_ptr<ExpressionContext>& expCtx) const final; + // TODO: SERVER-37090 bring mongos getIdleCursors to match mongod and pass back GenericCursors + // with all available fields. + std::vector<GenericCursor> getIdleCursors(const boost::intrusive_ptr<ExpressionContext>& expCtx, + CurrentOpUserMode userMode) const final; DBClientBase* directClient() final { MONGO_UNREACHABLE; diff --git a/src/mongo/db/pipeline/stub_mongo_process_interface.h b/src/mongo/db/pipeline/stub_mongo_process_interface.h index 24376600f5e..638419b2571 100644 --- a/src/mongo/db/pipeline/stub_mongo_process_interface.h +++ b/src/mongo/db/pipeline/stub_mongo_process_interface.h @@ -122,11 +122,12 @@ public: MONGO_UNREACHABLE; } - std::vector<BSONObj> getCurrentOps(OperationContext* opCtx, + std::vector<BSONObj> getCurrentOps(const boost::intrusive_ptr<ExpressionContext>& expCtx, CurrentOpConnectionsMode connMode, CurrentOpSessionsMode sessionMode, CurrentOpUserMode userMode, - CurrentOpTruncateMode truncateMode) const override { + CurrentOpTruncateMode truncateMode, + CurrentOpCursorMode cursorMode) const override { MONGO_UNREACHABLE; } @@ -148,8 +149,8 @@ public: MONGO_UNREACHABLE; } - std::vector<GenericCursor> getCursors( - const boost::intrusive_ptr<ExpressionContext>& expCtx) const { + std::vector<GenericCursor> getIdleCursors(const boost::intrusive_ptr<ExpressionContext>& expCtx, + CurrentOpUserMode userMode) const { MONGO_UNREACHABLE; } diff --git a/src/mongo/s/query/cluster_cursor_manager.cpp b/src/mongo/s/query/cluster_cursor_manager.cpp index 950ac3c22c6..a521bc13000 100644 --- a/src/mongo/s/query/cluster_cursor_manager.cpp +++ b/src/mongo/s/query/cluster_cursor_manager.cpp @@ -572,7 +572,7 @@ void ClusterCursorManager::appendActiveSessions(LogicalSessionIdSet* lsids) cons } } -std::vector<GenericCursor> ClusterCursorManager::getAllCursors() const { +std::vector<GenericCursor> ClusterCursorManager::getIdleCursors() const { std::vector<GenericCursor> cursors; stdx::lock_guard<stdx::mutex> lk(_mutex); @@ -581,14 +581,14 @@ std::vector<GenericCursor> ClusterCursorManager::getAllCursors() const { for (const auto& cursorIdEntryPair : nsContainerPair.second.entryMap) { const CursorEntry& entry = cursorIdEntryPair.second; - if (entry.isKillPending()) { - // Don't include sessions for killed cursors. + if (entry.isKillPending() || entry.getOperationUsingCursor()) { + // Don't include sessions for killed or pinned cursors. continue; } cursors.emplace_back(); auto& gc = cursors.back(); - gc.setId(cursorIdEntryPair.first); + gc.setCursorId(cursorIdEntryPair.first); gc.setNs(nsContainerPair.first); gc.setLsid(entry.getLsid()); } diff --git a/src/mongo/s/query/cluster_cursor_manager.h b/src/mongo/s/query/cluster_cursor_manager.h index d26e8a92243..611aa306543 100644 --- a/src/mongo/s/query/cluster_cursor_manager.h +++ b/src/mongo/s/query/cluster_cursor_manager.h @@ -404,9 +404,9 @@ public: void appendActiveSessions(LogicalSessionIdSet* lsids) const; /** - * Returns a list of GenericCursors for all cursors in the cursor manager. + * Returns a list of GenericCursors for all idle (non-pinned) cursors in the cursor manager. */ - std::vector<GenericCursor> getAllCursors() const; + std::vector<GenericCursor> getIdleCursors() const; std::pair<Status, int> killCursorsWithMatchingSessions(OperationContext* opCtx, const SessionKiller::Matcher& matcher); |