diff options
author | Ian Boros <ian.boros@10gen.com> | 2018-01-10 11:41:37 -0500 |
---|---|---|
committer | Ian Boros <ian.boros@10gen.com> | 2018-01-10 11:41:37 -0500 |
commit | da80e97d103434a6bc566c589a23af13477e1a28 (patch) | |
tree | ebf3227fcc30feb6d8c9a0742b20e956ecf25e58 /src/mongo/db | |
parent | ba1d0d901ef386310457b03eecf9f5c4afce9047 (diff) | |
download | mongo-da80e97d103434a6bc566c589a23af13477e1a28.tar.gz |
SERVER-21710 Add ability to kill pinned cursors on mongod
Diffstat (limited to 'src/mongo/db')
-rw-r--r-- | src/mongo/db/clientcursor.cpp | 20 | ||||
-rw-r--r-- | src/mongo/db/clientcursor.h | 55 | ||||
-rw-r--r-- | src/mongo/db/commands/killcursors_cmd.cpp | 11 | ||||
-rw-r--r-- | src/mongo/db/cursor_manager.cpp | 90 | ||||
-rw-r--r-- | src/mongo/db/cursor_manager.h | 20 | ||||
-rw-r--r-- | src/mongo/db/service_entry_point_mongod.cpp | 4 |
6 files changed, 123 insertions, 77 deletions
diff --git a/src/mongo/db/clientcursor.cpp b/src/mongo/db/clientcursor.cpp index 52007343a9c..a2ba154138c 100644 --- a/src/mongo/db/clientcursor.cpp +++ b/src/mongo/db/clientcursor.cpp @@ -78,20 +78,22 @@ long long ClientCursor::totalOpen() { ClientCursor::ClientCursor(ClientCursorParams params, CursorManager* cursorManager, CursorId cursorId, - boost::optional<LogicalSessionId> lsid, + OperationContext* operationUsingCursor, Date_t now) : _cursorid(cursorId), _nss(std::move(params.nss)), _authenticatedUsers(std::move(params.authenticatedUsers)), - _lsid(std::move(lsid)), + _lsid(operationUsingCursor->getLogicalSessionId()), _isReadCommitted(params.isReadCommitted), _cursorManager(cursorManager), _originatingCommand(params.originatingCommandObj), _queryOptions(params.queryOptions), _exec(std::move(params.exec)), + _operationUsingCursor(operationUsingCursor), _lastUseDate(now) { invariant(_cursorManager); invariant(_exec); + invariant(_operationUsingCursor); cursorStatsOpen.increment(); @@ -104,7 +106,7 @@ ClientCursor::ClientCursor(ClientCursorParams params, ClientCursor::~ClientCursor() { // Cursors must be unpinned and deregistered from their cursor manager before being deleted. - invariant(!_isPinned); + invariant(!_operationUsingCursor); invariant(_disposed); cursorStatsOpen.decrement(); @@ -150,7 +152,7 @@ void ClientCursor::updateSlaveLocation(OperationContext* opCtx) { ClientCursorPin::ClientCursorPin(OperationContext* opCtx, ClientCursor* cursor) : _opCtx(opCtx), _cursor(cursor) { invariant(_cursor); - invariant(_cursor->_isPinned); + invariant(_cursor->_operationUsingCursor); invariant(_cursor->_cursorManager); invariant(!_cursor->_disposed); @@ -166,7 +168,7 @@ ClientCursorPin::ClientCursorPin(ClientCursorPin&& other) // The pinned cursor is being transferred to us from another pin. The 'other' pin must have a // pinned cursor. invariant(other._cursor); - invariant(other._cursor->_isPinned); + invariant(other._cursor->_operationUsingCursor); // Be sure to set the 'other' pin's cursor to null in order to transfer ownership to ourself. other._cursor = nullptr; @@ -182,7 +184,7 @@ ClientCursorPin& ClientCursorPin::operator=(ClientCursorPin&& other) { // pinned cursor, and we must not have a cursor. invariant(!_cursor); invariant(other._cursor); - invariant(other._cursor->_isPinned); + invariant(other._cursor->_operationUsingCursor); // Copy the cursor pointer to ourselves, but also be sure to set the 'other' pin's cursor to // null so that it no longer has the cursor pinned. @@ -211,7 +213,7 @@ void ClientCursorPin::release() { _opCtx->lockState()->isCollectionLockedForMode(_cursor->_nss.ns(), MODE_IS); dassert(isLocked || _cursor->_cursorManager->isGlobalManager()); - invariant(_cursor->_isPinned); + invariant(_cursor->_operationUsingCursor); if (_cursor->getExecutor()->isMarkedAsKilled()) { // The ClientCursor was killed while we had it. Therefore, it is our responsibility to @@ -228,7 +230,7 @@ void ClientCursorPin::release() { void ClientCursorPin::deleteUnderlying() { invariant(_cursor); - invariant(_cursor->_isPinned); + invariant(_cursor->_operationUsingCursor); // Note the following subtleties of this method's implementation: // - We must unpin the cursor before destruction, since it is an error to delete a pinned // cursor. @@ -250,7 +252,7 @@ void ClientCursorPin::deleteUnderlying() { // Make sure the cursor is disposed and unpinned before being destroyed. _cursor->dispose(_opCtx); - _cursor->_isPinned = false; + _cursor->_operationUsingCursor = nullptr; delete _cursor; cursorStatsOpenPinned.decrement(); diff --git a/src/mongo/db/clientcursor.h b/src/mongo/db/clientcursor.h index 2e8d7894ad3..f6a97bfe4c9 100644 --- a/src/mongo/db/clientcursor.h +++ b/src/mongo/db/clientcursor.h @@ -255,7 +255,7 @@ private: ClientCursor(ClientCursorParams params, CursorManager* cursorManager, CursorId cursorId, - boost::optional<LogicalSessionId> lsid, + OperationContext* operationUsingCursor, Date_t now); /** @@ -287,10 +287,15 @@ private: // The ID of the ClientCursor. A value of 0 is used to mean that no cursor id has been assigned. CursorId _cursorid = 0; + // Threads may read from this field even if they don't have the cursor pinned, as long as they + // have the correct partition of the CursorManager locked (just like _authenticatedUsers). const NamespaceString _nss; - // The set of authenticated users when this cursor was created. - std::vector<UserName> _authenticatedUsers; + // The set of authenticated users when this cursor was created. Threads may read from this + // field (using the getter) even if they don't have the cursor pinned as long as they hold the + // correct partition's lock in the CursorManager. They must hold the lock to prevent the cursor + // from being freed by another thread during the read. + const std::vector<UserName> _authenticatedUsers; // A logical session id for this cursor, if it is running inside of a session. const boost::optional<LogicalSessionId> _lsid; @@ -333,29 +338,41 @@ private: // CursorManager, at which point it has sole ownership of the ClientCursor. // - // While a cursor is being used by a client, it is marked as "pinned". See ClientCursorPin - // below. + // While a cursor is being used by a client, it is marked as "pinned" by setting + // _operationUsingCursor to the current OperationContext. // - // Cursors always come into existence in a pinned state. - bool _isPinned = true; + // Cursors always come into existence in a pinned state (this must be non-null at construction). + // + // To write to this field one of the following must be true: + // 1) You have a lock on the appropriate partition in CursorManager and the cursor is unpinned + // (the field is null). + // 2) You own the cursor and the cursor manager it was associated with is gone (this can only + // happen in ClientCursorPin). In this case, nobody else will try to pin the cursor. + // + // To read this field one of the following must be true: + // 1) You have a lock on the appropriate partition in CursorManager. + // 2) You know you have the cursor pinned. + OperationContext* _operationUsingCursor; Date_t _lastUseDate; }; /** - * ClientCursorPin is an RAII class which must be used in order to access a cursor. On construction, - * the ClientCursorPin marks its cursor as in use, which is called "pinning" the cursor. On - * destructrution, the ClientCursorPin marks its cursor as no longer in use, which is called - * "unpinning" the cursor. Pinning is used to prevent multiple concurrent uses of the same cursor--- - * pinned cursors cannot be killed or timed out and cannot be used concurrently by other operations - * such as getMore or killCursors. A pin is obtained using the CursorManager. See cursor_manager.h - * for more details. + * ClientCursorPin is an RAII class which must be used in order to access a cursor. On + * construction, the ClientCursorPin marks its cursor as in use, which is called "pinning" the + * cursor. On destruction, the ClientCursorPin marks its cursor as no longer in use, which is + * called "unpinning" the cursor. Pinning is used to prevent multiple concurrent uses of the same + * cursor--- pinned cursors cannot be deleted or timed out and cannot be used concurrently by other + * operations such as getMore. They can however, be marked as interrupted and instructed to destroy + * themselves through killCursors. + * + * A pin is obtained using the CursorManager. See cursor_manager.h for more details. * - * A pin extends the lifetime of a ClientCursor object until the pin's release. Pinned - * ClientCursor objects cannot not be killed due to inactivity, and cannot be killed by user - * kill requests. When a CursorManager is destroyed (e.g. by a collection drop), ownership of - * any still-pinned ClientCursor objects is transferred to their managing ClientCursorPin - * objects. + * A pin extends the lifetime of a ClientCursor object until the pin's release. Pinned ClientCursor + * objects cannot not be killed due to inactivity, and cannot be immediately erased by user kill + * requests (though they can be marked as interrupted). When a CursorManager is destroyed (e.g. by + * a collection drop), ownership of any still-pinned ClientCursor objects is transferred to their + * managing ClientCursorPin objects. * * Example usage: * { diff --git a/src/mongo/db/commands/killcursors_cmd.cpp b/src/mongo/db/commands/killcursors_cmd.cpp index dbee95b4976..512ec7e0e94 100644 --- a/src/mongo/db/commands/killcursors_cmd.cpp +++ b/src/mongo/db/commands/killcursors_cmd.cpp @@ -51,14 +51,7 @@ private: Status _checkAuth(Client* client, const NamespaceString& nss, CursorId id) const final { auto opCtx = client->getOperationContext(); const auto check = [client, opCtx, id](CursorManager* manager) { - auto ccPin = manager->pinCursor(opCtx, id, CursorManager::kNoCheckSession); - if (!ccPin.isOK()) { - return ccPin.getStatus(); - } - - const auto* cursor = ccPin.getValue().getCursor(); - AuthorizationSession* as = AuthorizationSession::get(client); - return as->checkAuthForKillCursors(cursor->nss(), cursor->getAuthenticatedUsers()); + return manager->checkAuthForKillCursors(opCtx, id); }; return CursorManager::withCursorManager(opCtx, id, nss, check); @@ -80,7 +73,7 @@ private: return CursorManager::withCursorManager( opCtx, id, nss, [opCtx, id](CursorManager* manager) { - return manager->eraseCursor(opCtx, id, true /* shouldAudit */); + return manager->killCursor(opCtx, id, true /* shouldAudit */); }); } } killCursorsCmd; diff --git a/src/mongo/db/cursor_manager.cpp b/src/mongo/db/cursor_manager.cpp index 1b542bc27e7..6e077f0c8a6 100644 --- a/src/mongo/db/cursor_manager.cpp +++ b/src/mongo/db/cursor_manager.cpp @@ -93,7 +93,7 @@ public: /** * works globally */ - bool eraseCursor(OperationContext* opCtx, CursorId id, bool checkAuth); + bool killCursor(OperationContext* opCtx, CursorId id, bool checkAuth); void appendStats(BSONObjBuilder& builder); @@ -169,7 +169,7 @@ void GlobalCursorIdCache::deregisterCursorManager(uint32_t id, const NamespaceSt _idToNss.erase(id); } -bool GlobalCursorIdCache::eraseCursor(OperationContext* opCtx, CursorId id, bool checkAuth) { +bool GlobalCursorIdCache::killCursor(OperationContext* opCtx, CursorId id, bool checkAuth) { // Figure out what the namespace of this cursor is. NamespaceString nss; if (CursorManager::isGloballyManagedCursor(id)) { @@ -194,7 +194,7 @@ bool GlobalCursorIdCache::eraseCursor(OperationContext* opCtx, CursorId id, bool } invariant(nss.isValid()); - // Check if we are authorized to erase this cursor. + // Check if we are authorized to kill this cursor. if (checkAuth) { auto status = CursorManager::withCursorManager( opCtx, id, nss, [nss, id, opCtx](CursorManager* manager) { @@ -212,17 +212,17 @@ bool GlobalCursorIdCache::eraseCursor(OperationContext* opCtx, CursorId id, bool } } - // If this cursor is owned by the global cursor manager, ask it to erase the cursor for us. + // If this cursor is owned by the global cursor manager, ask it to kill the cursor for us. if (CursorManager::isGloballyManagedCursor(id)) { - Status eraseStatus = globalCursorManager->eraseCursor(opCtx, id, checkAuth); + Status killStatus = globalCursorManager->killCursor(opCtx, id, checkAuth); massert(28697, - eraseStatus.reason(), - eraseStatus.code() == ErrorCodes::OK || - eraseStatus.code() == ErrorCodes::CursorNotFound); - return eraseStatus.isOK(); + killStatus.reason(), + killStatus.code() == ErrorCodes::OK || + killStatus.code() == ErrorCodes::CursorNotFound); + return killStatus.isOK(); } - // If not, then the cursor must be owned by a collection. Erase the cursor under the + // If not, then the cursor must be owned by a collection. Kill the cursor under the // collection lock (to prevent the collection from going away during the erase). AutoGetCollectionForReadCommand ctx(opCtx, nss); Collection* collection = ctx.getCollection(); @@ -233,7 +233,7 @@ bool GlobalCursorIdCache::eraseCursor(OperationContext* opCtx, CursorId id, bool return false; } - Status eraseStatus = collection->getCursorManager()->eraseCursor(opCtx, id, checkAuth); + Status eraseStatus = collection->getCursorManager()->killCursor(opCtx, id, checkAuth); uassert(16089, eraseStatus.reason(), eraseStatus.code() == ErrorCodes::OK || @@ -328,7 +328,7 @@ std::vector<GenericCursor> CursorManager::getAllCursors(OperationContext* opCtx) std::pair<Status, int> CursorManager::killCursorsWithMatchingSessions( OperationContext* opCtx, const SessionKiller::Matcher& matcher) { auto eraser = [&](CursorManager& mgr, CursorId id) { - uassertStatusOK(mgr.eraseCursor(opCtx, id, true)); + uassertStatusOK(mgr.killCursor(opCtx, id, true)); }; auto visitor = makeKillSessionsCursorManagerVisitor(opCtx, matcher, std::move(eraser)); @@ -341,22 +341,22 @@ std::size_t CursorManager::timeoutCursorsGlobal(OperationContext* opCtx, Date_t return globalCursorIdCache->timeoutCursors(opCtx, now); } -int CursorManager::eraseCursorGlobalIfAuthorized(OperationContext* opCtx, int n, const char* _ids) { +int CursorManager::killCursorGlobalIfAuthorized(OperationContext* opCtx, int n, const char* _ids) { ConstDataCursor ids(_ids); int numDeleted = 0; for (int i = 0; i < n; i++) { - if (eraseCursorGlobalIfAuthorized(opCtx, ids.readAndAdvance<LittleEndian<int64_t>>())) + if (killCursorGlobalIfAuthorized(opCtx, ids.readAndAdvance<LittleEndian<int64_t>>())) numDeleted++; if (globalInShutdownDeprecated()) break; } return numDeleted; } -bool CursorManager::eraseCursorGlobalIfAuthorized(OperationContext* opCtx, CursorId id) { - return globalCursorIdCache->eraseCursor(opCtx, id, true); +bool CursorManager::killCursorGlobalIfAuthorized(OperationContext* opCtx, CursorId id) { + return globalCursorIdCache->killCursor(opCtx, id, true); } -bool CursorManager::eraseCursorGlobal(OperationContext* opCtx, CursorId id) { - return globalCursorIdCache->eraseCursor(opCtx, id, false); +bool CursorManager::killCursorGlobal(OperationContext* opCtx, CursorId id) { + return globalCursorIdCache->killCursor(opCtx, id, false); } Status CursorManager::withCursorManager(OperationContext* opCtx, @@ -433,9 +433,9 @@ void CursorManager::invalidateAll(OperationContext* opCtx, auto* cursor = it->second; cursor->markAsKilled(reason); - // If pinned, there is an active user of this cursor, who is now responsible for - // cleaning it up. Otherwise, we can immediately dispose of it. - if (cursor->_isPinned) { + // If there's an operation actively using the cursor, then that operation is now + // responsible for cleaning it up. Otherwise we can immediately dispose of it. + if (cursor->_operationUsingCursor) { it = partition.erase(it); continue; } @@ -481,7 +481,7 @@ void CursorManager::invalidateDocument(OperationContext* opCtx, } bool CursorManager::cursorShouldTimeout_inlock(const ClientCursor* cursor, Date_t now) { - if (cursor->isNoTimeout() || cursor->_isPinned) { + if (cursor->isNoTimeout() || cursor->_operationUsingCursor) { return false; } return (now - cursor->_lastUseDate) >= Milliseconds(getCursorTimeoutMillis()); @@ -538,7 +538,7 @@ StatusWith<ClientCursorPin> CursorManager::pinCursor(OperationContext* opCtx, ClientCursor* cursor = it->second; uassert(ErrorCodes::CursorInUse, str::stream() << "cursor id " << id << " is already in use", - !cursor->_isPinned); + !cursor->_operationUsingCursor); if (cursor->getExecutor()->isMarkedAsKilled()) { // This cursor was killed while it was idle. Status error{ErrorCodes::QueryPlanKilled, @@ -557,7 +557,7 @@ StatusWith<ClientCursorPin> CursorManager::pinCursor(OperationContext* opCtx, } } - cursor->_isPinned = true; + cursor->_operationUsingCursor = opCtx; // We use pinning of a cursor as a proxy for active, user-initiated use of a cursor. Therefor, // we pass down to the logical session cache and vivify the record (updating last use). @@ -573,8 +573,8 @@ void CursorManager::unpin(OperationContext* opCtx, ClientCursor* cursor) { auto now = opCtx->getServiceContext()->getPreciseClockSource()->now(); auto partitionLock = _cursorMap->lockOnePartition(cursor->cursorid()); - invariant(cursor->_isPinned); - cursor->_isPinned = false; + invariant(cursor->_operationUsingCursor); + cursor->_operationUsingCursor = nullptr; cursor->_lastUseDate = now; } @@ -673,8 +673,8 @@ ClientCursorPin CursorManager::registerCursor(OperationContext* opCtx, // we don't insert two cursors with the same cursor id. stdx::lock_guard<SimpleMutex> lock(_registrationLock); CursorId cursorId = allocateCursorId_inlock(); - std::unique_ptr<ClientCursor, ClientCursor::Deleter> clientCursor(new ClientCursor( - std::move(cursorParams), this, cursorId, opCtx->getLogicalSessionId(), now)); + std::unique_ptr<ClientCursor, ClientCursor::Deleter> clientCursor( + new ClientCursor(std::move(cursorParams), this, cursorId, opCtx, now)); // Transfer ownership of the cursor to '_cursorMap'. auto partition = _cursorMap->lockOnePartition(cursorId); @@ -687,7 +687,7 @@ void CursorManager::deregisterCursor(ClientCursor* cc) { _cursorMap->erase(cc->cursorid()); } -Status CursorManager::eraseCursor(OperationContext* opCtx, CursorId id, bool shouldAudit) { +Status CursorManager::killCursor(OperationContext* opCtx, CursorId id, bool shouldAudit) { auto lockedPartition = _cursorMap->lockOnePartition(id); auto it = lockedPartition->find(id); if (it == lockedPartition->end()) { @@ -699,12 +699,20 @@ Status CursorManager::eraseCursor(OperationContext* opCtx, CursorId id, bool sho } auto cursor = it->second; - if (cursor->_isPinned) { + if (cursor->_operationUsingCursor) { + // Rather than removing the cursor directly, kill the operation that's currently using the + // cursor. It will stop on its own (and remove the cursor) when it sees that it's been + // interrupted. + { + stdx::unique_lock<Client> lk(*cursor->_operationUsingCursor->getClient()); + cursor->_operationUsingCursor->getServiceContext()->killOperation( + cursor->_operationUsingCursor, ErrorCodes::CursorKilled); + } + if (shouldAudit) { - audit::logKillCursorsAuthzCheck( - opCtx->getClient(), _nss, id, ErrorCodes::OperationFailed); + audit::logKillCursorsAuthzCheck(opCtx->getClient(), _nss, id, ErrorCodes::OK); } - return {ErrorCodes::OperationFailed, str::stream() << "Cannot kill pinned cursor: " << id}; + return Status::OK(); } std::unique_ptr<ClientCursor, ClientCursor::Deleter> ownedCursor(cursor); @@ -717,4 +725,20 @@ Status CursorManager::eraseCursor(OperationContext* opCtx, CursorId id, bool sho return Status::OK(); } +Status CursorManager::checkAuthForKillCursors(OperationContext* opCtx, CursorId id) { + auto lockedPartition = _cursorMap->lockOnePartition(id); + auto it = lockedPartition->find(id); + if (it == lockedPartition->end()) { + return {ErrorCodes::CursorNotFound, str::stream() << "cursor id " << id << " not found"}; + } + + ClientCursor* cursor = it->second; + // Note that we're accessing the cursor without having pinned it! This is okay since we're only + // accessing nss() and getAuthenticatedUsers() both of which return values that don't change + // after the cursor's creation. We're guaranteed that the cursor won't get destroyed while we're + // reading from it because we hold the partition's lock. + AuthorizationSession* as = AuthorizationSession::get(opCtx->getClient()); + return as->checkAuthForKillCursors(cursor->nss(), cursor->getAuthenticatedUsers()); +} + } // namespace mongo diff --git a/src/mongo/db/cursor_manager.h b/src/mongo/db/cursor_manager.h index a292867cea1..f8dcebb803d 100644 --- a/src/mongo/db/cursor_manager.h +++ b/src/mongo/db/cursor_manager.h @@ -173,14 +173,24 @@ public: AuthCheck checkSessionAuth = kCheckSession); /** - * Returns an OK status if the cursor was successfully erased. + * Returns an OK status if the cursor was successfully killed, meaning either: + * (1) The cursor was erased from the cursor registry + * (2) The cursor's operation was interrupted, and the cursor will be cleaned up when the + * operation next checks for interruption. + * Case (2) will only occur if the cursor is pinned. * * Returns ErrorCodes::CursorNotFound if the cursor id is not owned by this manager. Returns * ErrorCodes::OperationFailed if attempting to erase a pinned cursor. * * If 'shouldAudit' is true, will perform audit logging. */ - Status eraseCursor(OperationContext* opCtx, CursorId id, bool shouldAudit); + Status killCursor(OperationContext* opCtx, CursorId id, bool shouldAudit); + + /** + * Returns an OK status if we're authorized to erase the cursor. Otherwise, returns + * ErrorCodes::Unauthorized. + */ + Status checkAuthForKillCursors(OperationContext* opCtx, CursorId id); void getCursorIds(std::set<CursorId>* openCursors) const; @@ -218,11 +228,11 @@ public: return (cursorId & mask) == (static_cast<long long>(0b01) << 62); } - static int eraseCursorGlobalIfAuthorized(OperationContext* opCtx, int n, const char* ids); + static int killCursorGlobalIfAuthorized(OperationContext* opCtx, int n, const char* ids); - static bool eraseCursorGlobalIfAuthorized(OperationContext* opCtx, CursorId id); + static bool killCursorGlobalIfAuthorized(OperationContext* opCtx, CursorId id); - static bool eraseCursorGlobal(OperationContext* opCtx, CursorId id); + static bool killCursorGlobal(OperationContext* opCtx, CursorId id); /** * Deletes inactive cursors from the global cursor manager and from all per-collection cursor diff --git a/src/mongo/db/service_entry_point_mongod.cpp b/src/mongo/db/service_entry_point_mongod.cpp index 4ef0b70ec81..1dfa4570fd4 100644 --- a/src/mongo/db/service_entry_point_mongod.cpp +++ b/src/mongo/db/service_entry_point_mongod.cpp @@ -957,7 +957,7 @@ void receivedKillCursors(OperationContext* opCtx, const Message& m) { const char* cursorArray = dbmessage.getArray(n); - int found = CursorManager::eraseCursorGlobalIfAuthorized(opCtx, n, cursorArray); + int found = CursorManager::killCursorGlobalIfAuthorized(opCtx, n, cursorArray); if (shouldLog(logger::LogSeverity::Debug(1)) || found != n) { LOG(found == n ? 1 : 0) << "killcursors: found " << found << " of " << n; @@ -1062,7 +1062,7 @@ DbResponse receivedGetMore(OperationContext* opCtx, // because it may now be out of sync with the client's iteration state. // SERVER-7952 // TODO Temporary code, see SERVER-4563 for a cleanup overview. - CursorManager::eraseCursorGlobal(opCtx, cursorid); + CursorManager::killCursorGlobal(opCtx, cursorid); } BSONObjBuilder err; |