diff options
author | Hari Khalsa <hkhalsa@10gen.com> | 2013-08-05 16:13:06 -0400 |
---|---|---|
committer | Hari Khalsa <hkhalsa@10gen.com> | 2013-08-07 17:20:58 -0400 |
commit | 3d3719bd3881b437f136d8ea8a8dfd22de2f3d52 (patch) | |
tree | 2922be571d85a034170b8bb5f0e0ff568d4a3d5d /src/mongo/db/clientcursor.h | |
parent | 3e50406c655679a5a3ed52cb64b5750c7518fe6f (diff) | |
download | mongo-3d3719bd3881b437f136d8ea8a8dfd22de2f3d52.tar.gz |
SERVER-10026 SERVER-10376 scrub clientcursor vigorously
Diffstat (limited to 'src/mongo/db/clientcursor.h')
-rw-r--r-- | src/mongo/db/clientcursor.h | 587 |
1 files changed, 275 insertions, 312 deletions
diff --git a/src/mongo/db/clientcursor.h b/src/mongo/db/clientcursor.h index 12a7ff9c5c3..4f739a126b3 100644 --- a/src/mongo/db/clientcursor.h +++ b/src/mongo/db/clientcursor.h @@ -1,26 +1,18 @@ -/* clientcursor.h */ - /** -* Copyright (C) 2008 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see <http://www.gnu.org/licenses/>. -*/ - -/* Cursor -- and its derived classes -- are our internal cursors. - - ClientCursor is a wrapper that represents a cursorid from our database - application's perspective. -*/ + * Copyright (C) 2008 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ #pragma once @@ -36,6 +28,7 @@ #include "mongo/db/keypattern.h" #include "mongo/db/matcher.h" #include "mongo/db/projection.h" +#include "mongo/db/query/runner.h" #include "mongo/s/collection_metadata.h" #include "mongo/util/net/message.h" #include "mongo/util/background.h" @@ -44,139 +37,149 @@ namespace mongo { typedef boost::recursive_mutex::scoped_lock recursive_scoped_lock; - class Cursor; /* internal server cursor base class */ class ClientCursor; class ParsedQuery; - /* todo: make this map be per connection. this will prevent cursor hijacking security attacks perhaps. - * ERH: 9/2010 this may not work since some drivers send getMore over a different connection - */ - typedef map<CursorId, ClientCursor*> CCById; - - extern BSONObj id_obj; - + /** + * ClientCursor is a wrapper that represents a cursorid from our database application's + * perspective. + */ class ClientCursor : private boost::noncopyable { - friend class CmdCursorInfo; public: - static void assertNoCursors(); + ClientCursor(int qopts, const shared_ptr<Cursor>& c, const string& ns, + BSONObj query = BSONObj()); - /* use this to assure we don't in the background time out cursor while it is under use. - if you are using noTimeout() already, there is no risk anyway. - Further, this mechanism guards against two getMore requests on the same cursor executing - at the same time - which might be bad. That should never happen, but if a client driver - had a bug, it could (or perhaps some sort of attack situation). - */ - class Pin : boost::noncopyable { - public: - Pin( long long cursorid ) : - _cursorid( INVALID_CURSOR_ID ) { - recursive_scoped_lock lock( ccmutex ); - ClientCursor *cursor = ClientCursor::find_inlock( cursorid, true ); - if ( cursor ) { - uassert( 12051, "clientcursor already in use? driver problem?", - cursor->_pinValue < 100 ); - cursor->_pinValue += 100; - _cursorid = cursorid; - } - } - void release() { - if ( _cursorid == INVALID_CURSOR_ID ) { - return; - } - ClientCursor *cursor = c(); - _cursorid = INVALID_CURSOR_ID; - if ( cursor ) { - verify( cursor->_pinValue >= 100 ); - cursor->_pinValue -= 100; - } - } - ~Pin() { DESTRUCTOR_GUARD( release(); ) } - ClientCursor *c() const { return ClientCursor::find( _cursorid ); } - private: - CursorId _cursorid; - }; + ClientCursor(int qopts, Runner* runner, const string& ns, BSONObj query = BSONObj()); - /** Assures safe and reliable cleanup of a ClientCursor. */ - class Holder : boost::noncopyable { - public: - Holder( ClientCursor *c = 0 ) : - _c( 0 ), - _id( INVALID_CURSOR_ID ) { - reset( c ); - } - void reset( ClientCursor *c = 0 ) { - if ( c == _c ) - return; - if ( _c ) { - // be careful in case cursor was deleted by someone else - ClientCursor::erase( _id ); - } - if ( c ) { - _c = c; - _id = c->_cursorid; - } - else { - _c = 0; - _id = INVALID_CURSOR_ID; - } - } - ~Holder() { - DESTRUCTOR_GUARD ( reset(); ); - } - ClientCursor* get() { return _c; } - operator bool() { return _c; } - ClientCursor * operator-> () { return _c; } - const ClientCursor * operator-> () const { return _c; } - /** Release ownership of the ClientCursor. */ - void release() { - _c = 0; - _id = INVALID_CURSOR_ID; - } - private: - ClientCursor *_c; - CursorId _id; - }; + ~ClientCursor(); /** - * Iterates through all ClientCursors, under its own ccmutex lock. - * Also supports deletion on the fly. + * Assert that there are no open cursors. + * Called from DatabaseHolder::closeAll. */ - class LockedIterator : boost::noncopyable { - public: - LockedIterator() : _lock( ccmutex ), _i( clientCursorsById.begin() ) {} - bool ok() const { return _i != clientCursorsById.end(); } - ClientCursor *current() const { return _i->second; } - void advance() { ++_i; } - /** - * Delete 'current' and advance. Properly handles cascading deletions that may occur - * when one ClientCursor is directly deleted. - */ - void deleteAndAdvance(); - private: - recursive_scoped_lock _lock; - CCById::const_iterator _i; - }; - - ClientCursor(int queryOptions, const shared_ptr<Cursor>& c, const string& ns, BSONObj query = BSONObj() ); - - ~ClientCursor(); + static void assertNoCursors(); - // *************** basic accessors ******************* + // + // Basic accessors + // CursorId cursorid() const { return _cursorid; } string ns() const { return _ns; } Database * db() const { return _db; } - const BSONObj& query() const { return _query; } - int queryOptions() const { return _queryOptions; } - DiskLoc lastLoc() const { return _lastLoc; } + // + // Invalidation of DiskLocs and dropping of namespaces + // - /* Get rid of cursors for namespaces 'ns'. When dropping a db, ns is "dbname." - Used by drop, dropIndexes, dropDatabase. - */ + /** + * Get rid of cursors for namespaces 'ns'. When dropping a db, ns is "dbname." Used by drop, + * dropIndexes, dropDatabase. + */ static void invalidate(const char *ns); /** + * Called when the provided DiskLoc is about to change state via a deletion or an update. + * All runners/cursors that might be using that DiskLoc must adapt. + */ + static void aboutToDelete(const StringData& ns, + const NamespaceDetails* nsd, + const DiskLoc& dl); + + // + // Yielding. + // + + static void staticYield(int micros, const StringData& ns, Record* rec ); + + // + // Static methods about all ClientCursors TODO: Document. + // + + static void appendStats( BSONObjBuilder& result ); + + // + // ClientCursor creation/deletion. + // + + static unsigned numCursors() { return clientCursorsById.size(); } + static void find( const string& ns , set<CursorId>& all ); + static ClientCursor* find(CursorId id, bool warn = true); + + // Same as erase but checks to make sure this thread has read permission on the cursor's + // namespace. This should be called when receiving killCursors from a client. This should + // not be called when ccmutex is held. + static int eraseIfAuthorized(int n, long long* ids); + static bool eraseIfAuthorized(CursorId id); + + /** + * @return number of cursors found + */ + static int erase(int n, long long* ids); + + /** + * Deletes the cursor with the provided @param 'id' if one exists. + * @throw if the cursor with the provided id is pinned. + * This does not do any auth checking and should be used only when erasing cursors as part + * of cleaning up internal operations. + */ + static bool erase(CursorId id); + + // + // Timing and timeouts + // + + /** + * called every 4 seconds. millis is amount of idle time passed since the last call -- + * could be zero + */ + static void idleTimeReport(unsigned millis); + + /** + * @param millis amount of idle passed time since last call + * note called outside of locks (other than ccmutex) so care must be exercised + */ + bool shouldTimeout( unsigned millis ); + unsigned idleTime() const { return _idleAgeMillis; } + + uint64_t getLeftoverMaxTimeMicros() const { return _leftoverMaxTimeMicros; } + void setLeftoverMaxTimeMicros( uint64_t leftoverMaxTimeMicros ) { + _leftoverMaxTimeMicros = leftoverMaxTimeMicros; + } + + // + // Sharding-specific data. TODO: Document. + // + + // future getMore. + void setCollMetadata( CollectionMetadataPtr metadata ){ _collMetadata = metadata; } + CollectionMetadataPtr getCollMetadata(){ return _collMetadata; } + + // + // Replication-related stuff. TODO: Document and clean. + // + + void storeOpForSlave( DiskLoc last ); + void updateSlaveLocation( CurOp& curop ); + void slaveReadTill( const OpTime& t ) { _slaveReadTill = t; } + /** Just for testing. */ + OpTime getSlaveReadTill() const { return _slaveReadTill; } + + // + // Query-specific functionality that may be adapted for the Runner. + // + + // Used by ops/query.cpp to stash how many results hvae been returned by a query. + int pos() const { return _pos; } + void incPos(int n) { _pos += n; } + void setPos(int n) { _pos = n; } + + // + // Yielding that is DEPRECATED. Will be removed when we use runners and they yield + // internally. + // + + /** + * DEPRECATED * @param microsToSleep -1 : ask client * 0 : pthread_yield or equivilant * >0 : sleep for that amount @@ -202,248 +205,208 @@ namespace mongo { * @return same as yield() */ bool yieldSometimes( RecordNeeds need, bool *yielded = 0 ); - - static int suggestYieldMicros(); - static void staticYield( int micros , const StringData& ns , Record * rec ); - struct YieldData { CursorId _id; bool _doingDeletes; }; bool prepareToYield( YieldData &data ); static bool recoverFromYield( const YieldData &data ); - - struct YieldLock : boost::noncopyable { - - explicit YieldLock( ptr<ClientCursor> cc ); - - ~YieldLock(); - - /** - * @return if the cursor is still ok - * if it is, we also relock - */ - bool stillOk(); - - void relock(); + static int suggestYieldMicros(); - private: - const bool _canYield; - YieldData _data; - scoped_ptr<dbtempreleasecond> _unlock; - }; + // + // Cursor-only DEPRECATED methods. + // - // --- some pass through helpers for Cursor --- + // Only used by ops/query.cpp, which will stop using them when queries are answered only by + // a runner. + const BSONObj& query() const { return _query; } + int queryOptions() const { return _queryOptions; } + shared_ptr<ParsedQuery> pq; + // This one is used also by pipeline/document_source_cursor.cpp + shared_ptr<Projection> fields; // which fields query wants returned + DiskLoc lastLoc() const { return _lastLoc; } Cursor* c() const { return _c.get(); } - int pos() const { return _pos; } - - void incPos( int n ) { _pos += n; } // TODO: this is bad - void setPos( int n ) { _pos = n; } // TODO : this is bad too - - BSONObj indexKeyPattern() { return _c->indexKeyPattern(); } - bool modifiedKeys() const { return _c->modifiedKeys(); } - bool isMultiKey() const { return _c->isMultiKey(); } - bool ok() { return _c->ok(); } bool advance() { return _c->advance(); } BSONObj current() { return _c->current(); } DiskLoc currLoc() { return _c->currLoc(); } BSONObj currKey() const { return _c->currKey(); } - /** - * same as BSONObj::getFieldsDotted - * if it can be retrieved from key, it is - * @param holder keeps the currKey in scope by keeping a reference to it here. generally you'll want - * holder and ret to destruct about the same time. - * @return if this was retrieved from key - */ - bool getFieldsDotted( const string& name, BSONElementSet &ret, BSONObj& holder ); - - /** - * same as BSONObj::getFieldDotted - * if it can be retrieved from key, it is - * @return if this was retrieved from key - */ - BSONElement getFieldDotted( const string& name , BSONObj& holder , bool * fromKey = 0 ) ; - - /** extract items from object which match a pattern object. - * e.g., if pattern is { x : 1, y : 1 }, builds an object with - * x and y elements of this object, if they are present. - * returns elements with original field names - * NOTE: copied from BSONObj::extractFields - */ - BSONObj extractFields(const BSONObj &pattern , bool fillWithNull = false) ; - - /** Extract elements from the object this cursor currently points to, using the expression - * specified in KeyPattern. Will use a covered index if the one in this cursor is usable. - * TODO: there are some cases where a covered index could be used but is not, for instance - * if both this index and the keyPattern are {a : "hashed"} - */ - BSONObj extractKey( const KeyPattern& usingKeyPattern ) const; - - void fillQueryResultFromObj( BufBuilder &b, const MatchDetails* details = NULL ) const; - bool currentIsDup() { return _c->getsetdup( _c->currLoc() ); } - bool currentMatches() { if ( ! _c->matcher() ) return true; return _c->matcher()->matchesCurrent( _c.get() ); } - void setCollMetadata( CollectionMetadataPtr metadata ){ _collMetadata = metadata; } - CollectionMetadataPtr getCollMetadata(){ return _collMetadata; } + void setDoingDeletes( bool doingDeletes ) {_doingDeletes = doingDeletes; } private: - void setLastLoc_inlock(DiskLoc); + friend class ClientCursorHolder; + friend class ClientCursorPin; + friend class ClientCursorYieldLock; + friend class CmdCursorInfo; - static ClientCursor* find_inlock(CursorId id, bool warn = true) { - CCById::iterator it = clientCursorsById.find(id); - if ( it == clientCursorsById.end() ) { - if ( warn ) { - OCCASIONALLY out() << "ClientCursor::find(): cursor not found in map '" << id - << "' (ok after a drop)" << endl; - } - return 0; - } - return it->second; - } - - /* call when cursor's location changes so that we can update the - cursorsbylocation map. if you are locked and internally iterating, only - need to call when you are ready to "unlock". - */ - void updateLocation(); + // A map from the CursorId to the ClientCursor behind it. + // TODO: Consider making this per-connection. + typedef map<CursorId, ClientCursor*> CCById; + static CCById clientCursorsById; - public: - static ClientCursor* find(CursorId id, bool warn = true) { - recursive_scoped_lock lock(ccmutex); - ClientCursor *c = find_inlock(id, warn); - // if this asserts, your code was not thread safe - you either need to set no timeout - // for the cursor or keep a ClientCursor::Pointer in scope for it. - massert( 12521, "internal error: use of an unlocked ClientCursor", c == 0 || c->_pinValue ); - return c; - } + // How many cursors have timed out? + static long long numberTimedOut; + + // This must be held when modifying any static member. + static boost::recursive_mutex& ccmutex; /** - * Deletes the cursor with the provided @param 'id' if one exists. - * @throw if the cursor with the provided id is pinned. - * This does not do any auth checking and should be used only when erasing cursors as part - * of cleaning up internal operations. + * Initialization common between Cursor and Runner. + * TODO: Remove when we're all-runner. */ - static bool erase(CursorId id); - // Same as erase but checks to make sure this thread has read permission on the cursor's - // namespace. This should be called when receiving killCursors from a client. This should - // not be called when ccmutex is held. - static bool eraseIfAuthorized(CursorId id); + void init(int qopts); /** - * @return number of cursors found + * Allocates a new CursorId. + * Called from init(...). Assumes ccmutex held. */ - static int erase(int n, long long* ids); - static int eraseIfAuthorized(int n, long long* ids); - - void mayUpgradeStorage() { - /* if ( !ids_.get() ) - return; - stringstream ss; - ss << ns << "." << cursorid; - ids_->mayUpgradeStorage( ss.str() );*/ - } + static CursorId allocCursorId_inlock(); /** - * @param millis amount of idle passed time since last call + * Find the ClientCursor with the provided ID. Optionally warn if it's not found. + * Assumes ccmutex is held. */ - bool shouldTimeout( unsigned millis ); - void storeOpForSlave( DiskLoc last ); - void updateSlaveLocation( CurOp& curop ); + static ClientCursor* find_inlock(CursorId id, bool warn = true); - unsigned idleTime() const { return _idleAgeMillis; } + /** + * Delete the ClientCursor with the provided ID. masserts if the cursor is pinned. + */ + static void _erase_inlock(ClientCursor* cursor); - uint64_t getLeftoverMaxTimeMicros() const { return _leftoverMaxTimeMicros; } - void setLeftoverMaxTimeMicros( uint64_t leftoverMaxTimeMicros ) { - _leftoverMaxTimeMicros = leftoverMaxTimeMicros; - } + // + // ClientCursor-specific data, independent of the underlying execution type. + // - void setDoingDeletes( bool doingDeletes ) {_doingDeletes = doingDeletes; } + // The ID of the ClientCursor. + CursorId _cursorid; - void slaveReadTill( const OpTime& t ) { _slaveReadTill = t; } - - /** Just for testing. */ - OpTime getSlaveReadTill() const { return _slaveReadTill; } + // A variable indicating the state of the ClientCursor. Possible values: + // 0: Normal behavior. May time out. + // 1: No timing out of this ClientCursor. + // 100: Currently in use (via ClientCursorPin). + unsigned _pinValue; - public: // static methods + // The namespace we're operating on. + const string _ns; - static void idleTimeReport(unsigned millis); + // The database we're operating on. + Database* _db; - static void appendStats( BSONObjBuilder& result ); - static unsigned numCursors() { return clientCursorsById.size(); } - static void aboutToDelete( const StringData& ns, - const NamespaceDetails* nsd, - const DiskLoc& dl ); - static void find( const string& ns , set<CursorId>& all ); + // How many objects have been returned by the find() so far? + int _pos; + // The query that prompted this ClientCursor. Only used for debugging. + const BSONObj _query; - private: // methods + // See the QueryOptions enum in dbclient.h + int _queryOptions; - // cursors normally timeout after an inactivity period to prevent excess memory use - // setting this prevents timeout of the cursor in question. - void noTimeout() { _pinValue++; } + // TODO: document better. + OpTime _slaveReadTill; - Record* _recordForYield( RecordNeeds need ); - static bool _erase_inlock(ClientCursor* cursor); + // How long has the cursor been idle? + unsigned _idleAgeMillis; - private: + // TODO: Document. + uint64_t _leftoverMaxTimeMicros; - CursorId _cursorid; + // TODO: document better. Somehow used in sharding. + CollectionMetadataPtr _collMetadata; - const string _ns; - Database * _db; + // + // The underlying execution machinery. + // - const shared_ptr<Cursor> _c; - map<string,int> _indexedFields; // map from indexed field to offset in key object - int _pos; // # objects into the cursor so far + // The new world: a runner. + scoped_ptr<Runner> _runner; - const BSONObj _query; // used for logging diags only; optional in constructor - int _queryOptions; // see enum QueryOptions dbclient.h + // + // Cursor-only private data and methods. DEPRECATED. + // - OpTime _slaveReadTill; + // The old world: a cursor. DEPRECATED. + const shared_ptr<Cursor> _c; + + /** + * call when cursor's location changes so that we can update the cursorsbylocation map. if + * you are locked and internally iterating, only need to call when you are ready to + * "unlock". + */ + void updateLocation(); + void setLastLoc_inlock(DiskLoc); + Record* _recordForYield( RecordNeeds need ); DiskLoc _lastLoc; // use getter and setter not this (important) - unsigned _idleAgeMillis; // how long has the cursor been around, relative to server idle time - - // For time-limited operations ($maxTimeMS): time remaining for future getmore operations - // on the cursor. 0 if original operation had no time limit set. - uint64_t _leftoverMaxTimeMicros; - - /* 0 = normal - 1 = no timeout allowed - 100 = in use (pinned) -- see Pointer class - */ - unsigned _pinValue; - bool _doingDeletes; // when true we are the delete and aboutToDelete shouldn't manipulate us - ElapsedTracker _yieldSometimesTracker; - CollectionMetadataPtr _collMetadata; + // TODO: This will be moved into the runner. + ElapsedTracker _yieldSometimesTracker; + }; + /** + * use this to assure we don't in the background time out cursor while it is under use. if you + * are using noTimeout() already, there is no risk anyway. Further, this mechanism guards + * against two getMore requests on the same cursor executing at the same time - which might be + * bad. That should never happen, but if a client driver had a bug, it could (or perhaps some + * sort of attack situation). + */ + class ClientCursorPin : boost::noncopyable { public: - shared_ptr<ParsedQuery> pq; - shared_ptr<Projection> fields; // which fields query wants returned - - private: // static members - - static CCById clientCursorsById; - static long long numberTimedOut; - static boost::recursive_mutex& ccmutex; // must use this for all statics above! - static CursorId allocCursorId_inlock(); + ClientCursorPin( long long cursorid ); + ~ClientCursorPin(); + void release(); + ClientCursor *c() const; + private: + CursorId _cursorid; + }; + /** Assures safe and reliable cleanup of a ClientCursor. */ + class ClientCursorHolder : boost::noncopyable { + public: + ClientCursorHolder( ClientCursor *c = 0 ); + ~ClientCursorHolder(); + void reset( ClientCursor *c = 0 ); + ClientCursor* get(); + operator bool() { return _c; } + ClientCursor * operator-> (); + const ClientCursor * operator-> () const; + /** Release ownership of the ClientCursor. */ + void release(); + private: + ClientCursor *_c; + CursorId _id; }; + /** thread for timing out old cursors */ class ClientCursorMonitor : public BackgroundJob { public: string name() const { return "ClientCursorMonitor"; } void run(); }; + + struct ClientCursorYieldLock : boost::noncopyable { + explicit ClientCursorYieldLock( ptr<ClientCursor> cc ); + ~ClientCursorYieldLock(); + + /** + * @return if the cursor is still ok + * if it is, we also relock + */ + bool stillOk(); + void relock(); + + private: + const bool _canYield; + ClientCursor::YieldData _data; + scoped_ptr<dbtempreleasecond> _unlock; + }; } // namespace mongo |