From 126069374744c62c65fac86a492531d658961f40 Mon Sep 17 00:00:00 2001 From: Randolph Tan Date: Mon, 6 Oct 2014 17:36:18 -0400 Subject: SERVER-15402 Improve ShardPtr/ConnectionString management for commands and setVersion Make Shard almost logically immutable. The only non-const method remaining is reset(), which will eventually be removed. --- src/mongo/s/chunk.cpp | 57 ++++++---------------- src/mongo/s/chunk.h | 24 ++++------ src/mongo/s/chunk_diff.h | 1 - src/mongo/s/chunk_diff_test.cpp | 1 - src/mongo/s/chunk_manager_targeter.cpp | 32 ++++++------- src/mongo/s/config.cpp | 16 +++++-- src/mongo/s/config.h | 6 ++- src/mongo/s/shard.cpp | 86 +++++++++++++++++++++------------- src/mongo/s/shard.h | 27 ++++++----- src/mongo/s/shard_test.cpp | 18 ++++--- src/mongo/s/version_manager.cpp | 31 ++++++++---- 11 files changed, 157 insertions(+), 142 deletions(-) (limited to 'src/mongo/s') diff --git a/src/mongo/s/chunk.cpp b/src/mongo/s/chunk.cpp index a9ffd904d89..8e75f8e3653 100644 --- a/src/mongo/s/chunk.cpp +++ b/src/mongo/s/chunk.cpp @@ -770,22 +770,7 @@ namespace mongo { _version = ChunkVersion::fromBSON( collDoc ); } - ChunkManager::ChunkManager( ChunkManagerPtr oldManager ) : - _ns( oldManager->getns() ), - _key( oldManager->getShardKey() ), - _unique( oldManager->isUnique() ), - _chunkRanges(), - _mutex("ChunkManager"), - _sequenceNumber(NextSequenceNumber.addAndFetch(1)) - { - // - // Sets up a chunk manager based on an older manager - // - - _oldManager = oldManager; - } - - void ChunkManager::loadExistingRanges( const string& config ){ + void ChunkManager::loadExistingRanges(const string& config, const ChunkManager* oldManager){ int tries = 3; while (tries--) { @@ -794,7 +779,7 @@ namespace mongo { ShardVersionMap shardVersions; Timer t; - bool success = _load( config, chunkMap, shards, shardVersions, _oldManager ); + bool success = _load(config, chunkMap, shards, shardVersions, oldManager); if( success ){ { @@ -803,7 +788,7 @@ namespace mongo { << " sequenceNumber: " << _sequenceNumber << " version: " << _version.toString() << " based on: " << - ( _oldManager.get() ? _oldManager->getVersion().toString() : "(empty)" ) + (oldManager ? oldManager->getVersion().toString() : "(empty)") << endl; } @@ -817,9 +802,6 @@ namespace mongo { const_cast(_shardVersions).swap(shardVersions); const_cast(_chunkRanges).reloadAll(_chunkMap); - // Once we load data, clear reference to old manager - _oldManager.reset(); - return; } } @@ -845,7 +827,7 @@ namespace mongo { * * The mongos adapter here tracks all shards, and stores ranges by (max, Chunk) in the map. */ - class CMConfigDiffTracker : public ConfigDiffTracker { + class CMConfigDiffTracker : public ConfigDiffTracker { public: CMConfigDiffTracker( ChunkManager* manager ) : _manager( manager ) {} @@ -865,11 +847,8 @@ namespace mongo { return make_pair( max, c ); } - virtual Shard shardFor( const string& name ) const { - return Shard::make( name ); - } - - virtual string nameFrom( const Shard& shard ) const { + virtual string shardFor(const string& hostName) const { + Shard shard = Shard::make(hostName); return shard.getName(); } @@ -877,11 +856,11 @@ namespace mongo { }; - bool ChunkManager::_load( const string& config, - ChunkMap& chunkMap, - set& shards, - ShardVersionMap& shardVersions, - ChunkManagerPtr oldManager) + bool ChunkManager::_load(const string& config, + ChunkMap& chunkMap, + set& shards, + ShardVersionMap& shardVersions, + const ChunkManager* oldManager) { // Reset the max version, but not the epoch, when we aren't loading from the oldManager @@ -1410,11 +1389,10 @@ namespace mongo { return bounds; } - bool ChunkManager::compatibleWith( const ChunkManager& other, const Shard& shard ) const { + bool ChunkManager::compatibleWith(const ChunkManager& other, const string& shardName) const { // Return true if the shard version is the same in the two chunk managers // TODO: This doesn't need to be so strong, just major vs - return other.getVersion( shard ).equals( getVersion( shard ) ); - + return other.getVersion(shardName).equals(getVersion(shardName)); } bool ChunkManager::compatibleWith( const Chunk& other ) const { @@ -1529,13 +1507,8 @@ namespace mongo { configServer.logChange( "dropCollection" , _ns , BSONObj() ); } - ChunkVersion ChunkManager::getVersion( const StringData& shardName ) const { - // NOTE: The empty-address Shard constructor is needed to avoid triggering a reload - return getVersion( Shard( shardName.toString(), "" ) ); - } - - ChunkVersion ChunkManager::getVersion( const Shard& shard ) const { - ShardVersionMap::const_iterator i = _shardVersions.find( shard ); + ChunkVersion ChunkManager::getVersion(const std::string& shardName) const { + ShardVersionMap::const_iterator i = _shardVersions.find(shardName); if ( i == _shardVersions.end() ) { // Shards without explicitly tracked shard versions (meaning they have // no chunks) always have a version of (0, 0, epoch). Note this is diff --git a/src/mongo/s/chunk.h b/src/mongo/s/chunk.h index 1b151704307..c77b1286519 100644 --- a/src/mongo/s/chunk.h +++ b/src/mongo/s/chunk.h @@ -362,7 +362,7 @@ namespace mongo { */ class ChunkManager { public: - typedef std::map ShardVersionMap; + typedef std::map ShardVersionMap; // Loads a new chunk manager from a collection document ChunkManager( const BSONObj& collDoc ); @@ -370,9 +370,6 @@ namespace mongo { // Creates an empty chunk manager for the namespace ChunkManager( const std::string& ns, const ShardKeyPattern& pattern, bool unique ); - // Updates a chunk manager based on an older manager - ChunkManager( ChunkManagerPtr oldManager ); - std::string getns() const { return _ns; } const ShardKeyPattern& getShardKey() const { return _key; } @@ -400,7 +397,7 @@ namespace mongo { const std::vector* initShards ); // Loads existing ranges based on info in chunk manager - void loadExistingRanges( const std::string& config ); + void loadExistingRanges(const std::string& config, const ChunkManager* oldManager); // Helpers for load @@ -465,16 +462,14 @@ namespace mongo { /** * Returns true if, for this shard, the chunks are identical in both chunk managers */ - bool compatibleWith( const ChunkManager& other, const Shard& shard ) const; - bool compatibleWith( ChunkManagerPtr other, const Shard& shard ) const { if( ! other ) return false; return compatibleWith( *other, shard ); } + bool compatibleWith(const ChunkManager& other, const std::string& shard) const; bool compatibleWith( const Chunk& other ) const; bool compatibleWith( ChunkPtr other ) const { if( ! other ) return false; return compatibleWith( *other ); } std::string toString() const; - ChunkVersion getVersion( const StringData& shardName ) const; - ChunkVersion getVersion( const Shard& shard ) const; + ChunkVersion getVersion(const std::string& shardName) const; ChunkVersion getVersion() const; void getInfo( BSONObjBuilder& b ) const; @@ -498,8 +493,11 @@ namespace mongo { // helpers for loading // returns true if load was consistent - bool _load( const std::string& config, ChunkMap& chunks, std::set& shards, - ShardVersionMap& shardVersions, ChunkManagerPtr oldManager); + bool _load(const std::string& config, + ChunkMap& chunks, + std::set& shards, + ShardVersionMap& shardVersions, + const ChunkManager* oldManager); static bool _isValid(const ChunkMap& chunks); // end helpers @@ -519,10 +517,6 @@ namespace mongo { // max version of any chunk ChunkVersion _version; - // the previous manager this was based on - // cleared after loading chunks - ChunkManagerPtr _oldManager; - mutable mutex _mutex; // only used with _nsLock const unsigned long long _sequenceNumber; diff --git a/src/mongo/s/chunk_diff.h b/src/mongo/s/chunk_diff.h index 70588694617..a4e7145ea47 100644 --- a/src/mongo/s/chunk_diff.h +++ b/src/mongo/s/chunk_diff.h @@ -124,7 +124,6 @@ namespace mongo { virtual std::pair rangeFor( const BSONObj& chunkDoc, const BSONObj& min, const BSONObj& max ) const = 0; virtual ShardType shardFor( const std::string& name ) const = 0; - virtual std::string nameFrom( const ShardType& shard ) const = 0; /// /// End adapter functions diff --git a/src/mongo/s/chunk_diff_test.cpp b/src/mongo/s/chunk_diff_test.cpp index 5a9b61068d5..97b650f8e34 100644 --- a/src/mongo/s/chunk_diff_test.cpp +++ b/src/mongo/s/chunk_diff_test.cpp @@ -64,7 +64,6 @@ namespace { } virtual string shardFor(const string& name) const { return name; } - virtual string nameFrom(const string& shard) const { return shard; } }; TEST(Basics, Simple) { diff --git a/src/mongo/s/chunk_manager_targeter.cpp b/src/mongo/s/chunk_manager_targeter.cpp index bbebd22f70c..1f3d24ab8c3 100644 --- a/src/mongo/s/chunk_manager_targeter.cpp +++ b/src/mongo/s/chunk_manager_targeter.cpp @@ -113,8 +113,8 @@ namespace mongo { } ChunkPtr chunk = _manager->findChunkForDoc( doc ); - *endpoint = new ShardEndpoint( chunk->getShard().getName(), - _manager->getVersion( chunk->getShard() ) ); + *endpoint = new ShardEndpoint(chunk->getShard().getName(), + _manager->getVersion(chunk->getShard().getName())); // Track autosplit stats for sharded collections _stats->chunkSizeDelta[chunk->getMin()] += doc.objsize(); @@ -330,10 +330,10 @@ namespace mongo { } for ( set::iterator it = shards.begin(); it != shards.end(); ++it ) { - endpoints->push_back( new ShardEndpoint( it->getName(), - _manager ? - _manager->getVersion( *it ) : - ChunkVersion::UNSHARDED() ) ); + endpoints->push_back(new ShardEndpoint(it->getName(), + _manager ? + _manager->getVersion(it->getName()) : + ChunkVersion::UNSHARDED())); } return Status::OK(); @@ -349,7 +349,7 @@ namespace mongo { Shard shard = chunk->getShard(); *endpoint = new ShardEndpoint(shard.getName(), - _manager->getVersion(StringData(shard.getName()))); + _manager->getVersion(shard.getName())); return Status::OK(); } @@ -372,10 +372,10 @@ namespace mongo { } for ( set::iterator it = shards.begin(); it != shards.end(); ++it ) { - endpoints->push_back( new ShardEndpoint( it->getName(), - _manager ? - _manager->getVersion( *it ) : - ChunkVersion::UNSHARDED() ) ); + endpoints->push_back(new ShardEndpoint(it->getName(), + _manager ? + _manager->getVersion(it->getName()) : + ChunkVersion::UNSHARDED())); } return Status::OK(); @@ -394,10 +394,10 @@ namespace mongo { Shard::getAllShards( shards ); for ( vector::iterator it = shards.begin(); it != shards.end(); ++it ) { - endpoints->push_back( new ShardEndpoint( it->getName(), - _manager ? - _manager->getVersion( *it ) : - ChunkVersion::UNSHARDED() ) ); + endpoints->push_back(new ShardEndpoint(it->getName(), + _manager ? + _manager->getVersion(it->getName()) : + ChunkVersion::UNSHARDED())); } return Status::OK(); @@ -445,7 +445,7 @@ namespace mongo { if ( primary ) return ChunkVersion::UNSHARDED(); - return manager->getVersion( shardName ); + return manager->getVersion(shardName.toString()); } /** diff --git a/src/mongo/s/config.cpp b/src/mongo/s/config.cpp index 16c491f3d86..23cc03a27e5 100644 --- a/src/mongo/s/config.cpp +++ b/src/mongo/s/config.cpp @@ -81,7 +81,7 @@ namespace mongo { void DBConfig::CollectionInfo::shard( ChunkManager* manager ){ // Do this *first* so we're invisible to everyone else - manager->loadExistingRanges( configServer.getPrimary().getConnString() ); + manager->loadExistingRanges(configServer.getPrimary().getConnString(), NULL); // // Collections with no chunks are unsharded, no matter what the collections entry says @@ -435,8 +435,10 @@ namespace mongo { } - temp.reset( new ChunkManager( oldManager ) ); - temp->loadExistingRanges( configServer.getPrimary().getConnString() ); + temp.reset(new ChunkManager(oldManager->getns(), + oldManager->getShardKey(), + oldManager->isUnique())); + temp->loadExistingRanges(configServer.getPrimary().getConnString(), oldManager.get()); if ( temp->numChunks() == 0 ) { // maybe we're not sharded any more @@ -826,7 +828,13 @@ namespace mongo { string fullString; joinStringDelim( configHosts, &fullString, ',' ); - _primary.setAddress( ConnectionString( fullString , ConnectionString::SYNC ) ); + _primary = Shard(_primary.getName(), + ConnectionString(fullString, ConnectionString::SYNC), + _primary.getMaxSize(), + _primary.isDraining(), + _primary.tags()); + Shard::installShard(_primary.getName(), _primary); + LOG(1) << " config string : " << fullString << endl; return true; diff --git a/src/mongo/s/config.h b/src/mongo/s/config.h index 06412248118..6dba0f3b9aa 100644 --- a/src/mongo/s/config.h +++ b/src/mongo/s/config.h @@ -103,7 +103,11 @@ namespace mongo { DBConfig( std::string name ) : _name( name ) , - _primary("config","") , + _primary("config", + "", + 0 /* maxSize */, + false /* draining */, + BSONArray() /* tags */), _shardingEnabled(false), _lock("DBConfig") , _hitConfigServerLock( "DBConfig::_hitConfigServerLock" ) { diff --git a/src/mongo/s/shard.cpp b/src/mongo/s/shard.cpp index 0c1521c23f9..a7de364a65a 100644 --- a/src/mongo/s/shard.cpp +++ b/src/mongo/s/shard.cpp @@ -55,6 +55,7 @@ #include "mongo/s/type_shard.h" #include "mongo/s/version_manager.h" #include "mongo/util/log.h" +#include "mongo/util/stacktrace.h" namespace mongo { @@ -116,34 +117,26 @@ namespace mongo { } _rsLookup.clear(); - for ( list::iterator i=all.begin(); i!=all.end(); ++i ) { - BSONObj o = *i; - string name = o[ ShardType::name() ].String(); - string host = o[ ShardType::host() ].String(); - - long long maxSize = 0; - BSONElement maxSizeElem = o[ ShardType::maxSize.name() ]; - if ( ! maxSizeElem.eoo() ) { - maxSize = maxSizeElem.numberLong(); - } - - bool isDraining = false; - BSONElement isDrainingElem = o[ ShardType::draining.name() ]; - if ( ! isDrainingElem.eoo() ) { - isDraining = isDrainingElem.Bool(); - } + for (list::const_iterator iter = all.begin(); iter != all.end(); ++iter) { + ShardType shardData; - ShardPtr s( new Shard( name , host , maxSize , isDraining ) ); - - if ( o[ ShardType::tags() ].type() == Array ) { - vector v = o[ ShardType::tags() ].Array(); - for ( unsigned j=0; jaddTag( v[j].String() ); - } + string errmsg; + if (!shardData.parseBSON(*iter, &errmsg) || !shardData.isValid(&errmsg)) { + uasserted(28530, errmsg); } - _lookup[name] = s; - _installHost( host , s ); + const long long maxSize = shardData.isMaxSizeSet() ? shardData.getMaxSize() : 0; + const bool isDraining = shardData.isDrainingSet() ? + shardData.getDraining() : false; + const BSONArray tags = shardData.isTagsSet() ? shardData.getTags() : BSONArray(); + ShardPtr shard = boost::make_shared(shardData.getName(), + shardData.getHost(), + maxSize, + isDraining, + tags); + + _lookup[shardData.getName()] = shard; + _installHost(shardData.getHost(), shard); } } @@ -329,7 +322,7 @@ namespace mongo { private: typedef map ShardMap; - ShardMap _lookup; + ShardMap _lookup; // Map of both shardName -> Shard and hostName -> Shard ShardMap _rsLookup; // Map from ReplSet name to shard mutable mongo::mutex _mutex; mutable mongo::mutex _rsMutex; @@ -355,6 +348,36 @@ namespace mongo { } } cmdGetShardMap; + Shard::Shard(const std::string& name, + const std::string& addr, + long long maxSize, + bool isDraining, + const BSONArray& tags): + _name(name), + _addr(addr), + _maxSize(maxSize), + _isDraining(isDraining) { + _setAddr(addr); + + BSONArrayIteratorSorted iter(tags); + while (iter.more()) { + BSONElement tag = iter.next(); + _tags.insert(tag.String()); + } + } + + Shard::Shard(const std::string& name, + const ConnectionString& connStr, + long long maxSize, + bool isDraining, + const set& tags): + _name(name), + _addr(connStr.toString()), + _cs(connStr), + _maxSize(maxSize), + _isDraining(isDraining), + _tags(tags) { + } Shard Shard::findIfExists( const string& shardName ) { ShardPtr shard = staticShardInfo.findIfExists( shardName ); @@ -368,13 +391,6 @@ namespace mongo { } } - void Shard::setAddress( const ConnectionString& cs) { - verify( _name.size() ); - _addr = cs.toString(); - _cs = cs; - staticShardInfo.set( _name , *this , true , false ); - } - void Shard::reset( const string& ident ) { *this = staticShardInfo.findCopy( ident ); } @@ -472,6 +488,10 @@ namespace mongo { return best.shard(); } + void Shard::installShard(const std::string& name, const Shard& shard) { + staticShardInfo.set(name, shard, true, false); + } + ShardStatus::ShardStatus( const Shard& shard , const BSONObj& obj ) : _shard( shard ) { _mapped = obj.getFieldDotted( "mem.mapped" ).numberLong(); diff --git a/src/mongo/s/shard.h b/src/mongo/s/shard.h index df0e59bd2f5..58108690685 100644 --- a/src/mongo/s/shard.h +++ b/src/mongo/s/shard.h @@ -49,10 +49,17 @@ namespace mongo { : _name("") , _addr("") , _maxSize(0) , _isDraining( false ) { } - Shard( const std::string& name , const std::string& addr, long long maxSize = 0 , bool isDraining = false ) - : _name(name) , _addr( addr ) , _maxSize( maxSize ) , _isDraining( isDraining ) { - _setAddr( addr ); - } + Shard(const std::string& name, + const std::string& addr, + long long maxSize, + bool isDraining, + const BSONArray& tags); + + Shard(const std::string& name, + const ConnectionString& connStr, + long long maxSize, + bool isDraining, + const std::set& tags); Shard( const std::string& ident ) { reset( ident ); @@ -64,11 +71,6 @@ namespace mongo { _tags( other._tags ) { } - Shard( const Shard* other ) - : _name( other->_name ) , _addr( other->_addr ), _cs( other->_cs ) , - _maxSize( other->_maxSize ) , _isDraining( other->_isDraining ) { - } - static Shard make( const std::string& ident ) { Shard s; s.reset( ident ); @@ -82,8 +84,6 @@ namespace mongo { */ void reset( const std::string& ident ); - void setAddress( const ConnectionString& cs ); - ConnectionString getAddress() const { return _cs; } std::string getName() const { @@ -152,7 +152,6 @@ namespace mongo { bool containsNode( const std::string& node ) const; const std::set& tags() const { return _tags; } - void addTag( const std::string& tag ) { _tags.insert( tag ); } static void getAllShards( std::vector& all ); static void printShardInfo( std::ostream& out ); @@ -172,10 +171,12 @@ namespace mongo { static Shard EMPTY; + static void installShard(const std::string& name, const Shard& shard); + private: void _setAddr( const std::string& addr ); - + std::string _name; std::string _addr; ConnectionString _cs; diff --git a/src/mongo/s/shard_test.cpp b/src/mongo/s/shard_test.cpp index 4b2e21707df..0e90f86e408 100644 --- a/src/mongo/s/shard_test.cpp +++ b/src/mongo/s/shard_test.cpp @@ -34,19 +34,23 @@ namespace mongo { TEST( Shard, EqualityRs ) { - Shard a( "foo", "bar/a,b" ); - Shard b( "foo", "bar/a,b" ); + Shard a("foo", "bar/a,b", 0, false, BSONArray()); + Shard b("foo", "bar/a,b", 0, false, BSONArray()); ASSERT_EQUALS( a, b ); - b = Shard( "foo", "bar/b,a" ); + b = Shard("foo", "bar/b,a", 0, false, BSONArray()); ASSERT_EQUALS( a, b ); } TEST( Shard, EqualitySingle ) { - ASSERT_EQUALS( Shard( "foo", "b.foo.com:123"), Shard( "foo", "b.foo.com:123") ); - ASSERT_NOT_EQUALS( Shard( "foo", "b.foo.com:123"), Shard( "foo", "a.foo.com:123") ); - ASSERT_NOT_EQUALS( Shard( "foo", "b.foo.com:123"), Shard( "foo", "b.foo.com:124") ); - ASSERT_NOT_EQUALS( Shard( "foo", "b.foo.com:123"), Shard( "foa", "b.foo.com:123") ); + ASSERT_EQUALS(Shard("foo", "b.foo.com:123", 0, false, BSONArray()), + Shard("foo", "b.foo.com:123", 0, false, BSONArray())); + ASSERT_NOT_EQUALS(Shard("foo", "b.foo.com:123", 0, false, BSONArray()), + Shard("foo", "a.foo.com:123", 0, false, BSONArray())); + ASSERT_NOT_EQUALS(Shard("foo", "b.foo.com:123", 0, false, BSONArray()), + Shard("foo", "b.foo.com:124", 0, false, BSONArray())); + ASSERT_NOT_EQUALS(Shard("foo", "b.foo.com:123", 0, false, BSONArray()), + Shard("foa", "b.foo.com:123", 0, false, BSONArray())); } TEST( Shard, EqualitySync ) { diff --git a/src/mongo/s/version_manager.cpp b/src/mongo/s/version_manager.cpp index e0be9a67c5e..9c94be9da9d 100644 --- a/src/mongo/s/version_manager.cpp +++ b/src/mongo/s/version_manager.cpp @@ -224,11 +224,21 @@ namespace mongo { if( isSharded && manager ){ Shard shard = Shard::make( conn->getServerAddress() ); - if( refManager && ! refManager->compatibleWith( manager, shard ) ){ - throw SendStaleConfigException( ns, str::stream() << "manager (" << manager->getVersion( shard ).toString() << " : " << manager->getSequenceNumber() << ") " - << "not compatible with reference manager (" << refManager->getVersion( shard ).toString() << " : " << refManager->getSequenceNumber() << ") " - << "on shard " << shard.getName() << " (" << shard.getAddress().toString() << ")", - refManager->getVersion( shard ), manager->getVersion( shard ) ); + if(refManager && !refManager->compatibleWith(*manager, shard.getName())) { + const ChunkVersion refVersion(refManager->getVersion(shard.getName())); + const ChunkVersion currentVersion(manager->getVersion(shard.getName())); + string msg(str::stream() << "manager (" + << currentVersion.toString() + << " : " << manager->getSequenceNumber() << ") " + << "not compatible with reference manager (" + << refVersion.toString() + << " : " << refManager->getSequenceNumber() << ") " + << "on shard " << shard.getName() + << " (" << shard.getAddress().toString() << ")"); + throw SendStaleConfigException(ns, + msg, + refVersion, + currentVersion); } } else if( refManager ){ @@ -241,8 +251,10 @@ namespace mongo { << "on conn " << conn->getServerAddress() << " (" << conn_in->getServerAddress() << ")" ); - throw SendStaleConfigException( ns, msg, - refManager->getVersion( shard ), ChunkVersion( 0, 0, OID() )); + throw SendStaleConfigException(ns, + msg, + refManager->getVersion(shard.getName()), + ChunkVersion::UNSHARDED()); } // has the ChunkManager been reloaded since the last time we updated the connection-level version? @@ -252,9 +264,10 @@ namespace mongo { return false; } - ChunkVersion version = ChunkVersion( 0, 0, OID() ); + ChunkVersion version(0, 0, OID()); if ( isSharded && manager ) { - version = manager->getVersion( Shard::make( conn->getServerAddress() ) ); + Shard shard(Shard::make(conn->getServerAddress())); + version = manager->getVersion(shard.getName()); } if( ! version.isSet() ){ -- cgit v1.2.1