SERVER-15375 lazily initShardVersion only when needed

6 files changed, 182 insertions, 145 deletions

diff --git a/src/mongo/s/chunk.cpp b/src/mongo/s/chunk.cpp
index 8e75f8e3653..d7a5eae8d81 100644
--- a/src/mongo/s/chunk.cpp
+++ b/src/mongo/s/chunk.cpp
@@ -1682,25 +1682,29 @@ namespace mongo {
         BSONObjBuilder cmdBuilder;
         cmdBuilder.append( "setShardVersion" , ns.c_str() );
         cmdBuilder.append( "configdb" , configServer.modelServer() );
-        version.addToBSON( cmdBuilder );
-        cmdBuilder.appendOID( "serverID" , &serverID );
-        if ( authoritative )
-            cmdBuilder.appendBool( "authoritative" , 1 );
-
-        Shard s = Shard::make( conn.getServerAddress() );
-        cmdBuilder.append( "shard" , s.getName() );
-        cmdBuilder.append( "shardHost" , s.getConnString() );
+
+        Shard s = Shard::make(conn.getServerAddress());
+        cmdBuilder.append("shard", s.getName());
+        cmdBuilder.append("shardHost", s.getConnString());
+
+        cmdBuilder.appendOID("serverID", &serverID);
+
+        if (ns.size() > 0) {
+            version.addToBSON(cmdBuilder);
+        }
+        else {
+            cmdBuilder.append("init", true);
+        }
+
+        if (authoritative)
+            cmdBuilder.appendBool("authoritative", 1);
+
         BSONObj cmd = cmdBuilder.obj();
 
-        LOG(1).stream()
-               << "    setShardVersion  " << s.getName()
-               << " " << conn.getServerAddress()
-               << "  " << ns
-               << "  " << cmd
-               << " " << &conn
-               << (manager.get() ? string(str::stream() << " " << manager->getSequenceNumber()) :
-                                   "")
-               << endl;
+        LOG(1) << "    setShardVersion  " << s.getName() << " " << conn.getServerAddress()
+               << "  " << ns << "  " << cmd
+               << (manager.get() ?
+                   string(str::stream() << " " << manager->getSequenceNumber()) : "");
 
         return conn.runCommand("admin", cmd, result, 0);
     }
diff --git a/src/mongo/s/default_version.cpp b/src/mongo/s/default_version.cpp
index 246a0fcca0c..90f7315ddce 100644
--- a/src/mongo/s/default_version.cpp
+++ b/src/mongo/s/default_version.cpp
@@ -45,10 +45,6 @@ namespace mongo {
         return false;
     }
 
-    bool VersionManager::initShardVersionCB( DBClientBase * conn_in, BSONObj& result ){
-        return false;
-    }
-
     bool VersionManager::forceRemoteCheckShardVersionCB( const string& ns ){
         return true;
     }
diff --git a/src/mongo/s/shard.cpp b/src/mongo/s/shard.cpp
index a7de364a65a..089407241f7 100644
--- a/src/mongo/s/shard.cpp
+++ b/src/mongo/s/shard.cpp
@@ -500,6 +500,9 @@ namespace mongo {
     }
 
     void ShardingConnectionHook::onCreate( DBClientBase * conn ) {
+
+        // Authenticate as the first thing we do
+        // NOTE: Replica set authentication allows authentication against *any* online host
         if(getGlobalAuthorizationManager()->isAuthEnabled()) {
             LOG(2) << "calling onCreate auth for " << conn->toString() << endl;
 
@@ -510,33 +513,23 @@ namespace mongo {
                      result );
         }
 
-        if ( _shardedConnections ) {
-            if ( versionManager.isVersionableCB( conn )) {
-                // We must initialize sharding on all connections, so that we get exceptions
-                // if sharding is enabled on the collection.
-                BSONObj result;
-                bool ok = versionManager.initShardVersionCB( conn, result );
-
-                // assert that we actually successfully setup sharding
-                uassert( 15907,
-                         str::stream() << "could not initialize sharding on connection "
-                             << conn->toString() << ( result["errmsg"].type() == String ?
-                                  causedBy( result["errmsg"].String() ) :
-                                  causedBy( (string)"unknown failure : " + result.toString() ) ),
-                         ok );
-            }
-            else {
-                // Initialize the wire protocol version of the connection to find out if we
-                // can send write commands to this connection.
-                string errMsg;
-                if ( !initWireVersion( conn, &errMsg )) {
-                    uasserted( 17363, errMsg );
-                }
+        // Initialize the wire version of single connections
+        if (conn->type() == ConnectionString::MASTER) {
+
+            LOG(2) << "checking wire version of new connection " << conn->toString();
+
+            // Initialize the wire protocol version of the connection to find out if we
+            // can send write commands to this connection.
+            string errMsg;
+            if (!initWireVersion(conn, &errMsg)) {
+                uasserted(17363, errMsg);
             }
+        }
 
+        if ( _shardedConnections ) {
             // For every DBClient created by mongos, add a hook that will capture the response from
-            // commands, so that we can target the correct node when subsequent getLastError calls
-            // are made by mongos.
+            // commands we pass along from the client, so that we can target the correct node when
+            // subsequent getLastError calls are made by mongos.
             conn->setPostRunCommandHook(stdx::bind(&saveGLEStats, stdx::placeholders::_1, stdx::placeholders::_2));
         }
 
diff --git a/src/mongo/s/shardconnection.cpp b/src/mongo/s/shardconnection.cpp
index b93ac07e37e..dd4b705a951 100644
--- a/src/mongo/s/shardconnection.cpp
+++ b/src/mongo/s/shardconnection.cpp
@@ -180,7 +180,13 @@ namespace mongo {
         }
 
         DBClientBase * get( const string& addr , const string& ns ) {
-            _check( ns );
+
+            {
+                // We want to report ns stats
+                scoped_spinlock lock(_lock);
+                if (ns.size() > 0)
+                    _seenNS.insert(ns);
+            }
 
             Status* s = _getStatus( addr );
 
@@ -286,19 +292,6 @@ namespace mongo {
             shardConnectionPool.release( addr , conn );
         }
 
-        void _check( const string& ns ) {
-
-            {
-                // We want to report ns stats too
-                scoped_spinlock lock( _lock );
-                if ( ns.size() == 0 || _seenNS.count( ns ) )
-                    return;
-                _seenNS.insert( ns );
-            }
-
-            checkVersions( ns );
-        }
-        
         /**
          * Appends info about the client connection pool to a BOBuilder
          * Safe to call with activeClientConnections lock
@@ -417,17 +410,17 @@ namespace mongo {
             return;
         _finishedInit = true;
 
-        if ( _ns.size() && versionManager.isVersionableCB( _conn ) ) {
+        if (versionManager.isVersionableCB(_conn)) {
             // Make sure we specified a manager for the correct namespace
-            if( _manager ) verify( _manager->getns() == _ns );
+            if (_ns.size() && _manager)
+                verify(_manager->getns() == _ns);
             _setVersion = versionManager.checkShardVersionCB( this , false , 1 );
         }
         else {
-            // Make sure we didn't specify a manager for an empty namespace
-            verify( ! _manager );
+            // Make sure we didn't specify a manager for a non-versionable connection (i.e. config)
+            verify(!_manager);
             _setVersion = false;
         }
-
     }
 
     void ShardConnection::done() {
diff --git a/src/mongo/s/version_manager.cpp b/src/mongo/s/version_manager.cpp
index 9c94be9da9d..0155c2d71ea 100644
--- a/src/mongo/s/version_manager.cpp
+++ b/src/mongo/s/version_manager.cpp
@@ -47,21 +47,44 @@ namespace mongo {
     // Global version manager
     VersionManager versionManager;
 
-    // when running in sharded mode, use chunk shard version control
+    /**
+     * Tracking information, per-connection, of the latest chunk manager iteration or sequence
+     * number that was used to send a shard version over this connection.
+     * When the chunk manager is replaced, implying new versions were loaded, the chunk manager
+     * sequence number is iterated by 1 and connections need to re-send shard versions.
+     */
     struct ConnectionShardStatus {
 
-        typedef unsigned long long S;
-
         ConnectionShardStatus()
             : _mutex( "ConnectionShardStatus" ) {
         }
 
-        S getSequence( DBClientBase * conn , const string& ns ) {
-            scoped_lock lk( _mutex );
-            return _map[conn->getConnectionId()][ns];
+        bool hasAnySequenceSet(DBClientBase* conn) {
+            scoped_lock lk(_mutex);
+
+            SequenceMap::const_iterator seenConnIt = _map.find(conn->getConnectionId());
+            return seenConnIt != _map.end() && seenConnIt->second.size() > 0;
+        }
+
+        bool getSequence(DBClientBase * conn,
+                         const string& ns,
+                         unsigned long long* sequence) {
+
+            scoped_lock lk(_mutex);
+
+            SequenceMap::const_iterator seenConnIt = _map.find(conn->getConnectionId());
+            if (seenConnIt == _map.end())
+                return false;
+
+            map<string, unsigned long long>::const_iterator seenNSIt = seenConnIt->second.find(ns);
+            if (seenNSIt == seenConnIt->second.end())
+                return false;
+
+            *sequence = seenNSIt->second;
+            return true;
         }
 
-        void setSequence( DBClientBase * conn , const string& ns , const S& s ) {
+        void setSequence( DBClientBase * conn , const string& ns , const unsigned long long& s ) {
             scoped_lock lk( _mutex );
             _map[conn->getConnectionId()][ns] = s;
         }
@@ -75,7 +98,8 @@ namespace mongo {
         mongo::mutex _mutex;
 
         // a map from a connection into ChunkManager's sequence number for each namespace
-        map<unsigned long long, map<string,unsigned long long> > _map;
+        typedef map<unsigned long long, map<string,unsigned long long> > SequenceMap;
+        SequenceMap _map;
 
     } connectionShardStatus;
 
@@ -113,34 +137,60 @@ namespace mongo {
         return NULL;
     }
 
-    extern OID serverID;
+    bool VersionManager::forceRemoteCheckShardVersionCB( const string& ns ){
+
+        DBConfigPtr conf = grid.getDBConfig( ns );
+        if ( ! conf ) return false;
+        conf->reload();
+
+        // If we don't have a collection, don't refresh the chunk manager
+        if( nsGetCollection( ns ).size() == 0 ) return false;
+
+        ChunkManagerPtr manager = conf->getChunkManagerIfExists( ns, true, true );
+        if( ! manager ) return false;
+
+        return true;
+
+    }
 
-    bool VersionManager::initShardVersionCB( DBClientBase * conn_in, BSONObj& result ){
+    /**
+     * Special internal logic to run reduced version handshake for empty namespace operations to
+     * shards.
+     *
+     * Eventually this should go completely away, but for now many commands rely on unversioned but
+     * mongos-specific behavior on mongod (auditing and replication information in commands)
+     */
+    static bool initShardVersionEmptyNS(DBClientBase * conn_in) {
 
         bool ok;
+        BSONObj result;
         DBClientBase* conn = NULL;
         try {
             // May throw if replica set primary is down
             conn = getVersionable( conn_in );
             dassert( conn ); // errors thrown above
 
-            BSONObjBuilder cmdBuilder;
+            // Check to see if we've already initialized this connection
+            if (connectionShardStatus.hasAnySequenceSet(conn))
+                return false;
 
-            cmdBuilder.append( "setShardVersion" , "" );
-            cmdBuilder.appendBool( "init", true );
-            cmdBuilder.append( "configdb" , configServer.modelServer() );
-            cmdBuilder.appendOID( "serverID" , &serverID );
-            cmdBuilder.appendBool( "authoritative" , true );
+            // Check to see if this is actually a shard and not a single config server
+            // NOTE: Config servers are registered only by the name "config" in the shard cache, not
+            // by host, so lookup by host will fail unless the host is also a shard.
+            Shard shard = Shard::findIfExists(conn->getServerAddress());
+            if (!shard.ok())
+                return false;
 
-            BSONObj cmd = cmdBuilder.obj();
+            LOG(1) << "initializing shard connection to " << shard.toString() << endl;
 
-            LOG(1) << "initializing shard connection to " << conn->toString() << endl;
-            LOG(2) << "initial sharding settings : " << cmd << endl;
-
-            ok = conn->runCommand("admin", cmd, result, 0);
+            ok = setShardVersion(*conn, "", ChunkVersion(), ChunkManagerPtr(), true, result);
         }
         catch( const DBException& ) {
 
+            // NOTE: Replica sets may fail to initShardVersion because future calls relying on
+            // correct versioning must later call checkShardVersion on the primary.
+            // Secondary queries and commands may not call checkShardVersion, but secondary ops
+            // aren't versioned at all.
             if ( conn_in->type() != ConnectionString::SET ) {
                 throw;
             }
@@ -154,15 +204,7 @@ namespace mongo {
                           << "will initialize on first use" << endl;
             }
 
-            return true;
-        }
-
-        // HACK for backwards compatibility with v1.8.x, v2.0.0 and v2.0.1
-        // Result is false, but will still initialize serverID and configdb
-        if( ! ok && ! result["errmsg"].eoo() && ( result["errmsg"].String() == "need to specify namespace"/* 2.0.1/2 */ ||
-                                                  result["errmsg"].String() == "need to speciy namespace" /* 1.8 */ ))
-        {
-            ok = true;
+            return false;
         }
 
         // Record the connection wire version if sent in the response, initShardVersion is a
@@ -176,32 +218,37 @@ namespace mongo {
 
         LOG(3) << "initial sharding result : " << result << endl;
 
-        return ok;
-
-    }
-
-    bool VersionManager::forceRemoteCheckShardVersionCB( const string& ns ){
-
-        DBConfigPtr conf = grid.getDBConfig( ns );
-        if ( ! conf ) return false;
-        conf->reload();
-
-        // If we don't have a collection, don't refresh the chunk manager
-        if( nsGetCollection( ns ).size() == 0 ) return false;
-
-        ChunkManagerPtr manager = conf->getChunkManagerIfExists( ns, true, true );
-        if( ! manager ) return false;
-
+        connectionShardStatus.setSequence(conn, "", 0);
         return true;
-
     }
 
     /**
-     * @return true if had to do something
+     * Updates the remote cached version on the remote shard host (primary, in the case of replica
+     * sets) if needed with a fully-qualified shard version for the given namespace:
+     *   config server(s) + shard name + shard version
+     *
+     * If no remote cached version has ever been set, an initial shard version is sent.
+     *
+     * If the namespace is empty and no version has ever been sent, the config server + shard name
+     * is sent to the remote shard host to initialize the connection as coming from mongos.
+     * NOTE: This initialization is *best-effort only*.  Operations which wish to correctly version
+     * must send the namespace.
+     *
+     * Config servers are special and are not (unless otherwise a shard) kept up to date with this
+     * protocol.  This is safe so long as config servers only contain unversioned collections.
+     *
+     * It is an error to call checkShardVersion with an unversionable connection (isVersionableCB).
+     *
+     * @return true if we contacted the remote host
      */
     bool checkShardVersion( DBClientBase * conn_in , const string& ns , ChunkManagerPtr refManager, bool authoritative , int tryNumber ) {
         // TODO: cache, optimize, etc...
 
+        // Empty namespaces are special - we require initialization but not versioning
+        if (ns.size() == 0) {
+            return initShardVersionEmptyNS(conn_in);
+        }
+
         DBConfigPtr conf = grid.getDBConfig( ns );
         if ( ! conf )
             return false;
@@ -211,20 +258,21 @@ namespace mongo {
 
         unsigned long long officialSequenceNumber = 0;
 
+        ShardPtr primary;
         ChunkManagerPtr manager;
-        const bool isSharded = conf->isSharded( ns );
-        if ( isSharded ) {
-            manager = conf->getChunkManagerIfExists( ns , authoritative );
-            // It's possible the chunk manager was reset since we checked whether sharded was true,
-            // so must check this here.
-            if( manager ) officialSequenceNumber = manager->getSequenceNumber();
-        }
+        if (authoritative)
+            conf->getChunkManagerIfExists(ns, true);
+
+        conf->getChunkManagerOrPrimary(ns, manager, primary);
+
+        if (manager)
+            officialSequenceNumber = manager->getSequenceNumber();
 
         // Check this manager against the reference manager
-        if( isSharded && manager ){
+        if( manager ){
 
             Shard shard = Shard::make( conn->getServerAddress() );
-            if(refManager && !refManager->compatibleWith(*manager, shard.getName())) {
+            if (refManager && !refManager->compatibleWith(*manager, shard.getName())) {
                 const ChunkVersion refVersion(refManager->getVersion(shard.getName()));
                 const ChunkVersion currentVersion(manager->getVersion(shard.getName()));
                 string msg(str::stream() << "manager ("
@@ -235,6 +283,7 @@ namespace mongo {
                         << " : " << refManager->getSequenceNumber() << ") "
                         << "on shard " << shard.getName()
                         << " (" << shard.getAddress().toString() << ")");
+
                 throw SendStaleConfigException(ns,
                                                msg,
                                                refVersion,
@@ -242,7 +291,8 @@ namespace mongo {
             }
         }
         else if( refManager ){
-            Shard shard = Shard::make( conn->getServerAddress() );
+
+            Shard shard = Shard::make(conn->getServerAddress());
             string msg( str::stream() << "not sharded ("
                         << ( (manager.get() == 0) ? string( "<none>" ) :
                                 str::stream() << manager->getSequenceNumber() )
@@ -257,33 +307,34 @@ namespace mongo {
                                            ChunkVersion::UNSHARDED());
         }
 
-        // has the ChunkManager been reloaded since the last time we updated the connection-level version?
-        // (ie., last time we issued the setShardVersions below)
-        unsigned long long sequenceNumber = connectionShardStatus.getSequence(conn,ns);
-        if ( sequenceNumber == officialSequenceNumber ) {
+        // Do not send setShardVersion to collections on the config servers - this causes problems
+        // when config servers are also shards and get SSV with conflicting names.
+        // TODO: Make config servers regular shards
+        if (primary && primary->getName() == "config") {
             return false;
         }
 
-        ChunkVersion version(0, 0, OID());
-        if ( isSharded && manager ) {
-            Shard shard(Shard::make(conn->getServerAddress()));
-            version = manager->getVersion(shard.getName());
+        // Has the ChunkManager been reloaded since the last time we updated the shard version over
+        // this connection?  If we've never updated the shard version, do so now.
+        unsigned long long sequenceNumber = 0;
+        if (connectionShardStatus.getSequence(conn, ns, &sequenceNumber)) {
+            if (sequenceNumber == officialSequenceNumber) {
+                return false;
+            }
         }
 
-        if( ! version.isSet() ){
-            LOG(0) << "resetting shard version of " << ns << " on " << conn->getServerAddress() << ", " <<
-                      ( ! isSharded ? "no longer sharded" :
-                      ( ! manager ? "no chunk manager found" :
-                                    "version is zero" ) ) << endl;
-        }
+        // Now that we're sure we're sending SSV and not to a single config server, get the shard
+        Shard shard = Shard::make(conn->getServerAddress());
+
+        ChunkVersion version = ChunkVersion(0, 0, OID());
+        if (manager)
+            version = manager->getVersion(shard.getName());
 
-        LOG(2).stream()
-            << " have to set shard version for conn: " << conn->getServerAddress() << " ns:" << ns
-            << " my last seq: " << sequenceNumber << "  current: " << officialSequenceNumber
-            << " version: " << version << " manager: " << manager.get()
-            << endl;
+        LOG(1) << "setting shard version of " << version << " for " << ns << " on shard "
+               << shard.toString();
 
-        const string versionableServerAddress(conn->getServerAddress());
+        LOG(3) << "last version sent with chunk manager iteration " << sequenceNumber
+               << ", current chunk manager iteration is " << officialSequenceNumber;
 
         BSONObj result;
         if ( setShardVersion( *conn , ns , version , manager , authoritative , result ) ) {
@@ -323,7 +374,7 @@ namespace mongo {
         const int maxNumTries = 7;
         if ( tryNumber < maxNumTries ) {
             LOG( tryNumber < ( maxNumTries / 2 ) ? 1 : 0 ) 
-                << "going to retry checkShardVersion host: " << versionableServerAddress << " " << result << endl;
+                << "going to retry checkShardVersion shard: " << shard.toString() << " " << result;
             sleepmillis( 10 * tryNumber );
             // use the original connection and get a fresh versionable connection
             // since conn can be invalidated (or worse, freed) after the failure
@@ -331,7 +382,8 @@ namespace mongo {
             return true;
         }
         
-        string errmsg = str::stream() << "setShardVersion failed host: " << versionableServerAddress << " " << result;
+        string errmsg = str::stream() << "setShardVersion failed shard: " << shard.toString()
+                                          << " " << result;
         log() << "     " << errmsg << endl;
         massert( 10429 , errmsg , 0 );
         return true;
diff --git a/src/mongo/s/version_manager.h b/src/mongo/s/version_manager.h
index f59e0e5a094..048cade8b7d 100644
--- a/src/mongo/s/version_manager.h
+++ b/src/mongo/s/version_manager.h
@@ -40,7 +40,6 @@ namespace mongo {
         VersionManager(){};
 
         bool isVersionableCB( DBClientBase* );
-        bool initShardVersionCB( DBClientBase*, BSONObj& );
         bool forceRemoteCheckShardVersionCB( const std::string& );
         bool checkShardVersionCB( DBClientBase*, const std::string&, bool, int );
         bool checkShardVersionCB( ShardConnection*, bool, int );