SERVER-5625 SERVER-7426 allow all connections in all states to tolerate downed shards and primaries

11 files changed, 798 insertions, 48 deletions

diff --git a/jstests/sharding/mongos_rs_shard_failure_tolerance.js b/jstests/sharding/mongos_rs_shard_failure_tolerance.js
new file mode 100644
index 00000000000..1538dc7ea50
--- /dev/null
+++ b/jstests/sharding/mongos_rs_shard_failure_tolerance.js
@@ -0,0 +1,320 @@
+//
+// Tests mongos's failure tolerance for replica set shards and slaveOk queries
+//
+// Sets up a cluster with three shards, the first shard of which has an unsharded collection and
+// half a sharded collection.  The second shard has the second half of the sharded collection, and
+// the third shard has nothing.  Progressively shuts down the primary of each shard to see the
+// impact on the cluster.
+//
+// Three different connection states are tested - active (connection is active through whole
+// sequence), idle (connection is connected but not used before a shard change), and new
+// (connection connected after shard change).
+//
+
+var options = {separateConfig : true,
+               rs : true,
+               rsOptions : { nodes : 2 }};
+
+var st = new ShardingTest({shards : 3, mongos : 1, other : options});
+st.stopBalancer();
+
+var mongos = st.s0;
+var admin = mongos.getDB( "admin" );
+var shards = mongos.getDB( "config" ).shards.find().toArray();
+
+assert.commandWorked( admin.runCommand({ setParameter : 1, traceExceptions : true }) );
+assert.commandWorked( admin.runCommand({ setParameter : 1, ignoreInitialVersionFailure : true }) );
+assert.commandWorked( admin.runCommand({ setParameter : 1, authOnPrimaryOnly : false }) );
+
+var collSharded = mongos.getCollection( "fooSharded.barSharded" );
+var collUnsharded = mongos.getCollection( "fooUnsharded.barUnsharded" );
+
+assert.commandWorked( admin.runCommand({ enableSharding : collSharded.getDB() + "" }) );
+printjson( admin.runCommand({ movePrimary : collSharded.getDB() + "", to : shards[0]._id }) );
+assert.commandWorked( admin.runCommand({ shardCollection : collSharded + "", key : { _id : 1 } }) );
+assert.commandWorked( admin.runCommand({ split : collSharded + "", middle : { _id : 0 } }) );
+assert.commandWorked( admin.runCommand({ moveChunk : collSharded + "",
+                                         find : { _id : 0 },
+                                         to : shards[1]._id }) );
+
+// Create the unsharded database
+collUnsharded.insert({ some : "doc" });
+assert.eq( null, collUnsharded.getDB().getLastError() );
+collUnsharded.remove({});
+assert.eq( null, collUnsharded.getDB().getLastError() );
+printjson( admin.runCommand({ movePrimary : collUnsharded.getDB() + "", to : shards[0]._id }) );
+
+// Make sure replica sets have a primary.
+st.rs0.getPrimary();
+st.rs1.getPrimary();
+st.rs2.getPrimary();
+
+st.printShardingStatus();
+
+// Needed b/c the GLE command itself can fail if the shard is down ("write result unknown") - we
+// don't care if this happens in this test, we only care that we did not get "write succeeded".
+// Depending on the connection pool state, we could get either.
+function gleErrorOrThrow(database, msg) {
+    var gle;
+    try {
+        gle = database.getLastErrorObj();
+    }
+    catch (ex) {
+        return;
+    }
+    if (!gle.err) doassert("getLastError is null: " + tojson(gle) + " :" + msg);
+    return;
+};
+
+//
+// Setup is complete
+//
+
+jsTest.log("Inserting initial data...");
+
+var mongosConnActive = new Mongo( mongos.host );
+var mongosConnIdle = null;
+var mongosConnNew = null;
+
+mongosConnActive.getCollection( collSharded + "" ).insert({ _id : -1 });
+mongosConnActive.getCollection( collSharded + "" ).insert({ _id : 1 });
+assert.eq(null, mongosConnActive.getCollection( collSharded + "" ).getDB().getLastError());
+
+mongosConnActive.getCollection( collUnsharded + "" ).insert({ _id : 1 });
+assert.eq(null, mongosConnActive.getCollection( collUnsharded + "" ).getDB().getLastError());
+
+jsTest.log("Stopping primary of third shard...");
+
+mongosConnIdle = new Mongo( mongos.host );
+
+st.rs2.stop(st.rs2.getPrimary(), true /*wait for stop*/ );
+
+jsTest.log("Testing active connection with third primary down...");
+
+assert.neq(null, mongosConnActive.getCollection( collSharded + "" ).findOne({ _id : -1 }));
+assert.neq(null, mongosConnActive.getCollection( collSharded + "" ).findOne({ _id : 1 }));
+assert.neq(null, mongosConnActive.getCollection( collUnsharded + "" ).findOne({ _id : 1 }));
+
+mongosConnActive.getCollection( collSharded + "" ).insert({ _id : -2 });
+assert.gleSuccess(mongosConnActive.getCollection( collSharded + "" ).getDB());
+mongosConnActive.getCollection( collSharded + "" ).insert({ _id : 2 });
+assert.gleSuccess(mongosConnActive.getCollection( collSharded + "" ).getDB());
+mongosConnActive.getCollection( collUnsharded + "" ).insert({ _id : 2 });
+assert.gleSuccess(mongosConnActive.getCollection( collUnsharded + "" ).getDB());
+
+jsTest.log("Testing idle connection with third primary down...");
+
+mongosConnIdle.getCollection( collSharded + "" ).insert({ _id : -3 });
+assert.gleSuccess(mongosConnIdle.getCollection( collSharded + "" ).getDB());
+mongosConnIdle.getCollection( collSharded + "" ).insert({ _id : 3 });
+assert.gleSuccess(mongosConnIdle.getCollection( collSharded + "" ).getDB());
+mongosConnIdle.getCollection( collUnsharded + "" ).insert({ _id : 3 });
+assert.gleSuccess(mongosConnIdle.getCollection( collUnsharded + "" ).getDB());
+
+assert.neq(null, mongosConnIdle.getCollection( collSharded + "" ).findOne({ _id : -1 }) );
+assert.neq(null, mongosConnIdle.getCollection( collSharded + "" ).findOne({ _id : 1 }) );
+assert.neq(null, mongosConnIdle.getCollection( collUnsharded + "" ).findOne({ _id : 1 }) );
+
+jsTest.log("Testing new connections with third primary down...");
+
+mongosConnNew = new Mongo( mongos.host );
+assert.neq(null, mongosConnNew.getCollection( collSharded + "" ).findOne({ _id : -1 }) );
+mongosConnNew = new Mongo( mongos.host );
+assert.neq(null, mongosConnNew.getCollection( collSharded + "" ).findOne({ _id : 1 }) );
+mongosConnNew = new Mongo( mongos.host );
+assert.neq(null, mongosConnNew.getCollection( collUnsharded + "" ).findOne({ _id : 1 }) );
+
+mongosConnNew = new Mongo( mongos.host );
+mongosConnNew.getCollection( collSharded + "" ).insert({ _id : -4 });
+assert.gleSuccess(mongosConnNew.getCollection( collSharded + "" ).getDB());
+mongosConnNew = new Mongo( mongos.host );
+mongosConnNew.getCollection( collSharded + "" ).insert({ _id : 4 });
+assert.gleSuccess(mongosConnNew.getCollection( collSharded + "" ).getDB());
+mongosConnNew = new Mongo( mongos.host );
+mongosConnNew.getCollection( collUnsharded + "" ).insert({ _id : 4 });
+assert.gleSuccess(mongosConnNew.getCollection( collUnsharded + "" ).getDB());
+
+gc(); // Clean up new connections
+
+mongosConnIdle = new Mongo( mongos.host );
+
+jsTest.log("Stopping primary of second shard...");
+
+mongosConnActive.setSlaveOk();
+mongosConnIdle = new Mongo( mongos.host );
+mongosConnIdle.setSlaveOk();
+
+// Need to save this node for later
+var rs1Secondary = st.rs1.getSecondary();
+
+st.rs1.stop(st.rs1.getPrimary(), true /* wait for stop */);
+
+jsTest.log("Testing active connection with second primary down...");
+
+assert.neq(null, mongosConnActive.getCollection( collSharded + "" ).findOne({ _id : -1 }));
+assert.neq(null, mongosConnActive.getCollection( collSharded + "" ).findOne({ _id : 1 }));
+assert.neq(null, mongosConnActive.getCollection( collUnsharded + "" ).findOne({ _id : 1 }));
+
+mongosConnActive.getCollection( collSharded + "" ).insert({ _id : -5 });
+assert.gleSuccess(mongosConnActive.getCollection( collSharded + "" ).getDB());
+mongosConnActive.getCollection( collSharded + "" ).insert({ _id : 5 });
+gleErrorOrThrow(mongosConnActive.getCollection( collSharded + "" ).getDB());
+mongosConnActive.getCollection( collUnsharded + "" ).insert({ _id : 5 });
+assert.gleSuccess(mongosConnActive.getCollection( collUnsharded + "" ).getDB());
+
+jsTest.log("Testing idle connection with second primary down...");
+
+mongosConnIdle.getCollection( collSharded + "" ).insert({ _id : -6 });
+assert.gleSuccess(mongosConnIdle.getCollection( collSharded + "" ).getDB());
+assert.commandWorked(admin.runCommand({ setParameter : 1, logLevel : 5 }));
+mongosConnIdle.getCollection( collSharded + "" ).insert({ _id : 6 });
+gleErrorOrThrow(mongosConnIdle.getCollection( collSharded + "" ).getDB());
+mongosConnIdle.getCollection( collUnsharded + "" ).insert({ _id : 6 });
+assert.gleSuccess(mongosConnIdle.getCollection( collUnsharded + "" ).getDB());
+
+assert.neq(null, mongosConnIdle.getCollection( collSharded + "" ).findOne({ _id : -1 }) );
+assert.neq(null, mongosConnIdle.getCollection( collSharded + "" ).findOne({ _id : 1 }) );
+assert.neq(null, mongosConnIdle.getCollection( collUnsharded + "" ).findOne({ _id : 1 }) );
+
+jsTest.log("Testing new connections with second primary down...");
+
+mongosConnNew = new Mongo( mongos.host );
+mongosConnNew.setSlaveOk();
+assert.neq(null, mongosConnNew.getCollection( collSharded + "" ).findOne({ _id : -1 }) );
+mongosConnNew = new Mongo( mongos.host );
+mongosConnNew.setSlaveOk();
+assert.neq(null, mongosConnNew.getCollection( collSharded + "" ).findOne({ _id : 1 }) );
+mongosConnNew = new Mongo( mongos.host );
+mongosConnNew.setSlaveOk();
+assert.neq(null, mongosConnNew.getCollection( collUnsharded + "" ).findOne({ _id : 1 }) );
+
+mongosConnNew = new Mongo( mongos.host );
+mongosConnNew.getCollection( collSharded + "" ).insert({ _id : -7 });
+assert.gleSuccess(mongosConnNew.getCollection( collSharded + "" ).getDB());
+mongosConnNew = new Mongo( mongos.host );
+mongosConnNew.getCollection( collSharded + "" ).insert({ _id : 7 });
+gleErrorOrThrow(mongosConnNew.getCollection( collSharded + "" ).getDB());
+mongosConnNew = new Mongo( mongos.host );
+mongosConnNew.getCollection( collUnsharded + "" ).insert({ _id : 7 });
+assert.gleSuccess(mongosConnNew.getCollection( collUnsharded + "" ).getDB());
+
+gc(); // Clean up new connections
+
+jsTest.log("Stopping primary of first shard...");
+
+mongosConnActive.setSlaveOk();
+mongosConnIdle = new Mongo( mongos.host );
+mongosConnIdle.setSlaveOk();
+
+st.rs0.stop(st.rs0.getPrimary(), true /*wait for stop*/ );
+
+jsTest.log("Testing active connection with first primary down...");
+
+assert.neq(null, mongosConnActive.getCollection( collSharded + "" ).findOne({ _id : -1 }));
+assert.neq(null, mongosConnActive.getCollection( collSharded + "" ).findOne({ _id : 1 }));
+assert.neq(null, mongosConnActive.getCollection( collUnsharded + "" ).findOne({ _id : 1 }));
+
+mongosConnActive.getCollection( collSharded + "" ).insert({ _id : -8 });
+gleErrorOrThrow(mongosConnActive.getCollection( collSharded + "" ).getDB());
+mongosConnActive.getCollection( collSharded + "" ).insert({ _id : 8 });
+gleErrorOrThrow(mongosConnActive.getCollection( collSharded + "" ).getDB());
+mongosConnActive.getCollection( collUnsharded + "" ).insert({ _id : 8 });
+gleErrorOrThrow(mongosConnActive.getCollection( collUnsharded + "" ).getDB());
+
+jsTest.log("Testing idle connection with first primary down...");
+
+mongosConnIdle.getCollection( collSharded + "" ).insert({ _id : -9 });
+gleErrorOrThrow(mongosConnIdle.getCollection( collSharded + "" ).getDB());
+mongosConnIdle.getCollection( collSharded + "" ).insert({ _id : 9 });
+gleErrorOrThrow(mongosConnIdle.getCollection( collSharded + "" ).getDB());
+mongosConnIdle.getCollection( collUnsharded + "" ).insert({ _id : 9 });
+gleErrorOrThrow(mongosConnIdle.getCollection( collUnsharded + "" ).getDB());
+
+assert.neq(null, mongosConnIdle.getCollection( collSharded + "" ).findOne({ _id : -1 }) );
+assert.neq(null, mongosConnIdle.getCollection( collSharded + "" ).findOne({ _id : 1 }) );
+assert.neq(null, mongosConnIdle.getCollection( collUnsharded + "" ).findOne({ _id : 1 }) );
+
+jsTest.log("Testing new connections with first primary down...");
+
+mongosConnNew = new Mongo( mongos.host );
+mongosConnNew.setSlaveOk();
+assert.neq(null, mongosConnNew.getCollection( collSharded + "" ).findOne({ _id : -1 }) );
+mongosConnNew = new Mongo( mongos.host );
+mongosConnNew.setSlaveOk();
+assert.neq(null, mongosConnNew.getCollection( collSharded + "" ).findOne({ _id : 1 }) );
+mongosConnNew = new Mongo( mongos.host );
+mongosConnNew.setSlaveOk();
+assert.neq(null, mongosConnNew.getCollection( collUnsharded + "" ).findOne({ _id : 1 }) );
+
+mongosConnNew = new Mongo( mongos.host );
+mongosConnNew.getCollection( collSharded + "" ).insert({ _id : -10 });
+gleErrorOrThrow(mongosConnNew.getCollection( collSharded + "" ).getDB());
+mongosConnNew = new Mongo( mongos.host );
+mongosConnNew.getCollection( collSharded + "" ).insert({ _id : 10 });
+gleErrorOrThrow(mongosConnNew.getCollection( collSharded + "" ).getDB());
+mongosConnNew = new Mongo( mongos.host );
+mongosConnNew.getCollection( collUnsharded + "" ).insert({ _id : 10 });
+gleErrorOrThrow(mongosConnNew.getCollection( collUnsharded + "" ).getDB());
+
+gc(); // Clean up new connections
+
+jsTest.log("Stopping second shard...");
+
+mongosConnActive.setSlaveOk();
+mongosConnIdle = new Mongo( mongos.host );
+mongosConnIdle.setSlaveOk();
+
+st.rs1.stop(rs1Secondary, true /* wait for stop */);
+
+jsTest.log("Testing active connection with second shard down...");
+
+assert.neq(null, mongosConnActive.getCollection( collSharded + "" ).findOne({ _id : -1 }));
+assert.neq(null, mongosConnActive.getCollection( collUnsharded + "" ).findOne({ _id : 1 }));
+
+mongosConnActive.getCollection( collSharded + "" ).insert({ _id : -8 });
+gleErrorOrThrow(mongosConnActive.getCollection( collSharded + "" ).getDB());
+mongosConnActive.getCollection( collSharded + "" ).insert({ _id : 8 });
+gleErrorOrThrow(mongosConnActive.getCollection( collSharded + "" ).getDB());
+mongosConnActive.getCollection( collUnsharded + "" ).insert({ _id : 8 });
+gleErrorOrThrow(mongosConnActive.getCollection( collUnsharded + "" ).getDB());
+
+jsTest.log("Testing idle connection with second shard down...");
+
+mongosConnIdle.getCollection( collSharded + "" ).insert({ _id : -9 });
+gleErrorOrThrow(mongosConnIdle.getCollection( collSharded + "" ).getDB());
+mongosConnIdle.getCollection( collSharded + "" ).insert({ _id : 9 });
+gleErrorOrThrow(mongosConnIdle.getCollection( collSharded + "" ).getDB());
+mongosConnIdle.getCollection( collUnsharded + "" ).insert({ _id : 9 });
+gleErrorOrThrow(mongosConnIdle.getCollection( collUnsharded + "" ).getDB());
+
+assert.neq(null, mongosConnIdle.getCollection( collSharded + "" ).findOne({ _id : -1 }) );
+assert.neq(null, mongosConnIdle.getCollection( collUnsharded + "" ).findOne({ _id : 1 }) );
+
+jsTest.log("Testing new connections with second shard down...");
+
+mongosConnNew = new Mongo( mongos.host );
+mongosConnNew.setSlaveOk();
+assert.neq(null, mongosConnNew.getCollection( collSharded + "" ).findOne({ _id : -1 }) );
+mongosConnNew = new Mongo( mongos.host );
+mongosConnNew.setSlaveOk();
+assert.neq(null, mongosConnNew.getCollection( collUnsharded + "" ).findOne({ _id : 1 }) );
+
+mongosConnNew = new Mongo( mongos.host );
+mongosConnNew.getCollection( collSharded + "" ).insert({ _id : -10 });
+gleErrorOrThrow(mongosConnNew.getCollection( collSharded + "" ).getDB());
+mongosConnNew = new Mongo( mongos.host );
+mongosConnNew.getCollection( collSharded + "" ).insert({ _id : 10 });
+gleErrorOrThrow(mongosConnNew.getCollection( collSharded + "" ).getDB());
+mongosConnNew = new Mongo( mongos.host );
+mongosConnNew.getCollection( collUnsharded + "" ).insert({ _id : 10 });
+gleErrorOrThrow(mongosConnNew.getCollection( collUnsharded + "" ).getDB());
+
+gc(); // Clean up new connections
+
+jsTest.log("DONE!");
+st.stop();
+
+
+
+
+
diff --git a/jstests/sharding/mongos_shard_failure_tolerance.js b/jstests/sharding/mongos_shard_failure_tolerance.js
new file mode 100644
index 00000000000..9f5ffefca28
--- /dev/null
+++ b/jstests/sharding/mongos_shard_failure_tolerance.js
@@ -0,0 +1,186 @@
+//
+// Tests mongos's failure tolerance for single-node shards
+//
+//
+// Sets up a cluster with three shards, the first shard of which has an unsharded collection and
+// half a sharded collection.  The second shard has the second half of the sharded collection, and
+// the third shard has nothing.  Progressively shuts down each shard to see the impact on the
+// cluster.
+//
+// Three different connection states are tested - active (connection is active through whole
+// sequence), idle (connection is connected but not used before a shard change), and new
+// (connection connected after shard change).
+//
+
+var options = {separateConfig : true};
+var st = new ShardingTest({shards : 3, mongos : 1, other : options});
+st.stopBalancer();
+
+var mongos = st.s0;
+var admin = mongos.getDB( "admin" );
+var shards = mongos.getDB( "config" ).shards.find().toArray();
+
+assert.commandWorked( admin.runCommand({ setParameter : 1, traceExceptions : true }) );
+assert.commandWorked( admin.runCommand({ setParameter : 1, ignoreInitialVersionFailure : true }) );
+
+var collSharded = mongos.getCollection( "fooSharded.barSharded" );
+var collUnsharded = mongos.getCollection( "fooUnsharded.barUnsharded" );
+
+assert.commandWorked( admin.runCommand({ enableSharding : collSharded.getDB() + "" }) );
+printjson( admin.runCommand({ movePrimary : collSharded.getDB() + "", to : shards[0]._id }) );
+assert.commandWorked( admin.runCommand({ shardCollection : collSharded + "", key : { _id : 1 } }) );
+assert.commandWorked( admin.runCommand({ split : collSharded + "", middle : { _id : 0 } }) );
+assert.commandWorked( admin.runCommand({ moveChunk : collSharded + "",
+                                         find : { _id : 0 },
+                                         to : shards[1]._id }) );
+
+// Create the unsharded database
+collUnsharded.insert({ some : "doc" });
+assert.eq( null, collUnsharded.getDB().getLastError() );
+collUnsharded.remove({});
+assert.eq( null, collUnsharded.getDB().getLastError() );
+printjson( admin.runCommand({ movePrimary : collUnsharded.getDB() + "", to : shards[0]._id }) );
+
+st.printShardingStatus();
+
+// Needed b/c the GLE command itself can fail if the shard is down ("write result unknown") - we
+// don't care if this happens in this test, we only care that we did not get "write succeeded".
+// Depending on the connection pool state, we could get either.
+function gleErrorOrThrow(database, msg) {
+    var gle;
+    try {
+        gle = database.getLastErrorObj();
+    }
+    catch (ex) {
+        return;
+    }
+    if (!gle.err) doassert("getLastError is null: " + tojson(gle) + " :" + msg);
+    return;
+};
+
+//
+// Setup is complete
+//
+
+jsTest.log("Inserting initial data...");
+
+var mongosConnActive = new Mongo( mongos.host );
+var mongosConnIdle = null;
+var mongosConnNew = null;
+
+mongosConnActive.getCollection( collSharded + "" ).insert({ _id : -1 });
+mongosConnActive.getCollection( collSharded + "" ).insert({ _id : 1 });
+assert.eq(null, mongosConnActive.getCollection( collSharded + "" ).getDB().getLastError());
+
+mongosConnActive.getCollection( collUnsharded + "" ).insert({ _id : 1 });
+assert.eq(null, mongosConnActive.getCollection( collUnsharded + "" ).getDB().getLastError());
+
+jsTest.log("Stopping third shard...");
+
+mongosConnIdle = new Mongo( mongos.host );
+
+MongoRunner.stopMongod( st.shard2 );
+
+jsTest.log("Testing active connection...");
+
+assert.neq(null, mongosConnActive.getCollection( collSharded + "" ).findOne({ _id : -1 }));
+assert.neq(null, mongosConnActive.getCollection( collSharded + "" ).findOne({ _id : 1 }));
+assert.neq(null, mongosConnActive.getCollection( collUnsharded + "" ).findOne({ _id : 1 }));
+
+mongosConnActive.getCollection( collSharded + "" ).insert({ _id : -2 });
+assert.gleSuccess(mongosConnActive.getCollection( collSharded + "" ).getDB());
+mongosConnActive.getCollection( collSharded + "" ).insert({ _id : 2 });
+assert.gleSuccess(mongosConnActive.getCollection( collSharded + "" ).getDB());
+mongosConnActive.getCollection( collUnsharded + "" ).insert({ _id : 2 });
+assert.gleSuccess(mongosConnActive.getCollection( collUnsharded + "" ).getDB());
+
+jsTest.log("Testing idle connection...");
+
+mongosConnIdle.getCollection( collSharded + "" ).insert({ _id : -3 });
+assert.gleSuccess(mongosConnIdle.getCollection( collSharded + "" ).getDB());
+mongosConnIdle.getCollection( collSharded + "" ).insert({ _id : 3 });
+assert.gleSuccess(mongosConnIdle.getCollection( collSharded + "" ).getDB());
+mongosConnIdle.getCollection( collUnsharded + "" ).insert({ _id : 3 });
+assert.gleSuccess(mongosConnIdle.getCollection( collUnsharded + "" ).getDB());
+
+assert.neq(null, mongosConnIdle.getCollection( collSharded + "" ).findOne({ _id : -1 }) );
+assert.neq(null, mongosConnIdle.getCollection( collSharded + "" ).findOne({ _id : 1 }) );
+assert.neq(null, mongosConnIdle.getCollection( collUnsharded + "" ).findOne({ _id : 1 }) );
+
+jsTest.log("Testing new connections...");
+
+mongosConnNew = new Mongo( mongos.host );
+assert.neq(null, mongosConnNew.getCollection( collSharded + "" ).findOne({ _id : -1 }) );
+mongosConnNew = new Mongo( mongos.host );
+assert.neq(null, mongosConnNew.getCollection( collSharded + "" ).findOne({ _id : 1 }) );
+mongosConnNew = new Mongo( mongos.host );
+assert.neq(null, mongosConnNew.getCollection( collUnsharded + "" ).findOne({ _id : 1 }) );
+
+mongosConnNew = new Mongo( mongos.host );
+mongosConnNew.getCollection( collSharded + "" ).insert({ _id : -4 });
+assert.gleSuccess(mongosConnNew.getCollection( collSharded + "" ).getDB());
+mongosConnNew = new Mongo( mongos.host );
+mongosConnNew.getCollection( collSharded + "" ).insert({ _id : 4 });
+assert.gleSuccess(mongosConnNew.getCollection( collSharded + "" ).getDB());
+mongosConnNew = new Mongo( mongos.host );
+mongosConnNew.getCollection( collUnsharded + "" ).insert({ _id : 4 });
+assert.gleSuccess(mongosConnNew.getCollection( collUnsharded + "" ).getDB());
+
+gc(); // Clean up new connections
+
+jsTest.log("Stopping second shard...");
+
+mongosConnIdle = new Mongo( mongos.host );
+
+MongoRunner.stopMongod( st.shard1 );
+
+jsTest.log("Testing active connection...");
+
+assert.neq(null, mongosConnActive.getCollection( collSharded + "" ).findOne({ _id : -1 }) );
+assert.neq(null, mongosConnActive.getCollection( collUnsharded + "" ).findOne({ _id : 1 }) );
+
+mongosConnActive.getCollection( collSharded + "" ).insert({ _id : -5 });
+assert.gleSuccess(mongosConnActive.getCollection( collSharded + "" ).getDB());
+mongosConnActive.getCollection( collSharded + "" ).insert({ _id : 5 });
+gleErrorOrThrow(mongosConnActive.getCollection( collSharded + "" ).getDB());
+mongosConnActive.getCollection( collUnsharded + "" ).insert({ _id : 5 });
+assert.gleSuccess(mongosConnActive.getCollection( collUnsharded + "" ).getDB());
+
+jsTest.log("Testing idle connection...");
+
+mongosConnIdle.getCollection( collSharded + "" ).insert({ _id : -6 });
+assert.gleSuccess(mongosConnIdle.getCollection( collSharded + "" ).getDB());
+mongosConnIdle.getCollection( collSharded + "" ).insert({ _id : 6 });
+gleErrorOrThrow(mongosConnIdle.getCollection( collSharded + "" ).getDB());
+mongosConnIdle.getCollection( collUnsharded + "" ).insert({ _id : 6 });
+assert.gleSuccess(mongosConnIdle.getCollection( collUnsharded + "" ).getDB());
+
+assert.neq(null, mongosConnIdle.getCollection( collSharded + "" ).findOne({ _id : -1 }) );
+assert.neq(null, mongosConnIdle.getCollection( collUnsharded + "" ).findOne({ _id : 1 }) );
+
+jsTest.log("Testing new connections...");
+
+mongosConnNew = new Mongo( mongos.host );
+assert.neq(null, mongosConnNew.getCollection( collSharded + "" ).findOne({ _id : -1 }) );
+mongosConnNew = new Mongo( mongos.host );
+assert.neq(null, mongosConnNew.getCollection( collUnsharded + "" ).findOne({ _id : 1 }) );
+
+mongosConnNew = new Mongo( mongos.host );
+mongosConnNew.getCollection( collSharded + "" ).insert({ _id : -7 });
+assert.gleSuccess(mongosConnNew.getCollection( collSharded + "" ).getDB());
+mongosConnNew = new Mongo( mongos.host );
+mongosConnNew.getCollection( collSharded + "" ).insert({ _id : 7 });
+gleErrorOrThrow(mongosConnNew.getCollection( collSharded + "" ).getDB());
+mongosConnNew = new Mongo( mongos.host );
+mongosConnNew.getCollection( collUnsharded + "" ).insert({ _id : 7 });
+assert.gleSuccess(mongosConnNew.getCollection( collUnsharded + "" ).getDB());
+
+gc(); // Clean up new connections
+
+jsTest.log("DONE!");
+st.stop();
+
+
+
+
+
diff --git a/src/mongo/base/error_codes.err b/src/mongo/base/error_codes.err
index 1dfa1702b64..5fa8cde17e2 100644
--- a/src/mongo/base/error_codes.err
+++ b/src/mongo/base/error_codes.err
@@ -27,4 +27,7 @@ error_code("AlreadyInitialized", 23)
 error_code("LockTimeout", 24)
 error_code("RemoteValidationError", 25)
 
+# Non-sequential error codes (for compatibility only)
+error_code("NodeNotFound", 74)
+
 error_class("NetworkError", ["HostUnreachable", "HostNotFound"])
diff --git a/src/mongo/client/dbclient_rs.cpp b/src/mongo/client/dbclient_rs.cpp
index 21a5b0810f4..62544aca668 100644
--- a/src/mongo/client/dbclient_rs.cpp
+++ b/src/mongo/client/dbclient_rs.cpp
@@ -1593,6 +1593,18 @@ namespace mongo {
         return _getMonitor()->isAnyNodeOk();
     }
 
+    void DBClientReplicaSet::authPrimary(const BSONObj& params) {
+        _auth(params);
+    }
+
+    bool DBClientReplicaSet::authPrimary( const string &dbname,
+                                          const string &username,
+                                          const string &password_text,
+                                          string& errmsg,
+                                          bool digestPassword ) {
+        return auth( dbname, username, password_text, errmsg, digestPassword );
+    }
+
     void DBClientReplicaSet::_auth(const BSONObj& params) {
         DBClientConnection * m = checkMaster();
 
@@ -1620,6 +1632,99 @@ namespace mongo {
         _auths[params[saslCommandPrincipalSourceFieldName].str()] = params.getOwned();
     }
 
+    bool DBClientReplicaSet::authAny( const string &dbname,
+                                      const string &username,
+                                      const string &password_text,
+                                      string& errmsg,
+                                      bool digestPassword ) {
+        try {
+            authAny(BSON(saslCommandMechanismFieldName << "MONGODB-CR" <<
+                         saslCommandPrincipalSourceFieldName << dbname <<
+                         saslCommandPrincipalFieldName << username <<
+                         saslCommandPasswordFieldName << password_text <<
+                         saslCommandDigestPasswordFieldName << digestPassword));
+            return true;
+        } catch(const UserException& ex) {
+            if (ex.getCode() != ErrorCodes::AuthenticationFailed)
+                throw;
+            errmsg = ex.what();
+            return false;
+        }
+    }
+
+    static bool isAuthException( const DBException& ex ) {
+        return ex.getCode() == ErrorCodes::AuthenticationFailed;
+    }
+
+    void DBClientReplicaSet::authAny( const BSONObj& params ) {
+
+        // We prefer to authenticate against a primary, but otherwise a secondary is ok too
+        // Empty tag matches every secondary
+        TagSet tags(BSON_ARRAY(BSONObj()));
+        shared_ptr<ReadPreferenceSetting> readPref(
+            new ReadPreferenceSetting( ReadPreference_PrimaryPreferred, tags ) );
+
+        LOG(3) << "dbclient_rs authentication of " << _getMonitor()->getName() << endl;
+
+        // NOTE that we retry MAX_RETRY + 1 times, since we're always primary preferred we don't
+        // fallback to the primary.
+        Status lastNodeStatus = Status::OK();
+        for ( size_t retry = 0; retry < MAX_RETRY + 1; retry++ ) {
+            try {
+                DBClientConnection* conn = selectNodeUsingTags( readPref );
+
+                if ( conn == NULL ) {
+                    break;
+                }
+
+                conn->auth( params );
+
+                // Cache the new auth information since we've now validated it's good
+                _auths[params[saslCommandPrincipalSourceFieldName].str()] = params.getOwned();
+
+                // Ensure the only child connection open is the one we authenticated against - other
+                // child connections may not have full authentication information.
+                // NOTE: _lastSlaveOkConn may or may not be the same as _master
+                dassert(_lastSlaveOkConn.get() == conn || _master.get() == conn);
+                if ( conn != _lastSlaveOkConn.get() ) {
+                    _lastSlaveOkHost = HostAndPort();
+                    _lastSlaveOkConn.reset();
+                }
+                if ( conn != _master.get() ) {
+                    _masterHost = HostAndPort();
+                    _master.reset();
+                }
+
+                return;
+            }
+            catch ( const DBException &ex ) {
+
+                // We care if we can't authenticate (i.e. bad password) in credential params.
+                // We shouldn't be unauthorized since we aren't doing anything yet.
+                if ( isAuthException( ex ) ) {
+                    throw;
+                }
+
+                StringBuilder errMsgB;
+                errMsgB << "can't authenticate against replica set node "
+                        << _lastSlaveOkHost.toString();
+                lastNodeStatus = ex.toStatus( errMsgB.str() );
+
+                LOG(1) << lastNodeStatus.reason() << endl;
+                invalidateLastSlaveOkCache();
+            }
+        }
+
+        if ( lastNodeStatus.isOK() ) {
+            StringBuilder assertMsgB;
+            assertMsgB << "Failed to authenticate, no good nodes in " << _getMonitor()->getName();
+            uasserted( ErrorCodes::NodeNotFound, assertMsgB.str() );
+        }
+        else {
+            uasserted( lastNodeStatus.code(), lastNodeStatus.reason() );
+        }
+    }
+
     void DBClientReplicaSet::logout(const string &dbname, BSONObj& info) {
         DBClientConnection* priConn = checkMaster();
 
diff --git a/src/mongo/client/dbclient_rs.h b/src/mongo/client/dbclient_rs.h
index 05f7faa9b2b..ddf986c732b 100644
--- a/src/mongo/client/dbclient_rs.h
+++ b/src/mongo/client/dbclient_rs.h
@@ -518,10 +518,51 @@ namespace mongo {
         virtual bool call( Message &toSend, Message &response, bool assertOk=true , string * actualServer = 0 );
         virtual bool callRead( Message& toSend , Message& response ) { return checkMaster()->callRead( toSend , response ); }
 
+        /**
+         *  Authenticate using supplied credentials.  Authenticates against the primary node, fails
+         *  if node is down.
+         *  Credentials are cached for future connections.
+         *
+         *  See DBClientWithCommands::auth() for more details.
+         *
+         *  This is the default authentication mode for DBClientReplicaSet connections.
+         */
+        void authPrimary(const BSONObj& params);
+
+        /**
+         * Same as above, but authorizes access to a particular database.
+         *
+         * See DBClientWithCommands::auth() for more details.
+         */
+        bool authPrimary(const string &dbname,
+                         const string &username,
+                         const string &password_text,
+                         string& errmsg,
+                         bool digestPassword);
+
+        /**
+         * Authenticate using supplied credentials.  Prefers authentication against the primary
+         * node, will fall back to a secondary and retry if the primary node is down but
+         * secondaries are still available.
+         * Credentials are cached for future connections.
+         *
+         * See DBClientWithCommands::auth() for more details.
+         */
+        void authAny(const BSONObj& params);
+
+        /**
+         * Same as above, but authorizes access to a particular database.
+         *
+         * See DBClientWithCommands::auth() for more details.
+         */
+        bool authAny( const string &dbname,
+                      const string &username,
+                      const string &password_text,
+                      string& errmsg,
+                      bool digestPassword );
 
     protected:
-        /** Authorize.  Authorizes all nodes as needed
-        */
+
         virtual void _auth(const BSONObj& params);
 
         virtual void sayPiggyBack( Message &toSend ) { checkMaster()->say( toSend ); }
diff --git a/src/mongo/client/parallel.cpp b/src/mongo/client/parallel.cpp
index 18649d9c6b9..d3b03425193 100644
--- a/src/mongo/client/parallel.cpp
+++ b/src/mongo/client/parallel.cpp
@@ -715,26 +715,27 @@ namespace mongo {
         }
 
         const DBClientBase* rawConn = state->conn->getRawConn();
-        if (( _options & QueryOption_SlaveOk ) &&
-                rawConn->type() == ConnectionString::SET &&
-                rawConn->isFailed() ) {
-            /* A side effect of this short circuiting is this will not be
-             * able figure out that the primary is now up on it's own and
-             * has to rely on other threads to refresh the node states.
-             */
+        bool allowShardVersionFailure = ( _options & QueryOption_SlaveOk )
+                                        && rawConn->type() == ConnectionString::SET;
+
+        if ( allowShardVersionFailure && rawConn->isFailed() ) {
+
+            state->conn->donotCheckVersion();
+
+            // A side effect of this short circuiting is the mongos will not be able figure out that
+            // the primary is now up on it's own and has to rely on other threads to refresh node
+            // states.
 
             OCCASIONALLY {
-                const DBClientReplicaSet* repl =
-                    dynamic_cast<const DBClientReplicaSet*>( rawConn );
+                const DBClientReplicaSet* repl = dynamic_cast<const DBClientReplicaSet*>( rawConn );
+                dassert(repl);
                 warning() << "Primary for " << repl->getServerAddress()
                           << " was down before, bypassing setShardVersion."
-                          << " Local config view can be stale." << endl;
+                          << " The local replica set view and targeting may be stale." << endl;
             }
-        } else {
+        }
+        else {
             try {
-                /* TODO: Undo SERVER-5797. This try-catch is a temporary hack until
-                 * secondaries can properly handle shard versioning
-                 */
                 if ( state->conn->setVersion() ) {
                     // It's actually okay if we set the version here, since either the
                     // manager will be verified as compatible, or if the manager doesn't
@@ -742,19 +743,20 @@ namespace mongo {
                     LOG( pc ) << "needed to set remote version on connection to value "
                               << "compatible with " << vinfo << endl;
                 }
-            } catch ( const DBException& dbEx ) {
-                if ( (dbEx.getCode() == 10009 /* no master */ &&
-                        ( _options & QueryOption_SlaveOk )) ) {
+            }
+            catch ( const DBException& dbEx ) {
+                if ( allowShardVersionFailure ) {
+
+                    // It's okay if we don't set the version when talking to a secondary, we can
+                    // be stale in any case.
 
                     OCCASIONALLY {
                         const DBClientReplicaSet* repl =
-                            dynamic_cast<const DBClientReplicaSet*>(
-                                    state->conn->getRawConn() );
-
-                        warning() << "Cannot contact primary for "
-                                  << repl->getServerAddress()
-                                  << " to check shard version. "
-                                  << "SlaveOk query can be sent to the wrong shard."
+                            dynamic_cast<const DBClientReplicaSet*>( state->conn->getRawConn() );
+                        dassert(repl);
+                        warning() << "Cannot contact primary for " << repl->getServerAddress()
+                                  << " to check shard version."
+                                  << " The local replica set view and targeting may be stale."
                                   << endl;
                     }
                 }
diff --git a/src/mongo/s/shard.cpp b/src/mongo/s/shard.cpp
index df620865603..28b281f0cbd 100644
--- a/src/mongo/s/shard.cpp
+++ b/src/mongo/s/shard.cpp
@@ -30,6 +30,7 @@
 #include "mongo/db/auth/privilege.h"
 #include "mongo/db/commands.h"
 #include "mongo/db/jsobj.h"
+#include "mongo/db/server_parameters.h"
 #include "mongo/s/client_info.h"
 #include "mongo/s/config.h"
 #include "mongo/s/request.h"
@@ -39,6 +40,8 @@
 
 namespace mongo {
 
+    MONGO_EXPORT_SERVER_PARAMETER(authOnPrimaryOnly, bool, true);
+
     class StaticShardInfo {
     public:
         StaticShardInfo() : _mutex("StaticShardInfo"), _rsMutex("RSNameMap") { }
@@ -411,14 +414,26 @@ namespace mongo {
 
     void ShardingConnectionHook::onCreate( DBClientBase * conn ) {
         if( !noauth ) {
+            bool result;
             string err;
             LOG(2) << "calling onCreate auth for " << conn->toString() << endl;
 
-            bool result = conn->auth( "local",
-                                      internalSecurity.user,
-                                      internalSecurity.pwd,
-                                      err,
-                                      false );
+            if ( conn->type() == ConnectionString::SET && !authOnPrimaryOnly ) {
+                DBClientReplicaSet* setConn = dynamic_cast<DBClientReplicaSet*>(conn);
+                verify(setConn);
+                result = setConn->authAny( "local",
+                                           internalSecurity.user,
+                                           internalSecurity.pwd,
+                                           err,
+                                           false );
+            }
+            else {
+                result = conn->auth( "local",
+                                     internalSecurity.user,
+                                     internalSecurity.pwd,
+                                     err,
+                                     false );
+            }
 
             uassert( 15847, str::stream() << "can't authenticate to server "
                                           << conn->getServerAddress() << causedBy( err ), result );
diff --git a/src/mongo/s/shard.h b/src/mongo/s/shard.h
index 39504311a65..41f819d3ae9 100644
--- a/src/mongo/s/shard.h
+++ b/src/mongo/s/shard.h
@@ -290,8 +290,12 @@ namespace mongo {
          */
         bool runCommand( const string& db , const BSONObj& cmd , BSONObj& res );
 
+        // Whether or not we release connections from the thread-local cache after a read
         static bool releaseConnectionsAfterResponse;
 
+        // Controls whether we throw on initially failing to set a version
+        static bool ignoreInitialVersionFailure;
+
         /** checks all of my thread local connections for the version of this ns */
         static void checkMyConnectionVersions( const string & ns );
 
diff --git a/src/mongo/s/shardconnection.cpp b/src/mongo/s/shardconnection.cpp
index 2ef03996367..66a105f05e8 100644
--- a/src/mongo/s/shardconnection.cpp
+++ b/src/mongo/s/shardconnection.cpp
@@ -22,6 +22,7 @@
 
 #include "mongo/db/client.h"
 #include "mongo/db/commands.h"
+#include "mongo/db/lasterror.h"
 #include "mongo/db/server_parameters.h"
 #include "mongo/s/config.h"
 #include "mongo/s/request.h"
@@ -33,7 +34,13 @@
 
 namespace mongo {
 
-    MONGO_EXPORT_SERVER_PARAMETER(ignoreInitialVersionFailure, bool, false);
+    bool ShardConnection::ignoreInitialVersionFailure( false );
+    ExportedServerParameter<bool>
+        _ignoreInitialVersionFailure( ServerParameterSet::getGlobal(),
+                                      "ignoreInitialVersionFailure",
+                                      &ShardConnection::ignoreInitialVersionFailure,
+                                      true,
+                                      true );
 
     DBConnectionPool shardConnectionPool;
 
@@ -237,6 +244,12 @@ namespace mongo {
             vector<Shard> all;
             Shard::getAllShards( all );
 
+            scoped_ptr<LastError::Disabled> ignoreForGLE;
+            if ( ShardConnection::ignoreInitialVersionFailure ) {
+                // Don't report exceptions here as errors in GetLastError if ignoring failures
+                ignoreForGLE.reset( new LastError::Disabled( lastError.get( false ) ) );
+            }
+
             // Now only check top-level shard connections
             for ( unsigned i=0; i<all.size(); i++ ) {
 
@@ -252,12 +265,12 @@ namespace mongo {
 
                     versionManager.checkShardVersionCB( s->avail, ns, false, 1 );
                 }
-                catch ( const std::exception& e ) {
+                catch ( const DBException& ex ) {
 
-                    warning() << "problem while initially checking shard versions on"
-                              << " " << shard.getName() << causedBy(e) << endl;
+                    warning() << "problem while initially checking shard versions on "
+                              << shard.getName() << causedBy( ex ) << endl;
                     
-                    if ( !ignoreInitialVersionFailure ) {
+                    if ( !ShardConnection::ignoreInitialVersionFailure ) {
                         throw;
                     }
                     else {
diff --git a/src/mongo/s/version_manager.cpp b/src/mongo/s/version_manager.cpp
index 1496ebc1407..3ba6254813b 100644
--- a/src/mongo/s/version_manager.cpp
+++ b/src/mongo/s/version_manager.cpp
@@ -103,23 +103,46 @@ namespace mongo {
 
         WriteBackListener::init( *conn_in );
 
-        DBClientBase* conn = getVersionable( conn_in );
-        verify( conn ); // errors thrown above
+        bool ok;
+        DBClientBase* conn = NULL;
+        try {
+            // May throw if replica set primary is down
+            conn = getVersionable( conn_in );
+            dassert( conn ); // errors thrown above
+
+            BSONObjBuilder cmdBuilder;
+
+            cmdBuilder.append( "setShardVersion" , "" );
+            cmdBuilder.appendBool( "init", true );
+            cmdBuilder.append( "configdb" , configServer.modelServer() );
+            cmdBuilder.appendOID( "serverID" , &serverID );
+            cmdBuilder.appendBool( "authoritative" , true );
 
-        BSONObjBuilder cmdBuilder;
+            BSONObj cmd = cmdBuilder.obj();
 
-        cmdBuilder.append( "setShardVersion" , "" );
-        cmdBuilder.appendBool( "init", true );
-        cmdBuilder.append( "configdb" , configServer.modelServer() );
-        cmdBuilder.appendOID( "serverID" , &serverID );
-        cmdBuilder.appendBool( "authoritative" , true );
+            LOG(1) << "initializing shard connection to " << conn->toString() << endl;
+            LOG(2) << "initial sharding settings : " << cmd << endl;
+
+            ok = conn->runCommand("admin", cmd, result, 0);
+        }
+        catch( const DBException& ex ) {
 
-        BSONObj cmd = cmdBuilder.obj();
+            bool ignoreFailure = ShardConnection::ignoreInitialVersionFailure
+                                 && conn_in->type() == ConnectionString::SET;
+            if ( !ignoreFailure )
+                throw;
 
-        LOG(1) << "initializing shard connection to " << conn->toString() << endl;
-        LOG(2) << "initial sharding settings : " << cmd << endl;
+            // Using initShardVersion is not strictly required when talking to replica sets - it is
+            // preferred to do so because it registers mongos early with the mongod.  This info is
+            // also sent by checkShardVersion before a connection is used for a write or read.
 
-        bool ok = conn->runCommand("admin", cmd, result, 0);
+            OCCASIONALLY {
+                warning() << "failed to initialize new replica set connection version, "
+                          << "will initialize on first use" << endl;
+            }
+
+            return true;
+        }
 
         // HACK for backwards compatibility with v1.8.x, v2.0.0 and v2.0.1
         // Result is false, but will still initialize serverID and configdb
diff --git a/src/mongo/shell/assert.js b/src/mongo/shell/assert.js
index b1279028307..0e84c52d11c 100644
--- a/src/mongo/shell/assert.js
+++ b/src/mongo/shell/assert.js
@@ -225,3 +225,41 @@ assert.close = function(a, b, msg, places){
     doassert(a + " is not equal to " + b + " within " + places +
               " places, diff: " + (a-b) + " : " + msg);
 };
+
+assert.gleSuccess = function(db, msg) {
+    var gle = db.getLastErrorObj();
+    if (gle.err) {
+        if (typeof(msg) == "function") 
+            msg = msg(gle);
+        doassert("getLastError not null:" + tojson(gle) + " :" + msg);
+    }
+}
+
+assert.gleError = function(db, msg) {
+    var gle = db.getLastErrorObj();
+    if (!gle.err) {
+        if (typeof(msg) == "function") 
+            msg = msg(gle);
+        doassert("getLastError is null: " + tojson(gle) + " :" + msg);
+    }
+}
+
+assert.gleErrorCode = function(db, code, msg) {
+    var gle = db.getLastErrorObj();
+    if (gle.err && (gle.code == code)) {
+        if (typeof(msg) == "function") 
+            msg = msg(gle);
+        doassert("getLastError not null or missing code( " + code + "): " 
+                 + tojson(gle) + " :" + msg);
+    }
+}
+
+assert.gleErrorRegex = function(db, regex, msg) {
+    var gle = db.getLastErrorObj();
+    if (!gle.err || !regex.test(gle.err)) {
+        if (typeof(msg) == "function") 
+            msg = msg(gle);
+        doassert("getLastError is null or doesn't match regex (" + regex + "): " 
+                 + tojson(gle) + " :" + msg);
+    }
+}