diff options
author | Greg Studer <greg@10gen.com> | 2013-11-22 16:51:20 -0500 |
---|---|---|
committer | Greg Studer <greg@10gen.com> | 2013-12-01 21:29:10 -0500 |
commit | 45662915c2f4e7e19b8333dc93d0d2526d8a34c5 (patch) | |
tree | 865e3c083bedd20942c356f3c5655ac4d712dbaf | |
parent | 38b3b8f7395d6717efd570a62dc1c2c085d1b049 (diff) | |
download | mongo-45662915c2f4e7e19b8333dc93d0d2526d8a34c5.tar.gz |
SERVER-5625 SERVER-7426 allow all connections in all states to tolerate downed shards and primaries
-rw-r--r-- | jstests/sharding/mongos_rs_shard_failure_tolerance.js | 320 | ||||
-rw-r--r-- | jstests/sharding/mongos_shard_failure_tolerance.js | 186 | ||||
-rw-r--r-- | src/mongo/base/error_codes.err | 3 | ||||
-rw-r--r-- | src/mongo/client/dbclient_rs.cpp | 105 | ||||
-rw-r--r-- | src/mongo/client/dbclient_rs.h | 45 | ||||
-rw-r--r-- | src/mongo/client/parallel.cpp | 50 | ||||
-rw-r--r-- | src/mongo/s/shard.cpp | 25 | ||||
-rw-r--r-- | src/mongo/s/shard.h | 4 | ||||
-rw-r--r-- | src/mongo/s/shardconnection.cpp | 23 | ||||
-rw-r--r-- | src/mongo/s/version_manager.cpp | 47 | ||||
-rw-r--r-- | src/mongo/shell/assert.js | 38 |
11 files changed, 798 insertions, 48 deletions
diff --git a/jstests/sharding/mongos_rs_shard_failure_tolerance.js b/jstests/sharding/mongos_rs_shard_failure_tolerance.js new file mode 100644 index 00000000000..1538dc7ea50 --- /dev/null +++ b/jstests/sharding/mongos_rs_shard_failure_tolerance.js @@ -0,0 +1,320 @@ +// +// Tests mongos's failure tolerance for replica set shards and slaveOk queries +// +// Sets up a cluster with three shards, the first shard of which has an unsharded collection and +// half a sharded collection. The second shard has the second half of the sharded collection, and +// the third shard has nothing. Progressively shuts down the primary of each shard to see the +// impact on the cluster. +// +// Three different connection states are tested - active (connection is active through whole +// sequence), idle (connection is connected but not used before a shard change), and new +// (connection connected after shard change). +// + +var options = {separateConfig : true, + rs : true, + rsOptions : { nodes : 2 }}; + +var st = new ShardingTest({shards : 3, mongos : 1, other : options}); +st.stopBalancer(); + +var mongos = st.s0; +var admin = mongos.getDB( "admin" ); +var shards = mongos.getDB( "config" ).shards.find().toArray(); + +assert.commandWorked( admin.runCommand({ setParameter : 1, traceExceptions : true }) ); +assert.commandWorked( admin.runCommand({ setParameter : 1, ignoreInitialVersionFailure : true }) ); +assert.commandWorked( admin.runCommand({ setParameter : 1, authOnPrimaryOnly : false }) ); + +var collSharded = mongos.getCollection( "fooSharded.barSharded" ); +var collUnsharded = mongos.getCollection( "fooUnsharded.barUnsharded" ); + +assert.commandWorked( admin.runCommand({ enableSharding : collSharded.getDB() + "" }) ); +printjson( admin.runCommand({ movePrimary : collSharded.getDB() + "", to : shards[0]._id }) ); +assert.commandWorked( admin.runCommand({ shardCollection : collSharded + "", key : { _id : 1 } }) ); +assert.commandWorked( admin.runCommand({ split : collSharded + "", middle : { _id : 0 } }) ); +assert.commandWorked( admin.runCommand({ moveChunk : collSharded + "", + find : { _id : 0 }, + to : shards[1]._id }) ); + +// Create the unsharded database +collUnsharded.insert({ some : "doc" }); +assert.eq( null, collUnsharded.getDB().getLastError() ); +collUnsharded.remove({}); +assert.eq( null, collUnsharded.getDB().getLastError() ); +printjson( admin.runCommand({ movePrimary : collUnsharded.getDB() + "", to : shards[0]._id }) ); + +// Make sure replica sets have a primary. +st.rs0.getPrimary(); +st.rs1.getPrimary(); +st.rs2.getPrimary(); + +st.printShardingStatus(); + +// Needed b/c the GLE command itself can fail if the shard is down ("write result unknown") - we +// don't care if this happens in this test, we only care that we did not get "write succeeded". +// Depending on the connection pool state, we could get either. +function gleErrorOrThrow(database, msg) { + var gle; + try { + gle = database.getLastErrorObj(); + } + catch (ex) { + return; + } + if (!gle.err) doassert("getLastError is null: " + tojson(gle) + " :" + msg); + return; +}; + +// +// Setup is complete +// + +jsTest.log("Inserting initial data..."); + +var mongosConnActive = new Mongo( mongos.host ); +var mongosConnIdle = null; +var mongosConnNew = null; + +mongosConnActive.getCollection( collSharded + "" ).insert({ _id : -1 }); +mongosConnActive.getCollection( collSharded + "" ).insert({ _id : 1 }); +assert.eq(null, mongosConnActive.getCollection( collSharded + "" ).getDB().getLastError()); + +mongosConnActive.getCollection( collUnsharded + "" ).insert({ _id : 1 }); +assert.eq(null, mongosConnActive.getCollection( collUnsharded + "" ).getDB().getLastError()); + +jsTest.log("Stopping primary of third shard..."); + +mongosConnIdle = new Mongo( mongos.host ); + +st.rs2.stop(st.rs2.getPrimary(), true /*wait for stop*/ ); + +jsTest.log("Testing active connection with third primary down..."); + +assert.neq(null, mongosConnActive.getCollection( collSharded + "" ).findOne({ _id : -1 })); +assert.neq(null, mongosConnActive.getCollection( collSharded + "" ).findOne({ _id : 1 })); +assert.neq(null, mongosConnActive.getCollection( collUnsharded + "" ).findOne({ _id : 1 })); + +mongosConnActive.getCollection( collSharded + "" ).insert({ _id : -2 }); +assert.gleSuccess(mongosConnActive.getCollection( collSharded + "" ).getDB()); +mongosConnActive.getCollection( collSharded + "" ).insert({ _id : 2 }); +assert.gleSuccess(mongosConnActive.getCollection( collSharded + "" ).getDB()); +mongosConnActive.getCollection( collUnsharded + "" ).insert({ _id : 2 }); +assert.gleSuccess(mongosConnActive.getCollection( collUnsharded + "" ).getDB()); + +jsTest.log("Testing idle connection with third primary down..."); + +mongosConnIdle.getCollection( collSharded + "" ).insert({ _id : -3 }); +assert.gleSuccess(mongosConnIdle.getCollection( collSharded + "" ).getDB()); +mongosConnIdle.getCollection( collSharded + "" ).insert({ _id : 3 }); +assert.gleSuccess(mongosConnIdle.getCollection( collSharded + "" ).getDB()); +mongosConnIdle.getCollection( collUnsharded + "" ).insert({ _id : 3 }); +assert.gleSuccess(mongosConnIdle.getCollection( collUnsharded + "" ).getDB()); + +assert.neq(null, mongosConnIdle.getCollection( collSharded + "" ).findOne({ _id : -1 }) ); +assert.neq(null, mongosConnIdle.getCollection( collSharded + "" ).findOne({ _id : 1 }) ); +assert.neq(null, mongosConnIdle.getCollection( collUnsharded + "" ).findOne({ _id : 1 }) ); + +jsTest.log("Testing new connections with third primary down..."); + +mongosConnNew = new Mongo( mongos.host ); +assert.neq(null, mongosConnNew.getCollection( collSharded + "" ).findOne({ _id : -1 }) ); +mongosConnNew = new Mongo( mongos.host ); +assert.neq(null, mongosConnNew.getCollection( collSharded + "" ).findOne({ _id : 1 }) ); +mongosConnNew = new Mongo( mongos.host ); +assert.neq(null, mongosConnNew.getCollection( collUnsharded + "" ).findOne({ _id : 1 }) ); + +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.getCollection( collSharded + "" ).insert({ _id : -4 }); +assert.gleSuccess(mongosConnNew.getCollection( collSharded + "" ).getDB()); +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.getCollection( collSharded + "" ).insert({ _id : 4 }); +assert.gleSuccess(mongosConnNew.getCollection( collSharded + "" ).getDB()); +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.getCollection( collUnsharded + "" ).insert({ _id : 4 }); +assert.gleSuccess(mongosConnNew.getCollection( collUnsharded + "" ).getDB()); + +gc(); // Clean up new connections + +mongosConnIdle = new Mongo( mongos.host ); + +jsTest.log("Stopping primary of second shard..."); + +mongosConnActive.setSlaveOk(); +mongosConnIdle = new Mongo( mongos.host ); +mongosConnIdle.setSlaveOk(); + +// Need to save this node for later +var rs1Secondary = st.rs1.getSecondary(); + +st.rs1.stop(st.rs1.getPrimary(), true /* wait for stop */); + +jsTest.log("Testing active connection with second primary down..."); + +assert.neq(null, mongosConnActive.getCollection( collSharded + "" ).findOne({ _id : -1 })); +assert.neq(null, mongosConnActive.getCollection( collSharded + "" ).findOne({ _id : 1 })); +assert.neq(null, mongosConnActive.getCollection( collUnsharded + "" ).findOne({ _id : 1 })); + +mongosConnActive.getCollection( collSharded + "" ).insert({ _id : -5 }); +assert.gleSuccess(mongosConnActive.getCollection( collSharded + "" ).getDB()); +mongosConnActive.getCollection( collSharded + "" ).insert({ _id : 5 }); +gleErrorOrThrow(mongosConnActive.getCollection( collSharded + "" ).getDB()); +mongosConnActive.getCollection( collUnsharded + "" ).insert({ _id : 5 }); +assert.gleSuccess(mongosConnActive.getCollection( collUnsharded + "" ).getDB()); + +jsTest.log("Testing idle connection with second primary down..."); + +mongosConnIdle.getCollection( collSharded + "" ).insert({ _id : -6 }); +assert.gleSuccess(mongosConnIdle.getCollection( collSharded + "" ).getDB()); +assert.commandWorked(admin.runCommand({ setParameter : 1, logLevel : 5 })); +mongosConnIdle.getCollection( collSharded + "" ).insert({ _id : 6 }); +gleErrorOrThrow(mongosConnIdle.getCollection( collSharded + "" ).getDB()); +mongosConnIdle.getCollection( collUnsharded + "" ).insert({ _id : 6 }); +assert.gleSuccess(mongosConnIdle.getCollection( collUnsharded + "" ).getDB()); + +assert.neq(null, mongosConnIdle.getCollection( collSharded + "" ).findOne({ _id : -1 }) ); +assert.neq(null, mongosConnIdle.getCollection( collSharded + "" ).findOne({ _id : 1 }) ); +assert.neq(null, mongosConnIdle.getCollection( collUnsharded + "" ).findOne({ _id : 1 }) ); + +jsTest.log("Testing new connections with second primary down..."); + +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.setSlaveOk(); +assert.neq(null, mongosConnNew.getCollection( collSharded + "" ).findOne({ _id : -1 }) ); +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.setSlaveOk(); +assert.neq(null, mongosConnNew.getCollection( collSharded + "" ).findOne({ _id : 1 }) ); +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.setSlaveOk(); +assert.neq(null, mongosConnNew.getCollection( collUnsharded + "" ).findOne({ _id : 1 }) ); + +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.getCollection( collSharded + "" ).insert({ _id : -7 }); +assert.gleSuccess(mongosConnNew.getCollection( collSharded + "" ).getDB()); +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.getCollection( collSharded + "" ).insert({ _id : 7 }); +gleErrorOrThrow(mongosConnNew.getCollection( collSharded + "" ).getDB()); +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.getCollection( collUnsharded + "" ).insert({ _id : 7 }); +assert.gleSuccess(mongosConnNew.getCollection( collUnsharded + "" ).getDB()); + +gc(); // Clean up new connections + +jsTest.log("Stopping primary of first shard..."); + +mongosConnActive.setSlaveOk(); +mongosConnIdle = new Mongo( mongos.host ); +mongosConnIdle.setSlaveOk(); + +st.rs0.stop(st.rs0.getPrimary(), true /*wait for stop*/ ); + +jsTest.log("Testing active connection with first primary down..."); + +assert.neq(null, mongosConnActive.getCollection( collSharded + "" ).findOne({ _id : -1 })); +assert.neq(null, mongosConnActive.getCollection( collSharded + "" ).findOne({ _id : 1 })); +assert.neq(null, mongosConnActive.getCollection( collUnsharded + "" ).findOne({ _id : 1 })); + +mongosConnActive.getCollection( collSharded + "" ).insert({ _id : -8 }); +gleErrorOrThrow(mongosConnActive.getCollection( collSharded + "" ).getDB()); +mongosConnActive.getCollection( collSharded + "" ).insert({ _id : 8 }); +gleErrorOrThrow(mongosConnActive.getCollection( collSharded + "" ).getDB()); +mongosConnActive.getCollection( collUnsharded + "" ).insert({ _id : 8 }); +gleErrorOrThrow(mongosConnActive.getCollection( collUnsharded + "" ).getDB()); + +jsTest.log("Testing idle connection with first primary down..."); + +mongosConnIdle.getCollection( collSharded + "" ).insert({ _id : -9 }); +gleErrorOrThrow(mongosConnIdle.getCollection( collSharded + "" ).getDB()); +mongosConnIdle.getCollection( collSharded + "" ).insert({ _id : 9 }); +gleErrorOrThrow(mongosConnIdle.getCollection( collSharded + "" ).getDB()); +mongosConnIdle.getCollection( collUnsharded + "" ).insert({ _id : 9 }); +gleErrorOrThrow(mongosConnIdle.getCollection( collUnsharded + "" ).getDB()); + +assert.neq(null, mongosConnIdle.getCollection( collSharded + "" ).findOne({ _id : -1 }) ); +assert.neq(null, mongosConnIdle.getCollection( collSharded + "" ).findOne({ _id : 1 }) ); +assert.neq(null, mongosConnIdle.getCollection( collUnsharded + "" ).findOne({ _id : 1 }) ); + +jsTest.log("Testing new connections with first primary down..."); + +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.setSlaveOk(); +assert.neq(null, mongosConnNew.getCollection( collSharded + "" ).findOne({ _id : -1 }) ); +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.setSlaveOk(); +assert.neq(null, mongosConnNew.getCollection( collSharded + "" ).findOne({ _id : 1 }) ); +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.setSlaveOk(); +assert.neq(null, mongosConnNew.getCollection( collUnsharded + "" ).findOne({ _id : 1 }) ); + +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.getCollection( collSharded + "" ).insert({ _id : -10 }); +gleErrorOrThrow(mongosConnNew.getCollection( collSharded + "" ).getDB()); +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.getCollection( collSharded + "" ).insert({ _id : 10 }); +gleErrorOrThrow(mongosConnNew.getCollection( collSharded + "" ).getDB()); +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.getCollection( collUnsharded + "" ).insert({ _id : 10 }); +gleErrorOrThrow(mongosConnNew.getCollection( collUnsharded + "" ).getDB()); + +gc(); // Clean up new connections + +jsTest.log("Stopping second shard..."); + +mongosConnActive.setSlaveOk(); +mongosConnIdle = new Mongo( mongos.host ); +mongosConnIdle.setSlaveOk(); + +st.rs1.stop(rs1Secondary, true /* wait for stop */); + +jsTest.log("Testing active connection with second shard down..."); + +assert.neq(null, mongosConnActive.getCollection( collSharded + "" ).findOne({ _id : -1 })); +assert.neq(null, mongosConnActive.getCollection( collUnsharded + "" ).findOne({ _id : 1 })); + +mongosConnActive.getCollection( collSharded + "" ).insert({ _id : -8 }); +gleErrorOrThrow(mongosConnActive.getCollection( collSharded + "" ).getDB()); +mongosConnActive.getCollection( collSharded + "" ).insert({ _id : 8 }); +gleErrorOrThrow(mongosConnActive.getCollection( collSharded + "" ).getDB()); +mongosConnActive.getCollection( collUnsharded + "" ).insert({ _id : 8 }); +gleErrorOrThrow(mongosConnActive.getCollection( collUnsharded + "" ).getDB()); + +jsTest.log("Testing idle connection with second shard down..."); + +mongosConnIdle.getCollection( collSharded + "" ).insert({ _id : -9 }); +gleErrorOrThrow(mongosConnIdle.getCollection( collSharded + "" ).getDB()); +mongosConnIdle.getCollection( collSharded + "" ).insert({ _id : 9 }); +gleErrorOrThrow(mongosConnIdle.getCollection( collSharded + "" ).getDB()); +mongosConnIdle.getCollection( collUnsharded + "" ).insert({ _id : 9 }); +gleErrorOrThrow(mongosConnIdle.getCollection( collUnsharded + "" ).getDB()); + +assert.neq(null, mongosConnIdle.getCollection( collSharded + "" ).findOne({ _id : -1 }) ); +assert.neq(null, mongosConnIdle.getCollection( collUnsharded + "" ).findOne({ _id : 1 }) ); + +jsTest.log("Testing new connections with second shard down..."); + +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.setSlaveOk(); +assert.neq(null, mongosConnNew.getCollection( collSharded + "" ).findOne({ _id : -1 }) ); +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.setSlaveOk(); +assert.neq(null, mongosConnNew.getCollection( collUnsharded + "" ).findOne({ _id : 1 }) ); + +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.getCollection( collSharded + "" ).insert({ _id : -10 }); +gleErrorOrThrow(mongosConnNew.getCollection( collSharded + "" ).getDB()); +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.getCollection( collSharded + "" ).insert({ _id : 10 }); +gleErrorOrThrow(mongosConnNew.getCollection( collSharded + "" ).getDB()); +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.getCollection( collUnsharded + "" ).insert({ _id : 10 }); +gleErrorOrThrow(mongosConnNew.getCollection( collUnsharded + "" ).getDB()); + +gc(); // Clean up new connections + +jsTest.log("DONE!"); +st.stop(); + + + + + diff --git a/jstests/sharding/mongos_shard_failure_tolerance.js b/jstests/sharding/mongos_shard_failure_tolerance.js new file mode 100644 index 00000000000..9f5ffefca28 --- /dev/null +++ b/jstests/sharding/mongos_shard_failure_tolerance.js @@ -0,0 +1,186 @@ +// +// Tests mongos's failure tolerance for single-node shards +// +// +// Sets up a cluster with three shards, the first shard of which has an unsharded collection and +// half a sharded collection. The second shard has the second half of the sharded collection, and +// the third shard has nothing. Progressively shuts down each shard to see the impact on the +// cluster. +// +// Three different connection states are tested - active (connection is active through whole +// sequence), idle (connection is connected but not used before a shard change), and new +// (connection connected after shard change). +// + +var options = {separateConfig : true}; +var st = new ShardingTest({shards : 3, mongos : 1, other : options}); +st.stopBalancer(); + +var mongos = st.s0; +var admin = mongos.getDB( "admin" ); +var shards = mongos.getDB( "config" ).shards.find().toArray(); + +assert.commandWorked( admin.runCommand({ setParameter : 1, traceExceptions : true }) ); +assert.commandWorked( admin.runCommand({ setParameter : 1, ignoreInitialVersionFailure : true }) ); + +var collSharded = mongos.getCollection( "fooSharded.barSharded" ); +var collUnsharded = mongos.getCollection( "fooUnsharded.barUnsharded" ); + +assert.commandWorked( admin.runCommand({ enableSharding : collSharded.getDB() + "" }) ); +printjson( admin.runCommand({ movePrimary : collSharded.getDB() + "", to : shards[0]._id }) ); +assert.commandWorked( admin.runCommand({ shardCollection : collSharded + "", key : { _id : 1 } }) ); +assert.commandWorked( admin.runCommand({ split : collSharded + "", middle : { _id : 0 } }) ); +assert.commandWorked( admin.runCommand({ moveChunk : collSharded + "", + find : { _id : 0 }, + to : shards[1]._id }) ); + +// Create the unsharded database +collUnsharded.insert({ some : "doc" }); +assert.eq( null, collUnsharded.getDB().getLastError() ); +collUnsharded.remove({}); +assert.eq( null, collUnsharded.getDB().getLastError() ); +printjson( admin.runCommand({ movePrimary : collUnsharded.getDB() + "", to : shards[0]._id }) ); + +st.printShardingStatus(); + +// Needed b/c the GLE command itself can fail if the shard is down ("write result unknown") - we +// don't care if this happens in this test, we only care that we did not get "write succeeded". +// Depending on the connection pool state, we could get either. +function gleErrorOrThrow(database, msg) { + var gle; + try { + gle = database.getLastErrorObj(); + } + catch (ex) { + return; + } + if (!gle.err) doassert("getLastError is null: " + tojson(gle) + " :" + msg); + return; +}; + +// +// Setup is complete +// + +jsTest.log("Inserting initial data..."); + +var mongosConnActive = new Mongo( mongos.host ); +var mongosConnIdle = null; +var mongosConnNew = null; + +mongosConnActive.getCollection( collSharded + "" ).insert({ _id : -1 }); +mongosConnActive.getCollection( collSharded + "" ).insert({ _id : 1 }); +assert.eq(null, mongosConnActive.getCollection( collSharded + "" ).getDB().getLastError()); + +mongosConnActive.getCollection( collUnsharded + "" ).insert({ _id : 1 }); +assert.eq(null, mongosConnActive.getCollection( collUnsharded + "" ).getDB().getLastError()); + +jsTest.log("Stopping third shard..."); + +mongosConnIdle = new Mongo( mongos.host ); + +MongoRunner.stopMongod( st.shard2 ); + +jsTest.log("Testing active connection..."); + +assert.neq(null, mongosConnActive.getCollection( collSharded + "" ).findOne({ _id : -1 })); +assert.neq(null, mongosConnActive.getCollection( collSharded + "" ).findOne({ _id : 1 })); +assert.neq(null, mongosConnActive.getCollection( collUnsharded + "" ).findOne({ _id : 1 })); + +mongosConnActive.getCollection( collSharded + "" ).insert({ _id : -2 }); +assert.gleSuccess(mongosConnActive.getCollection( collSharded + "" ).getDB()); +mongosConnActive.getCollection( collSharded + "" ).insert({ _id : 2 }); +assert.gleSuccess(mongosConnActive.getCollection( collSharded + "" ).getDB()); +mongosConnActive.getCollection( collUnsharded + "" ).insert({ _id : 2 }); +assert.gleSuccess(mongosConnActive.getCollection( collUnsharded + "" ).getDB()); + +jsTest.log("Testing idle connection..."); + +mongosConnIdle.getCollection( collSharded + "" ).insert({ _id : -3 }); +assert.gleSuccess(mongosConnIdle.getCollection( collSharded + "" ).getDB()); +mongosConnIdle.getCollection( collSharded + "" ).insert({ _id : 3 }); +assert.gleSuccess(mongosConnIdle.getCollection( collSharded + "" ).getDB()); +mongosConnIdle.getCollection( collUnsharded + "" ).insert({ _id : 3 }); +assert.gleSuccess(mongosConnIdle.getCollection( collUnsharded + "" ).getDB()); + +assert.neq(null, mongosConnIdle.getCollection( collSharded + "" ).findOne({ _id : -1 }) ); +assert.neq(null, mongosConnIdle.getCollection( collSharded + "" ).findOne({ _id : 1 }) ); +assert.neq(null, mongosConnIdle.getCollection( collUnsharded + "" ).findOne({ _id : 1 }) ); + +jsTest.log("Testing new connections..."); + +mongosConnNew = new Mongo( mongos.host ); +assert.neq(null, mongosConnNew.getCollection( collSharded + "" ).findOne({ _id : -1 }) ); +mongosConnNew = new Mongo( mongos.host ); +assert.neq(null, mongosConnNew.getCollection( collSharded + "" ).findOne({ _id : 1 }) ); +mongosConnNew = new Mongo( mongos.host ); +assert.neq(null, mongosConnNew.getCollection( collUnsharded + "" ).findOne({ _id : 1 }) ); + +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.getCollection( collSharded + "" ).insert({ _id : -4 }); +assert.gleSuccess(mongosConnNew.getCollection( collSharded + "" ).getDB()); +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.getCollection( collSharded + "" ).insert({ _id : 4 }); +assert.gleSuccess(mongosConnNew.getCollection( collSharded + "" ).getDB()); +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.getCollection( collUnsharded + "" ).insert({ _id : 4 }); +assert.gleSuccess(mongosConnNew.getCollection( collUnsharded + "" ).getDB()); + +gc(); // Clean up new connections + +jsTest.log("Stopping second shard..."); + +mongosConnIdle = new Mongo( mongos.host ); + +MongoRunner.stopMongod( st.shard1 ); + +jsTest.log("Testing active connection..."); + +assert.neq(null, mongosConnActive.getCollection( collSharded + "" ).findOne({ _id : -1 }) ); +assert.neq(null, mongosConnActive.getCollection( collUnsharded + "" ).findOne({ _id : 1 }) ); + +mongosConnActive.getCollection( collSharded + "" ).insert({ _id : -5 }); +assert.gleSuccess(mongosConnActive.getCollection( collSharded + "" ).getDB()); +mongosConnActive.getCollection( collSharded + "" ).insert({ _id : 5 }); +gleErrorOrThrow(mongosConnActive.getCollection( collSharded + "" ).getDB()); +mongosConnActive.getCollection( collUnsharded + "" ).insert({ _id : 5 }); +assert.gleSuccess(mongosConnActive.getCollection( collUnsharded + "" ).getDB()); + +jsTest.log("Testing idle connection..."); + +mongosConnIdle.getCollection( collSharded + "" ).insert({ _id : -6 }); +assert.gleSuccess(mongosConnIdle.getCollection( collSharded + "" ).getDB()); +mongosConnIdle.getCollection( collSharded + "" ).insert({ _id : 6 }); +gleErrorOrThrow(mongosConnIdle.getCollection( collSharded + "" ).getDB()); +mongosConnIdle.getCollection( collUnsharded + "" ).insert({ _id : 6 }); +assert.gleSuccess(mongosConnIdle.getCollection( collUnsharded + "" ).getDB()); + +assert.neq(null, mongosConnIdle.getCollection( collSharded + "" ).findOne({ _id : -1 }) ); +assert.neq(null, mongosConnIdle.getCollection( collUnsharded + "" ).findOne({ _id : 1 }) ); + +jsTest.log("Testing new connections..."); + +mongosConnNew = new Mongo( mongos.host ); +assert.neq(null, mongosConnNew.getCollection( collSharded + "" ).findOne({ _id : -1 }) ); +mongosConnNew = new Mongo( mongos.host ); +assert.neq(null, mongosConnNew.getCollection( collUnsharded + "" ).findOne({ _id : 1 }) ); + +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.getCollection( collSharded + "" ).insert({ _id : -7 }); +assert.gleSuccess(mongosConnNew.getCollection( collSharded + "" ).getDB()); +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.getCollection( collSharded + "" ).insert({ _id : 7 }); +gleErrorOrThrow(mongosConnNew.getCollection( collSharded + "" ).getDB()); +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.getCollection( collUnsharded + "" ).insert({ _id : 7 }); +assert.gleSuccess(mongosConnNew.getCollection( collUnsharded + "" ).getDB()); + +gc(); // Clean up new connections + +jsTest.log("DONE!"); +st.stop(); + + + + + diff --git a/src/mongo/base/error_codes.err b/src/mongo/base/error_codes.err index 1dfa1702b64..5fa8cde17e2 100644 --- a/src/mongo/base/error_codes.err +++ b/src/mongo/base/error_codes.err @@ -27,4 +27,7 @@ error_code("AlreadyInitialized", 23) error_code("LockTimeout", 24) error_code("RemoteValidationError", 25) +# Non-sequential error codes (for compatibility only) +error_code("NodeNotFound", 74) + error_class("NetworkError", ["HostUnreachable", "HostNotFound"]) diff --git a/src/mongo/client/dbclient_rs.cpp b/src/mongo/client/dbclient_rs.cpp index 21a5b0810f4..62544aca668 100644 --- a/src/mongo/client/dbclient_rs.cpp +++ b/src/mongo/client/dbclient_rs.cpp @@ -1593,6 +1593,18 @@ namespace mongo { return _getMonitor()->isAnyNodeOk(); } + void DBClientReplicaSet::authPrimary(const BSONObj& params) { + _auth(params); + } + + bool DBClientReplicaSet::authPrimary( const string &dbname, + const string &username, + const string &password_text, + string& errmsg, + bool digestPassword ) { + return auth( dbname, username, password_text, errmsg, digestPassword ); + } + void DBClientReplicaSet::_auth(const BSONObj& params) { DBClientConnection * m = checkMaster(); @@ -1620,6 +1632,99 @@ namespace mongo { _auths[params[saslCommandPrincipalSourceFieldName].str()] = params.getOwned(); } + bool DBClientReplicaSet::authAny( const string &dbname, + const string &username, + const string &password_text, + string& errmsg, + bool digestPassword ) { + try { + authAny(BSON(saslCommandMechanismFieldName << "MONGODB-CR" << + saslCommandPrincipalSourceFieldName << dbname << + saslCommandPrincipalFieldName << username << + saslCommandPasswordFieldName << password_text << + saslCommandDigestPasswordFieldName << digestPassword)); + return true; + } catch(const UserException& ex) { + if (ex.getCode() != ErrorCodes::AuthenticationFailed) + throw; + errmsg = ex.what(); + return false; + } + } + + static bool isAuthException( const DBException& ex ) { + return ex.getCode() == ErrorCodes::AuthenticationFailed; + } + + void DBClientReplicaSet::authAny( const BSONObj& params ) { + + // We prefer to authenticate against a primary, but otherwise a secondary is ok too + // Empty tag matches every secondary + TagSet tags(BSON_ARRAY(BSONObj())); + shared_ptr<ReadPreferenceSetting> readPref( + new ReadPreferenceSetting( ReadPreference_PrimaryPreferred, tags ) ); + + LOG(3) << "dbclient_rs authentication of " << _getMonitor()->getName() << endl; + + // NOTE that we retry MAX_RETRY + 1 times, since we're always primary preferred we don't + // fallback to the primary. + Status lastNodeStatus = Status::OK(); + for ( size_t retry = 0; retry < MAX_RETRY + 1; retry++ ) { + try { + DBClientConnection* conn = selectNodeUsingTags( readPref ); + + if ( conn == NULL ) { + break; + } + + conn->auth( params ); + + // Cache the new auth information since we've now validated it's good + _auths[params[saslCommandPrincipalSourceFieldName].str()] = params.getOwned(); + + // Ensure the only child connection open is the one we authenticated against - other + // child connections may not have full authentication information. + // NOTE: _lastSlaveOkConn may or may not be the same as _master + dassert(_lastSlaveOkConn.get() == conn || _master.get() == conn); + if ( conn != _lastSlaveOkConn.get() ) { + _lastSlaveOkHost = HostAndPort(); + _lastSlaveOkConn.reset(); + } + if ( conn != _master.get() ) { + _masterHost = HostAndPort(); + _master.reset(); + } + + return; + } + catch ( const DBException &ex ) { + + // We care if we can't authenticate (i.e. bad password) in credential params. + // We shouldn't be unauthorized since we aren't doing anything yet. + if ( isAuthException( ex ) ) { + throw; + } + + StringBuilder errMsgB; + errMsgB << "can't authenticate against replica set node " + << _lastSlaveOkHost.toString(); + lastNodeStatus = ex.toStatus( errMsgB.str() ); + + LOG(1) << lastNodeStatus.reason() << endl; + invalidateLastSlaveOkCache(); + } + } + + if ( lastNodeStatus.isOK() ) { + StringBuilder assertMsgB; + assertMsgB << "Failed to authenticate, no good nodes in " << _getMonitor()->getName(); + uasserted( ErrorCodes::NodeNotFound, assertMsgB.str() ); + } + else { + uasserted( lastNodeStatus.code(), lastNodeStatus.reason() ); + } + } + void DBClientReplicaSet::logout(const string &dbname, BSONObj& info) { DBClientConnection* priConn = checkMaster(); diff --git a/src/mongo/client/dbclient_rs.h b/src/mongo/client/dbclient_rs.h index 05f7faa9b2b..ddf986c732b 100644 --- a/src/mongo/client/dbclient_rs.h +++ b/src/mongo/client/dbclient_rs.h @@ -518,10 +518,51 @@ namespace mongo { virtual bool call( Message &toSend, Message &response, bool assertOk=true , string * actualServer = 0 ); virtual bool callRead( Message& toSend , Message& response ) { return checkMaster()->callRead( toSend , response ); } + /** + * Authenticate using supplied credentials. Authenticates against the primary node, fails + * if node is down. + * Credentials are cached for future connections. + * + * See DBClientWithCommands::auth() for more details. + * + * This is the default authentication mode for DBClientReplicaSet connections. + */ + void authPrimary(const BSONObj& params); + + /** + * Same as above, but authorizes access to a particular database. + * + * See DBClientWithCommands::auth() for more details. + */ + bool authPrimary(const string &dbname, + const string &username, + const string &password_text, + string& errmsg, + bool digestPassword); + + /** + * Authenticate using supplied credentials. Prefers authentication against the primary + * node, will fall back to a secondary and retry if the primary node is down but + * secondaries are still available. + * Credentials are cached for future connections. + * + * See DBClientWithCommands::auth() for more details. + */ + void authAny(const BSONObj& params); + + /** + * Same as above, but authorizes access to a particular database. + * + * See DBClientWithCommands::auth() for more details. + */ + bool authAny( const string &dbname, + const string &username, + const string &password_text, + string& errmsg, + bool digestPassword ); protected: - /** Authorize. Authorizes all nodes as needed - */ + virtual void _auth(const BSONObj& params); virtual void sayPiggyBack( Message &toSend ) { checkMaster()->say( toSend ); } diff --git a/src/mongo/client/parallel.cpp b/src/mongo/client/parallel.cpp index 18649d9c6b9..d3b03425193 100644 --- a/src/mongo/client/parallel.cpp +++ b/src/mongo/client/parallel.cpp @@ -715,26 +715,27 @@ namespace mongo { } const DBClientBase* rawConn = state->conn->getRawConn(); - if (( _options & QueryOption_SlaveOk ) && - rawConn->type() == ConnectionString::SET && - rawConn->isFailed() ) { - /* A side effect of this short circuiting is this will not be - * able figure out that the primary is now up on it's own and - * has to rely on other threads to refresh the node states. - */ + bool allowShardVersionFailure = ( _options & QueryOption_SlaveOk ) + && rawConn->type() == ConnectionString::SET; + + if ( allowShardVersionFailure && rawConn->isFailed() ) { + + state->conn->donotCheckVersion(); + + // A side effect of this short circuiting is the mongos will not be able figure out that + // the primary is now up on it's own and has to rely on other threads to refresh node + // states. OCCASIONALLY { - const DBClientReplicaSet* repl = - dynamic_cast<const DBClientReplicaSet*>( rawConn ); + const DBClientReplicaSet* repl = dynamic_cast<const DBClientReplicaSet*>( rawConn ); + dassert(repl); warning() << "Primary for " << repl->getServerAddress() << " was down before, bypassing setShardVersion." - << " Local config view can be stale." << endl; + << " The local replica set view and targeting may be stale." << endl; } - } else { + } + else { try { - /* TODO: Undo SERVER-5797. This try-catch is a temporary hack until - * secondaries can properly handle shard versioning - */ if ( state->conn->setVersion() ) { // It's actually okay if we set the version here, since either the // manager will be verified as compatible, or if the manager doesn't @@ -742,19 +743,20 @@ namespace mongo { LOG( pc ) << "needed to set remote version on connection to value " << "compatible with " << vinfo << endl; } - } catch ( const DBException& dbEx ) { - if ( (dbEx.getCode() == 10009 /* no master */ && - ( _options & QueryOption_SlaveOk )) ) { + } + catch ( const DBException& dbEx ) { + if ( allowShardVersionFailure ) { + + // It's okay if we don't set the version when talking to a secondary, we can + // be stale in any case. OCCASIONALLY { const DBClientReplicaSet* repl = - dynamic_cast<const DBClientReplicaSet*>( - state->conn->getRawConn() ); - - warning() << "Cannot contact primary for " - << repl->getServerAddress() - << " to check shard version. " - << "SlaveOk query can be sent to the wrong shard." + dynamic_cast<const DBClientReplicaSet*>( state->conn->getRawConn() ); + dassert(repl); + warning() << "Cannot contact primary for " << repl->getServerAddress() + << " to check shard version." + << " The local replica set view and targeting may be stale." << endl; } } diff --git a/src/mongo/s/shard.cpp b/src/mongo/s/shard.cpp index df620865603..28b281f0cbd 100644 --- a/src/mongo/s/shard.cpp +++ b/src/mongo/s/shard.cpp @@ -30,6 +30,7 @@ #include "mongo/db/auth/privilege.h" #include "mongo/db/commands.h" #include "mongo/db/jsobj.h" +#include "mongo/db/server_parameters.h" #include "mongo/s/client_info.h" #include "mongo/s/config.h" #include "mongo/s/request.h" @@ -39,6 +40,8 @@ namespace mongo { + MONGO_EXPORT_SERVER_PARAMETER(authOnPrimaryOnly, bool, true); + class StaticShardInfo { public: StaticShardInfo() : _mutex("StaticShardInfo"), _rsMutex("RSNameMap") { } @@ -411,14 +414,26 @@ namespace mongo { void ShardingConnectionHook::onCreate( DBClientBase * conn ) { if( !noauth ) { + bool result; string err; LOG(2) << "calling onCreate auth for " << conn->toString() << endl; - bool result = conn->auth( "local", - internalSecurity.user, - internalSecurity.pwd, - err, - false ); + if ( conn->type() == ConnectionString::SET && !authOnPrimaryOnly ) { + DBClientReplicaSet* setConn = dynamic_cast<DBClientReplicaSet*>(conn); + verify(setConn); + result = setConn->authAny( "local", + internalSecurity.user, + internalSecurity.pwd, + err, + false ); + } + else { + result = conn->auth( "local", + internalSecurity.user, + internalSecurity.pwd, + err, + false ); + } uassert( 15847, str::stream() << "can't authenticate to server " << conn->getServerAddress() << causedBy( err ), result ); diff --git a/src/mongo/s/shard.h b/src/mongo/s/shard.h index 39504311a65..41f819d3ae9 100644 --- a/src/mongo/s/shard.h +++ b/src/mongo/s/shard.h @@ -290,8 +290,12 @@ namespace mongo { */ bool runCommand( const string& db , const BSONObj& cmd , BSONObj& res ); + // Whether or not we release connections from the thread-local cache after a read static bool releaseConnectionsAfterResponse; + // Controls whether we throw on initially failing to set a version + static bool ignoreInitialVersionFailure; + /** checks all of my thread local connections for the version of this ns */ static void checkMyConnectionVersions( const string & ns ); diff --git a/src/mongo/s/shardconnection.cpp b/src/mongo/s/shardconnection.cpp index 2ef03996367..66a105f05e8 100644 --- a/src/mongo/s/shardconnection.cpp +++ b/src/mongo/s/shardconnection.cpp @@ -22,6 +22,7 @@ #include "mongo/db/client.h" #include "mongo/db/commands.h" +#include "mongo/db/lasterror.h" #include "mongo/db/server_parameters.h" #include "mongo/s/config.h" #include "mongo/s/request.h" @@ -33,7 +34,13 @@ namespace mongo { - MONGO_EXPORT_SERVER_PARAMETER(ignoreInitialVersionFailure, bool, false); + bool ShardConnection::ignoreInitialVersionFailure( false ); + ExportedServerParameter<bool> + _ignoreInitialVersionFailure( ServerParameterSet::getGlobal(), + "ignoreInitialVersionFailure", + &ShardConnection::ignoreInitialVersionFailure, + true, + true ); DBConnectionPool shardConnectionPool; @@ -237,6 +244,12 @@ namespace mongo { vector<Shard> all; Shard::getAllShards( all ); + scoped_ptr<LastError::Disabled> ignoreForGLE; + if ( ShardConnection::ignoreInitialVersionFailure ) { + // Don't report exceptions here as errors in GetLastError if ignoring failures + ignoreForGLE.reset( new LastError::Disabled( lastError.get( false ) ) ); + } + // Now only check top-level shard connections for ( unsigned i=0; i<all.size(); i++ ) { @@ -252,12 +265,12 @@ namespace mongo { versionManager.checkShardVersionCB( s->avail, ns, false, 1 ); } - catch ( const std::exception& e ) { + catch ( const DBException& ex ) { - warning() << "problem while initially checking shard versions on" - << " " << shard.getName() << causedBy(e) << endl; + warning() << "problem while initially checking shard versions on " + << shard.getName() << causedBy( ex ) << endl; - if ( !ignoreInitialVersionFailure ) { + if ( !ShardConnection::ignoreInitialVersionFailure ) { throw; } else { diff --git a/src/mongo/s/version_manager.cpp b/src/mongo/s/version_manager.cpp index 1496ebc1407..3ba6254813b 100644 --- a/src/mongo/s/version_manager.cpp +++ b/src/mongo/s/version_manager.cpp @@ -103,23 +103,46 @@ namespace mongo { WriteBackListener::init( *conn_in ); - DBClientBase* conn = getVersionable( conn_in ); - verify( conn ); // errors thrown above + bool ok; + DBClientBase* conn = NULL; + try { + // May throw if replica set primary is down + conn = getVersionable( conn_in ); + dassert( conn ); // errors thrown above + + BSONObjBuilder cmdBuilder; + + cmdBuilder.append( "setShardVersion" , "" ); + cmdBuilder.appendBool( "init", true ); + cmdBuilder.append( "configdb" , configServer.modelServer() ); + cmdBuilder.appendOID( "serverID" , &serverID ); + cmdBuilder.appendBool( "authoritative" , true ); - BSONObjBuilder cmdBuilder; + BSONObj cmd = cmdBuilder.obj(); - cmdBuilder.append( "setShardVersion" , "" ); - cmdBuilder.appendBool( "init", true ); - cmdBuilder.append( "configdb" , configServer.modelServer() ); - cmdBuilder.appendOID( "serverID" , &serverID ); - cmdBuilder.appendBool( "authoritative" , true ); + LOG(1) << "initializing shard connection to " << conn->toString() << endl; + LOG(2) << "initial sharding settings : " << cmd << endl; + + ok = conn->runCommand("admin", cmd, result, 0); + } + catch( const DBException& ex ) { - BSONObj cmd = cmdBuilder.obj(); + bool ignoreFailure = ShardConnection::ignoreInitialVersionFailure + && conn_in->type() == ConnectionString::SET; + if ( !ignoreFailure ) + throw; - LOG(1) << "initializing shard connection to " << conn->toString() << endl; - LOG(2) << "initial sharding settings : " << cmd << endl; + // Using initShardVersion is not strictly required when talking to replica sets - it is + // preferred to do so because it registers mongos early with the mongod. This info is + // also sent by checkShardVersion before a connection is used for a write or read. - bool ok = conn->runCommand("admin", cmd, result, 0); + OCCASIONALLY { + warning() << "failed to initialize new replica set connection version, " + << "will initialize on first use" << endl; + } + + return true; + } // HACK for backwards compatibility with v1.8.x, v2.0.0 and v2.0.1 // Result is false, but will still initialize serverID and configdb diff --git a/src/mongo/shell/assert.js b/src/mongo/shell/assert.js index b1279028307..0e84c52d11c 100644 --- a/src/mongo/shell/assert.js +++ b/src/mongo/shell/assert.js @@ -225,3 +225,41 @@ assert.close = function(a, b, msg, places){ doassert(a + " is not equal to " + b + " within " + places + " places, diff: " + (a-b) + " : " + msg); }; + +assert.gleSuccess = function(db, msg) { + var gle = db.getLastErrorObj(); + if (gle.err) { + if (typeof(msg) == "function") + msg = msg(gle); + doassert("getLastError not null:" + tojson(gle) + " :" + msg); + } +} + +assert.gleError = function(db, msg) { + var gle = db.getLastErrorObj(); + if (!gle.err) { + if (typeof(msg) == "function") + msg = msg(gle); + doassert("getLastError is null: " + tojson(gle) + " :" + msg); + } +} + +assert.gleErrorCode = function(db, code, msg) { + var gle = db.getLastErrorObj(); + if (gle.err && (gle.code == code)) { + if (typeof(msg) == "function") + msg = msg(gle); + doassert("getLastError not null or missing code( " + code + "): " + + tojson(gle) + " :" + msg); + } +} + +assert.gleErrorRegex = function(db, regex, msg) { + var gle = db.getLastErrorObj(); + if (!gle.err || !regex.test(gle.err)) { + if (typeof(msg) == "function") + msg = msg(gle); + doassert("getLastError is null or doesn't match regex (" + regex + "): " + + tojson(gle) + " :" + msg); + } +} |