diff options
author | Greg Studer <greg@10gen.com> | 2012-09-19 13:09:58 -0400 |
---|---|---|
committer | Greg Studer <greg@10gen.com> | 2012-09-19 18:40:27 -0400 |
commit | 0037700ed5f0d46a80ab8cb86075ae41813e83d6 (patch) | |
tree | 4d495b050b6c0d107f6fa278327ed3a2cd05d1fe | |
parent | 242597ae464698f8fc243ec869654cdc756dabb9 (diff) | |
download | mongo-0037700ed5f0d46a80ab8cb86075ae41813e83d6.tar.gz |
SERVER-6540 fix writeback hangs when messages are too big
-rw-r--r-- | jstests/sharding/writeback_bulk_insert.js | 115 | ||||
-rw-r--r-- | src/mongo/s/d_logic.cpp | 14 |
2 files changed, 127 insertions, 2 deletions
diff --git a/jstests/sharding/writeback_bulk_insert.js b/jstests/sharding/writeback_bulk_insert.js new file mode 100644 index 00000000000..a8ed53f880c --- /dev/null +++ b/jstests/sharding/writeback_bulk_insert.js @@ -0,0 +1,115 @@ +// +// Tests whether a writeback error during bulk insert hangs GLE +// + +jsTest.log("Starting sharded cluster...") + +var st = new ShardingTest({shards : 1, + mongos : 3, + verbose : 2, + separateConfig : 1}) + +st.stopBalancer() + +var mongosA = st.s0 +var mongosB = st.s1 +var mongosC = st.s2 + +jsTest.log("Adding new collection...") + +var collA = mongosA.getCollection(jsTestName() + ".coll") +collA.insert({hello : "world"}) +assert.eq(null, collA.getDB().getLastError()) + +var collB = mongosB.getCollection("" + collA) +collB.insert({hello : "world"}) +assert.eq(null, collB.getDB().getLastError()) + +var collC = mongosB.getCollection("" + collA) +collC.insert({hello : "world"}) +assert.eq(null, collC.getDB().getLastError()) + +jsTest.log("Enabling sharding...") + +printjson(mongosA.getDB("admin").runCommand({enableSharding : collA.getDB() + + ""})) +printjson(mongosA.getDB("admin").runCommand({shardCollection : collA + "", + key : {_id : 1}})) + +// MongoD doesn't know about the config shard version *until* MongoS tells it +collA.findOne() + +// Preparing insert of exactly 16MB + +jsTest.log("Preparing bulk insert...") + +var data1MB = "x" +while (data1MB.length < 1024 * 1024) + data1MB += data1MB; + +var data7MB = "" +// Data now at 7MB +for ( var i = 0; i < 7; i++) + data7MB += data1MB; + +print("7MB object size is : " + Object.bsonsize({_id : 0, + d : data7MB})) + +var dataCloseTo8MB = data7MB; +// WARNING - MAGIC NUMBERS HERE +// The idea is to exceed the 16MB limit by just enough so that the message gets passed in the +// shell, but adding additional writeback information fails. +for ( var i = 0; i < 1031 * 1024 + 862; i++) { + dataCloseTo8MB += "x" +} + +print("Object size is: " + Object.bsonsize([{_id : 0, + d : dataCloseTo8MB}, + {_id : 1, + d : dataCloseTo8MB}])) + +jsTest.log("Trigger wbl for mongosB...") + +collB.insert([{_id : 0, + d : dataCloseTo8MB}, + {_id : 1, + d : dataCloseTo8MB}]) + +// Will hang if overflow is not detected correctly +jsTest.log("Waiting for GLE...") + +assert.neq(null, collB.getDB().getLastError()) + +print("GLE correctly returned error...") + +assert.eq(3, collA.find().itcount()) +assert.eq(3, collB.find().itcount()) + +var data8MB = data8MB; +for ( var i = 0; i < 1024 * 1024; i++) { + data8MB += "x" +} + +print("Object size is: " + Object.bsonsize([{_id : 0, + d : data8MB}, + {_id : 1, + d : data8MB}])) + +jsTest.log("Trigger wbl for mongosC...") + +collC.insert([{_id : 0, + d : data8MB}, + {_id : 1, + d : data8MB}]) + +// Should succeed since our insert size is 16MB (plus very small overhead) +jsTest.log("Waiting for GLE...") + +assert.eq(null, collC.getDB().getLastError()) + +print("GLE Successful...") + +assert.eq(5, collA.find().itcount()) +assert.eq(5, collB.find().itcount()) + +st.stop() diff --git a/src/mongo/s/d_logic.cpp b/src/mongo/s/d_logic.cpp index 4d04eea2be9..dcf66033428 100644 --- a/src/mongo/s/d_logic.cpp +++ b/src/mongo/s/d_logic.cpp @@ -100,7 +100,7 @@ namespace mongo { dbresponse->responseTo = m.header()->id; return true; } - + uassert( 9517 , "writeback" , ( d.reservedField() & Reserved_FromWriteback ) == 0 ); OID writebackID; @@ -109,6 +109,13 @@ namespace mongo { const OID& clientID = ShardedConnectionInfo::get(false)->getID(); massert( 10422 , "write with bad shard config and no server id!" , clientID.isSet() ); + // We need to check this here, since otherwise we'll get errors wrapping the writeback - + // not just here, but also when returning as a command result. + // We choose 1/2 the overhead of the internal maximum so that we can still handle ops of + // 16MB exactly. + massert( 16437, "data size of operation is too large to queue for writeback", + m.dataSize() < BSONObjMaxInternalSize - (8 * 1024)); + LOG(1) << "writeback queued for " << m.toString() << endl; BSONObjBuilder b; @@ -123,11 +130,14 @@ namespace mongo { b.appendBinData( "msg" , m.header()->len , bdtCustom , (char*)(m.singleData()) ); LOG(2) << "writing back msg with len: " << m.header()->len << " op: " << m.operation() << endl; + // Convert to new BSONObj here just to be safe + BSONObj wbObj = b.obj(); + // Don't register the writeback until immediately before we queue it - // after this line, mongos will wait for an hour if we don't queue correctly lastError.getSafe()->writeback( writebackID ); - writeBackManager.queueWriteBack( clientID.str() , b.obj() ); + writeBackManager.queueWriteBack( clientID.str() , wbObj ); return true; } |