diff options
-rw-r--r-- | jstests/sharding/mongos_validate_backoff.js | 50 | ||||
-rw-r--r-- | src/mongo/dbtests/basictests.cpp | 53 | ||||
-rw-r--r-- | src/mongo/s/strategy_shard.cpp | 12 | ||||
-rw-r--r-- | src/mongo/util/time_support.cpp | 36 | ||||
-rw-r--r-- | src/mongo/util/time_support.h | 23 |
5 files changed, 171 insertions, 3 deletions
diff --git a/jstests/sharding/mongos_validate_backoff.js b/jstests/sharding/mongos_validate_backoff.js new file mode 100644 index 00000000000..9b1ecfc5c50 --- /dev/null +++ b/jstests/sharding/mongos_validate_backoff.js @@ -0,0 +1,50 @@ +// +// Ensures that single mongos shard-key errors are fast, but slow down when many are triggered +// + +var st = new ShardingTest({ shards : 1, mongos : 1 }) + +var mongos = st.s0 +var admin = mongos.getDB( "admin" ) +var coll = mongos.getCollection( "foo.bar" ) + +printjson( admin.runCommand({ enableSharding : coll.getDB() + "" }) ) + +coll.ensureIndex({ shardKey : 1 }) +printjson( admin.runCommand({ shardCollection : coll + "", key : { shardKey : 1 } }) ) + +var timeBadInsert = function(){ + + var start = new Date().getTime() + + // Bad insert, no shard key + coll.insert({ hello : "world" }) + assert.neq( null, coll.getDB().getLastError() ) + + var end = new Date().getTime() + + return end - start +} + +// Loop over this test a few times, to ensure that the error counters get reset if we don't have +// bad inserts over a long enough time. +for( var test = 0; test < 3; test++ ){ + + var firstWait = timeBadInsert() + var lastWait = 0 + + for( var i = 0; i < 20; i++ ){ + printjson( lastWait = timeBadInsert() ) + } + + // Kind a heuristic test, we want to make sure that the error wait after sleeping is much less + // than the error wait after a lot of errors + assert.gt( lastWait, firstWait * 2 * 2 * 2 * 2 ) + + // Sleeping for long enough to reset our exponential counter + sleep( 3000 ) +} + +jsTest.log( "DONE!" ) + +st.stop() diff --git a/src/mongo/dbtests/basictests.cpp b/src/mongo/dbtests/basictests.cpp index 9d1ff713875..ded62a13049 100644 --- a/src/mongo/dbtests/basictests.cpp +++ b/src/mongo/dbtests/basictests.cpp @@ -27,6 +27,7 @@ #include "../util/paths.h" #include "../util/stringutils.h" #include "../util/compress.h" +#include "../util/time_support.h" #include "../db/db.h" namespace BasicTests { @@ -260,6 +261,56 @@ namespace BasicTests { }; + class SleepBackoffTest { + public: + void run() { + + int maxSleepTimeMillis = 1000; + int lastSleepTimeMillis = -1; + int epsMillis = 50; // Allowable inprecision for timing + + Backoff backoff( maxSleepTimeMillis, maxSleepTimeMillis * 2 ); + + Timer t; + + // Make sure our backoff increases to the maximum value + int maxSleepCount = 0; + while( maxSleepCount < 3 ){ + + t.reset(); + + backoff.nextSleepMillis(); + + int elapsedMillis = t.millis(); + + log() << "Slept for " << elapsedMillis << endl; + + ASSERT( almostGTE( elapsedMillis, lastSleepTimeMillis, epsMillis ) ); + lastSleepTimeMillis = elapsedMillis; + + if( almostEq( elapsedMillis, maxSleepTimeMillis, epsMillis ) ) maxSleepCount++; + } + + // Make sure that our backoff gets reset if we wait much longer than the maximum wait + sleepmillis( maxSleepTimeMillis * 4 ); + + t.reset(); + backoff.nextSleepMillis(); + + ASSERT( almostEq( t.millis(), 0, epsMillis ) ); + + } + + bool almostEq( int a, int b, int eps ){ + return std::abs( a - b ) <= eps; + } + + bool almostGTE( int a, int b, int eps ){ + if( almostEq( a, b, eps ) ) return true; + return a > b; + } + }; + class AssertTests { public: @@ -723,7 +774,6 @@ namespace BasicTests { Tee _tee; }; - class All : public Suite { public: All() : Suite( "basic" ) { @@ -739,6 +789,7 @@ namespace BasicTests { add< stringbuildertests::reset2 >(); add< sleeptest >(); + add< SleepBackoffTest >(); add< AssertTests >(); add< ArrayTests::basic1 >(); diff --git a/src/mongo/s/strategy_shard.cpp b/src/mongo/s/strategy_shard.cpp index 23d2591c466..534defa8b17 100644 --- a/src/mongo/s/strategy_shard.cpp +++ b/src/mongo/s/strategy_shard.cpp @@ -216,6 +216,9 @@ namespace mongo { } } + static const int maxWaitMillis = 500; + boost::thread_specific_ptr<Backoff> perThreadBackoff; + /** * Invoked before mongos needs to throw an error relating to an operation which cannot * be performed on a sharded collection. @@ -223,10 +226,15 @@ namespace mongo { * This prevents mongos from refreshing config data too quickly in response to bad requests, * since doing so is expensive. * - * TODO: Can we restructure to make this simpler? + * Each thread gets its own backoff wait sequence, to avoid interfering with other valid + * operations. */ void _sleepForVerifiedLocalError(){ - sleepsecs( 1 ); + + if( ! perThreadBackoff.get() ) + perThreadBackoff.reset( new Backoff( maxWaitMillis, maxWaitMillis * 2 ) ); + + perThreadBackoff.get()->nextSleepMillis(); } void _handleRetries( const string& op, diff --git a/src/mongo/util/time_support.cpp b/src/mongo/util/time_support.cpp index 0021c577c3b..01cc9a1efcf 100644 --- a/src/mongo/util/time_support.cpp +++ b/src/mongo/util/time_support.cpp @@ -171,6 +171,42 @@ namespace mongo { } #endif + void Backoff::nextSleepMillis(){ + + // Get the current time + unsigned long long currTimeMillis = curTimeMillis64(); + + int lastSleepMillis = _lastSleepMillis; + + if( _lastErrorTimeMillis == 0 || _lastErrorTimeMillis > currTimeMillis /* VM bugs exist */ ) + _lastErrorTimeMillis = currTimeMillis; + unsigned long long lastErrorTimeMillis = _lastErrorTimeMillis; + _lastErrorTimeMillis = currTimeMillis; + + // Backoff logic + + // Get the time since the last error + unsigned long long timeSinceLastErrorMillis = currTimeMillis - lastErrorTimeMillis; + + // Makes the cast below safe + verify( _resetAfterMillis >= 0 ); + + // If we haven't seen another error recently (3x the max wait time), reset our + // wait counter. + if( timeSinceLastErrorMillis > (unsigned)( _resetAfterMillis ) ) lastSleepMillis = 0; + + // Makes the test below sane + verify( _maxSleepMillis > 0 ); + + // Wait a power of two millis + if( lastSleepMillis == 0 ) lastSleepMillis = 1; + else lastSleepMillis = std::min( lastSleepMillis * 2, _maxSleepMillis ); + + // Store the last slept time + _lastSleepMillis = lastSleepMillis; + sleepmillis( lastSleepMillis ); + } + extern long long jsTime_virtual_skew; extern boost::thread_specific_ptr<long long> jsTime_virtual_thread_skew; diff --git a/src/mongo/util/time_support.h b/src/mongo/util/time_support.h index 511bc092fb1..254401b01ad 100644 --- a/src/mongo/util/time_support.h +++ b/src/mongo/util/time_support.h @@ -43,6 +43,29 @@ namespace mongo { void sleepmillis(long long ms); void sleepmicros(long long micros); + class Backoff { + public: + + Backoff( int maxSleepMillis, int resetAfter ) : + _maxSleepMillis( maxSleepMillis ), + _resetAfterMillis( maxSleepMillis + resetAfter ), // Don't reset < the max sleep + _lastSleepMillis( 0 ), + _lastErrorTimeMillis( 0 ) + {} + + void nextSleepMillis(); + + private: + + // Parameters + int _maxSleepMillis; + int _resetAfterMillis; + + // Last sleep information + int _lastSleepMillis; + unsigned long long _lastErrorTimeMillis; + }; + // DO NOT TOUCH except for testing void jsTimeVirtualSkew( long long skew ); |