SERVER-16119: faster WT shutdown race avoidance

4 files changed, 30 insertions, 28 deletions

diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
index 01e4e6bbaa9..0ad0d32e9bf 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
@@ -330,7 +330,7 @@ namespace mongo {
             {
                 boost::mutex::scoped_lock lk( _identToDropMutex );
                 _identToDrop.insert( uri );
-                _epoch++;
+                _epoch.fetchAndAdd(1);
             }
             _sessionCache->closeAll();
             return false;
@@ -340,11 +340,14 @@ namespace mongo {
         return false;
     }
 
-    bool WiredTigerKVEngine::haveDropsQueued() const {
+    void WiredTigerKVEngine::syncSizeInfoOccasionally() const {
         if ( _sizeStorerSyncTracker.intervalHasElapsed() ) {
             _sizeStorerSyncTracker.resetLastTime();
             syncSizeInfo(false);
         }
+    }
+
+    bool WiredTigerKVEngine::haveDropsQueued() const {
         boost::mutex::scoped_lock lk( _identToDropMutex );
         return !_identToDrop.empty();
     }
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
index 0d2efc4cd44..04b030f51ad 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
@@ -112,10 +112,12 @@ namespace mongo {
         void dropAllQueued();
         bool haveDropsQueued() const;
 
-        int currentEpoch() const { return _epoch; }
+        int currentEpoch() const { return _epoch.loadRelaxed(); }
 
         void syncSizeInfo(bool sync) const;
 
+        void syncSizeInfoOccasionally() const;
+
     private:
 
         string _uri( const StringData& ident ) const;
@@ -132,7 +134,7 @@ namespace mongo {
         std::set<std::string> _identToDrop;
         mutable boost::mutex _identToDropMutex;
 
-        int _epoch; // this is how we keep track of if a session is too old
+        AtomicInt32 _epoch; // this is how we keep track of if a session is too old
 
         scoped_ptr<WiredTigerSizeStorer> _sizeStorer;
         string _sizeStorerUri;
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp
index 328af4f9905..d8dbf0b0d50 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp
@@ -112,11 +112,11 @@ namespace mongo {
     // -----------------------
 
     WiredTigerSessionCache::WiredTigerSessionCache( WiredTigerKVEngine* engine )
-        : _engine( engine ), _conn( engine->getConnection() ), _shuttingDown(0) {
+        : _engine( engine ), _conn( engine->getConnection() ), _inShutdown(false) {
     }
 
     WiredTigerSessionCache::WiredTigerSessionCache( WT_CONNECTION* conn )
-        : _engine( NULL ), _conn( conn ), _shuttingDown(0) {
+        : _engine( NULL ), _conn( conn ), _inShutdown(false) {
     }
 
     WiredTigerSessionCache::~WiredTigerSessionCache() {
@@ -124,14 +124,14 @@ namespace mongo {
     }
 
     void WiredTigerSessionCache::shuttingDown() {
-        if (_shuttingDown.load()) return;
-        _shuttingDown.store(1);
-
         {
             // This ensures that any calls, which are currently inside of getSession/releaseSession
             // will be able to complete before we start cleaning up the pool. Any others, which are
-            // about to enter will return immediately because of _shuttingDown == true.
-            boost::lock_guard<boost::shared_mutex> lk(_shutdownLock);
+            // about to enter will return immediately because of _inShutdown == true.
+            boost::mutex::scoped_lock lk(_sessionLock);
+            if ( _inShutdown )
+                return;
+            _inShutdown = true;
         }
 
         closeAll();
@@ -154,15 +154,13 @@ namespace mongo {
     }
 
     WiredTigerSession* WiredTigerSessionCache::getSession() {
-        boost::shared_lock<boost::shared_mutex> shutdownLock(_shutdownLock);
-
-        // We should never be able to get here after _shuttingDown is set, because no new
-        // operations should be allowed to start.
-        invariant(!_shuttingDown.loadRelaxed());
-
         {
             boost::mutex::scoped_lock lk( _sessionLock );
 
+            // We should never be able to get here after _inShutdown is set, because no new
+            // operations should be allowed to start.
+            invariant(!_inShutdown);
+
             if (!_sessionPool.empty()) {
                 WiredTigerSession* cachedSession = _sessionPool.back();
                 _sessionPool.pop_back();
@@ -175,11 +173,11 @@ namespace mongo {
     }
 
     void WiredTigerSessionCache::releaseSession( WiredTigerSession* session ) {
-        invariant( session );
+        invariant(session);
         invariant(session->cursorsOut() == 0);
 
-        boost::shared_lock<boost::shared_mutex> shutdownLock(_shutdownLock);
-        if (_shuttingDown.loadRelaxed()) {
+        boost::mutex::scoped_lock lk(_sessionLock);
+        if ( _inShutdown ) {
             // Leak the session in order to avoid race condition with clean shutdown, where the
             // storage engine is ripped from underneath transactions, which are not "active"
             // (i.e., do not have any locks), but are just about to delete the recovery unit.
@@ -189,20 +187,23 @@ namespace mongo {
 
         // This checks that we are only caching idle sessions and not something which might hold
         // locks or otherwise prevent truncation.
-        {
+        DEV {
             WT_SESSION* ss = session->getSession();
             uint64_t range;
             invariantWTOK(ss->transaction_pinned_range(ss, &range));
             invariant(range == 0);
         }
 
-        if (_engine && _engine->haveDropsQueued() && session->epoch() < _engine->currentEpoch()) {
+        if (_engine) {
+            _engine->syncSizeInfoOccasionally();
+        }
+
+        if (_engine && session->epoch() < _engine->currentEpoch() && _engine->haveDropsQueued()) {
             delete session;
             _engine->dropAllQueued();
             return;
         }
 
-        boost::mutex::scoped_lock lk(_sessionLock);
         _sessionPool.push_back( session );
     }
 
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.h b/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.h
index cfdb42b18f7..c245f86b1e6 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.h
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.h
@@ -111,11 +111,7 @@ namespace mongo {
         mutable boost::mutex _sessionLock;
         SessionPool _sessionPool;
 
-        // Regular operations take it in shared mode. Shutdown sets the _shuttingDown flag and
-        // then takes it in exclusive mode. This ensures that all threads, which would return
-        // sessions to the cache would leak them.
-        boost::shared_mutex _shutdownLock;
-        AtomicUInt32 _shuttingDown; // Used as boolean - 0 = false, 1 = true
+        bool _inShutdown;
     };
 
 }