SERVER-32424 Use hybrid cursor caching by default.

This mechanism is a hybrid between the current way of doing cursor caching
in the integration layer and doing cursor caching in WiredTiger.
This gives the best compromise between allowing exclusive operations
(drop/verify) to succeed, and good performance for typical workloads.
The old cursor caching scheme, as well as one that caches purely in
WiredTiger, are available as options when starting the server.
Added a performance improvement for capped collections, especially under
the new cursor caching schemes.

(cherry picked from commit c0d20e4fb98626143d772bb70a950b277685bbbd)

4 files changed, 61 insertions, 29 deletions

diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
index 6a7685c354c..90113cb7e96 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
@@ -357,8 +357,10 @@ WiredTigerKVEngine::WiredTigerKVEngine(const std::string& canonicalName,
     ss << "config_base=false,";
     ss << "statistics=(fast),";
 
-    // We are still using MongoDB's cursor cache, don't double up.
-    ss << "cache_cursors=false,";
+    if (!WiredTigerSessionCache::isEngineCachingCursors()) {
+        ss << "cache_cursors=false,";
+    }
+
     // Ensure WiredTiger creates data in the expected format and attempting to start with a
     // data directory created using a newer version will fail.
     ss << "compatibility=(release=\"3.0\",require_max=\"3.0\"),";
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp
index 8587faa7636..99183a6fb6f 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp
@@ -976,11 +976,17 @@ int64_t WiredTigerRecordStore::cappedDeleteAsNeeded_inlock(OperationContext* opC
             WiredTigerCursor startWrap(_uri, _tableId, true, opCtx);
             WT_CURSOR* truncateStart = startWrap.get();
 
-            // If we know where the start point is, set it for the truncate
             if (savedFirstKey != 0) {
+                // If we know where the start point is, set it for the truncate
                 setKey(truncateStart, RecordId(savedFirstKey));
             } else {
-                truncateStart = NULL;
+                // Position at the first record.  This is equivalent to
+                // providing a NULL argument to WT_SESSION->truncate, but
+                // in that case, truncate will need to open its own cursor.
+                // Since we already have a cursor, we can use it here to
+                // make the whole operation faster.
+                ret = WT_READ_CHECK(truncateStart->next(truncateStart));
+                invariantWTOK(ret);
             }
             ret = session->truncate(session, NULL, truncateStart, truncateEnd, NULL);
 
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp
index be58223212e..b3f042e287f 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp
@@ -50,29 +50,37 @@
 
 namespace mongo {
 
-AtomicInt32 kWiredTigerCursorCacheSize(10000);
-
-class WiredTigerCursorCacheSize
-    : public ExportedServerParameter<std::int32_t, ServerParameterType::kStartupAndRuntime> {
-public:
-    WiredTigerCursorCacheSize()
-        : ExportedServerParameter<std::int32_t, ServerParameterType::kStartupAndRuntime>(
-              ServerParameterSet::getGlobal(),
-              "wiredTigerCursorCacheSize",
-              &kWiredTigerCursorCacheSize) {}
-
-    virtual Status validate(const std::int32_t& potentialNewValue) {
-        if (potentialNewValue < 0) {
-            return Status(ErrorCodes::BadValue,
-                          str::stream()
-                              << "wiredTigerCursorCacheSize must be greater than or equal "
-                              << "to 0, but attempted to set to: "
-                              << potentialNewValue);
-        }
-
-        return Status::OK();
-    }
-} WiredTigerCursorCacheSizeSetting;
+// The "wiredTigerCursorCacheSize" parameter has the following meaning.
+//
+// wiredTigerCursorCacheSize == 0
+// For this setting, cursors are only cached in the WiredTiger storage engine
+// itself. Operations that need exclusive access such as drop or verify will
+// not be blocked by inactive cached cursors with this setting. However, this
+// setting may reduce the performance of certain workloads that normally
+// benefit from cursor caching above the storage engine.
+//
+// wiredTigerCursorCacheSize > 0
+// WiredTiger-level caching of cursors is disabled but cursor caching does
+// occur above the storage engine. The value of this setting represents the
+// maximum number of cursors that are cached. Setting the value to 10000 will
+// give the old (<= 3.6) behavior. Note that cursors remain cached, even when a
+// session is released back to the cache. Thus, exclusive operations may be
+// blocked temporarily, and in some cases, a long time. Drops that fail because
+// of exclusivity silently succeed and are queued for retries.
+//
+// wiredTigerCursorCacheSize < 0
+// This is a hybrid approach of the above two, and is the default. The the
+// absolute value of the setting is used as the number of cursors cached above
+// the storage engine. When a session is released, all cursors are closed, and
+// will be cached in WiredTiger. Exclusive operations should only be blocked
+// for a short time, except if a cursor is held by a long running session. This
+// is a good compromise for most workloads.
+AtomicInt32 kWiredTigerCursorCacheSize(-100);
+
+ExportedServerParameter<std::int32_t, ServerParameterType::kStartupAndRuntime>
+    WiredTigerCursorCacheSizeSetting(ServerParameterSet::getGlobal(),
+                                     "wiredTigerCursorCacheSize",
+                                     &kWiredTigerCursorCacheSize);
 
 WiredTigerSession::WiredTigerSession(WT_CONNECTION* conn, uint64_t epoch, uint64_t cursorEpoch)
     : _epoch(epoch), _cursorEpoch(cursorEpoch), _session(NULL), _cursorGen(0), _cursorsOut(0) {
@@ -147,8 +155,10 @@ void WiredTigerSession::releaseCursor(uint64_t id, WT_CURSOR* cursor) {
     // Cursors are pushed to the front of the list and removed from the back
     _cursors.push_front(WiredTigerCachedCursor(id, _cursorGen++, cursor));
 
-    std::uint64_t cursorCacheSize = static_cast<std::uint64_t>(kWiredTigerCursorCacheSize.load());
-    while (!_cursors.empty() && _cursorGen - _cursors.back()._gen > cursorCacheSize) {
+    // A negative value for wiredTigercursorCacheSize means to use hybrid caching.
+    std::uint32_t cacheSize = abs(kWiredTigerCursorCacheSize.load());
+
+    while (!_cursors.empty() && _cursorGen - _cursors.back()._gen > cacheSize) {
         cursor = _cursors.back()._cursor;
         _cursors.pop_back();
         invariantWTOK(cursor->close(cursor));
@@ -388,6 +398,11 @@ void WiredTigerSessionCache::releaseSession(WiredTigerSession* session) {
         invariant(range == 0);
 
         // Release resources in the session we're about to cache.
+        // If we are using hybrid caching, then close cursors now and let them
+        // be cached at the WiredTiger level.
+        if (kWiredTigerCursorCacheSize.load() < 0) {
+            session->closeAllCursors("");
+        }
         invariantWTOK(ss->reset(ss));
     }
 
@@ -426,6 +441,10 @@ void WiredTigerSessionCache::setJournalListener(JournalListener* jl) {
     _journalListener = jl;
 }
 
+bool WiredTigerSessionCache::isEngineCachingCursors() {
+    return kWiredTigerCursorCacheSize.load() <= 0;
+}
+
 void WiredTigerSessionCache::WiredTigerSessionDeleter::operator()(
     WiredTigerSession* session) const {
     session->_cache->releaseSession(session);
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.h b/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.h
index 4f77ef285c1..9b89e2f3ff9 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.h
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.h
@@ -189,6 +189,11 @@ public:
     };
 
     /**
+     * Indicates that WiredTiger should be configured to cache cursors.
+     */
+    static bool isEngineCachingCursors();
+
+    /**
      * Returns a smart pointer to a previously released session for reuse, or creates a new session.
      * This method must only be called while holding the global lock to avoid races with
      * shuttingDown, but otherwise is thread safe.