SERVER-32876 Don't stall ftdc due to WT cache full.

7 files changed, 66 insertions, 34 deletions

diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp
index 000f4228742..e405ab2e751 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp
@@ -782,7 +782,7 @@ int64_t WiredTigerRecordStore::storageSize(OperationContext* opCtx,
     if (_isEphemeral) {
         return dataSize(opCtx);
     }
-    WiredTigerSession* session = WiredTigerRecoveryUnit::get(opCtx)->getSession();
+    WiredTigerSession* session = WiredTigerRecoveryUnit::get(opCtx)->getSessionNoTxn();
     StatusWith<int64_t> result =
         WiredTigerUtil::getStatisticsValueAs<int64_t>(session->getSession(),
                                                       "statistics:" + getURI(),
@@ -1495,7 +1495,7 @@ void WiredTigerRecordStore::appendCustomStats(OperationContext* opCtx,
         result->appendIntOrLL("sleepCount", _cappedSleep.load());
         result->appendIntOrLL("sleepMS", _cappedSleepMS.load());
     }
-    WiredTigerSession* session = WiredTigerRecoveryUnit::get(opCtx)->getSession();
+    WiredTigerSession* session = WiredTigerRecoveryUnit::get(opCtx)->getSessionNoTxn();
     WT_SESSION* s = session->getSession();
     BSONObjBuilder bob(result->subobjStart(_engineName));
     {
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.cpp
index f8d409a31e9..d65570c5e5e 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.cpp
@@ -198,7 +198,12 @@ WiredTigerSession* WiredTigerRecoveryUnit::getSession() {
 
 WiredTigerSession* WiredTigerRecoveryUnit::getSessionNoTxn() {
     _ensureSession();
-    return _session.get();
+    WiredTigerSession* session = _session.get();
+
+    // Handling queued drops can be slow, which is not desired for internal operations like FTDC
+    // sampling. Disable handling of queued drops for such sessions.
+    session->dropQueuedIdentsAtSessionEndAllowed(false);
+    return session;
 }
 
 void WiredTigerRecoveryUnit::abandonSnapshot() {
@@ -309,14 +314,19 @@ void WiredTigerRecoveryUnit::_txnOpen() {
     // correctness. Also, note that we use the '_readAtTimestamp' to work around an oplog visibility
     // issue in cappedTruncateAfter by setting the timestamp to the maximum value.
     if (_readAtTimestamp != Timestamp::min()) {
+        invariantWTOK(session->begin_transaction(session, NULL));
+        auto rollbacker =
+            MakeGuard([&] { invariant(session->rollback_transaction(session, nullptr) == 0); });
         auto status =
-            _sessionCache->snapshotManager().beginTransactionAtTimestamp(_readAtTimestamp, session);
+            _sessionCache->snapshotManager().setTransactionReadTimestamp(_readAtTimestamp, session);
+
         if (!status.isOK() && status.code() == ErrorCodes::BadValue) {
             uasserted(ErrorCodes::SnapshotTooOld,
                       str::stream() << "Read timestamp " << _readAtTimestamp.toString()
                                     << " is older than the oldest available timestamp.");
         }
         uassertStatusOK(status);
+        rollbacker.Dismiss();
     } else if (_isReadingFromPointInTime()) {
         // We reset _majorityCommittedSnapshot to the actual read timestamp used when the
         // transaction was started.
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp
index e8358d7f141..30fa69d26cd 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp
@@ -391,6 +391,11 @@ void WiredTigerSessionCache::releaseSession(WiredTigerSession* session) {
 
     bool returnedToCache = false;
     uint64_t currentEpoch = _epoch.load();
+    bool dropQueuedIdentsAtSessionEnd = session->isDropQueuedIdentsAtSessionEndAllowed();
+
+    // Reset this session's flag for dropping queued idents to default, before returning it to
+    // session cache.
+    session->dropQueuedIdentsAtSessionEndAllowed(true);
 
     if (session->_getEpoch() == currentEpoch) {  // check outside of lock to reduce contention
         stdx::lock_guard<stdx::mutex> lock(_cacheLock);
@@ -404,7 +409,7 @@ void WiredTigerSessionCache::releaseSession(WiredTigerSession* session) {
     if (!returnedToCache)
         delete session;
 
-    if (_engine && _engine->haveDropsQueued())
+    if (dropQueuedIdentsAtSessionEnd && _engine && _engine->haveDropsQueued())
         _engine->dropSomeQueuedIdents();
 }
 
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.h b/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.h
index 287307b87cc..bed0bb72e5f 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.h
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.h
@@ -108,6 +108,14 @@ public:
         return _cursorsOut;
     }
 
+    bool isDropQueuedIdentsAtSessionEndAllowed() const {
+        return _dropQueuedIdentsAtSessionEnd;
+    }
+
+    void dropQueuedIdentsAtSessionEndAllowed(bool dropQueuedIdentsAtSessionEnd) {
+        _dropQueuedIdentsAtSessionEnd = dropQueuedIdentsAtSessionEnd;
+    }
+
     static uint64_t genTableId();
 
     /**
@@ -138,6 +146,7 @@ private:
     CursorCache _cursors;            // owned
     uint64_t _cursorGen;
     int _cursorsOut;
+    bool _dropQueuedIdentsAtSessionEnd = true;
 };
 
 /**
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_snapshot_manager.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_snapshot_manager.cpp
index 41c06ce2d5b..baad47cf5c6 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_snapshot_manager.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_snapshot_manager.cpp
@@ -64,9 +64,9 @@ boost::optional<Timestamp> WiredTigerSnapshotManager::getMinSnapshotForNextCommi
     return _committedSnapshot;
 }
 
-Status WiredTigerSnapshotManager::beginTransactionAtTimestamp(Timestamp pointInTime,
+Status WiredTigerSnapshotManager::setTransactionReadTimestamp(Timestamp pointInTime,
                                                               WT_SESSION* session) const {
-    char readTSConfigString[15 /* read_timestamp= */ + (8 * 2) /* 8 hexadecimal characters */ +
+    char readTSConfigString[15 /* read_timestamp= */ + 16 /* 16 hexadecimal digits */ +
                             1 /* trailing null */];
     auto size = std::snprintf(
         readTSConfigString, sizeof(readTSConfigString), "read_timestamp=%llx", pointInTime.asULL());
@@ -77,49 +77,47 @@ Status WiredTigerSnapshotManager::beginTransactionAtTimestamp(Timestamp pointInT
     }
     invariant(static_cast<std::size_t>(size) < sizeof(readTSConfigString));
 
-    return wtRCToStatus(session->begin_transaction(session, readTSConfigString));
+    return wtRCToStatus(session->timestamp_transaction(session, readTSConfigString));
 }
 
 Timestamp WiredTigerSnapshotManager::beginTransactionOnCommittedSnapshot(
     WT_SESSION* session) const {
-    stdx::lock_guard<stdx::mutex> lock(_mutex);
+    invariantWTOK(session->begin_transaction(session, nullptr));
+    auto rollbacker =
+        MakeGuard([&] { invariant(session->rollback_transaction(session, nullptr) == 0); });
 
+    stdx::lock_guard<stdx::mutex> lock(_mutex);
     uassert(ErrorCodes::ReadConcernMajorityNotAvailableYet,
             "Committed view disappeared while running operation",
             _committedSnapshot);
 
-    auto status = beginTransactionAtTimestamp(_committedSnapshot.get(), session);
+    auto status = setTransactionReadTimestamp(_committedSnapshot.get(), session);
     fassert(30635, status);
+    rollbacker.Dismiss();
     return *_committedSnapshot;
 }
 
 void WiredTigerSnapshotManager::beginTransactionOnOplog(WiredTigerOplogManager* oplogManager,
                                                         WT_SESSION* session) const {
+    invariantWTOK(session->begin_transaction(session, nullptr));
+    auto rollbacker =
+        MakeGuard([&] { invariant(session->rollback_transaction(session, nullptr) == 0); });
+
     stdx::lock_guard<stdx::mutex> lock(_mutex);
     auto allCommittedTimestamp = oplogManager->getOplogReadTimestamp();
-    char readTSConfigString[15 /* read_timestamp= */ + (8 * 2) /* 16 hexadecimal digits */ +
-                            1 /* trailing null */];
-    auto size = std::snprintf(readTSConfigString,
-                              sizeof(readTSConfigString),
-                              "read_timestamp=%llx",
-                              static_cast<unsigned long long>(allCommittedTimestamp));
-    if (size < 0) {
-        int e = errno;
-        error() << "error snprintf " << errnoWithDescription(e);
-        fassertFailedNoTrace(40663);
-    }
-    invariant(static_cast<std::size_t>(size) < sizeof(readTSConfigString));
-
-    int status = session->begin_transaction(session, readTSConfigString);
-
-    // If begin_transaction returns EINVAL, we will assume it is due to the oldest_timestamp
-    // racing ahead of the read_timestamp.  Rather than synchronizing for this rare case, throw a
-    // WriteConflictException which will presumably be retried.
-    if (status == EINVAL) {
+    invariant(Timestamp(static_cast<unsigned long long>(allCommittedTimestamp)).asULL() ==
+              allCommittedTimestamp);
+    auto status = setTransactionReadTimestamp(
+        Timestamp(static_cast<unsigned long long>(allCommittedTimestamp)), session);
+
+    // If we failed to set the read timestamp, we assume it is due to the oldest_timestamp racing
+    // ahead.  Rather than synchronizing for this rare case, if requested, throw a
+    // WriteConflictException which will be retried.
+    if (!status.isOK() && status.code() == ErrorCodes::BadValue) {
         throw WriteConflictException();
     }
-
-    invariantWTOK(status);
+    fassert(50771, status);
+    rollbacker.Dismiss();
 }
 
 }  // namespace mongo
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_snapshot_manager.h b/src/mongo/db/storage/wiredtiger/wiredtiger_snapshot_manager.h
index b14f0d2c34f..4eda95caa89 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_snapshot_manager.h
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_snapshot_manager.h
@@ -54,7 +54,12 @@ public:
     // WT-specific methods
     //
 
-    Status beginTransactionAtTimestamp(Timestamp pointInTime, WT_SESSION* session) const;
+    /**
+     * Sets the read timestamp on a transaction.
+     *
+     * Reads will be reflect the state of data as of the specified timestamp.
+     */
+    Status setTransactionReadTimestamp(Timestamp pointInTime, WT_SESSION* session) const;
 
     /**
      * Starts a transaction and returns the SnapshotName used.
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp
index e0e5e531a7f..b9f696975b6 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp
@@ -127,9 +127,14 @@ void WiredTigerUtil::fetchTypeAndSourceURI(OperationContext* opCtx,
 
 StatusWith<std::string> WiredTigerUtil::getMetadata(OperationContext* opCtx, StringData uri) {
     invariant(opCtx);
-    WiredTigerCursor curwrap("metadata:create", WiredTigerSession::kMetadataTableId, false, opCtx);
-    WT_CURSOR* cursor = curwrap.get();
+
+    auto session = WiredTigerRecoveryUnit::get(opCtx)->getSessionNoTxn();
+    WT_CURSOR* cursor =
+        session->getCursor("metadata:create", WiredTigerSession::kMetadataTableId, false);
     invariant(cursor);
+    auto releaser =
+        MakeGuard([&] { session->releaseCursor(WiredTigerSession::kMetadataTableId, cursor); });
+
     std::string strUri = uri.toString();
     cursor->set_key(cursor, strUri.c_str());
     int ret = cursor->search(cursor);