SERVER-45147: Ensure ghost timestamped transactions trigger the stable timestamp to advance when they commit/abort. Ensure the durable timestamp does not move backwards.

(cherry picked from commit 5ab625013bcf6438515dceb5fe7c2228257cb863)

3 files changed, 35 insertions, 4 deletions

diff --git a/src/mongo/db/catalog/index_timestamp_helper.cpp b/src/mongo/db/catalog/index_timestamp_helper.cpp
index bfdc902ba20..2e9f2cc7074 100644
--- a/src/mongo/db/catalog/index_timestamp_helper.cpp
+++ b/src/mongo/db/catalog/index_timestamp_helper.cpp
@@ -40,6 +40,21 @@
 namespace mongo {
 
 namespace {
+Status setGhostTimestamp(OperationContext* opCtx, Timestamp timestamp) {
+    if (auto status = opCtx->recoveryUnit()->setTimestamp(timestamp); !status.isOK()) {
+        return status;
+    }
+    opCtx->recoveryUnit()->setOrderedCommit(false);
+
+    auto replCoord = repl::ReplicationCoordinator::get(opCtx);
+    if (replCoord->isReplEnabled()) {
+        opCtx->recoveryUnit()->onCommit(
+            [replCoord](auto commitTime) { replCoord->attemptToAdvanceStableTimestamp(); });
+    }
+
+    return Status::OK();
+}
+
 bool requiresGhostCommitTimestampForWrite(OperationContext* opCtx, const NamespaceString& nss) {
     if (!nss.isReplicated()) {
         return false;
@@ -92,7 +107,7 @@ void IndexTimestampHelper::setGhostCommitTimestampForWrite(OperationContext* opC
                             << " cannot be older than current read timestamp "
                             << mySnapshot->toString());
 
-    auto status = opCtx->recoveryUnit()->setTimestamp(commitTimestamp);
+    auto status = setGhostTimestamp(opCtx, commitTimestamp);
     if (status.code() == ErrorCodes::BadValue) {
         LOGV2(20379,
               "Temporarily could not apply ghost commit timestamp. {status_reason}",
@@ -155,8 +170,8 @@ bool IndexTimestampHelper::setGhostCommitTimestampForCatalogWrite(OperationConte
     if (!requiresGhostCommitTimestampForCatalogWrite(opCtx, nss)) {
         return false;
     }
-    auto status = opCtx->recoveryUnit()->setTimestamp(
-        LogicalClock::get(opCtx)->getClusterTime().asTimestamp());
+    auto status =
+        setGhostTimestamp(opCtx, LogicalClock::get(opCtx)->getClusterTime().asTimestamp());
     if (status.code() == ErrorCodes::BadValue) {
         LOGV2(20381,
               "Temporarily could not timestamp the index build commit, retrying. {status_reason}",
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
index 1b963fe6fc7..4044e5481e5 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
@@ -2001,6 +2001,8 @@ void WiredTigerKVEngine::setStableTimestamp(Timestamp stableTimestamp, bool forc
         stableTSConfigString =
             "force=true,oldest_timestamp={0:x},commit_timestamp={0:x},stable_timestamp={0:x}"_format(
                 ts);
+        stdx::lock_guard<Latch> lk(_highestDurableTimestampMutex);
+        _highestSeenDurableTimestamp = ts;
     } else {
         stableTSConfigString = "stable_timestamp={:x}"_format(ts);
     }
@@ -2057,6 +2059,8 @@ void WiredTigerKVEngine::setOldestTimestamp(Timestamp newOldestTimestamp, bool f
                 newOldestTimestamp.asULL());
         invariantWTOK(_conn->set_timestamp(_conn, oldestTSConfigString.c_str()));
         _oldestTimestamp.store(newOldestTimestamp.asULL());
+        stdx::lock_guard<Latch> lk(_highestDurableTimestampMutex);
+        _highestSeenDurableTimestamp = newOldestTimestamp.asULL();
         LOGV2_DEBUG(22342,
                     2,
                     "oldest_timestamp and commit_timestamp force set to {newOldestTimestamp}",
@@ -2194,7 +2198,15 @@ StatusWith<Timestamp> WiredTigerKVEngine::recoverToStableTimestamp(OperationCont
 }
 
 Timestamp WiredTigerKVEngine::getAllDurableTimestamp() const {
-    return Timestamp(_oplogManager->fetchAllDurableValue(_conn));
+    auto ret = _oplogManager->fetchAllDurableValue(_conn);
+
+    stdx::lock_guard<Latch> lk(_highestDurableTimestampMutex);
+    if (ret < _highestSeenDurableTimestamp) {
+        ret = _highestSeenDurableTimestamp;
+    } else {
+        _highestSeenDurableTimestamp = ret;
+    }
+    return Timestamp(ret);
 }
 
 Timestamp WiredTigerKVEngine::getOldestOpenReadTimestamp() const {
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
index c3ec2ad6ad8..35d34b32d9e 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
@@ -501,5 +501,9 @@ private:
 
     std::unique_ptr<WiredTigerEngineRuntimeConfigParameter> _runTimeConfigParam;
     std::unique_ptr<WiredTigerMaxCacheOverflowSizeGBParameter> _maxCacheOverflowParam;
+
+    mutable Mutex _highestDurableTimestampMutex =
+        MONGO_MAKE_LATCH("WiredTigerKVEngine::_highestDurableTimestampMutex");
+    mutable unsigned long long _highestSeenDurableTimestamp = StorageEngine::kMinimumTimestamp;
 };
 }  // namespace mongo