SERVER-30310: Stage implementations for WT setStableTimestamp + setInitialDataTimestamp

8 files changed, 131 insertions, 4 deletions

diff --git a/src/mongo/db/storage/SConscript b/src/mongo/db/storage/SConscript
index 46c00c785e5..8a5f770d80f 100644
--- a/src/mongo/db/storage/SConscript
+++ b/src/mongo/db/storage/SConscript
@@ -219,3 +219,10 @@ env.CppUnitTest(
         '$BUILD_DIR/mongo/base',
         ]
 )
+
+env.CppUnitTest(
+    target='storage_snapshot_name_test',
+    source='storage_snapshot_name_test.cpp',
+    LIBDEPS=[],
+    LIBDEPS_PRIVATE=[],
+)
diff --git a/src/mongo/db/storage/kv/kv_engine.h b/src/mongo/db/storage/kv/kv_engine.h
index 6c15a9519e5..39f6110ff83 100644
--- a/src/mongo/db/storage/kv/kv_engine.h
+++ b/src/mongo/db/storage/kv/kv_engine.h
@@ -39,6 +39,7 @@
 #include "mongo/db/catalog/collection_options.h"
 #include "mongo/db/storage/kv/kv_prefix.h"
 #include "mongo/db/storage/record_store.h"
+#include "mongo/db/storage/snapshot_name.h"
 
 namespace mongo {
 
@@ -239,6 +240,16 @@ public:
     virtual void setJournalListener(JournalListener* jl) = 0;
 
     /**
+     * See `StorageEngine::setStableTimestamp`
+     */
+    virtual void setStableTimestamp(SnapshotName stableTimestamp) {}
+
+    /**
+     * See `StorageEngine::setInitialDataTimestamp`
+     */
+    virtual void setInitialDataTimestamp(SnapshotName initialDataTimestamp) {}
+
+    /**
      * The destructor will never be called from mongod, but may be called from tests.
      * Engines may assume that this will only be called in the case of clean shutdown, even if
      * cleanShutdown() hasn't been called.
diff --git a/src/mongo/db/storage/kv/kv_storage_engine.cpp b/src/mongo/db/storage/kv/kv_storage_engine.cpp
index f1538063192..89ffa97bbc6 100644
--- a/src/mongo/db/storage/kv/kv_storage_engine.cpp
+++ b/src/mongo/db/storage/kv/kv_storage_engine.cpp
@@ -300,4 +300,13 @@ Status KVStorageEngine::repairRecordStore(OperationContext* opCtx, const std::st
 void KVStorageEngine::setJournalListener(JournalListener* jl) {
     _engine->setJournalListener(jl);
 }
+
+void KVStorageEngine::setStableTimestamp(SnapshotName stableTimestamp) {
+    _engine->setStableTimestamp(stableTimestamp);
+}
+
+void KVStorageEngine::setInitialDataTimestamp(SnapshotName initialDataTimestamp) {
+    _engine->setInitialDataTimestamp(initialDataTimestamp);
+}
+
 }  // namespace mongo
diff --git a/src/mongo/db/storage/kv/kv_storage_engine.h b/src/mongo/db/storage/kv/kv_storage_engine.h
index 800f698b34a..dae157064a1 100644
--- a/src/mongo/db/storage/kv/kv_storage_engine.h
+++ b/src/mongo/db/storage/kv/kv_storage_engine.h
@@ -107,6 +107,10 @@ public:
 
     virtual void cleanShutdown();
 
+    virtual void setStableTimestamp(SnapshotName stableTimestamp) override;
+
+    virtual void setInitialDataTimestamp(SnapshotName initialDataTimestamp) override;
+
     SnapshotManager* getSnapshotManager() const final;
 
     void setJournalListener(JournalListener* jl) final;
diff --git a/src/mongo/db/storage/snapshot_name.h b/src/mongo/db/storage/snapshot_name.h
index 2805d7467e7..eddc06b48bd 100644
--- a/src/mongo/db/storage/snapshot_name.h
+++ b/src/mongo/db/storage/snapshot_name.h
@@ -29,10 +29,10 @@
 #pragma once
 
 #include <cstdint>
+#include <iostream>
 #include <limits>
 
 #include "mongo/bson/timestamp.h"
-#include "mongo/util/mongoutils/str.h"
 
 namespace mongo {
 
@@ -63,8 +63,11 @@ public:
     }
 
     std::string toString() const {
-        return (str::stream() << _value);
+        std::stringstream str;
+        str << std::hex << _value;
+        return str.str();
     }
+
     bool operator==(const SnapshotName& rhs) const {
         return _value == rhs._value;
     }
diff --git a/src/mongo/db/storage/storage_snapshot_name_test.cpp b/src/mongo/db/storage/storage_snapshot_name_test.cpp
new file mode 100644
index 00000000000..462dc12150a
--- /dev/null
+++ b/src/mongo/db/storage/storage_snapshot_name_test.cpp
@@ -0,0 +1,48 @@
+/**
+ *    Copyright (C) 2017 MongoDB Inc.
+ *
+ *    This program is free software: you can redistribute it and/or  modify
+ *    it under the terms of the GNU Affero General Public License, version 3,
+ *    as published by the Free Software Foundation.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Affero General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Affero General Public License
+ *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the GNU Affero General Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#include "mongo/platform/basic.h"
+
+#include <utility>
+#include <vector>
+
+#include "mongo/unittest/unittest.h"
+#include "snapshot_name.h"
+
+namespace mongo {
+TEST(SnapshotNameTest, TestToString) {
+    std::vector<std::pair<uint64_t, std::string>> tests = {
+        {1234, "4d2"},
+        {1501697192, "598214a8"},
+        {6449740328135032833, "598214a800000001"},
+        {6449740328135037744, "598214a800001330"}};
+    for (auto pair : tests) {
+        ASSERT_EQUALS(pair.second, SnapshotName(pair.first).toString());
+    }
+}
+}  // namespace mongo
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
index d9a4935cb90..a1dd058f69b 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
@@ -70,6 +70,7 @@
 #include "mongo/db/storage/wiredtiger/wiredtiger_session_cache.h"
 #include "mongo/db/storage/wiredtiger/wiredtiger_size_storer.h"
 #include "mongo/db/storage/wiredtiger/wiredtiger_util.h"
+#include "mongo/platform/atomic_word.h"
 #include "mongo/stdx/memory.h"
 #include "mongo/util/background.h"
 #include "mongo/util/concurrency/idle_thread_block.h"
@@ -136,7 +137,10 @@ private:
 class WiredTigerKVEngine::WiredTigerCheckpointThread : public BackgroundJob {
 public:
     explicit WiredTigerCheckpointThread(WiredTigerSessionCache* sessionCache)
-        : BackgroundJob(false /* deleteSelf */), _sessionCache(sessionCache) {}
+        : BackgroundJob(false /* deleteSelf */),
+          _sessionCache(sessionCache),
+          _stableTimestamp(0),
+          _initialDataTimestamp(0) {}
 
     virtual string name() const {
         return "WTCheckpointThread";
@@ -165,6 +169,14 @@ public:
         LOG(1) << "stopping " << name() << " thread";
     }
 
+    virtual void setStableTimestamp(SnapshotName stableTimestamp) {
+        _stableTimestamp.store(stableTimestamp.asU64());
+    }
+
+    virtual void setInitialDataTimestamp(SnapshotName initialDataTimestamp) {
+        _initialDataTimestamp.store(initialDataTimestamp.asU64());
+    }
+
     void shutdown() {
         _shuttingDown.store(true);
         _condvar.notify_one();
@@ -178,6 +190,8 @@ private:
     stdx::mutex _mutex;
     stdx::condition_variable _condvar;
     AtomicBool _shuttingDown{false};
+    AtomicWord<std::uint64_t> _stableTimestamp;
+    AtomicWord<std::uint64_t> _initialDataTimestamp;
 };
 
 namespace {
@@ -899,4 +913,31 @@ void WiredTigerKVEngine::setInitRsOplogBackgroundThreadCallback(
 bool WiredTigerKVEngine::initRsOplogBackgroundThread(StringData ns) {
     return initRsOplogBackgroundThreadCallback(ns);
 }
-}
+
+void WiredTigerKVEngine::setStableTimestamp(SnapshotName stableTimestamp) {
+    const bool keepOldBehavior = true;
+    // Communicate to WiredTiger what the "stable timestamp" is. Timestamp-aware checkpoints will
+    // only persist to disk transactions committed with a timestamp earlier than the "stable
+    // timestamp".
+    //
+    // After passing the "stable timestamp" to WiredTiger, communicate it to the
+    // `CheckpointThread`. It's not obvious a stale stable timestamp in the `CheckpointThread` is
+    // safe. Consider the following arguments:
+    //
+    // Setting the "stable timestamp" is only meaningful when the "initial data timestamp" is real
+    // (i.e: not `kAllowUnstableCheckpointsSentinel`). In this normal case, the `stableTimestamp`
+    // input must be greater than the current value. The only effect this can have in the
+    // `CheckpointThread` is to transition it from a state of not taking any checkpoints, to
+    // taking "stable checkpoints". In the transitioning case, it's imperative for the "stable
+    // timestamp" to have first been communicated to WiredTiger.
+    if (!keepOldBehavior) {
+        std::string conf = str::stream() << "stable_timestamp=" << stableTimestamp.toString();
+        _conn->set_timestamp(_conn, conf.c_str());
+    }
+    _checkpointThread->setStableTimestamp(stableTimestamp);
+}
+
+void WiredTigerKVEngine::setInitialDataTimestamp(SnapshotName initialDataTimestamp) {
+    _checkpointThread->setInitialDataTimestamp(initialDataTimestamp);
+}
+}  // namespace mongo
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
index c6078a06015..2849e8f3cf4 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
@@ -160,6 +160,10 @@ public:
 
     void setJournalListener(JournalListener* jl) final;
 
+    virtual void setStableTimestamp(SnapshotName stableTimestamp) override;
+
+    virtual void setInitialDataTimestamp(SnapshotName initialDataTimestamp) override;
+
     // wiredtiger specific
     // Calls WT_CONNECTION::reconfigure on the underlying WT_CONNECTION
     // held by this class