SERVER-28433 Implement KeysCollectionCacheReaderAndUpdater

8 files changed, 621 insertions, 2 deletions

diff --git a/src/mongo/db/SConscript b/src/mongo/db/SConscript
index 66049a93d88..6df38c3e2f8 100644
--- a/src/mongo/db/SConscript
+++ b/src/mongo/db/SConscript
@@ -1036,10 +1036,12 @@ env.Library(
     target='keys_collection_manager',
     source=[
         'keys_collection_cache_reader.cpp',
+        'keys_collection_cache_reader_and_updater.cpp',
     ],
     LIBDEPS=[
         'keys_collection_document',
         'logical_time',
+        '$BUILD_DIR/mongo/s/coreshard',
     ],
 )
 
@@ -1049,7 +1051,6 @@ env.Library(
         'logical_clock.cpp',
     ],
     LIBDEPS=[
-        'keys_collection_manager',
         'server_parameters',
         'service_context',
         'signed_logical_time',
@@ -1136,6 +1137,17 @@ env.CppUnitTest(
     ],
 )
 
+env.CppUnitTest(
+    target='keys_collection_cache_reader_and_updater_test',
+    source=[
+        'keys_collection_cache_reader_and_updater_test.cpp',
+    ],
+    LIBDEPS=[
+        'keys_collection_manager',
+        '$BUILD_DIR/mongo/s/config_server_test_fixture',
+    ],
+)
+
 env.Library(
     target= 'op_observer_noop',
     source= [
diff --git a/src/mongo/db/keys_collection_cache_reader_and_updater.cpp b/src/mongo/db/keys_collection_cache_reader_and_updater.cpp
new file mode 100644
index 00000000000..7c959c5f6c0
--- /dev/null
+++ b/src/mongo/db/keys_collection_cache_reader_and_updater.cpp
@@ -0,0 +1,147 @@
+/**
+ *    Copyright (C) 2017 MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or  modify
+ *    it under the terms of the GNU Affero General Public License, version 3,
+ *    as published by the Free Software Foundation.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Affero General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Affero General Public License
+ *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the GNU Affero General Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#include "mongo/platform/basic.h"
+
+#include "mongo/db/keys_collection_cache_reader_and_updater.h"
+
+#include "mongo/client/read_preference.h"
+#include "mongo/db/logical_clock.h"
+#include "mongo/db/operation_context.h"
+#include "mongo/s/catalog/sharding_catalog_client.h"
+#include "mongo/s/client/shard_registry.h"
+#include "mongo/s/grid.h"
+
+namespace mongo {
+
+namespace {
+
+/**
+ * Inserts a new key to the keys collection.
+ *
+ * Note: this relies on the fact that ShardRegistry returns a ShardLocal for config in config
+ * servers. In other words, it is relying on the fact that this will always execute the write
+ * locally and never remotely even if this node is no longer primary.
+ */
+Status insertNewKey(OperationContext* opCtx,
+                    long long keyId,
+                    const std::string& purpose,
+                    const LogicalTime& expiresAt) {
+    KeysCollectionDocument newKey(keyId, purpose, TimeProofService::generateRandomKey(), expiresAt);
+    return Grid::get(opCtx)->catalogClient(opCtx)->insertConfigDocument(
+        opCtx,
+        KeysCollectionDocument::ConfigNS,
+        newKey.toBSON(),
+        ShardingCatalogClient::kMajorityWriteConcern);
+}
+
+/**
+ * Returns a new LogicalTime with the seconds argument added to it.
+ */
+LogicalTime addSeconds(const LogicalTime& logicalTime, const Seconds& seconds) {
+    return LogicalTime(Timestamp(logicalTime.asTimestamp().getSecs() + seconds.count(), 0));
+}
+
+}  // unnamed namespace
+
+KeysCollectionCacheReaderAndUpdater::KeysCollectionCacheReaderAndUpdater(
+    std::string purpose, Seconds keyValidForInterval)
+    : KeysCollectionCacheReader(purpose),
+      _purpose(std::move(purpose)),
+      _keyValidForInterval(keyValidForInterval) {}
+
+StatusWith<KeysCollectionDocument> KeysCollectionCacheReaderAndUpdater::refresh(
+    OperationContext* opCtx) {
+
+    auto currentTime = LogicalClock::get(opCtx)->getClusterTime().getTime();
+    auto keyStatus = Grid::get(opCtx)->catalogClient(opCtx)->getNewKeys(
+        opCtx, _purpose, currentTime, repl::ReadConcernLevel::kLocalReadConcern);
+
+    if (!keyStatus.isOK()) {
+        return keyStatus.getStatus();
+    }
+
+    const auto& newKeys = keyStatus.getValue();
+    auto keyIter = newKeys.cbegin();
+
+    LogicalTime currentKeyExpiresAt;
+
+    long long keyId = currentTime.asTimestamp().asLL();
+
+    if (keyIter == newKeys.cend()) {
+        currentKeyExpiresAt = addSeconds(currentTime, _keyValidForInterval);
+        auto status = insertNewKey(opCtx, keyId, _purpose, currentKeyExpiresAt);
+
+        if (!status.isOK()) {
+            return status;
+        }
+
+        keyId++;
+    } else if (keyIter->getExpiresAt() < currentTime) {
+        currentKeyExpiresAt = addSeconds(currentTime, _keyValidForInterval);
+        auto status = insertNewKey(opCtx, keyId, _purpose, currentKeyExpiresAt);
+
+        if (!status.isOK()) {
+            return status;
+        }
+
+        keyId++;
+        ++keyIter;
+    } else {
+        currentKeyExpiresAt = keyIter->getExpiresAt();
+        ++keyIter;
+    }
+
+    // Create a new key in advance if we don't have a key on standby after the current one
+    // expires.
+    // Note: Convert this block into a loop if more reserved keys are desired.
+    if (keyIter == newKeys.cend()) {
+        auto reserveKeyExpiresAt = addSeconds(currentKeyExpiresAt, _keyValidForInterval);
+        auto status = insertNewKey(opCtx, keyId, _purpose, reserveKeyExpiresAt);
+
+        if (!status.isOK()) {
+            return status;
+        }
+    } else if (keyIter->getExpiresAt() < currentTime) {
+        currentKeyExpiresAt = addSeconds(currentKeyExpiresAt, _keyValidForInterval);
+        auto status = insertNewKey(opCtx, keyId, _purpose, currentKeyExpiresAt);
+
+        if (!status.isOK()) {
+            return status;
+        }
+    }
+
+    return KeysCollectionCacheReader::refresh(opCtx);
+}
+
+StatusWith<KeysCollectionDocument> KeysCollectionCacheReaderAndUpdater::getKey(
+    const LogicalTime& forThisTime) {
+    return KeysCollectionCacheReader::getKey(forThisTime);
+}
+
+}  // namespace mongo
diff --git a/src/mongo/db/keys_collection_cache_reader_and_updater.h b/src/mongo/db/keys_collection_cache_reader_and_updater.h
new file mode 100644
index 00000000000..e538a126bd3
--- /dev/null
+++ b/src/mongo/db/keys_collection_cache_reader_and_updater.h
@@ -0,0 +1,64 @@
+/**
+ *    Copyright (C) 2017 MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or  modify
+ *    it under the terms of the GNU Affero General Public License, version 3,
+ *    as published by the Free Software Foundation.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Affero General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Affero General Public License
+ *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the GNU Affero General Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#pragma once
+
+#include <string>
+
+#include "mongo/db/keys_collection_cache_reader.h"
+#include "mongo/util/duration.h"
+
+namespace mongo {
+
+/**
+ * Keeps a local cache of the keys with the ability to refresh. The refresh method also makes sure
+ * that there will be valid keys available to sign the current logical time and there will be
+ * another key ready after the current key expires.
+ *
+ * Assumptions and limitations:
+ * - assumes that user does not manually update the keys collection.
+ * - assumes that current process is the config primary.
+ */
+class KeysCollectionCacheReaderAndUpdater : public KeysCollectionCacheReader {
+public:
+    KeysCollectionCacheReaderAndUpdater(std::string purpose, Seconds keyValidForInterval);
+    ~KeysCollectionCacheReaderAndUpdater() = default;
+
+    /**
+     * Check if there are new documents expiresAt > latestKeyDoc.expiresAt.
+     */
+    StatusWith<KeysCollectionDocument> refresh(OperationContext* opCtx) override;
+
+    StatusWith<KeysCollectionDocument> getKey(const LogicalTime& forThisTime) override;
+
+private:
+    const std::string _purpose;
+    const Seconds _keyValidForInterval;
+};
+
+}  // namespace mongo
diff --git a/src/mongo/db/keys_collection_cache_reader_and_updater_test.cpp b/src/mongo/db/keys_collection_cache_reader_and_updater_test.cpp
new file mode 100644
index 00000000000..fdb8e150f07
--- /dev/null
+++ b/src/mongo/db/keys_collection_cache_reader_and_updater_test.cpp
@@ -0,0 +1,366 @@
+/**
+ *    Copyright (C) 2017 MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or  modify
+ *    it under the terms of the GNU Affero General Public License, version 3,
+ *    as published by the Free Software Foundation.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Affero General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Affero General Public License
+ *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the GNU Affero General Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#include "mongo/platform/basic.h"
+
+#include <set>
+#include <string>
+
+#include "mongo/db/jsobj.h"
+#include "mongo/db/keys_collection_cache_reader_and_updater.h"
+#include "mongo/db/keys_collection_document.h"
+#include "mongo/db/logical_clock.h"
+#include "mongo/db/namespace_string.h"
+#include "mongo/db/operation_context.h"
+#include "mongo/s/config_server_test_fixture.h"
+#include "mongo/stdx/memory.h"
+#include "mongo/unittest/unittest.h"
+#include "mongo/util/clock_source_mock.h"
+
+namespace mongo {
+
+class CacheUpdaterTest : public ConfigServerTestFixture {
+protected:
+    void setUp() override {
+        ConfigServerTestFixture::setUp();
+
+        auto clockSource = stdx::make_unique<ClockSourceMock>();
+        operationContext()->getServiceContext()->setFastClockSource(std::move(clockSource));
+    }
+};
+
+TEST_F(CacheUpdaterTest, ShouldCreate2KeysFromEmpty) {
+    KeysCollectionCacheReaderAndUpdater updater("dummy", Seconds(5));
+
+    const LogicalTime currentTime(LogicalTime(Timestamp(100, 2)));
+    LogicalClock::get(operationContext())->initClusterTimeFromTrustedSource(currentTime);
+
+    auto keyStatus = updater.refresh(operationContext());
+    // TODO: Inspect keyStatus once SERVER-28435 is implemented
+
+    auto allKeys = getKeys(operationContext());
+
+    ASSERT_EQ(2u, allKeys.size());
+
+    const auto& key1 = allKeys.front();
+    ASSERT_EQ(currentTime.asTimestamp().asLL(), key1.getKeyId());
+    ASSERT_EQ("dummy", key1.getPurpose());
+    ASSERT_EQ(Timestamp(105, 0), key1.getExpiresAt().asTimestamp());
+
+    const auto& key2 = allKeys.back();
+    ASSERT_EQ(currentTime.asTimestamp().asLL() + 1, key2.getKeyId());
+    ASSERT_EQ("dummy", key2.getPurpose());
+    ASSERT_EQ(Timestamp(110, 0), key2.getExpiresAt().asTimestamp());
+
+    ASSERT_NE(key1.getKey(), key2.getKey());
+}
+
+TEST_F(CacheUpdaterTest, ShouldCreateAnotherKeyIfOnlyOneKeyExists) {
+    KeysCollectionCacheReaderAndUpdater updater("dummy", Seconds(5));
+
+    LogicalClock::get(operationContext())
+        ->initClusterTimeFromTrustedSource(LogicalTime(Timestamp(100, 2)));
+
+    KeysCollectionDocument origKey1(
+        1, "dummy", TimeProofService::generateRandomKey(), LogicalTime(Timestamp(105, 0)));
+    ASSERT_OK(insertToConfigCollection(
+        operationContext(), NamespaceString(KeysCollectionDocument::ConfigNS), origKey1.toBSON()));
+
+    {
+        auto allKeys = getKeys(operationContext());
+
+        ASSERT_EQ(1u, allKeys.size());
+
+        const auto& key1 = allKeys.front();
+        ASSERT_EQ(1, key1.getKeyId());
+        ASSERT_EQ("dummy", key1.getPurpose());
+        ASSERT_EQ(Timestamp(105, 0), key1.getExpiresAt().asTimestamp());
+    }
+
+    auto currentTime = LogicalClock::get(operationContext())->getClusterTime().getTime();
+    auto keyStatus = updater.refresh(operationContext());
+    // TODO: Inspect keyStatus once SERVER-28435 is implemented
+
+    {
+        auto allKeys = getKeys(operationContext());
+
+        ASSERT_EQ(2u, allKeys.size());
+
+        const auto& key1 = allKeys.front();
+        ASSERT_EQ(1, key1.getKeyId());
+        ASSERT_EQ("dummy", key1.getPurpose());
+        ASSERT_EQ(origKey1.getKey(), key1.getKey());
+        ASSERT_EQ(Timestamp(105, 0), key1.getExpiresAt().asTimestamp());
+
+        const auto& key2 = allKeys.back();
+        ASSERT_EQ(currentTime.asTimestamp().asLL(), key2.getKeyId());
+        ASSERT_EQ("dummy", key2.getPurpose());
+        ASSERT_EQ(Timestamp(110, 0), key2.getExpiresAt().asTimestamp());
+
+        ASSERT_NE(key1.getKey(), key2.getKey());
+    }
+}
+
+TEST_F(CacheUpdaterTest, ShouldCreateAnotherKeyIfNoValidKeyAfterCurrent) {
+    KeysCollectionCacheReaderAndUpdater updater("dummy", Seconds(5));
+
+    LogicalClock::get(operationContext())
+        ->initClusterTimeFromTrustedSource(LogicalTime(Timestamp(108, 2)));
+
+    KeysCollectionDocument origKey1(
+        1, "dummy", TimeProofService::generateRandomKey(), LogicalTime(Timestamp(105, 0)));
+    ASSERT_OK(insertToConfigCollection(
+        operationContext(), NamespaceString(KeysCollectionDocument::ConfigNS), origKey1.toBSON()));
+
+    KeysCollectionDocument origKey2(
+        2, "dummy", TimeProofService::generateRandomKey(), LogicalTime(Timestamp(110, 0)));
+    ASSERT_OK(insertToConfigCollection(
+        operationContext(), NamespaceString(KeysCollectionDocument::ConfigNS), origKey2.toBSON()));
+
+    {
+        auto allKeys = getKeys(operationContext());
+
+        ASSERT_EQ(2u, allKeys.size());
+
+        const auto& key1 = allKeys.front();
+        ASSERT_EQ(1, key1.getKeyId());
+        ASSERT_EQ("dummy", key1.getPurpose());
+        ASSERT_EQ(Timestamp(105, 0), key1.getExpiresAt().asTimestamp());
+
+        const auto& key2 = allKeys.back();
+        ASSERT_EQ(2, key2.getKeyId());
+        ASSERT_EQ("dummy", key2.getPurpose());
+        ASSERT_EQ(Timestamp(110, 0), key2.getExpiresAt().asTimestamp());
+    }
+
+    auto currentTime = LogicalClock::get(operationContext())->getClusterTime().getTime();
+    auto keyStatus = updater.refresh(operationContext());
+    // TODO: Inspect keyStatus once SERVER-28435 is implemented
+
+    auto allKeys = getKeys(operationContext());
+
+    ASSERT_EQ(3u, allKeys.size());
+
+    auto citer = allKeys.cbegin();
+
+    std::set<std::string> seenKeys;
+
+    {
+        const auto& key = *citer;
+        ASSERT_EQ(1, key.getKeyId());
+        ASSERT_EQ("dummy", key.getPurpose());
+        ASSERT_EQ(origKey1.getKey(), key.getKey());
+        ASSERT_EQ(Timestamp(105, 0), key.getExpiresAt().asTimestamp());
+
+        bool inserted = false;
+        std::tie(std::ignore, inserted) = seenKeys.insert(key.getKey().toString());
+        ASSERT_TRUE(inserted);
+    }
+
+    {
+        ++citer;
+        const auto& key = *citer;
+        ASSERT_EQ(2, key.getKeyId());
+        ASSERT_EQ("dummy", key.getPurpose());
+        ASSERT_EQ(origKey2.getKey(), key.getKey());
+        ASSERT_EQ(Timestamp(110, 0), key.getExpiresAt().asTimestamp());
+
+        bool inserted = false;
+        std::tie(std::ignore, inserted) = seenKeys.insert(key.getKey().toString());
+        ASSERT_TRUE(inserted);
+    }
+
+    {
+        ++citer;
+        const auto& key = *citer;
+        ASSERT_EQ(currentTime.asTimestamp().asLL(), key.getKeyId());
+        ASSERT_EQ("dummy", key.getPurpose());
+        ASSERT_EQ(Timestamp(115, 0), key.getExpiresAt().asTimestamp());
+
+        bool inserted = false;
+        std::tie(std::ignore, inserted) = seenKeys.insert(key.getKey().toString());
+        ASSERT_TRUE(inserted);
+    }
+}
+
+TEST_F(CacheUpdaterTest, ShouldCreate2KeysIfAllKeysAreExpired) {
+    KeysCollectionCacheReaderAndUpdater updater("dummy", Seconds(5));
+
+    LogicalClock::get(operationContext())
+        ->initClusterTimeFromTrustedSource(LogicalTime(Timestamp(120, 2)));
+
+    KeysCollectionDocument origKey1(
+        1, "dummy", TimeProofService::generateRandomKey(), LogicalTime(Timestamp(105, 0)));
+    ASSERT_OK(insertToConfigCollection(
+        operationContext(), NamespaceString(KeysCollectionDocument::ConfigNS), origKey1.toBSON()));
+
+    KeysCollectionDocument origKey2(
+        2, "dummy", TimeProofService::generateRandomKey(), LogicalTime(Timestamp(110, 0)));
+    ASSERT_OK(insertToConfigCollection(
+        operationContext(), NamespaceString(KeysCollectionDocument::ConfigNS), origKey2.toBSON()));
+
+    {
+        auto allKeys = getKeys(operationContext());
+
+        ASSERT_EQ(2u, allKeys.size());
+
+        const auto& key1 = allKeys.front();
+        ASSERT_EQ(1, key1.getKeyId());
+        ASSERT_EQ("dummy", key1.getPurpose());
+        ASSERT_EQ(Timestamp(105, 0), key1.getExpiresAt().asTimestamp());
+
+        const auto& key2 = allKeys.back();
+        ASSERT_EQ(2, key2.getKeyId());
+        ASSERT_EQ("dummy", key2.getPurpose());
+        ASSERT_EQ(Timestamp(110, 0), key2.getExpiresAt().asTimestamp());
+    }
+
+    auto currentTime = LogicalClock::get(operationContext())->getClusterTime().getTime();
+    auto keyStatus = updater.refresh(operationContext());
+    // TODO: Inspect keyStatus once SERVER-28435 is implemented
+
+    auto allKeys = getKeys(operationContext());
+
+    ASSERT_EQ(4u, allKeys.size());
+
+    auto citer = allKeys.cbegin();
+
+    std::set<std::string> seenKeys;
+
+    {
+        const auto& key = *citer;
+        ASSERT_EQ(1, key.getKeyId());
+        ASSERT_EQ("dummy", key.getPurpose());
+        ASSERT_EQ(origKey1.getKey(), key.getKey());
+        ASSERT_EQ(Timestamp(105, 0), key.getExpiresAt().asTimestamp());
+
+        bool inserted = false;
+        std::tie(std::ignore, inserted) = seenKeys.insert(key.getKey().toString());
+        ASSERT_TRUE(inserted);
+    }
+
+    {
+        ++citer;
+        const auto& key = *citer;
+        ASSERT_EQ(2, key.getKeyId());
+        ASSERT_EQ("dummy", key.getPurpose());
+        ASSERT_EQ(origKey2.getKey(), key.getKey());
+        ASSERT_EQ(Timestamp(110, 0), key.getExpiresAt().asTimestamp());
+
+        bool inserted = false;
+        std::tie(std::ignore, inserted) = seenKeys.insert(key.getKey().toString());
+        ASSERT_TRUE(inserted);
+    }
+
+    {
+        ++citer;
+        const auto& key = *citer;
+        ASSERT_EQ(currentTime.asTimestamp().asLL(), key.getKeyId());
+        ASSERT_EQ("dummy", key.getPurpose());
+        ASSERT_EQ(Timestamp(125, 0), key.getExpiresAt().asTimestamp());
+
+        bool inserted = false;
+        std::tie(std::ignore, inserted) = seenKeys.insert(key.getKey().toString());
+        ASSERT_TRUE(inserted);
+    }
+
+    {
+        ++citer;
+        const auto& key = *citer;
+        ASSERT_EQ(currentTime.asTimestamp().asLL() + 1, key.getKeyId());
+        ASSERT_EQ("dummy", key.getPurpose());
+        ASSERT_NE(origKey1.getKey(), key.getKey());
+        ASSERT_NE(origKey2.getKey(), key.getKey());
+        ASSERT_EQ(Timestamp(130, 0), key.getExpiresAt().asTimestamp());
+
+        bool inserted = false;
+        std::tie(std::ignore, inserted) = seenKeys.insert(key.getKey().toString());
+        ASSERT_TRUE(inserted);
+    }
+}
+
+TEST_F(CacheUpdaterTest, ShouldNotCreateNewKeyIfThereAre2UnexpiredKeys) {
+    KeysCollectionCacheReaderAndUpdater updater("dummy", Seconds(5));
+
+    LogicalClock::get(operationContext())
+        ->initClusterTimeFromTrustedSource(LogicalTime(Timestamp(100, 2)));
+
+    KeysCollectionDocument origKey1(
+        1, "dummy", TimeProofService::generateRandomKey(), LogicalTime(Timestamp(105, 0)));
+    ASSERT_OK(insertToConfigCollection(
+        operationContext(), NamespaceString(KeysCollectionDocument::ConfigNS), origKey1.toBSON()));
+
+    KeysCollectionDocument origKey2(
+        2, "dummy", TimeProofService::generateRandomKey(), LogicalTime(Timestamp(110, 0)));
+    ASSERT_OK(insertToConfigCollection(
+        operationContext(), NamespaceString(KeysCollectionDocument::ConfigNS), origKey2.toBSON()));
+
+    {
+        auto allKeys = getKeys(operationContext());
+
+        ASSERT_EQ(2u, allKeys.size());
+
+        const auto& key1 = allKeys.front();
+        ASSERT_EQ(1, key1.getKeyId());
+        ASSERT_EQ("dummy", key1.getPurpose());
+        ASSERT_EQ(Timestamp(105, 0), key1.getExpiresAt().asTimestamp());
+
+        const auto& key2 = allKeys.back();
+        ASSERT_EQ(2, key2.getKeyId());
+        ASSERT_EQ("dummy", key2.getPurpose());
+        ASSERT_EQ(Timestamp(110, 0), key2.getExpiresAt().asTimestamp());
+    }
+
+    auto keyStatus = updater.refresh(operationContext());
+    // TODO: Inspect keyStatus once SERVER-28435 is implemented
+
+    auto allKeys = getKeys(operationContext());
+
+    ASSERT_EQ(2u, allKeys.size());
+
+    auto citer = allKeys.cbegin();
+
+    {
+        const auto& key = *citer;
+        ASSERT_EQ(1, key.getKeyId());
+        ASSERT_EQ("dummy", key.getPurpose());
+        ASSERT_EQ(origKey1.getKey(), key.getKey());
+        ASSERT_EQ(Timestamp(105, 0), key.getExpiresAt().asTimestamp());
+    }
+
+    {
+        ++citer;
+        const auto& key = *citer;
+        ASSERT_EQ(2, key.getKeyId());
+        ASSERT_EQ("dummy", key.getPurpose());
+        ASSERT_EQ(origKey2.getKey(), key.getKey());
+        ASSERT_EQ(Timestamp(110, 0), key.getExpiresAt().asTimestamp());
+    }
+}
+
+}  // namespace mongo
diff --git a/src/mongo/db/ops/insert.cpp b/src/mongo/db/ops/insert.cpp
index f856c227138..c448126eca7 100644
--- a/src/mongo/db/ops/insert.cpp
+++ b/src/mongo/db/ops/insert.cpp
@@ -241,6 +241,8 @@ Status userAllowedCreateNS(StringData db, StringData coll) {
                 return Status::OK();
             if (coll == "system.backup_users")
                 return Status::OK();
+            if (coll == "system.keys")
+                return Status::OK();
         }
         if (db == "local") {
             if (coll == "system.replset")
diff --git a/src/mongo/s/catalog/sharding_catalog_client_impl.cpp b/src/mongo/s/catalog/sharding_catalog_client_impl.cpp
index 8495c5f6b07..0c97211ca34 100644
--- a/src/mongo/s/catalog/sharding_catalog_client_impl.cpp
+++ b/src/mongo/s/catalog/sharding_catalog_client_impl.cpp
@@ -1518,7 +1518,7 @@ Status ShardingCatalogClientImpl::insertConfigDocument(OperationContext* opCtx,
                                                        const BSONObj& doc,
                                                        const WriteConcernOptions& writeConcern) {
     const NamespaceString nss(ns);
-    invariant(nss.db() == "config");
+    invariant(nss.db() == "config" || nss.db() == "admin");
 
     const BSONElement idField = doc.getField("_id");
     invariant(!idField.eoo());
diff --git a/src/mongo/s/config_server_test_fixture.cpp b/src/mongo/s/config_server_test_fixture.cpp
index 4f1e6246646..c5640333022 100644
--- a/src/mongo/s/config_server_test_fixture.cpp
+++ b/src/mongo/s/config_server_test_fixture.cpp
@@ -292,5 +292,27 @@ StatusWith<std::vector<BSONObj>> ConfigServerTestFixture::getIndexes(OperationCo
     return cursorResponse.getValue().getBatch();
 }
 
+std::vector<KeysCollectionDocument> ConfigServerTestFixture::getKeys(OperationContext* opCtx) {
+    auto config = getConfigShard();
+    auto findStatus =
+        config->exhaustiveFindOnConfig(opCtx,
+                                       kReadPref,
+                                       repl::ReadConcernLevel::kMajorityReadConcern,
+                                       NamespaceString(KeysCollectionDocument::ConfigNS),
+                                       BSONObj(),
+                                       BSON("expiresAt" << 1),
+                                       boost::none);
+    ASSERT_OK(findStatus.getStatus());
+
+    std::vector<KeysCollectionDocument> keys;
+    const auto& docs = findStatus.getValue().docs;
+    for (const auto& doc : docs) {
+        auto keyStatus = KeysCollectionDocument::fromBSON(doc);
+        ASSERT_OK(keyStatus.getStatus());
+        keys.push_back(keyStatus.getValue());
+    }
+
+    return keys;
+}
 
 }  // namespace mongo
diff --git a/src/mongo/s/config_server_test_fixture.h b/src/mongo/s/config_server_test_fixture.h
index 411b3672902..69d08afe17d 100644
--- a/src/mongo/s/config_server_test_fixture.h
+++ b/src/mongo/s/config_server_test_fixture.h
@@ -34,6 +34,7 @@ namespace mongo {
 
 class BSONObj;
 class ChunkType;
+class KeysCollectionDocument;
 class NamespaceString;
 class Shard;
 class ShardingCatalogClient;
@@ -109,6 +110,11 @@ public:
      */
     void onCommandForAddShard(executor::NetworkTestEnv::OnCommandFunction func);
 
+    /**
+     * Returns all the keys in admin.system.keys
+     */
+    std::vector<KeysCollectionDocument> getKeys(OperationContext* opCtx);
+
 protected:
     /**
      * Sets this node up as a mongod with sharding components for ClusterRole::ConfigServer.