SERVER-28435 Implement KeysCollectionCacheReader

5 files changed, 354 insertions, 15 deletions

diff --git a/src/mongo/base/error_codes.err b/src/mongo/base/error_codes.err
index 9311e88c70e..b01f2b09b30 100644
--- a/src/mongo/base/error_codes.err
+++ b/src/mongo/base/error_codes.err
@@ -208,6 +208,7 @@ error_code("InvalidUUID", 207)
 error_code("TooManyLocks", 208)
 error_code("StaleClusterTime", 209)
 error_code("CannotVerifyAndSignLogicalTime", 210)
+error_code("KeyNotFound", 211)
 
 # Error codes 4000-8999 are reserved.
 
diff --git a/src/mongo/db/SConscript b/src/mongo/db/SConscript
index b4159927562..2edd8f35d6a 100644
--- a/src/mongo/db/SConscript
+++ b/src/mongo/db/SConscript
@@ -1140,6 +1140,17 @@ env.CppUnitTest(
 )
 
 env.CppUnitTest(
+    target='keys_collection_cache_reader_test',
+    source=[
+        'keys_collection_cache_reader_test.cpp',
+    ],
+    LIBDEPS=[
+        'keys_collection_manager',
+        '$BUILD_DIR/mongo/s/config_server_test_fixture',
+    ],
+)
+
+env.CppUnitTest(
     target='keys_collection_cache_reader_and_updater_test',
     source=[
         'keys_collection_cache_reader_and_updater_test.cpp',
diff --git a/src/mongo/db/keys_collection_cache_reader.cpp b/src/mongo/db/keys_collection_cache_reader.cpp
index 5d90ba846e0..066527ef92a 100644
--- a/src/mongo/db/keys_collection_cache_reader.cpp
+++ b/src/mongo/db/keys_collection_cache_reader.cpp
@@ -30,21 +30,59 @@
 
 #include "mongo/db/keys_collection_cache_reader.h"
 
+#include "mongo/s/catalog/sharding_catalog_client.h"
+#include "mongo/s/grid.h"
+#include "mongo/util/mongoutils/str.h"
+
 namespace mongo {
 
 KeysCollectionCacheReader::KeysCollectionCacheReader(std::string purpose)
     : _purpose(std::move(purpose)) {}
 
 StatusWith<KeysCollectionDocument> KeysCollectionCacheReader::refresh(OperationContext* opCtx) {
-    // forThisTime = latest keyDoc.expiresAt or LogicalTime.MIN if _cache is empty
-    // read must be { level: 'majority' }.
-    // admin.system.keys.find({purpose: 'signLogicalTime', expiresAt: {$gt: <forThisTime>});
-    return {ErrorCodes::InternalError, "Not yet implemented"};
+    LogicalTime newerThanThis;
+
+    {
+        stdx::lock_guard<stdx::mutex> lk(_cacheMutex);
+        auto iter = _cache.crbegin();
+        if (iter != _cache.crend()) {
+            newerThanThis = iter->second.getExpiresAt();
+        }
+    }
+
+    auto refreshStatus = Grid::get(opCtx)->catalogClient(opCtx)->getNewKeys(
+        opCtx, _purpose, newerThanThis, repl::ReadConcernLevel::kMajorityReadConcern);
+
+    if (!refreshStatus.isOK()) {
+        return refreshStatus.getStatus();
+    }
+
+    auto& newKeys = refreshStatus.getValue();
+
+    stdx::lock_guard<stdx::mutex> lk(_cacheMutex);
+    for (auto&& key : newKeys) {
+        _cache.emplace(std::make_pair(key.getExpiresAt(), std::move(key)));
+    }
+
+    if (_cache.empty()) {
+        return {ErrorCodes::KeyNotFound, "No keys found after refresh"};
+    }
+
+    return _cache.crbegin()->second;
 }
 
 StatusWith<KeysCollectionDocument> KeysCollectionCacheReader::getKey(
     const LogicalTime& forThisTime) {
-    return {ErrorCodes::InternalError, "Not yet implemented"};
+    stdx::lock_guard<stdx::mutex> lk(_cacheMutex);
+
+    auto iter = _cache.upper_bound(forThisTime);
+
+    if (iter == _cache.cend()) {
+        return {ErrorCodes::KeyNotFound,
+                str::stream() << "No key found that is valid for " << forThisTime.toString()};
+    }
+
+    return iter->second;
 }
 
 }  // namespace mongo
diff --git a/src/mongo/db/keys_collection_cache_reader_and_updater_test.cpp b/src/mongo/db/keys_collection_cache_reader_and_updater_test.cpp
index 84a50dbe759..187cebd444a 100644
--- a/src/mongo/db/keys_collection_cache_reader_and_updater_test.cpp
+++ b/src/mongo/db/keys_collection_cache_reader_and_updater_test.cpp
@@ -67,8 +67,15 @@ TEST_F(CacheUpdaterTest, ShouldCreate2KeysFromEmpty) {
     const LogicalTime currentTime(LogicalTime(Timestamp(100, 2)));
     LogicalClock::get(operationContext())->initClusterTimeFromTrustedSource(currentTime);
 
-    auto keyStatus = updater.refresh(operationContext());
-    // TODO: Inspect keyStatus once SERVER-28435 is implemented
+    {
+        auto keyStatus = updater.refresh(operationContext());
+        ASSERT_OK(keyStatus.getStatus());
+
+        const auto key = keyStatus.getValue();
+        ASSERT_EQ(currentTime.asTimestamp().asLL() + 1, key.getKeyId());
+        ASSERT_EQ("dummy", key.getPurpose());
+        ASSERT_EQ(Timestamp(110, 0), key.getExpiresAt().asTimestamp());
+    }
 
     auto allKeys = getKeys(operationContext());
 
@@ -110,8 +117,16 @@ TEST_F(CacheUpdaterTest, ShouldCreateAnotherKeyIfOnlyOneKeyExists) {
     }
 
     auto currentTime = LogicalClock::get(operationContext())->getClusterTime().getTime();
-    auto keyStatus = updater.refresh(operationContext());
-    // TODO: Inspect keyStatus once SERVER-28435 is implemented
+
+    {
+        auto keyStatus = updater.refresh(operationContext());
+        ASSERT_OK(keyStatus.getStatus());
+
+        const auto key = keyStatus.getValue();
+        ASSERT_EQ(currentTime.asTimestamp().asLL(), key.getKeyId());
+        ASSERT_EQ("dummy", key.getPurpose());
+        ASSERT_EQ(Timestamp(110, 0), key.getExpiresAt().asTimestamp());
+    }
 
     {
         auto allKeys = getKeys(operationContext());
@@ -166,8 +181,17 @@ TEST_F(CacheUpdaterTest, ShouldCreateAnotherKeyIfNoValidKeyAfterCurrent) {
     }
 
     auto currentTime = LogicalClock::get(operationContext())->getClusterTime().getTime();
-    auto keyStatus = updater.refresh(operationContext());
-    // TODO: Inspect keyStatus once SERVER-28435 is implemented
+
+    {
+        auto keyStatus = updater.refresh(operationContext());
+        ASSERT_OK(keyStatus.getStatus());
+
+        const auto key = keyStatus.getValue();
+        ASSERT_EQ(currentTime.asTimestamp().asLL(), key.getKeyId());
+        ASSERT_EQ("dummy", key.getPurpose());
+        ASSERT_EQ(Timestamp(115, 0), key.getExpiresAt().asTimestamp());
+    }
+
 
     auto allKeys = getKeys(operationContext());
 
@@ -248,8 +272,16 @@ TEST_F(CacheUpdaterTest, ShouldCreate2KeysIfAllKeysAreExpired) {
     }
 
     auto currentTime = LogicalClock::get(operationContext())->getClusterTime().getTime();
-    auto keyStatus = updater.refresh(operationContext());
-    // TODO: Inspect keyStatus once SERVER-28435 is implemented
+
+    {
+        auto keyStatus = updater.refresh(operationContext());
+        ASSERT_OK(keyStatus.getStatus());
+
+        const auto key = keyStatus.getValue();
+        ASSERT_EQ(currentTime.asTimestamp().asLL() + 1, key.getKeyId());
+        ASSERT_EQ("dummy", key.getPurpose());
+        ASSERT_EQ(Timestamp(130, 0), key.getExpiresAt().asTimestamp());
+    }
 
     auto allKeys = getKeys(operationContext());
 
@@ -343,8 +375,15 @@ TEST_F(CacheUpdaterTest, ShouldNotCreateNewKeyIfThereAre2UnexpiredKeys) {
         ASSERT_EQ(Timestamp(110, 0), key2.getExpiresAt().asTimestamp());
     }
 
-    auto keyStatus = updater.refresh(operationContext());
-    // TODO: Inspect keyStatus once SERVER-28435 is implemented
+    {
+        auto keyStatus = updater.refresh(operationContext());
+        ASSERT_OK(keyStatus.getStatus());
+
+        const auto key = keyStatus.getValue();
+        ASSERT_EQ(2, key.getKeyId());
+        ASSERT_EQ("dummy", key.getPurpose());
+        ASSERT_EQ(Timestamp(110, 0), key.getExpiresAt().asTimestamp());
+    }
 
     auto allKeys = getKeys(operationContext());
 
diff --git a/src/mongo/db/keys_collection_cache_reader_test.cpp b/src/mongo/db/keys_collection_cache_reader_test.cpp
new file mode 100644
index 00000000000..e2e2594f00b
--- /dev/null
+++ b/src/mongo/db/keys_collection_cache_reader_test.cpp
@@ -0,0 +1,250 @@
+/**
+ *    Copyright (C) 2017 MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or  modify
+ *    it under the terms of the GNU Affero General Public License, version 3,
+ *    as published by the Free Software Foundation.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Affero General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Affero General Public License
+ *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the GNU Affero General Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#include "mongo/platform/basic.h"
+
+#include "mongo/db/jsobj.h"
+#include "mongo/db/keys_collection_cache_reader.h"
+#include "mongo/db/keys_collection_document.h"
+#include "mongo/db/operation_context.h"
+#include "mongo/s/config_server_test_fixture.h"
+#include "mongo/unittest/unittest.h"
+
+namespace mongo {
+
+using CacheReaderTest = ConfigServerTestFixture;
+
+TEST_F(CacheReaderTest, ErrorsIfCacheIsEmpty) {
+    KeysCollectionCacheReader reader("test");
+    auto status = reader.getKey(LogicalTime(Timestamp(1, 0))).getStatus();
+    ASSERT_EQ(ErrorCodes::KeyNotFound, status.code());
+    ASSERT_FALSE(status.reason().empty());
+}
+
+TEST_F(CacheReaderTest, RefreshErrorsIfCacheIsEmpty) {
+    KeysCollectionCacheReader reader("test");
+    auto status = reader.refresh(operationContext()).getStatus();
+    ASSERT_EQ(ErrorCodes::KeyNotFound, status.code());
+    ASSERT_FALSE(status.reason().empty());
+}
+
+TEST_F(CacheReaderTest, GetKeyShouldReturnCorrectKeyAfterRefresh) {
+    KeysCollectionCacheReader reader("test");
+
+    KeysCollectionDocument origKey1(
+        1, "test", TimeProofService::generateRandomKey(), LogicalTime(Timestamp(105, 0)));
+    ASSERT_OK(insertToConfigCollection(
+        operationContext(), NamespaceString(KeysCollectionDocument::ConfigNS), origKey1.toBSON()));
+
+    auto refreshStatus = reader.refresh(operationContext());
+    ASSERT_OK(refreshStatus.getStatus());
+
+    {
+        auto key = refreshStatus.getValue();
+        ASSERT_EQ(1, key.getKeyId());
+        ASSERT_EQ(origKey1.getKey(), key.getKey());
+        ASSERT_EQ("test", key.getPurpose());
+        ASSERT_EQ(Timestamp(105, 0), key.getExpiresAt().asTimestamp());
+    }
+
+    auto status = reader.getKey(LogicalTime(Timestamp(1, 0)));
+    ASSERT_OK(status.getStatus());
+
+    {
+        auto key = status.getValue();
+        ASSERT_EQ(1, key.getKeyId());
+        ASSERT_EQ(origKey1.getKey(), key.getKey());
+        ASSERT_EQ("test", key.getPurpose());
+        ASSERT_EQ(Timestamp(105, 0), key.getExpiresAt().asTimestamp());
+    }
+}
+
+TEST_F(CacheReaderTest, GetKeyShouldReturnErrorIfNoKeyIsValidForGivenTime) {
+    KeysCollectionCacheReader reader("test");
+
+    KeysCollectionDocument origKey1(
+        1, "test", TimeProofService::generateRandomKey(), LogicalTime(Timestamp(105, 0)));
+    ASSERT_OK(insertToConfigCollection(
+        operationContext(), NamespaceString(KeysCollectionDocument::ConfigNS), origKey1.toBSON()));
+
+    auto refreshStatus = reader.refresh(operationContext());
+    ASSERT_OK(refreshStatus.getStatus());
+
+    {
+        auto key = refreshStatus.getValue();
+        ASSERT_EQ(1, key.getKeyId());
+        ASSERT_EQ(origKey1.getKey(), key.getKey());
+        ASSERT_EQ("test", key.getPurpose());
+        ASSERT_EQ(Timestamp(105, 0), key.getExpiresAt().asTimestamp());
+    }
+
+    auto status = reader.getKey(LogicalTime(Timestamp(110, 0)));
+    ASSERT_EQ(ErrorCodes::KeyNotFound, status.getStatus());
+}
+
+TEST_F(CacheReaderTest, GetKeyShouldReturnOldestKeyPossible) {
+    KeysCollectionCacheReader reader("test");
+
+    KeysCollectionDocument origKey0(
+        0, "test", TimeProofService::generateRandomKey(), LogicalTime(Timestamp(100, 0)));
+    ASSERT_OK(insertToConfigCollection(
+        operationContext(), NamespaceString(KeysCollectionDocument::ConfigNS), origKey0.toBSON()));
+
+    KeysCollectionDocument origKey1(
+        1, "test", TimeProofService::generateRandomKey(), LogicalTime(Timestamp(105, 0)));
+    ASSERT_OK(insertToConfigCollection(
+        operationContext(), NamespaceString(KeysCollectionDocument::ConfigNS), origKey1.toBSON()));
+
+    KeysCollectionDocument origKey2(
+        2, "test", TimeProofService::generateRandomKey(), LogicalTime(Timestamp(110, 0)));
+    ASSERT_OK(insertToConfigCollection(
+        operationContext(), NamespaceString(KeysCollectionDocument::ConfigNS), origKey2.toBSON()));
+
+    auto refreshStatus = reader.refresh(operationContext());
+    ASSERT_OK(refreshStatus.getStatus());
+
+    {
+        auto key = refreshStatus.getValue();
+        ASSERT_EQ(2, key.getKeyId());
+        ASSERT_EQ(origKey2.getKey(), key.getKey());
+        ASSERT_EQ("test", key.getPurpose());
+        ASSERT_EQ(Timestamp(110, 0), key.getExpiresAt().asTimestamp());
+    }
+
+    auto keyStatus = reader.getKey(LogicalTime(Timestamp(103, 1)));
+    ASSERT_OK(keyStatus.getStatus());
+
+    {
+        auto key = keyStatus.getValue();
+        ASSERT_EQ(1, key.getKeyId());
+        ASSERT_EQ(origKey1.getKey(), key.getKey());
+        ASSERT_EQ("test", key.getPurpose());
+        ASSERT_EQ(Timestamp(105, 0), key.getExpiresAt().asTimestamp());
+    }
+}
+
+TEST_F(CacheReaderTest, RefreshShouldNotGetKeysForOtherPurpose) {
+    KeysCollectionCacheReader reader("test");
+
+    KeysCollectionDocument origKey0(
+        0, "dummy", TimeProofService::generateRandomKey(), LogicalTime(Timestamp(100, 0)));
+    ASSERT_OK(insertToConfigCollection(
+        operationContext(), NamespaceString(KeysCollectionDocument::ConfigNS), origKey0.toBSON()));
+
+    {
+        auto refreshStatus = reader.refresh(operationContext());
+        ASSERT_EQ(ErrorCodes::KeyNotFound, refreshStatus.getStatus());
+
+        auto emptyKeyStatus = reader.getKey(LogicalTime(Timestamp(50, 0)));
+        ASSERT_EQ(ErrorCodes::KeyNotFound, emptyKeyStatus.getStatus());
+    }
+
+    KeysCollectionDocument origKey1(
+        1, "test", TimeProofService::generateRandomKey(), LogicalTime(Timestamp(105, 0)));
+    ASSERT_OK(insertToConfigCollection(
+        operationContext(), NamespaceString(KeysCollectionDocument::ConfigNS), origKey1.toBSON()));
+
+    {
+        auto refreshStatus = reader.refresh(operationContext());
+        ASSERT_OK(refreshStatus.getStatus());
+
+        auto key = refreshStatus.getValue();
+        ASSERT_EQ(1, key.getKeyId());
+        ASSERT_EQ(origKey1.getKey(), key.getKey());
+        ASSERT_EQ("test", key.getPurpose());
+        ASSERT_EQ(Timestamp(105, 0), key.getExpiresAt().asTimestamp());
+    }
+
+    auto keyStatus = reader.getKey(LogicalTime(Timestamp(60, 1)));
+    ASSERT_OK(keyStatus.getStatus());
+
+    {
+        auto key = keyStatus.getValue();
+        ASSERT_EQ(1, key.getKeyId());
+        ASSERT_EQ(origKey1.getKey(), key.getKey());
+        ASSERT_EQ("test", key.getPurpose());
+        ASSERT_EQ(Timestamp(105, 0), key.getExpiresAt().asTimestamp());
+    }
+}
+
+TEST_F(CacheReaderTest, RefreshCanIncrementallyGetNewKeys) {
+    KeysCollectionCacheReader reader("test");
+
+    KeysCollectionDocument origKey0(
+        0, "test", TimeProofService::generateRandomKey(), LogicalTime(Timestamp(100, 0)));
+    ASSERT_OK(insertToConfigCollection(
+        operationContext(), NamespaceString(KeysCollectionDocument::ConfigNS), origKey0.toBSON()));
+
+    {
+        auto refreshStatus = reader.refresh(operationContext());
+        ASSERT_OK(refreshStatus.getStatus());
+
+
+        auto key = refreshStatus.getValue();
+        ASSERT_EQ(0, key.getKeyId());
+        ASSERT_EQ(origKey0.getKey(), key.getKey());
+        ASSERT_EQ("test", key.getPurpose());
+        ASSERT_EQ(Timestamp(100, 0), key.getExpiresAt().asTimestamp());
+
+        auto keyStatus = reader.getKey(LogicalTime(Timestamp(112, 1)));
+        ASSERT_EQ(ErrorCodes::KeyNotFound, keyStatus.getStatus());
+    }
+
+    KeysCollectionDocument origKey1(
+        1, "test", TimeProofService::generateRandomKey(), LogicalTime(Timestamp(105, 0)));
+    ASSERT_OK(insertToConfigCollection(
+        operationContext(), NamespaceString(KeysCollectionDocument::ConfigNS), origKey1.toBSON()));
+
+    KeysCollectionDocument origKey2(
+        2, "test", TimeProofService::generateRandomKey(), LogicalTime(Timestamp(110, 0)));
+    ASSERT_OK(insertToConfigCollection(
+        operationContext(), NamespaceString(KeysCollectionDocument::ConfigNS), origKey2.toBSON()));
+
+    {
+        auto refreshStatus = reader.refresh(operationContext());
+        ASSERT_OK(refreshStatus.getStatus());
+
+        auto key = refreshStatus.getValue();
+        ASSERT_EQ(2, key.getKeyId());
+        ASSERT_EQ(origKey2.getKey(), key.getKey());
+        ASSERT_EQ("test", key.getPurpose());
+        ASSERT_EQ(Timestamp(110, 0), key.getExpiresAt().asTimestamp());
+    }
+
+    {
+        auto keyStatus = reader.getKey(LogicalTime(Timestamp(108, 1)));
+
+        auto key = keyStatus.getValue();
+        ASSERT_EQ(2, key.getKeyId());
+        ASSERT_EQ(origKey2.getKey(), key.getKey());
+        ASSERT_EQ("test", key.getPurpose());
+        ASSERT_EQ(Timestamp(110, 0), key.getExpiresAt().asTimestamp());
+    }
+}
+
+}  // namespace mongo