summaryrefslogtreecommitdiff
path: root/src/mongo
diff options
context:
space:
mode:
Diffstat (limited to 'src/mongo')
-rw-r--r--src/mongo/db/keys_collection_cache_reader_and_updater.cpp7
-rw-r--r--src/mongo/db/keys_collection_cache_reader_and_updater_test.cpp19
-rw-r--r--src/mongo/db/keys_collection_manager.cpp27
-rw-r--r--src/mongo/db/keys_collection_manager.h11
-rw-r--r--src/mongo/db/keys_collection_manager_test.cpp48
-rw-r--r--src/mongo/db/logical_time_validator.cpp4
-rw-r--r--src/mongo/db/logical_time_validator.h6
-rw-r--r--src/mongo/db/logical_time_validator_test.cpp41
-rw-r--r--src/mongo/s/commands/strategy.cpp28
-rw-r--r--src/mongo/shell/shardingtest.js20
10 files changed, 198 insertions, 13 deletions
diff --git a/src/mongo/db/keys_collection_cache_reader_and_updater.cpp b/src/mongo/db/keys_collection_cache_reader_and_updater.cpp
index f816d95bec7..f81f5afabef 100644
--- a/src/mongo/db/keys_collection_cache_reader_and_updater.cpp
+++ b/src/mongo/db/keys_collection_cache_reader_and_updater.cpp
@@ -35,11 +35,14 @@
#include "mongo/db/operation_context.h"
#include "mongo/s/catalog/sharding_catalog_client.h"
#include "mongo/s/client/shard_registry.h"
+#include "mongo/util/fail_point_service.h"
namespace mongo {
namespace {
+MONGO_FP_DECLARE(disableKeyGeneration);
+
/**
* Inserts a new key to the keys collection.
*
@@ -78,6 +81,10 @@ KeysCollectionCacheReaderAndUpdater::KeysCollectionCacheReaderAndUpdater(
StatusWith<KeysCollectionDocument> KeysCollectionCacheReaderAndUpdater::refresh(
OperationContext* opCtx) {
+ if (MONGO_FAIL_POINT(disableKeyGeneration)) {
+ return {ErrorCodes::FailPointEnabled, "key generation disabled"};
+ }
+
auto currentTime = LogicalClock::get(opCtx)->getClusterTime();
auto keyStatus = _catalogClient->getNewKeys(
opCtx, _purpose, currentTime, repl::ReadConcernLevel::kLocalReadConcern);
diff --git a/src/mongo/db/keys_collection_cache_reader_and_updater_test.cpp b/src/mongo/db/keys_collection_cache_reader_and_updater_test.cpp
index 91b5cf2915e..03e337175ab 100644
--- a/src/mongo/db/keys_collection_cache_reader_and_updater_test.cpp
+++ b/src/mongo/db/keys_collection_cache_reader_and_updater_test.cpp
@@ -429,4 +429,23 @@ TEST_F(CacheUpdaterTest, ShouldNotCreateNewKeyIfThereAre2UnexpiredKeys) {
}
}
+TEST_F(CacheUpdaterTest, ShouldNotCreateKeysWithDisableKeyGenerationFailPoint) {
+ auto catalogClient = Grid::get(operationContext())->catalogClient(operationContext());
+ KeysCollectionCacheReaderAndUpdater updater("dummy", catalogClient, Seconds(5));
+
+ const LogicalTime currentTime(LogicalTime(Timestamp(100, 0)));
+ LogicalClock::get(operationContext())->setClusterTimeFromTrustedSource(currentTime);
+
+ {
+ FailPointEnableBlock failKeyGenerationBlock("disableKeyGeneration");
+
+ auto keyStatus = updater.refresh(operationContext());
+ ASSERT_EQ(ErrorCodes::FailPointEnabled, keyStatus.getStatus());
+ }
+
+ auto allKeys = getKeys(operationContext());
+
+ ASSERT_EQ(0U, allKeys.size());
+}
+
} // namespace mongo
diff --git a/src/mongo/db/keys_collection_manager.cpp b/src/mongo/db/keys_collection_manager.cpp
index 3d4599e2533..591665e3564 100644
--- a/src/mongo/db/keys_collection_manager.cpp
+++ b/src/mongo/db/keys_collection_manager.cpp
@@ -38,6 +38,7 @@
#include "mongo/db/service_context.h"
#include "mongo/stdx/memory.h"
#include "mongo/util/concurrency/idle_thread_block.h"
+#include "mongo/util/fail_point_service.h"
#include "mongo/util/mongoutils/str.h"
#include "mongo/util/time_support.h"
@@ -49,6 +50,10 @@ Milliseconds kDefaultRefreshWaitTime(30 * 1000);
Milliseconds kRefreshIntervalIfErrored(200);
Milliseconds kMaxRefreshWaitTime(10 * 60 * 1000);
+// Prevents the refresher thread from waiting longer than the given number of milliseconds, even on
+// a successful refresh.
+MONGO_FP_DECLARE(maxKeyRefreshWaitTimeOverrideMS);
+
/**
* Returns the amount of time to wait until the monitoring thread should attempt to refresh again.
*/
@@ -170,6 +175,10 @@ void KeysCollectionManager::enableKeyGenerator(OperationContext* opCtx, bool doE
}
}
+bool KeysCollectionManager::hasSeenKeys() {
+ return _refresher.hasSeenKeys();
+}
+
void KeysCollectionManager::PeriodicRunner::refreshNow(OperationContext* opCtx) {
auto refreshRequest = [this]() {
stdx::lock_guard<stdx::mutex> lk(_mutex);
@@ -227,6 +236,11 @@ void KeysCollectionManager::PeriodicRunner::_doPeriodicRefresh(ServiceContext* s
const auto& latestKey = latestKeyStatusWith.getValue();
auto currentTime = LogicalClock::get(service)->getClusterTime();
+ {
+ stdx::unique_lock<stdx::mutex> lock(_mutex);
+ _hasSeenKeys = true;
+ }
+
nextWakeup =
howMuchSleepNeedFor(currentTime, latestKey.getExpiresAt(), refreshInterval);
} else {
@@ -237,6 +251,14 @@ void KeysCollectionManager::PeriodicRunner::_doPeriodicRefresh(ServiceContext* s
}
}
+ MONGO_FAIL_POINT_BLOCK(maxKeyRefreshWaitTimeOverrideMS, data) {
+ const BSONObj& dataObj = data.getData();
+ auto overrideMS = Milliseconds(dataObj["overrideMS"].numberInt());
+ if (nextWakeup > overrideMS) {
+ nextWakeup = overrideMS;
+ }
+ }
+
stdx::unique_lock<stdx::mutex> lock(_mutex);
if (_refreshRequest) {
@@ -308,4 +330,9 @@ void KeysCollectionManager::PeriodicRunner::stop() {
_backgroundThread.join();
}
+bool KeysCollectionManager::PeriodicRunner::hasSeenKeys() {
+ stdx::lock_guard<stdx::mutex> lock(_mutex);
+ return _hasSeenKeys;
+}
+
} // namespace mongo
diff --git a/src/mongo/db/keys_collection_manager.h b/src/mongo/db/keys_collection_manager.h
index bfe4a27a408..9333ba284d9 100644
--- a/src/mongo/db/keys_collection_manager.h
+++ b/src/mongo/db/keys_collection_manager.h
@@ -98,6 +98,11 @@ public:
*/
void enableKeyGenerator(OperationContext* opCtx, bool doEnable);
+ /**
+ * Returns true if the refresher has ever successfully returned keys from the config server.
+ */
+ bool hasSeenKeys();
+
private:
/**
* This is responsible for periodically performing refresh in the background.
@@ -143,6 +148,11 @@ private:
*/
void stop();
+ /**
+ * Returns true if keys have ever successfully been returned from the config server.
+ */
+ bool hasSeenKeys();
+
private:
void _doPeriodicRefresh(ServiceContext* service,
std::string threadName,
@@ -155,6 +165,7 @@ private:
stdx::thread _backgroundThread;
std::shared_ptr<RefreshFunc> _doRefresh;
+ bool _hasSeenKeys = false;
bool _inShutdown = false;
};
diff --git a/src/mongo/db/keys_collection_manager_test.cpp b/src/mongo/db/keys_collection_manager_test.cpp
index aae1db1a6a4..31a0b6bc57b 100644
--- a/src/mongo/db/keys_collection_manager_test.cpp
+++ b/src/mongo/db/keys_collection_manager_test.cpp
@@ -314,4 +314,52 @@ TEST_F(KeysManagerTest, ShouldStillBeAbleToUpdateCacheEvenIfItCantCreateKeys) {
ASSERT_EQ(Timestamp(105, 0), key.getExpiresAt().asTimestamp());
}
+TEST_F(KeysManagerTest, ShouldNotCreateKeysWithDisableKeyGenerationFailPoint) {
+ const LogicalTime currentTime(Timestamp(100, 0));
+ LogicalClock::get(operationContext())->setClusterTimeFromTrustedSource(currentTime);
+
+ {
+ FailPointEnableBlock failKeyGenerationBlock("disableKeyGeneration");
+ keyManager()->startMonitoring(getServiceContext());
+ keyManager()->enableKeyGenerator(operationContext(), true);
+
+ keyManager()->refreshNow(operationContext());
+ auto keyStatus = keyManager()->getKeyForValidation(
+ operationContext(), 1, LogicalTime(Timestamp(100, 0)));
+ ASSERT_EQ(ErrorCodes::KeyNotFound, keyStatus.getStatus());
+ }
+
+ // Once the failpoint is disabled, the generator can make keys again.
+ keyManager()->refreshNow(operationContext());
+ auto keyStatus = keyManager()->getKeyForSigning(LogicalTime(Timestamp(100, 0)));
+ ASSERT_OK(keyStatus.getStatus());
+}
+
+TEST_F(KeysManagerTest, HasSeenKeysIsFalseUntilKeysAreFound) {
+ const LogicalTime currentTime(Timestamp(100, 0));
+ LogicalClock::get(operationContext())->setClusterTimeFromTrustedSource(currentTime);
+
+ ASSERT_EQ(false, keyManager()->hasSeenKeys());
+
+ {
+ FailPointEnableBlock failKeyGenerationBlock("disableKeyGeneration");
+ keyManager()->startMonitoring(getServiceContext());
+ keyManager()->enableKeyGenerator(operationContext(), true);
+
+ keyManager()->refreshNow(operationContext());
+ auto keyStatus = keyManager()->getKeyForValidation(
+ operationContext(), 1, LogicalTime(Timestamp(100, 0)));
+ ASSERT_EQ(ErrorCodes::KeyNotFound, keyStatus.getStatus());
+
+ ASSERT_EQ(false, keyManager()->hasSeenKeys());
+ }
+
+ // Once the failpoint is disabled, the generator can make keys again.
+ keyManager()->refreshNow(operationContext());
+ auto keyStatus = keyManager()->getKeyForSigning(LogicalTime(Timestamp(100, 0)));
+ ASSERT_OK(keyStatus.getStatus());
+
+ ASSERT_EQ(true, keyManager()->hasSeenKeys());
+}
+
} // namespace mongo
diff --git a/src/mongo/db/logical_time_validator.cpp b/src/mongo/db/logical_time_validator.cpp
index 8387198f6f9..2248145ce3d 100644
--- a/src/mongo/db/logical_time_validator.cpp
+++ b/src/mongo/db/logical_time_validator.cpp
@@ -181,4 +181,8 @@ bool LogicalTimeValidator::isAuthorizedToAdvanceClock(OperationContext* opCtx) {
advanceLogicalClockPrivilege);
}
+bool LogicalTimeValidator::shouldGossipLogicalTime() {
+ return _keyManager->hasSeenKeys();
+}
+
} // namespace mongo
diff --git a/src/mongo/db/logical_time_validator.h b/src/mongo/db/logical_time_validator.h
index 24be173d2dd..d46f1787ebf 100644
--- a/src/mongo/db/logical_time_validator.h
+++ b/src/mongo/db/logical_time_validator.h
@@ -91,6 +91,12 @@ public:
*/
static bool isAuthorizedToAdvanceClock(OperationContext* opCtx);
+ /**
+ * Returns true if the server should gossip, sign, and validate logical times. False until there
+ * are keys in the config server.
+ */
+ bool shouldGossipLogicalTime();
+
private:
SignedLogicalTime _getProof(const KeysCollectionDocument& keyDoc, LogicalTime newTime);
diff --git a/src/mongo/db/logical_time_validator_test.cpp b/src/mongo/db/logical_time_validator_test.cpp
index 2aeb11462a8..fca439630fb 100644
--- a/src/mongo/db/logical_time_validator_test.cpp
+++ b/src/mongo/db/logical_time_validator_test.cpp
@@ -85,6 +85,20 @@ protected:
}
/**
+ * Replaces the test's LogicalTimeValidator with a new one with a disabled keyGenerator.
+ */
+ void resetValidator() {
+ _validator->shutDown();
+
+ auto catalogClient = Grid::get(operationContext())->catalogClient(operationContext());
+ auto keyManager =
+ stdx::make_unique<KeysCollectionManager>("dummy", catalogClient, Seconds(1000));
+ _keyManager = keyManager.get();
+ _validator = stdx::make_unique<LogicalTimeValidator>(std::move(keyManager));
+ _validator->init(operationContext()->getServiceContext());
+ }
+
+ /**
* Forces KeyManager to refresh cache and generate new keys.
*/
void refreshKeyManager() {
@@ -150,5 +164,32 @@ TEST_F(LogicalTimeValidatorTest, ValidateErrorsOnInvalidTimeWithImplicitRefresh)
ASSERT_EQ(ErrorCodes::TimeProofMismatch, status);
}
+TEST_F(LogicalTimeValidatorTest, ShouldGossipLogicalTimeIsFalseUntilKeysAreFound) {
+ // Use a new validator with a disabled key generator.
+ resetValidator();
+
+ // shouldGossipLogicalTime initially returns false.
+ ASSERT_EQ(false, validator()->shouldGossipLogicalTime());
+
+ // Enable key generation.
+ validator()->enableKeyGenerator(operationContext(), true);
+
+ // shouldGossipLogicalTime still returns false after an unsuccessful refresh.
+ validator()->enableKeyGenerator(operationContext(), false);
+ refreshKeyManager();
+
+ LogicalTime t1(Timestamp(20, 0));
+ validator()->trySignLogicalTime(t1);
+ ASSERT_EQ(false, validator()->shouldGossipLogicalTime());
+
+ // Once keys are successfully found, shouldGossipLogicalTime returns true.
+ validator()->enableKeyGenerator(operationContext(), true);
+ refreshKeyManager();
+ auto newTime = validator()->signLogicalTime(operationContext(), t1);
+
+ ASSERT_EQ(true, validator()->shouldGossipLogicalTime());
+ ASSERT_OK(validator()->validate(operationContext(), newTime));
+}
+
} // unnamed namespace
} // namespace mongo
diff --git a/src/mongo/s/commands/strategy.cpp b/src/mongo/s/commands/strategy.cpp
index afa4f04e55a..79cd0f44a0b 100644
--- a/src/mongo/s/commands/strategy.cpp
+++ b/src/mongo/s/commands/strategy.cpp
@@ -126,20 +126,22 @@ Status processCommandMetadata(OperationContext* opCtx, const BSONObj& cmdObj) {
* Append required fields to command response.
*/
void appendRequiredFieldsToResponse(OperationContext* opCtx, BSONObjBuilder* responseBuilder) {
- // Add $logicalTime.
auto validator = LogicalTimeValidator::get(opCtx);
- auto currentTime =
- validator->signLogicalTime(opCtx, LogicalClock::get(opCtx)->getClusterTime());
- rpc::LogicalTimeMetadata(currentTime).writeToMetadata(responseBuilder);
-
- // Add operationTime.
- if (auto tracker = OperationTimeTracker::get(opCtx)) {
- auto operationTime = OperationTimeTracker::get(opCtx)->getMaxOperationTime();
- responseBuilder->append(kOperationTime, operationTime.asTimestamp());
- } else if (currentTime.getTime() != LogicalTime::kUninitialized) {
- // If we don't know the actual operation time, use the cluster time instead. This is safe
- // but not optimal because we can always return a later operation time than actual.
- responseBuilder->append(kOperationTime, currentTime.getTime().asTimestamp());
+ if (validator->shouldGossipLogicalTime()) {
+ // Add $logicalTime.
+ auto currentTime =
+ validator->signLogicalTime(opCtx, LogicalClock::get(opCtx)->getClusterTime());
+ rpc::LogicalTimeMetadata(currentTime).writeToMetadata(responseBuilder);
+
+ // Add operationTime.
+ if (auto tracker = OperationTimeTracker::get(opCtx)) {
+ auto operationTime = OperationTimeTracker::get(opCtx)->getMaxOperationTime();
+ responseBuilder->append(kOperationTime, operationTime.asTimestamp());
+ } else if (currentTime.getTime() != LogicalTime::kUninitialized) {
+ // If we don't know the actual operation time, use the cluster time instead. This is
+ // safe but not optimal because we can always return a later operation time than actual.
+ responseBuilder->append(kOperationTime, currentTime.getTime().asTimestamp());
+ }
}
}
diff --git a/src/mongo/shell/shardingtest.js b/src/mongo/shell/shardingtest.js
index 90222f329ca..55c76c18bb8 100644
--- a/src/mongo/shell/shardingtest.js
+++ b/src/mongo/shell/shardingtest.js
@@ -17,6 +17,9 @@
* mongos {number|Object|Array.<Object>}: number of mongos or mongos
* configuration object(s)(*). @see MongoRunner.runMongos
*
+ * mongosWaitsForKeys {boolean}: if true, wait for mongos to discover keys from the config
+ * server and to start sending logical times.
+ *
* rs {Object|Array.<Object>}: replica set configuration object. Can
* contain:
* {
@@ -1482,4 +1485,21 @@ var ShardingTest = function(params) {
jsTest.authenticateNodes(this._configServers);
jsTest.authenticateNodes(this._mongos);
}
+
+ // Mongos does not block for keys from the config servers at startup, so it may not initially
+ // return logical times. If mongosWaitsForKeys is set, block until all mongos servers have found
+ // the keys and begun to send logical times. Retry every 500 milliseconds and timeout after 60
+ // seconds.
+ if (params.mongosWaitsForKeys) {
+ assert.soon(function() {
+ for (let i = 0; i < numMongos; i++) {
+ const res = self._mongos[i].adminCommand({isMaster: 1});
+ if (!res.hasOwnProperty("$logicalTime")) {
+ print("Waiting for mongos #" + i + " to start sending logical times.");
+ return false;
+ }
+ }
+ return true;
+ }, "waiting for all mongos servers to return logical times", 60 * 1000, 500);
+ }
};
View Lorry Controller Status