diff options
author | Gabriel Russell <gabriel.russell@mongodb.com> | 2020-02-13 11:49:46 -0500 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2020-02-13 18:16:35 +0000 |
commit | a84c09a19720b73cedb2e8ef7c5cfeedfa1c9761 (patch) | |
tree | 85ac46cd5f4ea6d5134560bf764fb9e6cf11fe4e /src/mongo/db/repl | |
parent | 6df40e01f7b6899affc4536e7e73a35802cabf98 (diff) | |
download | mongo-a84c09a19720b73cedb2e8ef7c5cfeedfa1c9761.tar.gz |
SERVER-45869 automatically converted structured logging
Diffstat (limited to 'src/mongo/db/repl')
52 files changed, 2918 insertions, 1364 deletions
diff --git a/src/mongo/db/repl/abstract_oplog_fetcher.cpp b/src/mongo/db/repl/abstract_oplog_fetcher.cpp index 56a5fb710dd..c9bfaace0e9 100644 --- a/src/mongo/db/repl/abstract_oplog_fetcher.cpp +++ b/src/mongo/db/repl/abstract_oplog_fetcher.cpp @@ -40,6 +40,7 @@ #include "mongo/db/commands/server_status_metric.h" #include "mongo/db/jsobj.h" #include "mongo/db/repl/repl_server_parameters_gen.h" +#include "mongo/logv2/log.h" #include "mongo/platform/mutex.h" #include "mongo/util/assert_util.h" #include "mongo/util/log.h" @@ -211,8 +212,12 @@ void AbstractOplogFetcher::_callback(const Fetcher::QueryResponseStatus& result, Status responseStatus = _checkForShutdownAndConvertStatus(result.getStatus(), "error in fetcher batch callback"); if (ErrorCodes::CallbackCanceled == responseStatus) { - LOG(1) << _getComponentName() << " oplog query cancelled to " << _getSource() << ": " - << redact(responseStatus); + LOGV2_DEBUG(21032, + 1, + "{getComponentName} oplog query cancelled to {getSource}: {responseStatus}", + "getComponentName"_attr = _getComponentName(), + "getSource"_attr = _getSource(), + "responseStatus"_attr = redact(responseStatus)); _finishCallback(responseStatus); return; } @@ -235,11 +240,16 @@ void AbstractOplogFetcher::_callback(const Fetcher::QueryResponseStatus& result, auto scheduleStatus = _scheduleFetcher_inlock(); if (scheduleStatus.isOK()) { - log() << "Scheduled new oplog query " << _fetcher->toString(); + LOGV2(21033, + "Scheduled new oplog query {fetcher}", + "fetcher"_attr = _fetcher->toString()); return; } - error() << "Error scheduling new oplog query: " << redact(scheduleStatus) - << ". Returning current oplog query error: " << redact(responseStatus); + LOGV2_ERROR(21037, + "Error scheduling new oplog query: {scheduleStatus}. Returning current " + "oplog query error: {responseStatus}", + "scheduleStatus"_attr = redact(scheduleStatus), + "responseStatus"_attr = redact(responseStatus)); } } _finishCallback(responseStatus); @@ -290,8 +300,11 @@ void AbstractOplogFetcher::_callback(const Fetcher::QueryResponseStatus& result, return; } auto lastDoc = lastDocRes.getValue(); - LOG(3) << _getComponentName() - << " setting last fetched optime ahead after batch: " << lastDoc; + LOGV2_DEBUG(21034, + 3, + "{getComponentName} setting last fetched optime ahead after batch: {lastDoc}", + "getComponentName"_attr = _getComponentName(), + "lastDoc"_attr = lastDoc); stdx::lock_guard<Latch> lock(_mutex); _lastFetched = lastDoc; @@ -348,13 +361,18 @@ std::unique_ptr<Fetcher> AbstractOplogFetcher::_makeFetcher(const BSONObj& findC bool AbstractOplogFetcher::OplogFetcherRestartDecisionDefault::shouldContinue( AbstractOplogFetcher* fetcher, Status status) { if (_fetcherRestarts == _maxFetcherRestarts) { - log() << "Error returned from oplog query (no more query restarts left): " - << redact(status); + LOGV2(21035, + "Error returned from oplog query (no more query restarts left): {status}", + "status"_attr = redact(status)); return false; } - log() << "Restarting oplog query due to error: " << redact(status) - << ". Last fetched optime: " << fetcher->_getLastOpTimeFetched() - << ". Restarts remaining: " << (_maxFetcherRestarts - _fetcherRestarts); + LOGV2( + 21036, + "Restarting oplog query due to error: {status}. Last fetched optime: " + "{fetcher_getLastOpTimeFetched}. Restarts remaining: {maxFetcherRestarts_fetcherRestarts}", + "status"_attr = redact(status), + "fetcher_getLastOpTimeFetched"_attr = fetcher->_getLastOpTimeFetched(), + "maxFetcherRestarts_fetcherRestarts"_attr = (_maxFetcherRestarts - _fetcherRestarts)); _fetcherRestarts++; return true; } diff --git a/src/mongo/db/repl/abstract_oplog_fetcher_test.cpp b/src/mongo/db/repl/abstract_oplog_fetcher_test.cpp index 7c35e08ab76..5fe87c8530c 100644 --- a/src/mongo/db/repl/abstract_oplog_fetcher_test.cpp +++ b/src/mongo/db/repl/abstract_oplog_fetcher_test.cpp @@ -27,6 +27,8 @@ * it in the license file. */ +#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kDefault + #include "mongo/platform/basic.h" #include <memory> @@ -35,8 +37,10 @@ #include "mongo/db/repl/abstract_oplog_fetcher_test_fixture.h" #include "mongo/db/repl/oplog_entry.h" #include "mongo/db/repl/task_executor_mock.h" +#include "mongo/logv2/log.h" #include "mongo/unittest/task_executor_proxy.h" #include "mongo/unittest/unittest.h" +#include "mongo/util/log.h" #include "mongo/util/scopeguard.h" namespace { @@ -300,20 +304,20 @@ TEST_F(AbstractOplogFetcherTest, OplogFetcherStopsRestartingFetcherIfRestartLimi ASSERT_OK(oplogFetcher.startup()); - unittest::log() << "processing find request from first fetcher"; + LOGV2(21038, "processing find request from first fetcher"); _assertFindCommandTimestampEquals( ops[0], processNetworkResponse({makeCursorResponse(1, {ops[0], ops[1], ops[2]})}, true)); - unittest::log() << "sending error response to getMore request from first fetcher"; + LOGV2(21039, "sending error response to getMore request from first fetcher"); assertRemoteCommandNameEquals( "getMore", processNetworkResponse({ErrorCodes::CappedPositionLost, "fail 1"}, true)); - unittest::log() << "sending error response to find request from second fetcher"; + LOGV2(21040, "sending error response to find request from second fetcher"); _assertFindCommandTimestampEquals( ops[2], processNetworkResponse({ErrorCodes::IllegalOperation, "fail 2"}, true)); - unittest::log() << "sending error response to find request from third fetcher"; + LOGV2(21041, "sending error response to find request from third fetcher"); _assertFindCommandTimestampEquals( ops[2], processNetworkResponse({ErrorCodes::OperationFailed, "fail 3"}, false)); @@ -335,28 +339,28 @@ TEST_F(AbstractOplogFetcherTest, OplogFetcherResetsRestartCounterOnSuccessfulFet ASSERT_OK(oplogFetcher.startup()); - unittest::log() << "processing find request from first fetcher"; + LOGV2(21042, "processing find request from first fetcher"); _assertFindCommandTimestampEquals( ops[0], processNetworkResponse({makeCursorResponse(1, {ops[0], ops[1], ops[2]})}, true)); - unittest::log() << "sending error response to getMore request from first fetcher"; + LOGV2(21043, "sending error response to getMore request from first fetcher"); assertRemoteCommandNameEquals( "getMore", processNetworkResponse({ErrorCodes::CappedPositionLost, "fail 1"}, true)); - unittest::log() << "processing find request from second fetcher"; + LOGV2(21044, "processing find request from second fetcher"); _assertFindCommandTimestampEquals( ops[2], processNetworkResponse({makeCursorResponse(1, {ops[2], ops[3], ops[4]})}, true)); - unittest::log() << "sending error response to getMore request from second fetcher"; + LOGV2(21045, "sending error response to getMore request from second fetcher"); assertRemoteCommandNameEquals( "getMore", processNetworkResponse({ErrorCodes::IllegalOperation, "fail 2"}, true)); - unittest::log() << "sending error response to find request from third fetcher"; + LOGV2(21046, "sending error response to find request from third fetcher"); _assertFindCommandTimestampEquals( ops[4], processNetworkResponse({ErrorCodes::InternalError, "fail 3"}, true)); - unittest::log() << "sending error response to find request from fourth fetcher"; + LOGV2(21047, "sending error response to find request from fourth fetcher"); _assertFindCommandTimestampEquals( ops[4], processNetworkResponse({ErrorCodes::OperationFailed, "fail 4"}, false)); @@ -406,12 +410,12 @@ TEST_F(AbstractOplogFetcherTest, ASSERT_OK(oplogFetcher.startup()); ASSERT_TRUE(oplogFetcher.isActive()); - unittest::log() << "processing find request from first fetcher"; + LOGV2(21048, "processing find request from first fetcher"); _assertFindCommandTimestampEquals( ops[0], processNetworkResponse({makeCursorResponse(1, {ops[0], ops[1], ops[2]})}, true)); - unittest::log() << "sending error response to getMore request from first fetcher"; + LOGV2(21049, "sending error response to getMore request from first fetcher"); shouldFailSchedule = true; assertRemoteCommandNameEquals( "getMore", processNetworkResponse({ErrorCodes::CappedPositionLost, "dead cursor"}, false)); diff --git a/src/mongo/db/repl/abstract_oplog_fetcher_test_fixture.cpp b/src/mongo/db/repl/abstract_oplog_fetcher_test_fixture.cpp index 9f748b283c5..68d8e861ddd 100644 --- a/src/mongo/db/repl/abstract_oplog_fetcher_test_fixture.cpp +++ b/src/mongo/db/repl/abstract_oplog_fetcher_test_fixture.cpp @@ -27,13 +27,17 @@ * it in the license file. */ +#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kDefault + #include "mongo/platform/basic.h" #include "mongo/db/repl/abstract_oplog_fetcher_test_fixture.h" #include "mongo/db/repl/oplog_entry.h" #include "mongo/executor/thread_pool_task_executor_test_fixture.h" +#include "mongo/logv2/log.h" #include "mongo/unittest/unittest.h" +#include "mongo/util/log.h" namespace mongo { namespace repl { @@ -119,13 +123,13 @@ executor::RemoteCommandRequest AbstractOplogFetcherTest::processNetworkResponse( auto net = getNet(); executor::NetworkInterfaceMock::InNetworkGuard guard(net); - unittest::log() << "scheduling response."; + LOGV2(21050, "scheduling response."); auto request = net->scheduleSuccessfulResponse(response); - unittest::log() << "running network ops."; + LOGV2(21051, "running network ops."); net->runReadyNetworkOperations(); - unittest::log() << "checking for more requests"; + LOGV2(21052, "checking for more requests"); ASSERT_EQUALS(expectReadyRequestsAfterProcessing, net->hasReadyRequests()); - unittest::log() << "returning consumed request"; + LOGV2(21053, "returning consumed request"); return request; } diff --git a/src/mongo/db/repl/all_database_cloner.cpp b/src/mongo/db/repl/all_database_cloner.cpp index c9c1345e982..ca47cac6aef 100644 --- a/src/mongo/db/repl/all_database_cloner.cpp +++ b/src/mongo/db/repl/all_database_cloner.cpp @@ -35,6 +35,7 @@ #include "mongo/base/string_data.h" #include "mongo/db/repl/all_database_cloner.h" +#include "mongo/logv2/log.h" #include "mongo/util/assert_util.h" #include "mongo/util/log.h" namespace mongo { @@ -56,7 +57,7 @@ BaseCloner::ClonerStages AllDatabaseCloner::getStages() { Status AllDatabaseCloner::ensurePrimaryOrSecondary( const executor::RemoteCommandResponse& isMasterReply) { if (!isMasterReply.isOK()) { - log() << "Cannot reconnect because isMaster command failed."; + LOGV2(21054, "Cannot reconnect because isMaster command failed."); return isMasterReply.status; } if (isMasterReply.data["ismaster"].trueValue() || isMasterReply.data["secondary"].trueValue()) @@ -104,14 +105,19 @@ BaseCloner::AfterStageBehavior AllDatabaseCloner::listDatabasesStage() { auto databasesArray = getClient()->getDatabaseInfos(BSONObj(), true /* nameOnly */); for (const auto& dbBSON : databasesArray) { if (!dbBSON.hasField("name")) { - LOG(1) << "Excluding database due to the 'listDatabases' response not containing a " - "'name' field for this entry: " - << dbBSON; + LOGV2_DEBUG(21055, + 1, + "Excluding database due to the 'listDatabases' response not containing a " + "'name' field for this entry: {dbBSON}", + "dbBSON"_attr = dbBSON); continue; } const auto& dbName = dbBSON["name"].str(); if (dbName == "local") { - LOG(1) << "Excluding database from the 'listDatabases' response: " << dbBSON; + LOGV2_DEBUG(21056, + 1, + "Excluding database from the 'listDatabases' response: {dbBSON}", + "dbBSON"_attr = dbBSON); continue; } else { _databases.emplace_back(dbName); @@ -146,15 +152,24 @@ void AllDatabaseCloner::postStage() { } auto dbStatus = _currentDatabaseCloner->run(); if (dbStatus.isOK()) { - LOG(1) << "Database clone for '" << dbName << "' finished: " << dbStatus; + LOGV2_DEBUG(21057, + 1, + "Database clone for '{dbName}' finished: {dbStatus}", + "dbName"_attr = dbName, + "dbStatus"_attr = dbStatus); } else { - warning() << "database '" << dbName << "' (" << (_stats.databasesCloned + 1) << " of " - << _databases.size() << ") clone failed due to " << dbStatus.toString(); + LOGV2_WARNING(21060, + "database '{dbName}' ({stats_databasesCloned_1} of {databases_size}) " + "clone failed due to {dbStatus}", + "dbName"_attr = dbName, + "stats_databasesCloned_1"_attr = (_stats.databasesCloned + 1), + "databases_size"_attr = _databases.size(), + "dbStatus"_attr = dbStatus.toString()); setInitialSyncFailedStatus(dbStatus); return; } if (StringData(dbName).equalCaseInsensitive("admin")) { - LOG(1) << "Finished the 'admin' db, now validating it."; + LOGV2_DEBUG(21058, 1, "Finished the 'admin' db, now validating it."); // Do special checks for the admin database because of auth. collections. auto adminStatus = Status(ErrorCodes::NotYetInitialized, ""); { @@ -167,7 +182,10 @@ void AllDatabaseCloner::postStage() { adminStatus = getStorageInterface()->isAdminDbValid(opCtx); } if (!adminStatus.isOK()) { - LOG(1) << "Validation failed on 'admin' db due to " << adminStatus; + LOGV2_DEBUG(21059, + 1, + "Validation failed on 'admin' db due to {adminStatus}", + "adminStatus"_attr = adminStatus); setInitialSyncFailedStatus(adminStatus); return; } diff --git a/src/mongo/db/repl/all_database_cloner_test.cpp b/src/mongo/db/repl/all_database_cloner_test.cpp index ffc72c8d2fa..bbbc8a66c67 100644 --- a/src/mongo/db/repl/all_database_cloner_test.cpp +++ b/src/mongo/db/repl/all_database_cloner_test.cpp @@ -27,6 +27,8 @@ * it in the license file. */ +#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kDefault + #include "mongo/platform/basic.h" #include "mongo/db/repl/all_database_cloner.h" @@ -35,9 +37,11 @@ #include "mongo/db/repl/storage_interface_mock.h" #include "mongo/db/service_context_test_fixture.h" #include "mongo/dbtests/mock/mock_dbclient_connection.h" +#include "mongo/logv2/log.h" #include "mongo/unittest/unittest.h" #include "mongo/util/clock_source_mock.h" #include "mongo/util/concurrency/thread_pool.h" +#include "mongo/util/log.h" namespace mongo { namespace repl { @@ -114,7 +118,7 @@ TEST_F(AllDatabaseClonerTest, RetriesConnect) { ASSERT_EQ(2, _sharedData->getTotalRetries(WithLock::withoutLock())); // Bring the server up. - unittest::log() << "Bringing mock server back up."; + LOGV2(21061, "Bringing mock server back up."); _mockServer->reboot(); // Allow the cloner to finish. @@ -229,7 +233,7 @@ TEST_F(AllDatabaseClonerTest, RetriesListDatabases) { ASSERT_EQ(2, _sharedData->getTotalRetries(WithLock::withoutLock())); // Bring the server up. - unittest::log() << "Bringing mock server back up."; + LOGV2(21062, "Bringing mock server back up."); _mockServer->reboot(); // Allow the cloner to finish. @@ -277,7 +281,7 @@ TEST_F(AllDatabaseClonerTest, RetriesListDatabasesButRollBackIdChanges) { _mockServer->setCommandReply("replSetGetRBID", fromjson("{ok:1, rbid:2}")); // Bring the server up. - unittest::log() << "Bringing mock server back up."; + LOGV2(21063, "Bringing mock server back up."); _mockServer->reboot(); // Allow the cloner to finish. diff --git a/src/mongo/db/repl/apply_ops.cpp b/src/mongo/db/repl/apply_ops.cpp index 2df9c7428be..8f5be92b221 100644 --- a/src/mongo/db/repl/apply_ops.cpp +++ b/src/mongo/db/repl/apply_ops.cpp @@ -55,6 +55,7 @@ #include "mongo/db/service_context.h" #include "mongo/db/session_catalog_mongod.h" #include "mongo/db/transaction_participant.h" +#include "mongo/logv2/log.h" #include "mongo/rpc/get_status_from_command_result.h" #include "mongo/util/fail_point.h" #include "mongo/util/log.h" @@ -266,7 +267,7 @@ Status _applyOps(OperationContext* opCtx, ab.append(status.isOK()); if (!status.isOK()) { - log() << "applyOps error applying: " << status; + LOGV2(21064, "applyOps error applying: {status}", "status"_attr = status); errors++; } diff --git a/src/mongo/db/repl/base_cloner.cpp b/src/mongo/db/repl/base_cloner.cpp index 68411039509..99b38605c4f 100644 --- a/src/mongo/db/repl/base_cloner.cpp +++ b/src/mongo/db/repl/base_cloner.cpp @@ -32,6 +32,7 @@ #include "mongo/platform/basic.h" #include "mongo/db/repl/base_cloner.h" +#include "mongo/logv2/log.h" #include "mongo/util/log.h" #include "mongo/util/scopeguard.h" @@ -91,8 +92,10 @@ Status BaseCloner::run() { } stdx::lock_guard<InitialSyncSharedData> lk(*_sharedData); if (!_sharedData->getInitialSyncStatus(lk).isOK()) { - log() << "Failing data clone because initial sync failed outside data clone: " - << _sharedData->getInitialSyncStatus(lk); + LOGV2(21065, + "Failing data clone because initial sync failed outside data clone: " + "{sharedData_getInitialSyncStatus_lk}", + "sharedData_getInitialSyncStatus_lk"_attr = _sharedData->getInitialSyncStatus(lk)); } return _sharedData->getInitialSyncStatus(lk); } @@ -116,13 +119,14 @@ void BaseCloner::pauseForFuzzer(BaseClonerStage* stage) { // nb: This log message is specifically checked for in // initial_sync_test_fixture_test.js, so if you change it here you will need to change // it there. - log() << "Collection Cloner scheduled a remote command on the " - << describeForFuzzer(stage); - log() << "initialSyncFuzzerSynchronizationPoint1 fail point enabled."; + LOGV2(21066, + "Collection Cloner scheduled a remote command on the {describeForFuzzer_stage}", + "describeForFuzzer_stage"_attr = describeForFuzzer(stage)); + LOGV2(21067, "initialSyncFuzzerSynchronizationPoint1 fail point enabled."); initialSyncFuzzerSynchronizationPoint1.pauseWhileSet(); if (MONGO_unlikely(initialSyncFuzzerSynchronizationPoint2.shouldFail())) { - log() << "initialSyncFuzzerSynchronizationPoint2 fail point enabled."; + LOGV2(21068, "initialSyncFuzzerSynchronizationPoint2 fail point enabled."); initialSyncFuzzerSynchronizationPoint2.pauseWhileSet(); } } @@ -130,15 +134,21 @@ void BaseCloner::pauseForFuzzer(BaseClonerStage* stage) { } BaseCloner::AfterStageBehavior BaseCloner::runStage(BaseClonerStage* stage) { - LOG(1) << "Cloner " << getClonerName() << " running stage " << stage->getName(); + LOGV2_DEBUG(21069, + 1, + "Cloner {getClonerName} running stage {stage_getName}", + "getClonerName"_attr = getClonerName(), + "stage_getName"_attr = stage->getName()); pauseForFuzzer(stage); auto isThisStageFailPoint = [this, stage](const BSONObj& data) { return data["stage"].str() == stage->getName() && isMyFailPoint(data); }; hangBeforeClonerStage.executeIf( [&](const BSONObj& data) { - log() << "Cloner " << getClonerName() << " hanging before running stage " - << stage->getName(); + LOGV2(21070, + "Cloner {getClonerName} hanging before running stage {stage_getName}", + "getClonerName"_attr = getClonerName(), + "stage_getName"_attr = stage->getName()); while (!mustExit() && hangBeforeClonerStage.shouldFail(isThisStageFailPoint)) { sleepmillis(100); } @@ -147,14 +157,20 @@ BaseCloner::AfterStageBehavior BaseCloner::runStage(BaseClonerStage* stage) { auto afterStageBehavior = runStageWithRetries(stage); hangAfterClonerStage.executeIf( [&](const BSONObj& data) { - log() << "Cloner " << getClonerName() << " hanging after running stage " - << stage->getName(); + LOGV2(21071, + "Cloner {getClonerName} hanging after running stage {stage_getName}", + "getClonerName"_attr = getClonerName(), + "stage_getName"_attr = stage->getName()); while (!mustExit() && hangAfterClonerStage.shouldFail(isThisStageFailPoint)) { sleepmillis(100); } }, isThisStageFailPoint); - LOG(1) << "Cloner " << getClonerName() << " finished running stage " << stage->getName(); + LOGV2_DEBUG(21072, + 1, + "Cloner {getClonerName} finished running stage {stage_getName}", + "getClonerName"_attr = getClonerName(), + "stage_getName"_attr = stage->getName()); return afterStageBehavior; } @@ -170,7 +186,7 @@ Status BaseCloner::checkRollBackIdIsUnchanged() { if (ErrorCodes::isRetriableError(e)) { auto status = e.toStatus().withContext( ": failed while attempting to retrieve rollBackId after re-connect"); - LOG(1) << status; + LOGV2_DEBUG(21073, 1, "{status}", "status"_attr = status); return status; } throw; @@ -200,16 +216,23 @@ BaseCloner::AfterStageBehavior BaseCloner::runStageWithRetries(BaseClonerStage* // If lastError is set, this is a retry. hangBeforeRetryingClonerStage.executeIf( [&](const BSONObj& data) { - log() << "Cloner " << getClonerName() << " hanging before retrying stage " - << stage->getName(); + LOGV2( + 21074, + "Cloner {getClonerName} hanging before retrying stage {stage_getName}", + "getClonerName"_attr = getClonerName(), + "stage_getName"_attr = stage->getName()); while (!mustExit() && hangBeforeRetryingClonerStage.shouldFail(isThisStageFailPoint)) { sleepmillis(100); } }, isThisStageFailPoint); - log() << "Initial Sync retrying " << getClonerName() << " stage " - << stage->getName() << " due to " << lastError; + LOGV2(21075, + "Initial Sync retrying {getClonerName} stage {stage_getName} due to " + "{lastError}", + "getClonerName"_attr = getClonerName(), + "stage_getName"_attr = stage->getName(), + "lastError"_attr = lastError); bool shouldRetry = [&] { stdx::lock_guard<InitialSyncSharedData> lk(*_sharedData); return _sharedData->shouldRetryOperation(lk, &_retryableOp); @@ -225,9 +248,11 @@ BaseCloner::AfterStageBehavior BaseCloner::runStageWithRetries(BaseClonerStage* } hangBeforeCheckingRollBackIdClonerStage.executeIf( [&](const BSONObj& data) { - log() << "Cloner " << getClonerName() - << " hanging before checking rollBackId for stage " - << stage->getName(); + LOGV2(21076, + "Cloner {getClonerName} hanging before checking rollBackId for stage " + "{stage_getName}", + "getClonerName"_attr = getClonerName(), + "stage_getName"_attr = stage->getName()); while (!mustExit() && hangBeforeCheckingRollBackIdClonerStage.shouldFail( isThisStageFailPoint)) { @@ -248,12 +273,21 @@ BaseCloner::AfterStageBehavior BaseCloner::runStageWithRetries(BaseClonerStage* } catch (DBException& e) { lastError = e.toStatus(); if (!stage->isTransientError(lastError)) { - log() << "Non-retryable error occured during cloner " << getClonerName() - << " stage " + stage->getName() << ": " << lastError; + LOGV2(21077, + "Non-retryable error occured during cloner " + "{getClonerName}{stage_stage_getName}: {lastError}", + "getClonerName"_attr = getClonerName(), + "stage_stage_getName"_attr = " stage " + stage->getName(), + "lastError"_attr = lastError); throw; } - LOG(1) << "Transient error occured during cloner " << getClonerName() - << " stage " + stage->getName() << ": " << lastError; + LOGV2_DEBUG(21078, + 1, + "Transient error occured during cloner " + "{getClonerName}{stage_stage_getName}: {lastError}", + "getClonerName"_attr = getClonerName(), + "stage_stage_getName"_attr = " stage " + stage->getName(), + "lastError"_attr = lastError); } } } diff --git a/src/mongo/db/repl/bgsync.cpp b/src/mongo/db/repl/bgsync.cpp index bbe81147ee0..270dc41626c 100644 --- a/src/mongo/db/repl/bgsync.cpp +++ b/src/mongo/db/repl/bgsync.cpp @@ -58,6 +58,7 @@ #include "mongo/db/repl/rs_rollback.h" #include "mongo/db/repl/storage_interface.h" #include "mongo/db/s/shard_identity_rollback_notifier.h" +#include "mongo/logv2/log.h" #include "mongo/rpc/get_status_from_command_result.h" #include "mongo/rpc/metadata/repl_set_metadata.h" #include "mongo/util/log.h" @@ -211,12 +212,13 @@ void BackgroundSync::_run() { _runProducer(); } catch (const DBException& e) { std::string msg(str::stream() << "sync producer problem: " << redact(e)); - error() << msg; + LOGV2_ERROR(21125, "{msg}", "msg"_attr = msg); _replCoord->setMyHeartbeatMessage(msg); sleepmillis(100); // sleep a bit to keep from hammering this thread with temp. errors. } catch (const std::exception& e2) { // redact(std::exception&) doesn't work - severe() << "sync producer exception: " << redact(e2.what()); + LOGV2_FATAL( + 21127, "sync producer exception: {e2_what}", "e2_what"_attr = redact(e2.what())); fassertFailed(28546); } } @@ -253,8 +255,9 @@ void BackgroundSync::_runProducer() { void BackgroundSync::_produce() { if (MONGO_unlikely(stopReplProducer.shouldFail())) { // This log output is used in js tests so please leave it. - log() << "bgsync - stopReplProducer fail point " - "enabled. Blocking until fail point is disabled."; + LOGV2(21079, + "bgsync - stopReplProducer fail point " + "enabled. Blocking until fail point is disabled."); mongo::sleepsecs(1); return; } @@ -296,7 +299,9 @@ void BackgroundSync::_produce() { const auto requiredOpTime = (minValidSaved > _lastOpTimeFetched) ? minValidSaved : OpTime(); lastOpTimeFetched = _lastOpTimeFetched; if (!_syncSourceHost.empty()) { - log() << "Clearing sync source " << _syncSourceHost << " to choose a new one."; + LOGV2(21080, + "Clearing sync source {syncSourceHost} to choose a new one.", + "syncSourceHost"_attr = _syncSourceHost); } _syncSourceHost = HostAndPort(); _syncSourceResolver = std::make_unique<SyncSourceResolver>( @@ -328,14 +333,22 @@ void BackgroundSync::_produce() { if (syncSourceResp.syncSourceStatus == ErrorCodes::OplogStartMissing) { // All (accessible) sync sources are too far ahead of us. if (_replCoord->getMemberState().primary()) { - warning() << "Too stale to catch up."; - log() << "Our newest OpTime : " << lastOpTimeFetched; - log() << "Earliest OpTime available is " << syncSourceResp.earliestOpTimeSeen - << " from " << syncSourceResp.getSyncSource(); + LOGV2_WARNING(21115, "Too stale to catch up."); + LOGV2(21081, + "Our newest OpTime : {lastOpTimeFetched}", + "lastOpTimeFetched"_attr = lastOpTimeFetched); + LOGV2(21082, + "Earliest OpTime available is {syncSourceResp_earliestOpTimeSeen} from " + "{syncSourceResp_getSyncSource}", + "syncSourceResp_earliestOpTimeSeen"_attr = syncSourceResp.earliestOpTimeSeen, + "syncSourceResp_getSyncSource"_attr = syncSourceResp.getSyncSource()); auto status = _replCoord->abortCatchupIfNeeded( ReplicationCoordinator::PrimaryCatchUpConclusionReason::kFailedWithError); if (!status.isOK()) { - LOG(1) << "Aborting catch-up failed with status: " << status; + LOGV2_DEBUG(21083, + 1, + "Aborting catch-up failed with status: {status}", + "status"_attr = status); } return; } @@ -349,22 +362,33 @@ void BackgroundSync::_produce() { auto opCtx = cc().makeOperationContext(); ReplicationStateTransitionLockGuard transitionGuard(opCtx.get(), MODE_X); - error() << "too stale to catch up -- entering maintenance mode"; - log() << "Our newest OpTime : " << lastOpTimeFetched; - log() << "Earliest OpTime available is " << syncSourceResp.earliestOpTimeSeen; - log() << "See http://dochub.mongodb.org/core/resyncingaverystalereplicasetmember"; + LOGV2_ERROR(21126, "too stale to catch up -- entering maintenance mode"); + LOGV2(21084, + "Our newest OpTime : {lastOpTimeFetched}", + "lastOpTimeFetched"_attr = lastOpTimeFetched); + LOGV2(21085, + "Earliest OpTime available is {syncSourceResp_earliestOpTimeSeen}", + "syncSourceResp_earliestOpTimeSeen"_attr = syncSourceResp.earliestOpTimeSeen); + LOGV2(21086, "See http://dochub.mongodb.org/core/resyncingaverystalereplicasetmember"); // Activate maintenance mode and transition to RECOVERING. auto status = _replCoord->setMaintenanceMode(true); if (!status.isOK()) { - warning() << "Failed to transition into maintenance mode: " << status; + LOGV2_WARNING(21116, + "Failed to transition into maintenance mode: {status}", + "status"_attr = status); // Do not mark ourselves too stale on errors so we can try again next time. return; } status = _replCoord->setFollowerMode(MemberState::RS_RECOVERING); if (!status.isOK()) { - warning() << "Failed to transition into " << MemberState(MemberState::RS_RECOVERING) - << ". Current state: " << _replCoord->getMemberState() << causedBy(status); + LOGV2_WARNING(21117, + "Failed to transition into {MemberState_MemberState_RS_RECOVERING}. " + "Current state: {replCoord_getMemberState}{causedBy_status}", + "MemberState_MemberState_RS_RECOVERING"_attr = + MemberState(MemberState::RS_RECOVERING), + "replCoord_getMemberState"_attr = _replCoord->getMemberState(), + "causedBy_status"_attr = causedBy(status)); // Do not mark ourselves too stale on errors so we can try again next time. return; } @@ -384,22 +408,31 @@ void BackgroundSync::_produce() { forceBgSyncSyncSourceRetryWaitMS.execute( [&](const BSONObj& data) { sleepMS = data["sleepMS"].numberInt(); }); - log() << "Chose same sync source candidate as last time, " << source - << ". Sleeping for " << sleepMS - << "ms to avoid immediately choosing a new sync source for the same reason as " - "last time."; + LOGV2( + 21087, + "Chose same sync source candidate as last time, {source}. Sleeping for {sleepMS}ms " + "to avoid immediately choosing a new sync source for the same reason as " + "last time.", + "source"_attr = source, + "sleepMS"_attr = sleepMS); numTimesChoseSameSyncSource.increment(1); mongo::sleepmillis(sleepMS); } else { - log() << "Changed sync source from " - << (oldSource.empty() ? std::string("empty") : oldSource.toString()) << " to " - << source; + LOGV2( + 21088, + "Changed sync source from {oldSource_empty_std_string_empty_oldSource} to {source}", + "oldSource_empty_std_string_empty_oldSource"_attr = + (oldSource.empty() ? std::string("empty") : oldSource.toString()), + "source"_attr = source); numTimesChoseDifferentSyncSource.increment(1); } } else { if (!syncSourceResp.isOK()) { - log() << "failed to find sync source, received error " - << syncSourceResp.syncSourceStatus.getStatus(); + LOGV2(21089, + "failed to find sync source, received error " + "{syncSourceResp_syncSourceStatus_getStatus}", + "syncSourceResp_syncSourceStatus_getStatus"_attr = + syncSourceResp.syncSourceStatus.getStatus()); } long long sleepMS = 1000; @@ -407,8 +440,10 @@ void BackgroundSync::_produce() { [&](const BSONObj& data) { sleepMS = data["sleepMS"].numberInt(); }); // No sync source found. - LOG(1) << "Could not find a sync source. Sleeping for " << sleepMS - << "ms before trying again."; + LOGV2_DEBUG(21090, + 1, + "Could not find a sync source. Sleeping for {sleepMS}ms before trying again.", + "sleepMS"_attr = sleepMS); numTimesCouldNotFindSyncSource.increment(1); mongo::sleepmillis(sleepMS); return; @@ -417,10 +452,11 @@ void BackgroundSync::_produce() { // If we find a good sync source after having gone too stale, disable maintenance mode so we can // transition to SECONDARY. if (_tooStale.swap(false)) { - log() << "No longer too stale. Able to sync from " << source; + LOGV2(21091, "No longer too stale. Able to sync from {source}", "source"_attr = source); auto status = _replCoord->setMaintenanceMode(false); if (!status.isOK()) { - warning() << "Failed to leave maintenance mode: " << status; + LOGV2_WARNING( + 21118, "Failed to leave maintenance mode: {status}", "status"_attr = status); } } @@ -484,24 +520,34 @@ void BackgroundSync::_produce() { } const auto logLevel = getTestCommandsEnabled() ? 0 : 1; - LOG(logLevel) << "scheduling fetcher to read remote oplog on " << source << " starting at " - << oplogFetcher->getFindQuery_forTest()["filter"]; + LOGV2_DEBUG(21092, + logSeverityV1toV2(logLevel).toInt(), + "scheduling fetcher to read remote oplog on {source} starting at " + "{oplogFetcher_getFindQuery_forTest_filter}", + "source"_attr = source, + "oplogFetcher_getFindQuery_forTest_filter"_attr = + oplogFetcher->getFindQuery_forTest()["filter"]); auto scheduleStatus = oplogFetcher->startup(); if (!scheduleStatus.isOK()) { - warning() << "unable to schedule fetcher to read remote oplog on " << source << ": " - << scheduleStatus; + LOGV2_WARNING( + 21119, + "unable to schedule fetcher to read remote oplog on {source}: {scheduleStatus}", + "source"_attr = source, + "scheduleStatus"_attr = scheduleStatus); return; } oplogFetcher->join(); - LOG(1) << "fetcher stopped reading remote oplog on " << source; + LOGV2_DEBUG( + 21093, 1, "fetcher stopped reading remote oplog on {source}", "source"_attr = source); // If the background sync is stopped after the fetcher is started, we need to // re-evaluate our sync source and oplog common point. if (getState() != ProducerState::Running) { - log() << "Replication producer stopped after oplog fetcher finished returning a batch from " - "our sync source. Abandoning this batch of oplog entries and re-evaluating our " - "sync source."; + LOGV2(21094, + "Replication producer stopped after oplog fetcher finished returning a batch from " + "our sync source. Abandoning this batch of oplog entries and re-evaluating our " + "sync source."); return; } @@ -509,7 +555,9 @@ void BackgroundSync::_produce() { // This is bad because it means that our source // has not returned oplog entries in ascending ts order, and they need to be. - warning() << redact(fetcherReturnStatus); + LOGV2_WARNING(21120, + "{fetcherReturnStatus}", + "fetcherReturnStatus"_attr = redact(fetcherReturnStatus)); // Do not blacklist the server here, it will be blacklisted when we try to reuse it, // if it can't return a matching oplog start from the last fetch oplog ts field. return; @@ -520,19 +568,23 @@ void BackgroundSync::_produce() { opCtx.get(), fetcherReturnStatus, source, syncSourceResp.rbid, storageInterface); if (bgSyncHangAfterRunRollback.shouldFail()) { - log() << "bgSyncHangAfterRunRollback failpoint is set."; + LOGV2(21095, "bgSyncHangAfterRunRollback failpoint is set."); while (MONGO_unlikely(bgSyncHangAfterRunRollback.shouldFail()) && !inShutdown()) { mongo::sleepmillis(100); } } } else if (fetcherReturnStatus == ErrorCodes::InvalidBSON) { Seconds blacklistDuration(60); - warning() << "Fetcher got invalid BSON while querying oplog. Blacklisting sync source " - << source << " for " << blacklistDuration << "."; + LOGV2_WARNING(21121, + "Fetcher got invalid BSON while querying oplog. Blacklisting sync source " + "{source} for {blacklistDuration}.", + "source"_attr = source, + "blacklistDuration"_attr = blacklistDuration); _replCoord->blacklistSyncSource(source, Date_t::now() + blacklistDuration); } else if (!fetcherReturnStatus.isOK()) { - warning() << "Fetcher stopped querying remote oplog with error: " - << redact(fetcherReturnStatus); + LOGV2_WARNING(21122, + "Fetcher stopped querying remote oplog with error: {fetcherReturnStatus}", + "fetcherReturnStatus"_attr = redact(fetcherReturnStatus)); } } @@ -564,7 +616,10 @@ Status BackgroundSync::_enqueueDocuments(Fetcher::Documents::const_iterator begi // Update last fetched info. _lastOpTimeFetched = info.lastDocument; - LOG(3) << "batch resetting _lastOpTimeFetched: " << _lastOpTimeFetched; + LOGV2_DEBUG(21096, + 3, + "batch resetting _lastOpTimeFetched: {lastOpTimeFetched}", + "lastOpTimeFetched"_attr = _lastOpTimeFetched); } // Check some things periodically (whenever we run out of items in the current cursor batch). @@ -587,11 +642,13 @@ void BackgroundSync::_runRollback(OperationContext* opCtx, int requiredRBID, StorageInterface* storageInterface) { if (_replCoord->getMemberState().primary()) { - warning() << "Rollback situation detected in catch-up mode. Aborting catch-up mode."; + LOGV2_WARNING(21123, + "Rollback situation detected in catch-up mode. Aborting catch-up mode."); auto status = _replCoord->abortCatchupIfNeeded( ReplicationCoordinator::PrimaryCatchUpConclusionReason::kFailedWithError); if (!status.isOK()) { - LOG(1) << "Aborting catch-up failed with status: " << status; + LOGV2_DEBUG( + 21097, 1, "Aborting catch-up failed with status: {status}", "status"_attr = status); } return; } @@ -610,8 +667,12 @@ void BackgroundSync::_runRollback(OperationContext* opCtx, lastOpTimeFetched = _lastOpTimeFetched; } - log() << "Starting rollback due to " << redact(fetcherReturnStatus); - log() << "Replication commit point: " << _replCoord->getLastCommittedOpTime(); + LOGV2(21098, + "Starting rollback due to {fetcherReturnStatus}", + "fetcherReturnStatus"_attr = redact(fetcherReturnStatus)); + LOGV2(21099, + "Replication commit point: {replCoord_getLastCommittedOpTime}", + "replCoord_getLastCommittedOpTime"_attr = _replCoord->getLastCommittedOpTime()); // TODO: change this to call into the Applier directly to block until the applier is // drained. @@ -619,8 +680,11 @@ void BackgroundSync::_runRollback(OperationContext* opCtx, // Wait till all buffered oplog entries have drained and been applied. auto lastApplied = _replCoord->getMyLastAppliedOpTime(); if (lastApplied != lastOpTimeFetched) { - log() << "Waiting for all operations from " << lastApplied << " until " << lastOpTimeFetched - << " to be applied before starting rollback."; + LOGV2(21100, + "Waiting for all operations from {lastApplied} until {lastOpTimeFetched} to be " + "applied before starting rollback.", + "lastApplied"_attr = lastApplied, + "lastOpTimeFetched"_attr = lastOpTimeFetched); while (lastOpTimeFetched > (lastApplied = _replCoord->getMyLastAppliedOpTime())) { sleepmillis(10); if (getState() != ProducerState::Running) { @@ -631,8 +695,9 @@ void BackgroundSync::_runRollback(OperationContext* opCtx, if (MONGO_unlikely(rollbackHangBeforeStart.shouldFail())) { // This log output is used in js tests so please leave it. - log() << "rollback - rollbackHangBeforeStart fail point " - "enabled. Blocking until fail point is disabled."; + LOGV2(21101, + "rollback - rollbackHangBeforeStart fail point " + "enabled. Blocking until fail point is disabled."); while (MONGO_unlikely(rollbackHangBeforeStart.shouldFail()) && !inShutdown()) { mongo::sleepsecs(1); } @@ -659,11 +724,11 @@ void BackgroundSync::_runRollback(OperationContext* opCtx, auto storageEngine = opCtx->getServiceContext()->getStorageEngine(); if (!forceRollbackViaRefetch.load() && storageEngine->supportsRecoverToStableTimestamp()) { - log() << "Rollback using 'recoverToStableTimestamp' method."; + LOGV2(21102, "Rollback using 'recoverToStableTimestamp' method."); _runRollbackViaRecoverToCheckpoint( opCtx, source, &localOplog, storageInterface, getConnection); } else { - log() << "Rollback using the 'rollbackViaRefetch' method."; + LOGV2(21103, "Rollback using the 'rollbackViaRefetch' method."); _fallBackOnRollbackViaRefetch( opCtx, source, abortedIndexBuilds, requiredRBID, &localOplog, getConnection); } @@ -695,15 +760,17 @@ void BackgroundSync::_runRollbackViaRecoverToCheckpoint( _rollback = std::make_unique<RollbackImpl>( localOplog, &remoteOplog, storageInterface, _replicationProcess, _replCoord); - log() << "Scheduling rollback (sync source: " << source << ")"; + LOGV2(21104, "Scheduling rollback (sync source: {source})", "source"_attr = source); auto status = _rollback->runRollback(opCtx); if (status.isOK()) { - log() << "Rollback successful."; + LOGV2(21105, "Rollback successful."); } else if (status == ErrorCodes::UnrecoverableRollbackError) { - severe() << "Rollback failed with unrecoverable error: " << status; + LOGV2_FATAL( + 21128, "Rollback failed with unrecoverable error: {status}", "status"_attr = status); fassertFailedWithStatusNoTrace(50666, status); } else { - warning() << "Rollback failed with retryable error: " << status; + LOGV2_WARNING( + 21124, "Rollback failed with retryable error: {status}", "status"_attr = status); } } @@ -736,7 +803,9 @@ HostAndPort BackgroundSync::getSyncTarget() const { void BackgroundSync::clearSyncTarget() { stdx::unique_lock<Latch> lock(_mutex); - log() << "Resetting sync source to empty, which was " << _syncSourceHost; + LOGV2(21106, + "Resetting sync source to empty, which was {syncSourceHost}", + "syncSourceHost"_attr = _syncSourceHost); _syncSourceHost = HostAndPort(); } @@ -744,13 +813,13 @@ void BackgroundSync::stop(bool resetLastFetchedOptime) { stdx::lock_guard<Latch> lock(_mutex); _state = ProducerState::Stopped; - log() << "Stopping replication producer"; + LOGV2(21107, "Stopping replication producer"); _syncSourceHost = HostAndPort(); if (resetLastFetchedOptime) { invariant(_oplogApplier->getBuffer()->isEmpty()); _lastOpTimeFetched = OpTime(); - log() << "Resetting last fetched optimes in bgsync"; + LOGV2(21108, "Resetting last fetched optimes in bgsync"); } if (_syncSourceResolver) { @@ -779,21 +848,28 @@ void BackgroundSync::start(OperationContext* opCtx) { // If a node steps down during drain mode, then the buffer may not be empty at the beginning // of secondary state. if (!_oplogApplier->getBuffer()->isEmpty()) { - log() << "going to start syncing, but buffer is not empty"; + LOGV2(21109, "going to start syncing, but buffer is not empty"); } _state = ProducerState::Running; // When a node steps down during drain mode, the last fetched optime would be newer than // the last applied. if (_lastOpTimeFetched <= lastAppliedOpTime) { - LOG(1) << "Setting bgsync _lastOpTimeFetched=" << lastAppliedOpTime - << ". Previous _lastOpTimeFetched: " << _lastOpTimeFetched; + LOGV2_DEBUG(21110, + 1, + "Setting bgsync _lastOpTimeFetched={lastAppliedOpTime}. Previous " + "_lastOpTimeFetched: {lastOpTimeFetched}", + "lastAppliedOpTime"_attr = lastAppliedOpTime, + "lastOpTimeFetched"_attr = _lastOpTimeFetched); _lastOpTimeFetched = lastAppliedOpTime; } // Reload the last applied optime from disk if it has been changed. } while (lastAppliedOpTime != _replCoord->getMyLastAppliedOpTime()); - LOG(1) << "bgsync fetch queue set to: " << _lastOpTimeFetched; + LOGV2_DEBUG(21111, + 1, + "bgsync fetch queue set to: {lastOpTimeFetched}", + "lastOpTimeFetched"_attr = _lastOpTimeFetched); } OpTime BackgroundSync::_readLastAppliedOpTime(OperationContext* opCtx) { @@ -812,27 +888,35 @@ OpTime BackgroundSync::_readLastAppliedOpTime(OperationContext* opCtx) { } catch (const ExceptionForCat<ErrorCategory::ShutdownError>&) { throw; } catch (const DBException& ex) { - severe() << "Problem reading " << NamespaceString::kRsOplogNamespace.ns() << ": " - << redact(ex); + LOGV2_FATAL(21129, + "Problem reading {NamespaceString_kRsOplogNamespace_ns}: {ex}", + "NamespaceString_kRsOplogNamespace_ns"_attr = + NamespaceString::kRsOplogNamespace.ns(), + "ex"_attr = redact(ex)); fassertFailed(18904); } OplogEntry parsedEntry(oplogEntry); - LOG(1) << "Successfully read last entry of oplog while starting bgsync: " << redact(oplogEntry); + LOGV2_DEBUG(21112, + 1, + "Successfully read last entry of oplog while starting bgsync: {oplogEntry}", + "oplogEntry"_attr = redact(oplogEntry)); return parsedEntry.getOpTime(); } bool BackgroundSync::shouldStopFetching() const { // Check if we have been stopped. if (getState() != ProducerState::Running) { - LOG(2) << "Stopping oplog fetcher due to stop request."; + LOGV2_DEBUG(21113, 2, "Stopping oplog fetcher due to stop request."); return true; } // Check current sync source. if (getSyncTarget().empty()) { - LOG(1) << "Stopping oplog fetcher; canceling oplog query because we have no valid sync " - "source."; + LOGV2_DEBUG(21114, + 1, + "Stopping oplog fetcher; canceling oplog query because we have no valid sync " + "source."); return true; } diff --git a/src/mongo/db/repl/collection_bulk_loader_impl.cpp b/src/mongo/db/repl/collection_bulk_loader_impl.cpp index 944776aee8f..08393acd715 100644 --- a/src/mongo/db/repl/collection_bulk_loader_impl.cpp +++ b/src/mongo/db/repl/collection_bulk_loader_impl.cpp @@ -45,6 +45,7 @@ #include "mongo/db/operation_context.h" #include "mongo/db/repl/collection_bulk_loader_impl.h" #include "mongo/db/repl/repl_server_parameters_gen.h" +#include "mongo/logv2/log.h" #include "mongo/util/destructor_guard.h" #include "mongo/util/log.h" #include "mongo/util/scopeguard.h" @@ -212,7 +213,7 @@ Status CollectionBulkLoaderImpl::insertDocuments(const std::vector<BSONObj>::con Status CollectionBulkLoaderImpl::commit() { return _runTaskReleaseResourcesOnFailure([&] { _stats.startBuildingIndexes = Date_t::now(); - LOG(2) << "Creating indexes for ns: " << _nss.ns(); + LOGV2_DEBUG(21130, 2, "Creating indexes for ns: {nss_ns}", "nss_ns"_attr = _nss.ns()); UnreplicatedWritesBlock uwb(_opCtx.get()); // Commit before deleting dups, so the dups will be removed from secondary indexes when @@ -309,7 +310,11 @@ Status CollectionBulkLoaderImpl::commit() { } _stats.endBuildingIndexes = Date_t::now(); - LOG(2) << "Done creating indexes for ns: " << _nss.ns() << ", stats: " << _stats.toString(); + LOGV2_DEBUG(21131, + 2, + "Done creating indexes for ns: {nss_ns}, stats: {stats}", + "nss_ns"_attr = _nss.ns(), + "stats"_attr = _stats.toString()); _releaseResources(); return Status::OK(); diff --git a/src/mongo/db/repl/collection_cloner.cpp b/src/mongo/db/repl/collection_cloner.cpp index 119239e1acf..8ad5135f5eb 100644 --- a/src/mongo/db/repl/collection_cloner.cpp +++ b/src/mongo/db/repl/collection_cloner.cpp @@ -38,6 +38,7 @@ #include "mongo/db/repl/database_cloner_gen.h" #include "mongo/db/repl/repl_server_parameters_gen.h" #include "mongo/db/wire_version.h" +#include "mongo/logv2/log.h" #include "mongo/util/assert_util.h" #include "mongo/util/log.h" @@ -117,9 +118,12 @@ BaseCloner::AfterStageBehavior CollectionCloner::CollectionClonerStage::run() { try { return ClonerStage<CollectionCloner>::run(); } catch (const ExceptionFor<ErrorCodes::NamespaceNotFound>&) { - log() << "CollectionCloner ns: '" << getCloner()->getSourceNss() << "' uuid: UUID(\"" - << getCloner()->getSourceUuid() - << "\") stopped because collection was dropped on source."; + LOGV2( + 21132, + "CollectionCloner ns: '{getCloner_getSourceNss}' uuid: " + "UUID(\"{getCloner_getSourceUuid}\") stopped because collection was dropped on source.", + "getCloner_getSourceNss"_attr = getCloner()->getSourceNss(), + "getCloner_getSourceUuid"_attr = getCloner()->getSourceUuid()); getCloner()->waitForDatabaseWorkToComplete(); return kSkipRemainingStages; } catch (const DBException&) { @@ -135,8 +139,9 @@ BaseCloner::AfterStageBehavior CollectionCloner::countStage() { // so we set it to zero here to avoid aborting the collection clone. // Note that this count value is only used for reporting purposes. if (count < 0) { - warning() << "Count command returned negative value. Updating to 0 to allow progress " - "meter to function properly. "; + LOGV2_WARNING(21142, + "Count command returned negative value. Updating to 0 to allow progress " + "meter to function properly. "); count = 0; } @@ -151,8 +156,11 @@ BaseCloner::AfterStageBehavior CollectionCloner::countStage() { BaseCloner::AfterStageBehavior CollectionCloner::listIndexesStage() { auto indexSpecs = getClient()->getIndexSpecs(_sourceDbAndUuid, QueryOption_SlaveOk); if (indexSpecs.empty()) { - warning() << "No indexes found for collection " << _sourceNss.ns() << " while cloning from " - << getSource(); + LOGV2_WARNING( + 21143, + "No indexes found for collection {sourceNss_ns} while cloning from {getSource}", + "sourceNss_ns"_attr = _sourceNss.ns(), + "getSource"_attr = getSource()); } for (auto&& spec : indexSpecs) { if (spec["name"].str() == "_id_"_sd) { @@ -167,9 +175,10 @@ BaseCloner::AfterStageBehavior CollectionCloner::listIndexesStage() { }; if (!_idIndexSpec.isEmpty() && _collectionOptions.autoIndexId == CollectionOptions::NO) { - warning() - << "Found the _id_ index spec but the collection specified autoIndexId of false on ns:" - << this->_sourceNss; + LOGV2_WARNING(21144, + "Found the _id_ index spec but the collection specified autoIndexId of false " + "on ns:{this_sourceNss}", + "this_sourceNss"_attr = this->_sourceNss); } return kContinueNormally; } @@ -200,12 +209,12 @@ void CollectionCloner::runQuery() { if (_resumeSupported) { if (_resumeToken) { // Resume the query from where we left off. - LOG(1) << "Collection cloner will resume the last successful query"; + LOGV2_DEBUG(21133, 1, "Collection cloner will resume the last successful query"); query = QUERY("query" << BSONObj() << "$readOnce" << true << "$_requestResumeToken" << true << "$_resumeAfter" << _resumeToken.get()); } else { // New attempt at a resumable query. - LOG(1) << "Collection cloner will run a new query"; + LOGV2_DEBUG(21134, 1, "Collection cloner will run a new query"); query = QUERY("query" << BSONObj() << "$readOnce" << true << "$_requestResumeToken" << true); } @@ -247,7 +256,9 @@ void CollectionCloner::runQuery() { // Collection has changed upstream. This will trigger the code block above next round, // (unless we find out the collection was dropped via getting a NamespaceNotFound). if (_queryStage.isCursorError(status)) { - log() << "Lost cursor during non-resumable query: " << status; + LOGV2(21135, + "Lost cursor during non-resumable query: {status}", + "status"_attr = status); _lostNonResumableCursor = true; throw; } @@ -268,7 +279,7 @@ void CollectionCloner::handleNextBatch(DBClientCursorBatchIterator& iter) { std::string message = str::stream() << "Collection cloning cancelled due to initial sync failure: " << getSharedData()->getInitialSyncStatus(lk).toString(); - log() << message; + LOGV2(21136, "{message}", "message"_attr = message); uasserted(ErrorCodes::CallbackCanceled, message); } } @@ -316,9 +327,10 @@ void CollectionCloner::handleNextBatch(DBClientCursorBatchIterator& iter) { while (MONGO_unlikely( initialSyncHangCollectionClonerAfterHandlingBatchResponse.shouldFail()) && !mustExit()) { - log() << "initialSyncHangCollectionClonerAfterHandlingBatchResponse fail point " - "enabled for " - << _sourceNss.toString() << ". Blocking until fail point is disabled."; + LOGV2(21137, + "initialSyncHangCollectionClonerAfterHandlingBatchResponse fail point " + "enabled for {sourceNss}. Blocking until fail point is disabled.", + "sourceNss"_attr = _sourceNss.toString()); mongo::sleepsecs(1); } }, @@ -336,8 +348,9 @@ void CollectionCloner::insertDocumentsCallback(const executor::TaskExecutor::Cal stdx::lock_guard<Latch> lk(_mutex); std::vector<BSONObj> docs; if (_documentsToInsert.size() == 0) { - warning() << "insertDocumentsCallback, but no documents to insert for ns:" - << _sourceNss; + LOGV2_WARNING(21145, + "insertDocumentsCallback, but no documents to insert for ns:{sourceNss}", + "sourceNss"_attr = _sourceNss); return; } _documentsToInsert.swap(docs); @@ -353,8 +366,9 @@ void CollectionCloner::insertDocumentsCallback(const executor::TaskExecutor::Cal initialSyncHangDuringCollectionClone.executeIf( [&](const BSONObj&) { - log() << "initial sync - initialSyncHangDuringCollectionClone fail point " - "enabled. Blocking until fail point is disabled."; + LOGV2(21138, + "initial sync - initialSyncHangDuringCollectionClone fail point " + "enabled. Blocking until fail point is disabled."); while (MONGO_unlikely(initialSyncHangDuringCollectionClone.shouldFail()) && !mustExit()) { mongo::sleepsecs(1); @@ -386,11 +400,11 @@ void CollectionCloner::killOldQueryCursor() { auto id = _remoteCursorId; auto cmdObj = BSON("killCursors" << nss.coll() << "cursors" << BSON_ARRAY(id)); - LOG(1) << "Attempting to kill old remote cursor with id: " << id; + LOGV2_DEBUG(21139, 1, "Attempting to kill old remote cursor with id: {id}", "id"_attr = id); try { getClient()->runCommand(nss.db().toString(), cmdObj, infoObj); } catch (...) { - log() << "Error while trying to kill remote cursor after transient query error"; + LOGV2(21140, "Error while trying to kill remote cursor after transient query error"); } // Clear the stored cursorId on success. @@ -404,7 +418,7 @@ void CollectionCloner::forgetOldQueryCursor() { // Throws. void CollectionCloner::abortNonResumableClone(const Status& status) { invariant(!_resumeSupported); - log() << "Error during non-resumable clone: " << status; + LOGV2(21141, "Error during non-resumable clone: {status}", "status"_attr = status); std::string message = str::stream() << "Collection clone failed and is not resumable. nss: " << _sourceNss; uasserted(ErrorCodes::InitialSyncFailure, message); diff --git a/src/mongo/db/repl/data_replicator_external_state_impl.cpp b/src/mongo/db/repl/data_replicator_external_state_impl.cpp index 33ea1ea0d16..bed866625c6 100644 --- a/src/mongo/db/repl/data_replicator_external_state_impl.cpp +++ b/src/mongo/db/repl/data_replicator_external_state_impl.cpp @@ -43,6 +43,7 @@ #include "mongo/db/repl/replication_coordinator_external_state.h" #include "mongo/db/repl/replication_process.h" #include "mongo/db/repl/storage_interface.h" +#include "mongo/logv2/log.h" #include "mongo/util/log.h" namespace mongo { @@ -101,16 +102,22 @@ bool DataReplicatorExternalStateImpl::shouldStopFetching( // If OplogQueryMetadata was provided, its values were used to determine if we should // change sync sources. if (oqMetadata) { - log() << "Canceling oplog query due to OplogQueryMetadata. We have to choose a new " - "sync source. Current source: " - << source << ", OpTime " << oqMetadata->getLastOpApplied() - << ", its sync source index:" << oqMetadata->getSyncSourceIndex(); + LOGV2(21150, + "Canceling oplog query due to OplogQueryMetadata. We have to choose a new " + "sync source. Current source: {source}, OpTime {oqMetadata_getLastOpApplied}, " + "its sync source index:{oqMetadata_getSyncSourceIndex}", + "source"_attr = source, + "oqMetadata_getLastOpApplied"_attr = oqMetadata->getLastOpApplied(), + "oqMetadata_getSyncSourceIndex"_attr = oqMetadata->getSyncSourceIndex()); } else { - log() << "Canceling oplog query due to ReplSetMetadata. We have to choose a new sync " - "source. Current source: " - << source << ", OpTime " << replMetadata.getLastOpVisible() - << ", its sync source index:" << replMetadata.getSyncSourceIndex(); + LOGV2(21151, + "Canceling oplog query due to ReplSetMetadata. We have to choose a new sync " + "source. Current source: {source}, OpTime {replMetadata_getLastOpVisible}, its " + "sync source index:{replMetadata_getSyncSourceIndex}", + "source"_attr = source, + "replMetadata_getLastOpVisible"_attr = replMetadata.getLastOpVisible(), + "replMetadata_getSyncSourceIndex"_attr = replMetadata.getSyncSourceIndex()); } return true; } diff --git a/src/mongo/db/repl/database_cloner.cpp b/src/mongo/db/repl/database_cloner.cpp index d896ee08d0e..5dbabe1cb92 100644 --- a/src/mongo/db/repl/database_cloner.cpp +++ b/src/mongo/db/repl/database_cloner.cpp @@ -35,6 +35,7 @@ #include "mongo/db/commands/list_collections_filter.h" #include "mongo/db/repl/database_cloner.h" #include "mongo/db/repl/database_cloner_gen.h" +#include "mongo/logv2/log.h" #include "mongo/util/assert_util.h" #include "mongo/util/log.h" @@ -88,10 +89,13 @@ BaseCloner::AfterStageBehavior DatabaseCloner::listCollectionsStage() { } NamespaceString collectionNamespace(_dbName, result.getName()); if (collectionNamespace.isSystem() && !collectionNamespace.isLegalClientSystemNS()) { - LOG(1) << "Skipping 'system' collection: " << collectionNamespace.ns(); + LOGV2_DEBUG(21146, + 1, + "Skipping 'system' collection: {collectionNamespace_ns}", + "collectionNamespace_ns"_attr = collectionNamespace.ns()); continue; } - LOG(2) << "Allowing cloning of collectionInfo: " << info; + LOGV2_DEBUG(21147, 2, "Allowing cloning of collectionInfo: {info}", "info"_attr = info); bool isDuplicate = seen.insert(result.getName().toString()).second; uassert(51005, @@ -137,10 +141,13 @@ void DatabaseCloner::postStage() { } auto collStatus = _currentCollectionCloner->run(); if (collStatus.isOK()) { - LOG(1) << "collection clone finished: " << sourceNss; + LOGV2_DEBUG( + 21148, 1, "collection clone finished: {sourceNss}", "sourceNss"_attr = sourceNss); } else { - error() << "collection clone for '" << sourceNss << "' failed due to " - << collStatus.toString(); + LOGV2_ERROR(21149, + "collection clone for '{sourceNss}' failed due to {collStatus}", + "sourceNss"_attr = sourceNss, + "collStatus"_attr = collStatus.toString()); setInitialSyncFailedStatus( {ErrorCodes::InitialSyncFailure, collStatus diff --git a/src/mongo/db/repl/drop_pending_collection_reaper.cpp b/src/mongo/db/repl/drop_pending_collection_reaper.cpp index 4c380e4c8c5..cf3e1e54843 100644 --- a/src/mongo/db/repl/drop_pending_collection_reaper.cpp +++ b/src/mongo/db/repl/drop_pending_collection_reaper.cpp @@ -40,6 +40,7 @@ #include "mongo/db/operation_context.h" #include "mongo/db/repl/storage_interface.h" #include "mongo/db/service_context.h" +#include "mongo/logv2/log.h" #include "mongo/util/log.h" namespace mongo { @@ -89,8 +90,11 @@ void DropPendingCollectionReaper::addDropPendingNamespace( }; if (std::find_if(lowerBound, upperBound, matcher) != upperBound) { - severe() << "Failed to add drop-pending collection " << dropPendingNamespace - << " with drop optime " << dropOpTime << ": duplicate optime and namespace pair."; + LOGV2_FATAL(21156, + "Failed to add drop-pending collection {dropPendingNamespace} with drop optime " + "{dropOpTime}: duplicate optime and namespace pair.", + "dropPendingNamespace"_attr = dropPendingNamespace, + "dropOpTime"_attr = dropOpTime); fassertFailedNoTrace(40448); } @@ -136,16 +140,23 @@ bool DropPendingCollectionReaper::rollBackDropPendingCollection( auto matcher = [&pendingNss](const auto& pair) { return pair.second == pendingNss; }; auto it = std::find_if(lowerBound, upperBound, matcher); if (it == upperBound) { - warning() << "Cannot find drop-pending namespace at OpTime " << opTime - << " for collection " << collectionNamespace << " to roll back."; + LOGV2_WARNING(21154, + "Cannot find drop-pending namespace at OpTime {opTime} for collection " + "{collectionNamespace} to roll back.", + "opTime"_attr = opTime, + "collectionNamespace"_attr = collectionNamespace); return false; } _dropPendingNamespaces.erase(it); } - log() << "Rolling back collection drop for " << pendingNss << " with drop OpTime " << opTime - << " to namespace " << collectionNamespace; + LOGV2(21152, + "Rolling back collection drop for {pendingNss} with drop OpTime {opTime} to namespace " + "{collectionNamespace}", + "pendingNss"_attr = pendingNss, + "opTime"_attr = opTime, + "collectionNamespace"_attr = collectionNamespace); return true; } @@ -174,8 +185,12 @@ void DropPendingCollectionReaper::dropCollectionsOlderThan(OperationContext* opC for (const auto& opTimeAndNamespace : toDrop) { const auto& dropOpTime = opTimeAndNamespace.first; const auto& nss = opTimeAndNamespace.second; - log() << "Completing collection drop for " << nss << " with drop optime " << dropOpTime - << " (notification optime: " << opTime << ")"; + LOGV2(21153, + "Completing collection drop for {nss} with drop optime {dropOpTime} " + "(notification optime: {opTime})", + "nss"_attr = nss, + "dropOpTime"_attr = dropOpTime, + "opTime"_attr = opTime); Status status = Status::OK(); try { // dropCollection could throw an interrupt exception, since it acquires db locks. @@ -184,9 +199,13 @@ void DropPendingCollectionReaper::dropCollectionsOlderThan(OperationContext* opC status = exceptionToStatus(); } if (!status.isOK()) { - warning() << "Failed to remove drop-pending collection " << nss - << " with drop optime " << dropOpTime - << " (notification optime: " << opTime << "): " << status; + LOGV2_WARNING(21155, + "Failed to remove drop-pending collection {nss} with drop optime " + "{dropOpTime} (notification optime: {opTime}): {status}", + "nss"_attr = nss, + "dropOpTime"_attr = dropOpTime, + "opTime"_attr = opTime, + "status"_attr = status); } } } diff --git a/src/mongo/db/repl/idempotency_test.cpp b/src/mongo/db/repl/idempotency_test.cpp index 4eacc99fb52..89b3a37de5b 100644 --- a/src/mongo/db/repl/idempotency_test.cpp +++ b/src/mongo/db/repl/idempotency_test.cpp @@ -27,6 +27,8 @@ * it in the license file. */ +#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kDefault + #include "mongo/platform/basic.h" #include "mongo/db/catalog/index_catalog.h" @@ -40,7 +42,9 @@ #include "mongo/db/repl/idempotency_update_sequence.h" #include "mongo/db/repl/replication_coordinator.h" #include "mongo/db/server_options.h" +#include "mongo/logv2/log.h" #include "mongo/unittest/unittest.h" +#include "mongo/util/log.h" namespace mongo { namespace repl { @@ -114,7 +118,10 @@ std::vector<OplogEntry> RandomizedIdempotencyTest::createUpdateSequence( std::string RandomizedIdempotencyTest::getStatesString(const std::vector<CollectionState>& state1, const std::vector<CollectionState>& state2, const std::vector<OplogEntry>& ops) { - unittest::log() << IdempotencyTest::getStatesString(state1, state2, ops); + LOGV2(21157, + "{IdempotencyTest_getStatesString_state1_state2_ops}", + "IdempotencyTest_getStatesString_state1_state2_ops"_attr = + IdempotencyTest::getStatesString(state1, state2, ops)); StringBuilder sb; sb << "Ran update ops: "; sb << "[ "; diff --git a/src/mongo/db/repl/initial_syncer.cpp b/src/mongo/db/repl/initial_syncer.cpp index 13f2f099ded..2d33f06f66e 100644 --- a/src/mongo/db/repl/initial_syncer.cpp +++ b/src/mongo/db/repl/initial_syncer.cpp @@ -63,6 +63,7 @@ #include "mongo/db/session_txn_record_gen.h" #include "mongo/executor/task_executor.h" #include "mongo/executor/thread_pool_task_executor.h" +#include "mongo/logv2/log.h" #include "mongo/rpc/metadata/repl_set_metadata.h" #include "mongo/util/assert_util.h" #include "mongo/util/destructor_guard.h" @@ -174,13 +175,13 @@ void pauseAtInitialSyncFuzzerSyncronizationPoints(std::string msg) { // Set and unset by the InitialSyncTest fixture to cause initial sync to pause so that the // Initial Sync Fuzzer can run commands on the sync source. if (MONGO_unlikely(initialSyncFuzzerSynchronizationPoint1.shouldFail())) { - log() << msg; - log() << "initialSyncFuzzerSynchronizationPoint1 fail point enabled."; + LOGV2(21158, "{msg}", "msg"_attr = msg); + LOGV2(21159, "initialSyncFuzzerSynchronizationPoint1 fail point enabled."); initialSyncFuzzerSynchronizationPoint1.pauseWhileSet(); } if (MONGO_unlikely(initialSyncFuzzerSynchronizationPoint2.shouldFail())) { - log() << "initialSyncFuzzerSynchronizationPoint2 fail point enabled."; + LOGV2(21160, "initialSyncFuzzerSynchronizationPoint2 fail point enabled."); initialSyncFuzzerSynchronizationPoint2.pauseWhileSet(); } } @@ -446,7 +447,7 @@ BSONObj InitialSyncer::_getInitialSyncProgress_inlock() const { } return bob.obj(); } catch (const DBException& e) { - log() << "Error creating initial sync progress object: " << e.toString(); + LOGV2(21161, "Error creating initial sync progress object: {e}", "e"_attr = e.toString()); } BSONObjBuilder bob; _appendInitialSyncProgressMinimal_inlock(&bob); @@ -495,7 +496,7 @@ void InitialSyncer::_setUp_inlock(OperationContext* opCtx, std::uint32_t initial _storage->setInitialDataTimestamp(serviceCtx, Timestamp::kAllowUnstableCheckpointsSentinel); _storage->setStableTimestamp(serviceCtx, Timestamp::min()); - LOG(1) << "Creating oplogBuffer."; + LOGV2_DEBUG(21162, 1, "Creating oplogBuffer."); _oplogBuffer = _dataReplicatorExternalState->makeInitialSyncOplogBuffer(opCtx); _oplogBuffer->startup(opCtx); @@ -542,8 +543,11 @@ void InitialSyncer::_tearDown_inlock(OperationContext* opCtx, invariant(currentLastAppliedOpTime == lastAppliedOpTime); } - log() << "initial sync done; took " - << duration_cast<Seconds>(_stats.initialSyncEnd - _stats.initialSyncStart) << "."; + LOGV2(21163, + "initial sync done; took " + "{duration_cast_Seconds_stats_initialSyncEnd_stats_initialSyncStart}.", + "duration_cast_Seconds_stats_initialSyncEnd_stats_initialSyncStart"_attr = + duration_cast<Seconds>(_stats.initialSyncEnd - _stats.initialSyncStart)); initialSyncCompletes.increment(); } @@ -560,8 +564,10 @@ void InitialSyncer::_startInitialSyncAttemptCallback( return; } - log() << "Starting initial sync (attempt " << (initialSyncAttempt + 1) << " of " - << initialSyncMaxAttempts << ")"; + LOGV2(21164, + "Starting initial sync (attempt {initialSyncAttempt_1} of {initialSyncMaxAttempts})", + "initialSyncAttempt_1"_attr = (initialSyncAttempt + 1), + "initialSyncMaxAttempts"_attr = initialSyncMaxAttempts); // This completion guard invokes _finishInitialSyncAttempt on destruction. auto cancelRemainingWorkInLock = [this]() { _cancelRemainingWork_inlock(); }; @@ -577,15 +583,18 @@ void InitialSyncer::_startInitialSyncAttemptCallback( _oplogApplier = {}; - LOG(2) << "Resetting sync source so a new one can be chosen for this initial sync attempt."; + LOGV2_DEBUG(21165, + 2, + "Resetting sync source so a new one can be chosen for this initial sync attempt."); _syncSource = HostAndPort(); - LOG(2) << "Resetting all optimes before starting this initial sync attempt."; + LOGV2_DEBUG(21166, 2, "Resetting all optimes before starting this initial sync attempt."); _opts.resetOptimes(); _lastApplied = {OpTime(), Date_t()}; _lastFetched = {}; - LOG(2) << "Resetting the oldest timestamp before starting this initial sync attempt."; + LOGV2_DEBUG( + 21167, 2, "Resetting the oldest timestamp before starting this initial sync attempt."); auto storageEngine = getGlobalServiceContext()->getStorageEngine(); if (storageEngine) { // Set the oldestTimestamp to one because WiredTiger does not allow us to set it to zero @@ -595,9 +604,11 @@ void InitialSyncer::_startInitialSyncAttemptCallback( storageEngine->setOldestTimestamp(kTimestampOne); } - LOG(2) << "Resetting feature compatibility version to last-stable. If the sync source is in " - "latest feature compatibility version, we will find out when we clone the " - "server configuration collection (admin.system.version)."; + LOGV2_DEBUG(21168, + 2, + "Resetting feature compatibility version to last-stable. If the sync source is in " + "latest feature compatibility version, we will find out when we clone the " + "server configuration collection (admin.system.version)."); serverGlobalParams.featureCompatibility.reset(); // Clear the oplog buffer. @@ -656,9 +667,17 @@ void InitialSyncer::_chooseSyncSourceCallback( } auto when = _exec->now() + _opts.syncSourceRetryWait; - LOG(1) << "Error getting sync source: '" << syncSource.getStatus() << "', trying again in " - << _opts.syncSourceRetryWait << " at " << when.toString() << ". Attempt " - << (chooseSyncSourceAttempt + 1) << " of " << numInitialSyncConnectAttempts.load(); + LOGV2_DEBUG(21169, + 1, + "Error getting sync source: '{syncSource_getStatus}', trying again in " + "{opts_syncSourceRetryWait} at {when}. Attempt {chooseSyncSourceAttempt_1} of " + "{numInitialSyncConnectAttempts_load}", + "syncSource_getStatus"_attr = syncSource.getStatus(), + "opts_syncSourceRetryWait"_attr = _opts.syncSourceRetryWait, + "when"_attr = when.toString(), + "chooseSyncSourceAttempt_1"_attr = (chooseSyncSourceAttempt + 1), + "numInitialSyncConnectAttempts_load"_attr = + numInitialSyncConnectAttempts.load()); auto status = _scheduleWorkAtAndSaveHandle_inlock( when, [=](const executor::TaskExecutor::CallbackArgs& args) { @@ -678,8 +697,9 @@ void InitialSyncer::_chooseSyncSourceCallback( if (MONGO_unlikely(initialSyncHangBeforeCreatingOplog.shouldFail())) { // This log output is used in js tests so please leave it. - log() << "initial sync - initialSyncHangBeforeCreatingOplog fail point " - "enabled. Blocking until fail point is disabled."; + LOGV2(21170, + "initial sync - initialSyncHangBeforeCreatingOplog fail point " + "enabled. Blocking until fail point is disabled."); lock.unlock(); while (MONGO_unlikely(initialSyncHangBeforeCreatingOplog.shouldFail()) && !_isShuttingDown()) { @@ -714,8 +734,11 @@ void InitialSyncer::_chooseSyncSourceCallback( Status InitialSyncer::_truncateOplogAndDropReplicatedDatabases() { // truncate oplog; drop user databases. - LOG(1) << "About to truncate the oplog, if it exists, ns:" << _opts.localOplogNS - << ", and drop all user databases (so that we can clone them)."; + LOGV2_DEBUG(21171, + 1, + "About to truncate the oplog, if it exists, ns:{opts_localOplogNS}, and drop all " + "user databases (so that we can clone them).", + "opts_localOplogNS"_attr = _opts.localOplogNS); auto opCtx = makeOpCtx(); @@ -723,13 +746,21 @@ Status InitialSyncer::_truncateOplogAndDropReplicatedDatabases() { UnreplicatedWritesBlock unreplicatedWritesBlock(opCtx.get()); // 1.) Truncate the oplog. - LOG(2) << "Truncating the existing oplog: " << _opts.localOplogNS; + LOGV2_DEBUG(21172, + 2, + "Truncating the existing oplog: {opts_localOplogNS}", + "opts_localOplogNS"_attr = _opts.localOplogNS); Timer timer; auto status = _storage->truncateCollection(opCtx.get(), _opts.localOplogNS); - log() << "Initial syncer oplog truncation finished in: " << timer.millis() << "ms"; + LOGV2(21173, + "Initial syncer oplog truncation finished in: {timer_millis}ms", + "timer_millis"_attr = timer.millis()); if (!status.isOK()) { // 1a.) Create the oplog. - LOG(2) << "Creating the oplog: " << _opts.localOplogNS; + LOGV2_DEBUG(21174, + 2, + "Creating the oplog: {opts_localOplogNS}", + "opts_localOplogNS"_attr = _opts.localOplogNS); status = _storage->createOplog(opCtx.get(), _opts.localOplogNS); if (!status.isOK()) { return status; @@ -737,7 +768,7 @@ Status InitialSyncer::_truncateOplogAndDropReplicatedDatabases() { } // 2.) Drop user databases. - LOG(2) << "Dropping user databases"; + LOGV2_DEBUG(21175, 2, "Dropping user databases"); return _storage->dropReplicatedDatabases(opCtx.get()); } @@ -895,7 +926,7 @@ void InitialSyncer::_getBeginFetchingOpTimeCallback( pauseAtInitialSyncFuzzerSyncronizationPoints(logMsg); if (MONGO_unlikely(initialSyncHangAfterGettingBeginFetchingTimestamp.shouldFail())) { - log() << "initialSyncHangAfterGettingBeginFetchingTimestamp fail point enabled."; + LOGV2(21176, "initialSyncHangAfterGettingBeginFetchingTimestamp fail point enabled."); initialSyncHangAfterGettingBeginFetchingTimestamp.pauseWhileSet(); } @@ -1064,10 +1095,16 @@ void InitialSyncer::_fcvFetcherCallback(const StatusWith<Fetcher::QueryResponse> << _initialSyncState->beginFetchingTimestamp.toBSON()); invariant(!result.getValue().documents.empty()); - LOG(2) << "Setting begin applying timestamp to " << _initialSyncState->beginApplyingTimestamp - << " using last oplog entry: " << redact(result.getValue().documents.front()) - << ", ns: " << _opts.localOplogNS << " and the begin fetching timestamp to " - << _initialSyncState->beginFetchingTimestamp; + LOGV2_DEBUG( + 21177, + 2, + "Setting begin applying timestamp to {initialSyncState_beginApplyingTimestamp} using last " + "oplog entry: {result_getValue_documents_front}, ns: {opts_localOplogNS} and the begin " + "fetching timestamp to {initialSyncState_beginFetchingTimestamp}", + "initialSyncState_beginApplyingTimestamp"_attr = _initialSyncState->beginApplyingTimestamp, + "result_getValue_documents_front"_attr = redact(result.getValue().documents.front()), + "opts_localOplogNS"_attr = _opts.localOplogNS, + "initialSyncState_beginFetchingTimestamp"_attr = _initialSyncState->beginFetchingTimestamp); const auto configResult = _dataReplicatorExternalState->getCurrentConfig(); status = configResult.getStatus(); @@ -1098,7 +1135,10 @@ void InitialSyncer::_fcvFetcherCallback(const StatusWith<Fetcher::QueryResponse> initialSyncOplogFetcherBatchSize, OplogFetcher::StartingPoint::kEnqueueFirstDoc); - LOG(2) << "Starting OplogFetcher: " << _oplogFetcher->toString(); + LOGV2_DEBUG(21178, + 2, + "Starting OplogFetcher: {oplogFetcher}", + "oplogFetcher"_attr = _oplogFetcher->toString()); // _startupComponent_inlock is shutdown-aware. status = _startupComponent_inlock(_oplogFetcher); @@ -1113,8 +1153,9 @@ void InitialSyncer::_fcvFetcherCallback(const StatusWith<Fetcher::QueryResponse> // This could have been done with a scheduleWorkAt but this is used only by JS tests where // we run with multiple threads so it's fine to spin on this thread. // This log output is used in js tests so please leave it. - log() << "initial sync - initialSyncHangBeforeCopyingDatabases fail point " - "enabled. Blocking until fail point is disabled."; + LOGV2(21179, + "initial sync - initialSyncHangBeforeCopyingDatabases fail point " + "enabled. Blocking until fail point is disabled."); while (MONGO_unlikely(initialSyncHangBeforeCopyingDatabases.shouldFail()) && !_isShuttingDown()) { mongo::sleepsecs(1); @@ -1122,7 +1163,11 @@ void InitialSyncer::_fcvFetcherCallback(const StatusWith<Fetcher::QueryResponse> lock.lock(); } - LOG(2) << "Starting AllDatabaseCloner: " << _initialSyncState->allDatabaseCloner->toString(); + LOGV2_DEBUG(21180, + 2, + "Starting AllDatabaseCloner: {initialSyncState_allDatabaseCloner}", + "initialSyncState_allDatabaseCloner"_attr = + _initialSyncState->allDatabaseCloner->toString()); auto [startClonerFuture, startCloner] = _initialSyncState->allDatabaseCloner->runOnExecutorEvent(_clonerExec); @@ -1166,8 +1211,11 @@ void InitialSyncer::_fcvFetcherCallback(const StatusWith<Fetcher::QueryResponse> void InitialSyncer::_oplogFetcherCallback(const Status& oplogFetcherFinishStatus, std::shared_ptr<OnCompletionGuard> onCompletionGuard) { stdx::lock_guard<Latch> lock(_mutex); - log() << "Finished fetching oplog during initial sync: " << redact(oplogFetcherFinishStatus) - << ". Last fetched optime: " << _lastFetched.toString(); + LOGV2(21181, + "Finished fetching oplog during initial sync: {oplogFetcherFinishStatus}. Last fetched " + "optime: {lastFetched}", + "oplogFetcherFinishStatus"_attr = redact(oplogFetcherFinishStatus), + "lastFetched"_attr = _lastFetched.toString()); auto status = _checkForShutdownAndConvertStatus_inlock( oplogFetcherFinishStatus, "error fetching oplog during initial sync"); @@ -1182,8 +1230,9 @@ void InitialSyncer::_oplogFetcherCallback(const Status& oplogFetcherFinishStatus // an OK status is when the 'stopReplProducer' fail point is enabled, which causes the // OplogFetcher to ignore the current sync source response and return early. if (status.isOK()) { - log() << "Finished fetching oplog fetching early. Last fetched optime: " - << _lastFetched.toString(); + LOGV2(21182, + "Finished fetching oplog fetching early. Last fetched optime: {lastFetched}", + "lastFetched"_attr = _lastFetched.toString()); return; } @@ -1199,16 +1248,18 @@ void InitialSyncer::_oplogFetcherCallback(const Status& oplogFetcherFinishStatus void InitialSyncer::_allDatabaseClonerCallback( const Status& databaseClonerFinishStatus, std::shared_ptr<OnCompletionGuard> onCompletionGuard) { - log() << "Finished cloning data: " << redact(databaseClonerFinishStatus) - << ". Beginning oplog replay."; + LOGV2(21183, + "Finished cloning data: {databaseClonerFinishStatus}. Beginning oplog replay.", + "databaseClonerFinishStatus"_attr = redact(databaseClonerFinishStatus)); _client->shutdownAndDisallowReconnect(); if (MONGO_unlikely(initialSyncHangAfterDataCloning.shouldFail())) { // This could have been done with a scheduleWorkAt but this is used only by JS tests where // we run with multiple threads so it's fine to spin on this thread. // This log output is used in js tests so please leave it. - log() << "initial sync - initialSyncHangAfterDataCloning fail point " - "enabled. Blocking until fail point is disabled."; + LOGV2(21184, + "initial sync - initialSyncHangAfterDataCloning fail point " + "enabled. Blocking until fail point is disabled."); while (MONGO_unlikely(initialSyncHangAfterDataCloning.shouldFail()) && !_isShuttingDown()) { mongo::sleepsecs(1); } @@ -1309,7 +1360,10 @@ void InitialSyncer::_lastOplogEntryFetcherCallbackForStopTimestamp( const auto& documents = result.getValue().documents; invariant(!documents.empty()); const BSONObj oplogSeedDoc = documents.front(); - LOG(2) << "Inserting oplog seed document: " << oplogSeedDoc; + LOGV2_DEBUG(21185, + 2, + "Inserting oplog seed document: {oplogSeedDoc}", + "oplogSeedDoc"_attr = oplogSeedDoc); auto opCtx = makeOpCtx(); // StorageInterface::insertDocument() has to be called outside the lock because we may @@ -1333,8 +1387,9 @@ void InitialSyncer::_lastOplogEntryFetcherCallbackForStopTimestamp( stdx::lock_guard<Latch> lock(_mutex); _lastApplied = resultOpTimeAndWallTime; - log() << "No need to apply operations. (currently at " - << _initialSyncState->stopTimestamp.toBSON() << ")"; + LOGV2(21186, + "No need to apply operations. (currently at {initialSyncState_stopTimestamp})", + "initialSyncState_stopTimestamp"_attr = _initialSyncState->stopTimestamp.toBSON()); // This sets the error in 'onCompletionGuard' and shuts down the OplogFetcher on error. _scheduleRollbackCheckerCheckForRollback_inlock(lock, onCompletionGuard); @@ -1353,7 +1408,9 @@ void InitialSyncer::_getNextApplierBatchCallback( auto batchResult = _getNextApplierBatch_inlock(); if (!batchResult.isOK()) { - warning() << "Failure creating next apply batch: " << redact(batchResult.getStatus()); + LOGV2_WARNING(21196, + "Failure creating next apply batch: {batchResult_getStatus}", + "batchResult_getStatus"_attr = redact(batchResult.getStatus())); onCompletionGuard->setResultAndCancelRemainingWork_inlock(lock, batchResult.getStatus()); return; } @@ -1364,8 +1421,9 @@ void InitialSyncer::_getNextApplierBatchCallback( pauseAtInitialSyncFuzzerSyncronizationPoints(logMsg); if (MONGO_unlikely(failInitialSyncBeforeApplyingBatch.shouldFail())) { - log() << "initial sync - failInitialSyncBeforeApplyingBatch fail point enabled. Pausing" - << "until fail point is disabled, then will fail initial sync."; + LOGV2(21187, + "initial sync - failInitialSyncBeforeApplyingBatch fail point enabled. Pausinguntil " + "fail point is disabled, then will fail initial sync."); failInitialSyncBeforeApplyingBatch.pauseWhileSet(); status = Status(ErrorCodes::CallbackCanceled, "failInitialSyncBeforeApplyingBatch fail point enabled"); @@ -1409,7 +1467,7 @@ void InitialSyncer::_getNextApplierBatchCallback( "in the oplog buffer. Aborting this initial sync attempt. Last applied: " << _lastApplied.opTime.toString() << ". Last fetched: " << _lastFetched.toString() << ". Number of operations applied: " << _initialSyncState->appliedOps; - log() << msg; + LOGV2(21188, "{msg}", "msg"_attr = msg); status = Status(ErrorCodes::RemoteResultsUnavailable, msg); onCompletionGuard->setResultAndCancelRemainingWork_inlock(lock, status); return; @@ -1441,12 +1499,13 @@ void InitialSyncer::_multiApplierCallback(const Status& multiApplierStatus, // Set to cause initial sync to fassert instead of restart if applying a batch fails, so that // tests can be robust to network errors but not oplog idempotency errors. if (MONGO_unlikely(initialSyncFassertIfApplyingBatchFails.shouldFail())) { - log() << "initialSyncFassertIfApplyingBatchFails fail point enabled."; + LOGV2(21189, "initialSyncFassertIfApplyingBatchFails fail point enabled."); fassert(31210, status); } if (!status.isOK()) { - error() << "Failed to apply batch due to '" << redact(status) << "'"; + LOGV2_ERROR( + 21199, "Failed to apply batch due to '{status}'", "status"_attr = redact(status)); onCompletionGuard->setResultAndCancelRemainingWork_inlock(lock, status); return; } @@ -1475,7 +1534,10 @@ void InitialSyncer::_rollbackCheckerCheckForRollbackCallback( auto status = _checkForShutdownAndConvertStatus_inlock(result.getStatus(), "error while getting last rollback ID"); if (_shouldRetryError(lock, status)) { - LOG(1) << "Retrying rollback checker because of network error " << status; + LOGV2_DEBUG(21190, + 1, + "Retrying rollback checker because of network error {status}", + "status"_attr = status); _scheduleRollbackCheckerCheckForRollback_inlock(lock, onCompletionGuard); return; } @@ -1519,17 +1581,20 @@ void InitialSyncer::_finishInitialSyncAttempt(const StatusWith<OpTimeAndWallTime auto scheduleResult = _exec->scheduleWork( [=](const mongo::executor::TaskExecutor::CallbackArgs&) { _finishCallback(result); }); if (!scheduleResult.isOK()) { - warning() << "Unable to schedule initial syncer completion task due to " - << redact(scheduleResult.getStatus()) - << ". Running callback on current thread."; + LOGV2_WARNING(21197, + "Unable to schedule initial syncer completion task due to " + "{scheduleResult_getStatus}. Running callback on current thread.", + "scheduleResult_getStatus"_attr = redact(scheduleResult.getStatus())); _finishCallback(result); } }); - log() << "Initial sync attempt finishing up."; + LOGV2(21191, "Initial sync attempt finishing up."); stdx::lock_guard<Latch> lock(_mutex); - log() << "Initial Sync Attempt Statistics: " << redact(_getInitialSyncProgress_inlock()); + LOGV2(21192, + "Initial Sync Attempt Statistics: {getInitialSyncProgress_inlock}", + "getInitialSyncProgress_inlock"_attr = redact(_getInitialSyncProgress_inlock())); auto runTime = _initialSyncState ? _initialSyncState->timer.millis() : 0; int rollBackId = -1; @@ -1552,7 +1617,7 @@ void InitialSyncer::_finishInitialSyncAttempt(const StatusWith<OpTimeAndWallTime totalTimeUnreachableMillis}); if (MONGO_unlikely(failAndHangInitialSync.shouldFail())) { - log() << "failAndHangInitialSync fail point enabled."; + LOGV2(21193, "failAndHangInitialSync fail point enabled."); failAndHangInitialSync.pauseWhileSet(); result = Status(ErrorCodes::InternalError, "failAndHangInitialSync fail point enabled"); } @@ -1570,15 +1635,19 @@ void InitialSyncer::_finishInitialSyncAttempt(const StatusWith<OpTimeAndWallTime // startup. initialSyncFailedAttempts.increment(); - error() << "Initial sync attempt failed -- attempts left: " - << (_stats.maxFailedInitialSyncAttempts - _stats.failedInitialSyncAttempts) - << " cause: " << redact(result.getStatus()); + LOGV2_ERROR(21200, + "Initial sync attempt failed -- attempts left: " + "{stats_maxFailedInitialSyncAttempts_stats_failedInitialSyncAttempts} cause: " + "{result_getStatus}", + "stats_maxFailedInitialSyncAttempts_stats_failedInitialSyncAttempts"_attr = + (_stats.maxFailedInitialSyncAttempts - _stats.failedInitialSyncAttempts), + "result_getStatus"_attr = redact(result.getStatus())); // Check if need to do more retries. if (_stats.failedInitialSyncAttempts >= _stats.maxFailedInitialSyncAttempts) { const std::string err = "The maximum number of retries have been exhausted for initial sync."; - severe() << err; + LOGV2_FATAL(21202, "{err}", "err"_attr = err); initialSyncFailures.increment(); @@ -1626,8 +1695,9 @@ void InitialSyncer::_finishCallback(StatusWith<OpTimeAndWallTime> lastApplied) { if (MONGO_unlikely(initialSyncHangBeforeFinish.shouldFail())) { // This log output is used in js tests so please leave it. - log() << "initial sync - initialSyncHangBeforeFinish fail point " - "enabled. Blocking until fail point is disabled."; + LOGV2(21194, + "initial sync - initialSyncHangBeforeFinish fail point " + "enabled. Blocking until fail point is disabled."); while (MONGO_unlikely(initialSyncHangBeforeFinish.shouldFail()) && !_isShuttingDown()) { mongo::sleepsecs(1); } @@ -1637,8 +1707,9 @@ void InitialSyncer::_finishCallback(StatusWith<OpTimeAndWallTime> lastApplied) { try { onCompletion(lastApplied); } catch (...) { - warning() << "initial syncer finish callback threw exception: " - << redact(exceptionToStatus()); + LOGV2_WARNING(21198, + "initial syncer finish callback threw exception: {exceptionToStatus}", + "exceptionToStatus"_attr = redact(exceptionToStatus())); } // Destroy the remaining reference to the completion callback before we transition the state to @@ -1704,7 +1775,7 @@ void InitialSyncer::_checkApplierProgressAndScheduleGetNextApplierBatch_inlock( << "Possible rollback on sync source " << _syncSource.toString() << ". Currently at " << _initialSyncState->stopTimestamp.toBSON() << ". Started at " << _initialSyncState->beginApplyingTimestamp.toBSON(); - error() << msg; + LOGV2_ERROR(21201, "{msg}", "msg"_attr = msg); onCompletionGuard->setResultAndCancelRemainingWork_inlock( lock, Status(ErrorCodes::OplogOutOfOrder, msg)); return; @@ -1713,11 +1784,16 @@ void InitialSyncer::_checkApplierProgressAndScheduleGetNextApplierBatch_inlock( if (_lastApplied.opTime.isNull()) { // Check if any ops occurred while cloning or any ops need to be fetched. invariant(_initialSyncState->beginFetchingTimestamp < _initialSyncState->stopTimestamp); - log() << "Writing to the oplog and applying operations until " - << _initialSyncState->stopTimestamp.toBSON() - << " before initial sync can complete. (started fetching at " - << _initialSyncState->beginFetchingTimestamp.toBSON() << " and applying at " - << _initialSyncState->beginApplyingTimestamp.toBSON() << ")"; + LOGV2(21195, + "Writing to the oplog and applying operations until {initialSyncState_stopTimestamp} " + "before initial sync can complete. (started fetching at " + "{initialSyncState_beginFetchingTimestamp} and applying at " + "{initialSyncState_beginApplyingTimestamp})", + "initialSyncState_stopTimestamp"_attr = _initialSyncState->stopTimestamp.toBSON(), + "initialSyncState_beginFetchingTimestamp"_attr = + _initialSyncState->beginFetchingTimestamp.toBSON(), + "initialSyncState_beginApplyingTimestamp"_attr = + _initialSyncState->beginApplyingTimestamp.toBSON()); // Fall through to scheduling _getNextApplierBatchCallback(). } else if (_lastApplied.opTime.getTimestamp() >= _initialSyncState->stopTimestamp) { // Check for rollback if we have applied far enough to be consistent. diff --git a/src/mongo/db/repl/insert_group.cpp b/src/mongo/db/repl/insert_group.cpp index 24acf95b0d2..931ca0d3367 100644 --- a/src/mongo/db/repl/insert_group.cpp +++ b/src/mongo/db/repl/insert_group.cpp @@ -40,6 +40,7 @@ #include "mongo/db/ops/write_ops.h" #include "mongo/db/repl/oplog_applier_impl.h" #include "mongo/db/repl/oplog_entry.h" +#include "mongo/logv2/log.h" #include "mongo/util/assert_util.h" #include "mongo/util/log.h" @@ -139,9 +140,9 @@ StatusWith<InsertGroup::ConstIterator> InsertGroup::groupAndApplyInserts(ConstIt // It's not an error during initial sync to encounter DuplicateKey errors. if (Mode::kInitialSync == _mode && ErrorCodes::DuplicateKey == status) { - LOG(2) << status; + LOGV2_DEBUG(21203, 2, "{status}", "status"_attr = status); } else { - error() << status; + LOGV2_ERROR(21204, "{status}", "status"_attr = status); } // Avoid quadratic run time from failed insert by not retrying until we diff --git a/src/mongo/db/repl/isself.cpp b/src/mongo/db/repl/isself.cpp index b50a893efb8..b4af71d7fdb 100644 --- a/src/mongo/db/repl/isself.cpp +++ b/src/mongo/db/repl/isself.cpp @@ -44,6 +44,7 @@ #include "mongo/db/auth/privilege.h" #include "mongo/db/commands.h" #include "mongo/db/service_context.h" +#include "mongo/logv2/log.h" #include "mongo/util/log.h" #include "mongo/util/net/socket_utils.h" #include "mongo/util/scopeguard.h" @@ -119,8 +120,10 @@ std::vector<std::string> getAddrsForHost(const std::string& iporhost, int err = getaddrinfo(iporhost.c_str(), portNum.c_str(), &hints, &addrs); if (err) { - warning() << "getaddrinfo(\"" << iporhost << "\") failed: " << stringifyError(err) - << std::endl; + LOGV2_WARNING(21207, + "getaddrinfo(\"{iporhost}\") failed: {stringifyError_err}", + "iporhost"_attr = iporhost, + "stringifyError_err"_attr = stringifyError(err)); return out; } @@ -134,7 +137,9 @@ std::vector<std::string> getAddrsForHost(const std::string& iporhost, err = getnameinfo( addr->ai_addr, addr->ai_addrlen, host, NI_MAXHOST, nullptr, 0, NI_NUMERICHOST); if (err) { - warning() << "getnameinfo() failed: " << stringifyError(err) << std::endl; + LOGV2_WARNING(21208, + "getnameinfo() failed: {stringifyError_err}", + "stringifyError_err"_attr = stringifyError(err)); continue; } out.push_back(host); @@ -147,7 +152,7 @@ std::vector<std::string> getAddrsForHost(const std::string& iporhost, for (std::vector<std::string>::const_iterator o = out.begin(); o != out.end(); ++o) { builder << " [ " << *o << "]"; } - LOG(2) << builder.str(); + LOGV2_DEBUG(21205, 2, "{builder_str}", "builder_str"_attr = builder.str()); } return out; @@ -210,7 +215,10 @@ bool isSelf(const HostAndPort& hostAndPort, ServiceContext* const ctx) { return me; } catch (const std::exception& e) { - warning() << "couldn't check isSelf (" << hostAndPort << ") " << e.what() << std::endl; + LOGV2_WARNING(21209, + "couldn't check isSelf ({hostAndPort}) {e_what}", + "hostAndPort"_attr = hostAndPort, + "e_what"_attr = e.what()); } return false; @@ -229,7 +237,9 @@ std::vector<std::string> getBoundAddrs(const bool ipv6enabled) { int err = getifaddrs(&addrs); if (err) { - warning() << "getifaddrs failure: " << errnoWithDescription(err) << std::endl; + LOGV2_WARNING(21210, + "getifaddrs failure: {errnoWithDescription_err}", + "errnoWithDescription_err"_attr = errnoWithDescription(err)); return out; } ON_BLOCK_EXIT([&] { freeifaddrs(addrs); }); @@ -251,7 +261,9 @@ std::vector<std::string> getBoundAddrs(const bool ipv6enabled) { 0, NI_NUMERICHOST); if (err) { - warning() << "getnameinfo() failed: " << gai_strerror(err) << std::endl; + LOGV2_WARNING(21211, + "getnameinfo() failed: {gai_strerror_err}", + "gai_strerror_err"_attr = gai_strerror(err)); continue; } out.push_back(host); @@ -288,7 +300,9 @@ std::vector<std::string> getBoundAddrs(const bool ipv6enabled) { } if (err != NO_ERROR) { - warning() << "GetAdaptersAddresses() failed: " << errnoWithDescription(err) << std::endl; + LOGV2_WARNING(21212, + "GetAdaptersAddresses() failed: {errnoWithDescription_err}", + "errnoWithDescription_err"_attr = errnoWithDescription(err)); return out; } @@ -306,8 +320,9 @@ std::vector<std::string> getBoundAddrs(const bool ipv6enabled) { boost::asio::detail::socket_ops::inet_ntop( AF_INET, &(sock->sin_addr), addrstr, INET_ADDRSTRLEN, 0, ec); if (ec) { - warning() << "inet_ntop failed during IPv4 address conversion: " << ec.message() - << std::endl; + LOGV2_WARNING(21213, + "inet_ntop failed during IPv4 address conversion: {ec_message}", + "ec_message"_attr = ec.message()); continue; } out.push_back(addrstr); @@ -319,8 +334,9 @@ std::vector<std::string> getBoundAddrs(const bool ipv6enabled) { boost::asio::detail::socket_ops::inet_ntop( AF_INET6, &(sock->sin6_addr), addrstr, INET6_ADDRSTRLEN, 0, ec); if (ec) { - warning() << "inet_ntop failed during IPv6 address conversion: " << ec.message() - << std::endl; + LOGV2_WARNING(21214, + "inet_ntop failed during IPv6 address conversion: {ec_message}", + "ec_message"_attr = ec.message()); continue; } out.push_back(addrstr); @@ -336,7 +352,7 @@ std::vector<std::string> getBoundAddrs(const bool ipv6enabled) { for (std::vector<std::string>::const_iterator o = out.begin(); o != out.end(); ++o) { builder << " [ " << *o << "]"; } - LOG(2) << builder.str(); + LOGV2_DEBUG(21206, 2, "{builder_str}", "builder_str"_attr = builder.str()); } return out; } diff --git a/src/mongo/db/repl/member_data.cpp b/src/mongo/db/repl/member_data.cpp index 571e83d3a1c..56cc13619e3 100644 --- a/src/mongo/db/repl/member_data.cpp +++ b/src/mongo/db/repl/member_data.cpp @@ -35,6 +35,7 @@ #include "mongo/db/repl/member_data.h" #include "mongo/db/repl/rslog.h" +#include "mongo/logv2/log.h" #include "mongo/util/log.h" namespace mongo { @@ -69,8 +70,11 @@ bool MemberData::setUpValues(Date_t now, ReplSetHeartbeatResponse&& hbResponse) } // Log if the state changes if (_lastResponse.getState() != hbResponse.getState()) { - log() << "Member " << _hostAndPort.toString() << " is now in state " - << hbResponse.getState().toString() << rsLog; + LOGV2_OPTIONS(21215, + {logv2::LogTag::kRS}, + "Member {hostAndPort} is now in state {hbResponse_getState}", + "hostAndPort"_attr = _hostAndPort.toString(), + "hbResponse_getState"_attr = hbResponse.getState().toString()); } bool opTimeAdvanced = @@ -93,8 +97,11 @@ void MemberData::setDownValues(Date_t now, const std::string& heartbeatMessage) _lastHeartbeatMessage = heartbeatMessage; if (_lastResponse.getState() != MemberState::RS_DOWN) { - log() << "Member " << _hostAndPort.toString() << " is now in state RS_DOWN - " - << redact(heartbeatMessage) << rsLog; + LOGV2_OPTIONS(21216, + {logv2::LogTag::kRS}, + "Member {hostAndPort} is now in state RS_DOWN - {heartbeatMessage}", + "hostAndPort"_attr = _hostAndPort.toString(), + "heartbeatMessage"_attr = redact(heartbeatMessage)); } _lastResponse = ReplSetHeartbeatResponse(); @@ -116,8 +123,11 @@ void MemberData::setAuthIssue(Date_t now) { _lastHeartbeatMessage.clear(); if (_lastResponse.getState() != MemberState::RS_UNKNOWN) { - log() << "Member " << _hostAndPort.toString() - << " is now in state RS_UNKNOWN due to authentication issue." << rsLog; + LOGV2_OPTIONS( + 21217, + {logv2::LogTag::kRS}, + "Member {hostAndPort} is now in state RS_UNKNOWN due to authentication issue.", + "hostAndPort"_attr = _hostAndPort.toString()); } _lastResponse = ReplSetHeartbeatResponse(); @@ -142,12 +152,16 @@ void MemberData::setLastDurableOpTimeAndWallTime(OpTimeAndWallTime opTime, Date_ if (_lastAppliedOpTime < opTime.opTime) { // TODO(russotto): We think this should never happen, rollback or no rollback. Make this an // invariant and see what happens. - log() << "Durable progress (" << opTime.opTime << ") is ahead of the applied progress (" - << _lastAppliedOpTime - << ". This is likely due to a " - "rollback." - << " memberid: " << _memberId << _hostAndPort.toString() - << " previous durable progress: " << _lastDurableOpTime; + LOGV2(21218, + "Durable progress ({opTime_opTime}) is ahead of the applied progress " + "({lastAppliedOpTime}. This is likely due to a " + "rollback. memberid: {memberId}{hostAndPort} previous durable progress: " + "{lastDurableOpTime}", + "opTime_opTime"_attr = opTime.opTime, + "lastAppliedOpTime"_attr = _lastAppliedOpTime, + "memberId"_attr = _memberId, + "hostAndPort"_attr = _hostAndPort.toString(), + "lastDurableOpTime"_attr = _lastDurableOpTime); } else { _lastDurableOpTime = opTime.opTime; _lastDurableWallTime = opTime.wallTime; diff --git a/src/mongo/db/repl/noop_writer.cpp b/src/mongo/db/repl/noop_writer.cpp index 29ee8017bfb..67fba097925 100644 --- a/src/mongo/db/repl/noop_writer.cpp +++ b/src/mongo/db/repl/noop_writer.cpp @@ -43,6 +43,7 @@ #include "mongo/db/repl/noop_writer.h" #include "mongo/db/repl/oplog.h" #include "mongo/db/repl/repl_server_parameters_gen.h" +#include "mongo/logv2/log.h" #include "mongo/util/concurrency/idle_thread_block.h" #include "mongo/util/log.h" @@ -150,14 +151,14 @@ void NoopWriter::_writeNoop(OperationContext* opCtx) { Lock::GlobalLock lock( opCtx, MODE_IX, Date_t::now() + Milliseconds(1), Lock::InterruptBehavior::kLeaveUnlocked); if (!lock.isLocked()) { - LOG(1) << "Global lock is not available skipping noopWrite"; + LOGV2_DEBUG(21219, 1, "Global lock is not available skipping noopWrite"); return; } auto replCoord = ReplicationCoordinator::get(opCtx); // Its a proxy for being a primary if (!replCoord->canAcceptWritesForDatabase(opCtx, "admin")) { - LOG(1) << "Not a primary, skipping the noop write"; + LOGV2_DEBUG(21220, 1, "Not a primary, skipping the noop write"); return; } @@ -165,14 +166,20 @@ void NoopWriter::_writeNoop(OperationContext* opCtx) { // _lastKnownOpTime is not protected by lock as its used only by one thread. if (lastAppliedOpTime != _lastKnownOpTime) { - LOG(1) << "Not scheduling a noop write. Last known OpTime: " << _lastKnownOpTime - << " != last primary OpTime: " << lastAppliedOpTime; + LOGV2_DEBUG(21221, + 1, + "Not scheduling a noop write. Last known OpTime: {lastKnownOpTime} != last " + "primary OpTime: {lastAppliedOpTime}", + "lastKnownOpTime"_attr = _lastKnownOpTime, + "lastAppliedOpTime"_attr = lastAppliedOpTime); } else { if (writePeriodicNoops.load()) { const auto logLevel = getTestCommandsEnabled() ? 0 : 1; - LOG(logLevel) - << "Writing noop to oplog as there has been no writes to this replica set in over " - << _writeInterval; + LOGV2_DEBUG(21222, + logSeverityV1toV2(logLevel).toInt(), + "Writing noop to oplog as there has been no writes to this replica set in " + "over {writeInterval}", + "writeInterval"_attr = _writeInterval); writeConflictRetry( opCtx, "writeNoop", NamespaceString::kRsOplogNamespace.ns(), [&opCtx] { WriteUnitOfWork uow(opCtx); @@ -184,7 +191,10 @@ void NoopWriter::_writeNoop(OperationContext* opCtx) { } _lastKnownOpTime = replCoord->getMyLastAppliedOpTime(); - LOG(1) << "Set last known op time to " << _lastKnownOpTime; + LOGV2_DEBUG(21223, + 1, + "Set last known op time to {lastKnownOpTime}", + "lastKnownOpTime"_attr = _lastKnownOpTime); } } // namespace repl diff --git a/src/mongo/db/repl/oplog.cpp b/src/mongo/db/repl/oplog.cpp index 7e74528cf6d..c792ba15917 100644 --- a/src/mongo/db/repl/oplog.cpp +++ b/src/mongo/db/repl/oplog.cpp @@ -127,8 +127,11 @@ bool shouldBuildInForeground(OperationContext* opCtx, const NamespaceString& indexNss, repl::OplogApplication::Mode mode) { if (mode == OplogApplication::Mode::kRecovering) { - LOG(3) << "apply op: building background index " << index - << " in the foreground because the node is in recovery"; + LOGV2_DEBUG(21241, + 3, + "apply op: building background index {index} in the foreground because the " + "node is in recovery", + "index"_attr = index); return true; } @@ -137,8 +140,11 @@ bool shouldBuildInForeground(OperationContext* opCtx, const bool isPrimary = repl::ReplicationCoordinator::get(opCtx)->canAcceptWritesFor(opCtx, indexNss); if (isPrimary) { - LOG(3) << "apply op: not building background index " << index - << " in a background thread because this is a primary"; + LOGV2_DEBUG(21242, + 3, + "apply op: not building background index {index} in a background thread " + "because this is a primary", + "index"_attr = index); return true; } @@ -262,7 +268,7 @@ void _logOpsInner(OperationContext* opCtx, Status result = oplogCollection->insertDocumentsForOplog(opCtx, records, timestamps); if (!result.isOK()) { - severe() << "write to oplog failed: " << result.toString(); + LOGV2_FATAL(21263, "write to oplog failed: {result}", "result"_attr = result.toString()); fassertFailed(17322); } @@ -279,7 +285,7 @@ void _logOpsInner(OperationContext* opCtx, // Optionally hang before advancing lastApplied. if (MONGO_unlikely(hangBeforeLogOpAdvancesLastApplied.shouldFail())) { - log() << "hangBeforeLogOpAdvancesLastApplied fail point enabled."; + LOGV2(21243, "hangBeforeLogOpAdvancesLastApplied fail point enabled."); hangBeforeLogOpAdvancesLastApplied.pauseWhileSet(opCtx); } @@ -417,8 +423,13 @@ std::vector<OpTime> logInsertOps(OperationContext* opCtx, sleepBetweenInsertOpTimeGenerationAndLogOp.execute([&](const BSONObj& data) { auto numMillis = data["waitForMillis"].numberInt(); - log() << "Sleeping for " << numMillis << "ms after receiving " << count << " optimes from " - << opTimes.front() << " to " << opTimes.back(); + LOGV2(21244, + "Sleeping for {numMillis}ms after receiving {count} optimes from {opTimes_front} to " + "{opTimes_back}", + "numMillis"_attr = numMillis, + "count"_attr = count, + "opTimes_front"_attr = opTimes.front(), + "opTimes_back"_attr = opTimes.back()); sleepmillis(numMillis); }); @@ -470,7 +481,7 @@ long long getNewOplogSizeBytes(OperationContext* opCtx, const ReplSettings& repl ProcessInfo pi; if (pi.getAddrSize() == 32) { const auto sz = 50LL * 1024LL * 1024LL; - LOG(3) << "32bit system; choosing " << sz << " bytes oplog"; + LOGV2_DEBUG(21245, 3, "32bit system; choosing {sz} bytes oplog", "sz"_attr = sz); return sz; } // First choose a minimum size. @@ -478,7 +489,7 @@ long long getNewOplogSizeBytes(OperationContext* opCtx, const ReplSettings& repl #if defined(__APPLE__) // typically these are desktops (dev machines), so keep it smallish const auto sz = 192 * 1024 * 1024; - LOG(3) << "Apple system; choosing " << sz << " bytes oplog"; + LOGV2_DEBUG(21246, 3, "Apple system; choosing {sz} bytes oplog", "sz"_attr = sz); return sz; #else long long lowerBound = 0; @@ -487,14 +498,22 @@ long long getNewOplogSizeBytes(OperationContext* opCtx, const ReplSettings& repl // in memory: 50MB minimum size lowerBound = 50LL * 1024 * 1024; bytes = pi.getMemSizeMB() * 1024 * 1024; - LOG(3) << "Ephemeral storage system; lowerBound: " << lowerBound << " bytes, " << bytes - << " bytes total memory"; + LOGV2_DEBUG( + 21247, + 3, + "Ephemeral storage system; lowerBound: {lowerBound} bytes, {bytes} bytes total memory", + "lowerBound"_attr = lowerBound, + "bytes"_attr = bytes); } else { // disk: 990MB minimum size lowerBound = 990LL * 1024 * 1024; bytes = File::freeSpace(storageGlobalParams.dbpath); //-1 if call not supported. - LOG(3) << "Disk storage system; lowerBound: " << lowerBound << " bytes, " << bytes - << " bytes free space on device"; + LOGV2_DEBUG(21248, + 3, + "Disk storage system; lowerBound: {lowerBound} bytes, {bytes} bytes free space " + "on device", + "lowerBound"_attr = lowerBound, + "bytes"_attr = bytes); } long long fivePct = static_cast<long long>(bytes * 0.05); auto sz = std::max(fivePct, lowerBound); @@ -531,7 +550,7 @@ void createOplog(OperationContext* opCtx, stringstream ss; ss << "cmdline oplogsize (" << n << ") different than existing (" << o << ") see: http://dochub.mongodb.org/core/increase-oplog"; - log() << ss.str() << endl; + LOGV2(21249, "{ss_str}", "ss_str"_attr = ss.str()); uasserted(13257, ss.str()); } } @@ -544,8 +563,10 @@ void createOplog(OperationContext* opCtx, /* create an oplog collection, if it doesn't yet exist. */ const auto sz = getNewOplogSizeBytes(opCtx, replSettings); - log() << "******" << endl; - log() << "creating replication oplog of size: " << (int)(sz / (1024 * 1024)) << "MB..." << endl; + LOGV2(21250, "******"); + LOGV2(21251, + "creating replication oplog of size: {int_sz_1024_1024}MB...", + "int_sz_1024_1024"_attr = (int)(sz / (1024 * 1024))); CollectionOptions options; options.capped = true; @@ -565,7 +586,7 @@ void createOplog(OperationContext* opCtx, /* sync here so we don't get any surprising lag later when we try to sync */ service->getStorageEngine()->flushAllFiles(opCtx, /*callerHoldsReadLock*/ false); - log() << "******" << endl; + LOGV2(21252, "******"); } void createOplog(OperationContext* opCtx) { @@ -778,10 +799,11 @@ const StringMap<ApplyOpMetadata> kOpsMap = { const auto& cmd = entry.getObject(); auto nss = extractNsFromUUIDorNs(opCtx, entry.getNss(), entry.getUuid(), cmd); if (nss.isDropPendingNamespace()) { - log() - << "applyCommand: " << nss - << " : collection is already in a drop-pending state: ignoring collection drop: " - << redact(cmd); + LOGV2(21253, + "applyCommand: {nss} : collection is already in a drop-pending state: ignoring " + "collection drop: {cmd}", + "nss"_attr = nss, + "cmd"_attr = redact(cmd)); return Status::OK(); } // Parse optime from oplog entry unless we are applying this command in standalone or on a @@ -920,8 +942,12 @@ Status applyOperation_inlock(OperationContext* opCtx, IncrementOpsAppliedStatsFn incrementOpsAppliedStats) { // Get the single oplog entry to be applied or the first oplog entry of grouped inserts. auto op = opOrGroupedInserts.getOp(); - LOG(3) << "applying op (or grouped inserts): " << redact(opOrGroupedInserts.toBSON()) - << ", oplog application mode: " << OplogApplication::modeToString(mode); + LOGV2_DEBUG(21254, + 3, + "applying op (or grouped inserts): {opOrGroupedInserts}, oplog application mode: " + "{OplogApplication_modeToString_mode}", + "opOrGroupedInserts"_attr = redact(opOrGroupedInserts.toBSON()), + "OplogApplication_modeToString_mode"_attr = OplogApplication::modeToString(mode)); // Choose opCounters based on running on standalone/primary or secondary by checking // whether writes are replicated. Atomic applyOps command is an exception, which runs @@ -1167,8 +1193,9 @@ Status applyOperation_inlock(OperationContext* opCtx, UpdateResult res = update(opCtx, db, request); if (res.numMatched == 0 && res.upserted.isEmpty()) { - error() << "No document was updated even though we got a DuplicateKey " - "error when inserting"; + LOGV2_ERROR(21257, + "No document was updated even though we got a DuplicateKey " + "error when inserting"); fassertFailedNoTrace(28750); } wuow.commit(); @@ -1225,7 +1252,7 @@ Status applyOperation_inlock(OperationContext* opCtx, // was a simple { _id : ... } update criteria string msg = str::stream() << "failed to apply update: " << redact(op.toBSON()); - error() << msg; + LOGV2_ERROR(21258, "{msg}", "msg"_attr = msg); return Status(ErrorCodes::UpdateOperationFailed, msg); } @@ -1242,7 +1269,7 @@ Status applyOperation_inlock(OperationContext* opCtx, Helpers::findOne(opCtx, collection, updateCriteria, false).isNull())) { string msg = str::stream() << "couldn't find doc: " << redact(op.toBSON()); - error() << msg; + LOGV2_ERROR(21259, "{msg}", "msg"_attr = msg); return Status(ErrorCodes::UpdateOperationFailed, msg); } @@ -1255,7 +1282,7 @@ Status applyOperation_inlock(OperationContext* opCtx, if (!upsert) { string msg = str::stream() << "update of non-mod failed: " << redact(op.toBSON()); - error() << msg; + LOGV2_ERROR(21260, "{msg}", "msg"_attr = msg); return Status(ErrorCodes::UpdateOperationFailed, msg); } } @@ -1324,8 +1351,12 @@ Status applyOperation_inlock(OperationContext* opCtx, Status applyCommand_inlock(OperationContext* opCtx, const OplogEntry& entry, OplogApplication::Mode mode) { - LOG(3) << "applying command op: " << redact(entry.toBSON()) - << ", oplog application mode: " << OplogApplication::modeToString(mode); + LOGV2_DEBUG(21255, + 3, + "applying command op: {entry}, oplog application mode: " + "{OplogApplication_modeToString_mode}", + "entry"_attr = redact(entry.toBSON()), + "OplogApplication_modeToString_mode"_attr = OplogApplication::modeToString(mode)); // Only commands are processed here. invariant(entry.getOpType() == OpTypeEnum::kCommand); @@ -1472,8 +1503,11 @@ Status applyCommand_inlock(OperationContext* opCtx, cmd->parse(opCtx, OpMsgRequest::fromDBAndBody(nss.db(), o))->ns().toString(); auto swUUID = entry.getUuid(); if (!swUUID) { - error() << "Failed command " << redact(o) << " on " << ns - << "during oplog application. Expected a UUID."; + LOGV2_ERROR( + 21261, + "Failed command {o} on {ns}during oplog application. Expected a UUID.", + "o"_attr = redact(o), + "ns"_attr = ns); } BackgroundOperation::awaitNoBgOpInProgForNs(ns); IndexBuildsCoordinator::get(opCtx)->awaitNoIndexBuildInProgressForCollection( @@ -1492,8 +1526,12 @@ Status applyCommand_inlock(OperationContext* opCtx, } default: { if (!curOpToApply.acceptableErrors.count(status.code())) { - error() << "Failed command " << redact(o) << " on " << nss.db() - << " with status " << status << " during oplog application"; + LOGV2_ERROR(21262, + "Failed command {o} on {nss_db} with status {status} during oplog " + "application", + "o"_attr = redact(o), + "nss_db"_attr = nss.db(), + "status"_attr = status); return status; } @@ -1527,7 +1565,7 @@ void initTimestampFromOplog(OperationContext* opCtx, const NamespaceString& oplo c.findOne(oplogNss.ns(), Query().sort(reverseNaturalObj), nullptr, QueryOption_SlaveOk); if (!lastOp.isEmpty()) { - LOG(1) << "replSet setting last Timestamp"; + LOGV2_DEBUG(21256, 1, "replSet setting last Timestamp"); const OpTime opTime = fassert(28696, OpTime::parseFromOplogEntry(lastOp)); setNewTimestamp(opCtx->getServiceContext(), opTime.getTimestamp()); } diff --git a/src/mongo/db/repl/oplog_applier.cpp b/src/mongo/db/repl/oplog_applier.cpp index f505933bbdc..294930161ec 100644 --- a/src/mongo/db/repl/oplog_applier.cpp +++ b/src/mongo/db/repl/oplog_applier.cpp @@ -36,6 +36,7 @@ #include "mongo/db/auth/authorization_session.h" #include "mongo/db/namespace_string.h" #include "mongo/db/repl/repl_server_parameters_gen.h" +#include "mongo/logv2/log.h" #include "mongo/util/log.h" #include "mongo/util/time_support.h" @@ -63,9 +64,9 @@ Future<void> OplogApplier::startup() { auto callback = [ this, promise = std::move(pf.promise) ](const CallbackArgs& args) mutable noexcept { invariant(args.status); - log() << "Starting oplog application"; + LOGV2(21224, "Starting oplog application"); _run(_oplogBuffer); - log() << "Finished oplog application"; + LOGV2(21225, "Finished oplog application"); promise.setWith([] {}); }; invariant(_executor->scheduleWork(std::move(callback)).getStatus()); @@ -75,7 +76,7 @@ Future<void> OplogApplier::startup() { void OplogApplier::shutdown() { // Shutdown will hang if this failpoint is enabled. if (globalFailPointRegistry().find("rsSyncApplyStop")->shouldFail()) { - severe() << "Turn off rsSyncApplyStop before attempting clean shutdown"; + LOGV2_FATAL(21227, "Turn off rsSyncApplyStop before attempting clean shutdown"); fassertFailedNoTrace(40304); } @@ -110,7 +111,10 @@ void OplogApplier::enqueue(OperationContext* opCtx, OplogBuffer::Batch::const_iterator end) { static Occasionally sampler; if (sampler.tick()) { - LOG(2) << "oplog buffer has " << _oplogBuffer->getSize() << " bytes"; + LOGV2_DEBUG(21226, + 2, + "oplog buffer has {oplogBuffer_getSize} bytes", + "oplogBuffer_getSize"_attr = _oplogBuffer->getSize()); } _oplogBuffer->push(opCtx, begin, end); } diff --git a/src/mongo/db/repl/oplog_applier_impl.cpp b/src/mongo/db/repl/oplog_applier_impl.cpp index 1ac70f2a05e..e54bc128219 100644 --- a/src/mongo/db/repl/oplog_applier_impl.cpp +++ b/src/mongo/db/repl/oplog_applier_impl.cpp @@ -45,6 +45,7 @@ #include "mongo/db/repl/insert_group.h" #include "mongo/db/repl/transaction_oplog_application.h" #include "mongo/db/stats/timer_stats.h" +#include "mongo/logv2/log.h" #include "mongo/platform/basic.h" #include "mongo/util/fail_point.h" #include "mongo/util/log.h" @@ -125,7 +126,7 @@ Status finishAndLogApply(OperationContext* opCtx, s << redact(entryOrGroupedInserts.toBSON()); s << ", took " << opDuration << "ms"; - log() << s.str(); + LOGV2(21228, "{s_str}", "s_str"_attr = s.str()); } } return finalStatus; @@ -434,8 +435,9 @@ void OplogApplierImpl::_run(OplogBuffer* oplogBuffer) { // For pausing replication in tests. if (MONGO_unlikely(rsSyncApplyStop.shouldFail())) { - log() << "Oplog Applier - rsSyncApplyStop fail point enabled. Blocking until fail " - "point is disabled."; + LOGV2(21229, + "Oplog Applier - rsSyncApplyStop fail point enabled. Blocking until fail " + "point is disabled."); rsSyncApplyStop.pauseWhileSet(&opCtx); } @@ -602,7 +604,7 @@ StatusWith<OpTime> OplogApplierImpl::_applyOplogBatch(OperationContext* opCtx, std::vector<OplogEntry> ops) { invariant(!ops.empty()); - LOG(2) << "replication batch size is " << ops.size(); + LOGV2_DEBUG(21230, 2, "replication batch size is {ops_size}", "ops_size"_attr = ops.size()); // Stop all readers until we're done. This also prevents doc-locking engines from deleting old // entries from the oplog until we finish writing. @@ -610,7 +612,7 @@ StatusWith<OpTime> OplogApplierImpl::_applyOplogBatch(OperationContext* opCtx, invariant(_replCoord); if (_replCoord->getApplierState() == ReplicationCoordinator::ApplierState::Stopped) { - severe() << "attempting to replicate ops while primary"; + LOGV2_FATAL(21234, "attempting to replicate ops while primary"); return {ErrorCodes::CannotApplyOplogWhilePrimary, "attempting to replicate ops while primary"}; } @@ -653,8 +655,9 @@ StatusWith<OpTime> OplogApplierImpl::_applyOplogBatch(OperationContext* opCtx, // Use this fail point to hold the PBWM lock after we have written the oplog entries but // before we have applied them. if (MONGO_unlikely(pauseBatchApplicationAfterWritingOplogEntries.shouldFail())) { - log() << "pauseBatchApplicationAfterWritingOplogEntries fail point enabled. Blocking " - "until fail point is disabled."; + LOGV2(21231, + "pauseBatchApplicationAfterWritingOplogEntries fail point enabled. Blocking " + "until fail point is disabled."); pauseBatchApplicationAfterWritingOplogEntries.pauseWhileSet(opCtx); } @@ -699,12 +702,17 @@ StatusWith<OpTime> OplogApplierImpl::_applyOplogBatch(OperationContext* opCtx, for (auto it = statusVector.cbegin(); it != statusVector.cend(); ++it) { const auto& status = *it; if (!status.isOK()) { - severe() - << "Failed to apply batch of operations. Number of operations in batch: " - << ops.size() << ". First operation: " << redact(ops.front().toBSON()) - << ". Last operation: " << redact(ops.back().toBSON()) - << ". Oplog application failed in writer thread " - << std::distance(statusVector.cbegin(), it) << ": " << redact(status); + LOGV2_FATAL(21235, + "Failed to apply batch of operations. Number of operations in " + "batch: {ops_size}. First operation: {ops_front}. Last operation: " + "{ops_back}. Oplog application failed in writer thread " + "{std_distance_statusVector_cbegin_it}: {status}", + "ops_size"_attr = ops.size(), + "ops_front"_attr = redact(ops.front().toBSON()), + "ops_back"_attr = redact(ops.back().toBSON()), + "std_distance_statusVector_cbegin_it"_attr = + std::distance(statusVector.cbegin(), it), + "status"_attr = redact(status)); return status; } } @@ -721,12 +729,14 @@ StatusWith<OpTime> OplogApplierImpl::_applyOplogBatch(OperationContext* opCtx, // Use this fail point to hold the PBWM lock and prevent the batch from completing. if (MONGO_unlikely(pauseBatchApplicationBeforeCompletion.shouldFail())) { - log() << "pauseBatchApplicationBeforeCompletion fail point enabled. Blocking until fail " - "point is disabled."; + LOGV2(21232, + "pauseBatchApplicationBeforeCompletion fail point enabled. Blocking until fail " + "point is disabled."); while (MONGO_unlikely(pauseBatchApplicationBeforeCompletion.shouldFail())) { if (inShutdown()) { - severe() << "Turn off pauseBatchApplicationBeforeCompletion before attempting " - "clean shutdown"; + LOGV2_FATAL(21236, + "Turn off pauseBatchApplicationBeforeCompletion before attempting " + "clean shutdown"); fassertFailedNoTrace(50798); } sleepmillis(100); @@ -904,10 +914,10 @@ Status applyOplogEntryOrGroupedInserts(OperationContext* opCtx, auto applyStartTime = clockSource->now(); if (MONGO_unlikely(hangAfterRecordingOpApplicationStartTime.shouldFail())) { - log() << "applyOplogEntryOrGroupedInserts - fail point " - "hangAfterRecordingOpApplicationStartTime " - "enabled. " - << "Blocking until fail point is disabled. "; + LOGV2(21233, + "applyOplogEntryOrGroupedInserts - fail point " + "hangAfterRecordingOpApplicationStartTime " + "enabled. Blocking until fail point is disabled. "); hangAfterRecordingOpApplicationStartTime.pauseWhileSet(); } @@ -1046,8 +1056,10 @@ Status OplogApplierImpl::applyOplogBatchPerWorker(OperationContext* opCtx, continue; } - severe() << "Error applying operation (" << redact(entry.toBSON()) - << "): " << causedBy(redact(status)); + LOGV2_FATAL(21237, + "Error applying operation ({entry}): {causedBy_status}", + "entry"_attr = redact(entry.toBSON()), + "causedBy_status"_attr = causedBy(redact(status))); return status; } } catch (const DBException& e) { @@ -1058,8 +1070,10 @@ Status OplogApplierImpl::applyOplogBatchPerWorker(OperationContext* opCtx, continue; } - severe() << "writer worker caught exception: " << redact(e) - << " on: " << redact(entry.toBSON()); + LOGV2_FATAL(21238, + "writer worker caught exception: {e} on: {entry}", + "e"_attr = redact(e), + "entry"_attr = redact(entry.toBSON())); return e.toStatus(); } } diff --git a/src/mongo/db/repl/oplog_batcher.cpp b/src/mongo/db/repl/oplog_batcher.cpp index 80f0ffbbec6..3d2348ad8a1 100644 --- a/src/mongo/db/repl/oplog_batcher.cpp +++ b/src/mongo/db/repl/oplog_batcher.cpp @@ -35,6 +35,7 @@ #include "mongo/db/commands/txn_cmds_gen.h" #include "mongo/db/repl/oplog_applier.h" #include "mongo/db/repl/repl_server_parameters_gen.h" +#include "mongo/logv2/log.h" #include "mongo/util/log.h" namespace mongo { @@ -173,7 +174,7 @@ StatusWith<std::vector<OplogEntry>> OplogBatcher::getNextApplierBatch( std::string message = str::stream() << "expected oplog version " << OplogEntry::kOplogVersion << " but found version " << entry.getVersion() << " in oplog entry: " << redact(entry.toBSON()); - severe() << message; + LOGV2_FATAL(21240, "{message}", "message"_attr = message); return {ErrorCodes::BadValue, message}; } @@ -319,7 +320,9 @@ void OplogBatcher::_run(StorageInterface* storageInterface) { // Check the oplog buffer after the applier state to ensure the producer is stopped. if (isDraining && _oplogBuffer->isEmpty()) { ops.setTermWhenExhausted(termWhenBufferIsEmpty); - log() << "Oplog buffer has been drained in term " << termWhenBufferIsEmpty; + LOGV2(21239, + "Oplog buffer has been drained in term {termWhenBufferIsEmpty}", + "termWhenBufferIsEmpty"_attr = termWhenBufferIsEmpty); } else { // Don't emit empty batches. continue; diff --git a/src/mongo/db/repl/oplog_fetcher.cpp b/src/mongo/db/repl/oplog_fetcher.cpp index afff88413c3..0609e46c634 100644 --- a/src/mongo/db/repl/oplog_fetcher.cpp +++ b/src/mongo/db/repl/oplog_fetcher.cpp @@ -41,6 +41,7 @@ #include "mongo/db/repl/replication_auth.h" #include "mongo/db/repl/replication_coordinator.h" #include "mongo/db/stats/timer_stats.h" +#include "mongo/logv2/log.h" #include "mongo/rpc/metadata/oplog_query_metadata.h" #include "mongo/util/assert_util.h" #include "mongo/util/fail_point.h" @@ -467,7 +468,7 @@ StatusWith<BSONObj> OplogFetcher::_onSuccessfulBatch(const Fetcher::QueryRespons [&](auto&&) { status = {ErrorCodes::FailPointEnabled, "stopReplProducerOnDocument fail point is enabled."}; - log() << status.reason(); + LOGV2(21264, "{status_reason}", "status_reason"_attr = status.reason()); }, [&](const BSONObj& data) { auto opCtx = cc().makeOperationContext(); @@ -485,17 +486,25 @@ StatusWith<BSONObj> OplogFetcher::_onSuccessfulBatch(const Fetcher::QueryRespons auto firstDocToApply = documents.cbegin(); if (!documents.empty()) { - LOG(2) << "oplog fetcher read " << documents.size() - << " operations from remote oplog starting at " << documents.front()["ts"] - << " and ending at " << documents.back()["ts"]; + LOGV2_DEBUG(21265, + 2, + "oplog fetcher read {documents_size} operations from remote oplog starting at " + "{documents_front_ts} and ending at {documents_back_ts}", + "documents_size"_attr = documents.size(), + "documents_front_ts"_attr = documents.front()["ts"], + "documents_back_ts"_attr = documents.back()["ts"]); } else { - LOG(2) << "oplog fetcher read 0 operations from remote oplog"; + LOGV2_DEBUG(21266, 2, "oplog fetcher read 0 operations from remote oplog"); } auto oqMetadataResult = parseOplogQueryMetadata(queryResponse); if (!oqMetadataResult.isOK()) { - error() << "invalid oplog query metadata from sync source " << _getSource() << ": " - << oqMetadataResult.getStatus() << ": " << queryResponse.otherFields.metadata; + LOGV2_ERROR(21276, + "invalid oplog query metadata from sync source {getSource}: " + "{oqMetadataResult_getStatus}: {queryResponse_otherFields_metadata}", + "getSource"_attr = _getSource(), + "oqMetadataResult_getStatus"_attr = oqMetadataResult.getStatus(), + "queryResponse_otherFields_metadata"_attr = queryResponse.otherFields.metadata); return oqMetadataResult.getStatus(); } auto oqMetadata = oqMetadataResult.getValue(); @@ -519,7 +528,10 @@ StatusWith<BSONObj> OplogFetcher::_onSuccessfulBatch(const Fetcher::QueryRespons return status; } - LOG(1) << "oplog fetcher successfully fetched from " << _getSource(); + LOGV2_DEBUG(21267, + 1, + "oplog fetcher successfully fetched from {getSource}", + "getSource"_attr = _getSource()); // We do not always enqueue the first document. We elect to skip it for the following // reasons: @@ -553,8 +565,12 @@ StatusWith<BSONObj> OplogFetcher::_onSuccessfulBatch(const Fetcher::QueryRespons const auto& metadataObj = queryResponse.otherFields.metadata; auto metadataResult = rpc::ReplSetMetadata::readFromMetadata(metadataObj); if (!metadataResult.isOK()) { - error() << "invalid replication metadata from sync source " << _getSource() << ": " - << metadataResult.getStatus() << ": " << metadataObj; + LOGV2_ERROR(21277, + "invalid replication metadata from sync source {getSource}: " + "{metadataResult_getStatus}: {metadataObj}", + "getSource"_attr = _getSource(), + "metadataResult_getStatus"_attr = metadataResult.getStatus(), + "metadataObj"_attr = metadataObj); return metadataResult.getStatus(); } replSetMetadata = metadataResult.getValue(); @@ -838,7 +854,7 @@ void NewOplogFetcher::_runQuery(const executor::TaskExecutor::CallbackArgs& call if (MONGO_unlikely(logAfterOplogFetcherConnCreated.shouldFail())) { // Used in tests that wait for this failpoint to be entered to ensure the DBClientConnection // was created. - log() << "logAfterOplogFetcherConnCreated failpoint enabled."; + LOGV2(21268, "logAfterOplogFetcherConnCreated failpoint enabled."); } hangAfterOplogFetcherCallbackScheduled.pauseWhileSet(); @@ -1074,7 +1090,7 @@ Status NewOplogFetcher::_onSuccessfulBatch(const Documents& documents) { [&](auto&&) { status = {ErrorCodes::FailPointEnabled, "stopReplProducerOnDocument fail point is enabled."}; - log() << status.reason(); + LOGV2(21269, "{status_reason}", "status_reason"_attr = status.reason()); }, [&](const BSONObj& data) { auto opCtx = cc().makeOperationContext(); @@ -1091,17 +1107,25 @@ Status NewOplogFetcher::_onSuccessfulBatch(const Documents& documents) { auto firstDocToApply = documents.cbegin(); if (!documents.empty()) { - LOG(2) << "oplog fetcher read " << documents.size() - << " operations from remote oplog starting at " << documents.front()["ts"] - << " and ending at " << documents.back()["ts"]; + LOGV2_DEBUG(21270, + 2, + "oplog fetcher read {documents_size} operations from remote oplog starting at " + "{documents_front_ts} and ending at {documents_back_ts}", + "documents_size"_attr = documents.size(), + "documents_front_ts"_attr = documents.front()["ts"], + "documents_back_ts"_attr = documents.back()["ts"]); } else { - LOG(2) << "oplog fetcher read 0 operations from remote oplog"; + LOGV2_DEBUG(21271, 2, "oplog fetcher read 0 operations from remote oplog"); } auto oqMetadataResult = parseOplogQueryMetadata(_metadataObj); if (!oqMetadataResult.isOK()) { - error() << "invalid oplog query metadata from sync source " << _source << ": " - << oqMetadataResult.getStatus() << ": " << _metadataObj; + LOGV2_ERROR(21278, + "invalid oplog query metadata from sync source {source}: " + "{oqMetadataResult_getStatus}: {metadataObj}", + "source"_attr = _source, + "oqMetadataResult_getStatus"_attr = oqMetadataResult.getStatus(), + "metadataObj"_attr = _metadataObj); return oqMetadataResult.getStatus(); } auto oqMetadata = oqMetadataResult.getValue(); @@ -1124,7 +1148,8 @@ Status NewOplogFetcher::_onSuccessfulBatch(const Documents& documents) { return status; } - LOG(1) << "oplog fetcher successfully fetched from " << _source; + LOGV2_DEBUG( + 21272, 1, "oplog fetcher successfully fetched from {source}", "source"_attr = _source); // We do not always enqueue the first document. We elect to skip it for the following // reasons: @@ -1156,8 +1181,12 @@ Status NewOplogFetcher::_onSuccessfulBatch(const Documents& documents) { if (receivedReplMetadata) { auto metadataResult = rpc::ReplSetMetadata::readFromMetadata(_metadataObj); if (!metadataResult.isOK()) { - error() << "invalid replication metadata from sync source " << _source << ": " - << metadataResult.getStatus() << ": " << _metadataObj; + LOGV2_ERROR(21279, + "invalid replication metadata from sync source {source}: " + "{metadataResult_getStatus}: {metadataObj}", + "source"_attr = _source, + "metadataResult_getStatus"_attr = metadataResult.getStatus(), + "metadataObj"_attr = _metadataObj); return metadataResult.getStatus(); } replSetMetadata = metadataResult.getValue(); @@ -1210,7 +1239,10 @@ Status NewOplogFetcher::_onSuccessfulBatch(const Documents& documents) { } auto lastDocOpTime = lastDocOpTimeRes.getValue(); - LOG(3) << "Oplog fetcher setting last fetched optime ahead after batch: " << lastDocOpTime; + LOGV2_DEBUG(21273, + 3, + "Oplog fetcher setting last fetched optime ahead after batch: {lastDocOpTime}", + "lastDocOpTime"_attr = lastDocOpTime); stdx::lock_guard<Latch> lock(_mutex); _lastFetched = lastDocOpTime; @@ -1223,13 +1255,17 @@ Status NewOplogFetcher::_onSuccessfulBatch(const Documents& documents) { bool NewOplogFetcher::OplogFetcherRestartDecisionDefault::shouldContinue(NewOplogFetcher* fetcher, Status status) { if (_numRestarts == _maxRestarts) { - log() << "Error returned from oplog query (no more query restarts left): " - << redact(status); + LOGV2(21274, + "Error returned from oplog query (no more query restarts left): {status}", + "status"_attr = redact(status)); return false; } - log() << "Recreating cursor for oplog fetcher due to error: " << redact(status) - << ". Last fetched optime: " << fetcher->_getLastOpTimeFetched() - << ". Attempts remaining: " << (_maxRestarts - _numRestarts); + LOGV2(21275, + "Recreating cursor for oplog fetcher due to error: {status}. Last fetched optime: " + "{fetcher_getLastOpTimeFetched}. Attempts remaining: {maxRestarts_numRestarts}", + "status"_attr = redact(status), + "fetcher_getLastOpTimeFetched"_attr = fetcher->_getLastOpTimeFetched(), + "maxRestarts_numRestarts"_attr = (_maxRestarts - _numRestarts)); _numRestarts++; return true; } diff --git a/src/mongo/db/repl/repl_client_info.cpp b/src/mongo/db/repl/repl_client_info.cpp index 0690e7b483e..119ec569a65 100644 --- a/src/mongo/db/repl/repl_client_info.cpp +++ b/src/mongo/db/repl/repl_client_info.cpp @@ -36,6 +36,7 @@ #include "mongo/db/client.h" #include "mongo/db/operation_context.h" #include "mongo/db/repl/replication_coordinator.h" +#include "mongo/logv2/log.h" #include "mongo/util/decorable.h" #include "mongo/util/log.h" @@ -103,10 +104,13 @@ void ReplClientInfo::setLastOpToSystemLastOpTime(OperationContext* opCtx) { if (systemOpTime >= _lastOp) { _lastOp = systemOpTime; } else { - log() << "Not setting the last OpTime for this Client from " << _lastOp - << " to the current system time of " << systemOpTime - << " as that would be moving the OpTime backwards. This should only happen if " - "there was a rollback recently"; + LOGV2(21280, + "Not setting the last OpTime for this Client from {lastOp} to the current system " + "time of {systemOpTime} as that would be moving the OpTime backwards. This " + "should only happen if " + "there was a rollback recently", + "lastOp"_attr = _lastOp, + "systemOpTime"_attr = systemOpTime); } lastOpInfo(opCtx).lastOpSetExplicitly = true; @@ -122,7 +126,10 @@ void ReplClientInfo::setLastOpToSystemLastOpTimeIgnoringInterrupt(OperationConte } catch (const ExceptionForCat<ErrorCategory::Interruption>& e) { // In most cases, it is safe to ignore interruption errors because we cannot use the same // OperationContext to wait for writeConcern anyways. - LOG(2) << "Ignoring set last op interruption error: " << e.toStatus(); + LOGV2_DEBUG(21281, + 2, + "Ignoring set last op interruption error: {e_toStatus}", + "e_toStatus"_attr = e.toStatus()); } } diff --git a/src/mongo/db/repl/repl_set_commands.cpp b/src/mongo/db/repl/repl_set_commands.cpp index a9c7bdc5336..8203c36f567 100644 --- a/src/mongo/db/repl/repl_set_commands.cpp +++ b/src/mongo/db/repl/repl_set_commands.cpp @@ -62,6 +62,7 @@ #include "mongo/db/service_context.h" #include "mongo/db/storage/storage_engine.h" #include "mongo/executor/network_interface.h" +#include "mongo/logv2/log.h" #include "mongo/transport/session.h" #include "mongo/transport/transport_layer.h" #include "mongo/util/decimal_counter.h" @@ -103,7 +104,7 @@ public: const string&, const BSONObj& cmdObj, BSONObjBuilder& result) { - log() << "replSetTest command received: " << cmdObj.toString(); + LOGV2(21573, "replSetTest command received: {cmdObj}", "cmdObj"_attr = cmdObj.toString()); auto replCoord = ReplicationCoordinator::get(getGlobalServiceContext()); @@ -120,8 +121,10 @@ public: status = bsonExtractIntegerField(cmdObj, "timeoutMillis", &timeoutMillis); uassertStatusOK(status); Milliseconds timeout(timeoutMillis); - log() << "replSetTest: waiting " << timeout << " for member state to become " - << expectedState; + LOGV2(21574, + "replSetTest: waiting {timeout} for member state to become {expectedState}", + "timeout"_attr = timeout, + "expectedState"_attr = expectedState); status = replCoord->waitForMemberState(expectedState, timeout); @@ -132,7 +135,9 @@ public: auto status = bsonExtractIntegerField(cmdObj, "waitForDrainFinish", &timeoutMillis); uassertStatusOK(status); Milliseconds timeout(timeoutMillis); - log() << "replSetTest: waiting " << timeout << " for applier buffer to finish draining"; + LOGV2(21575, + "replSetTest: waiting {timeout} for applier buffer to finish draining", + "timeout"_attr = timeout); status = replCoord->waitForDrainFinish(timeout); @@ -275,7 +280,7 @@ void parseReplSetSeedList(ReplicationCoordinatorExternalState* externalState, seedSet.insert(m); // uassert(13101, "can't use localhost in replset host list", !m.isLocalHost()); if (externalState->isSelf(m, getGlobalServiceContext())) { - LOG(1) << "ignoring seed " << m.toString() << " (=self)"; + LOGV2_DEBUG(21576, 1, "ignoring seed {m} (=self)", "m"_attr = m.toString()); } else { seeds->push_back(m); } @@ -315,7 +320,7 @@ public: "no configuration specified. " "Using a default configuration for the set"; result.append("info2", noConfigMessage); - log() << "initiate : " << noConfigMessage; + LOGV2(21577, "initiate : {noConfigMessage}", "noConfigMessage"_attr = noConfigMessage); ReplicationCoordinatorExternalStateImpl externalState( opCtx->getServiceContext(), @@ -346,7 +351,9 @@ public: } b.appendArray("members", members.obj()); configObj = b.obj(); - log() << "created this configuration for initiation : " << configObj.toString(); + LOGV2(21578, + "created this configuration for initiation : {configObj}", + "configObj"_attr = configObj.toString()); } if (configObj.getField("version").eoo()) { @@ -522,12 +529,12 @@ public: uassertStatusOK(status); } - log() << "Attempting to step down in response to replSetStepDown command"; + LOGV2(21579, "Attempting to step down in response to replSetStepDown command"); ReplicationCoordinator::get(opCtx)->stepDown( opCtx, force, Seconds(secondaryCatchUpPeriodSecs), Seconds(stepDownForSecs)); - log() << "replSetStepDown command completed"; + LOGV2(21580, "replSetStepDown command completed"); onExitGuard.dismiss(); return true; @@ -726,13 +733,15 @@ public: Status status = ReplicationCoordinator::get(opCtx)->checkReplEnabledForCommand(&result); uassertStatusOK(status); - log() << "Received replSetStepUp request"; + LOGV2(21581, "Received replSetStepUp request"); const bool skipDryRun = cmdObj["skipDryRun"].trueValue(); status = ReplicationCoordinator::get(opCtx)->stepUpIfEligible(skipDryRun); if (!status.isOK()) { - log() << "replSetStepUp request failed" << causedBy(status); + LOGV2(21582, + "replSetStepUp request failed{causedBy_status}", + "causedBy_status"_attr = causedBy(status)); } uassertStatusOK(status); @@ -761,13 +770,15 @@ public: BSONObjBuilder& result) override { Status status = ReplicationCoordinator::get(opCtx)->checkReplEnabledForCommand(&result); uassertStatusOK(status); - log() << "Received replSetAbortPrimaryCatchUp request"; + LOGV2(21583, "Received replSetAbortPrimaryCatchUp request"); status = ReplicationCoordinator::get(opCtx)->abortCatchupIfNeeded( ReplicationCoordinator::PrimaryCatchUpConclusionReason:: kFailedWithReplSetAbortPrimaryCatchUpCmd); if (!status.isOK()) { - log() << "replSetAbortPrimaryCatchUp request failed" << causedBy(status); + LOGV2(21584, + "replSetAbortPrimaryCatchUp request failed{causedBy_status}", + "causedBy_status"_attr = causedBy(status)); } uassertStatusOK(status); return true; diff --git a/src/mongo/db/repl/replication_consistency_markers_impl.cpp b/src/mongo/db/repl/replication_consistency_markers_impl.cpp index 8d7046a89f2..6464b0311cc 100644 --- a/src/mongo/db/repl/replication_consistency_markers_impl.cpp +++ b/src/mongo/db/repl/replication_consistency_markers_impl.cpp @@ -39,6 +39,7 @@ #include "mongo/db/repl/optime.h" #include "mongo/db/repl/replication_coordinator.h" #include "mongo/db/repl/storage_interface.h" +#include "mongo/logv2/log.h" #include "mongo/util/log.h" namespace mongo { @@ -93,7 +94,7 @@ void ReplicationConsistencyMarkersImpl::_updateMinValidDocument( } void ReplicationConsistencyMarkersImpl::initializeMinValidDocument(OperationContext* opCtx) { - LOG(3) << "Initializing minValid document"; + LOGV2_DEBUG(21282, 3, "Initializing minValid document"); // This initializes the values of the required fields if they are not already set. // If one of the fields is already set, the $max will prefer the existing value since it @@ -114,22 +115,25 @@ void ReplicationConsistencyMarkersImpl::initializeMinValidDocument(OperationCont bool ReplicationConsistencyMarkersImpl::getInitialSyncFlag(OperationContext* opCtx) const { auto doc = _getMinValidDocument(opCtx); if (!doc) { - LOG(3) << "No min valid document found, returning initial sync flag value of false."; + LOGV2_DEBUG( + 21283, 3, "No min valid document found, returning initial sync flag value of false."); return false; } boost::optional<bool> flag = doc->getInitialSyncFlag(); if (!flag) { - LOG(3) << "No initial sync flag set, returning initial sync flag value of false."; + LOGV2_DEBUG( + 21284, 3, "No initial sync flag set, returning initial sync flag value of false."); return false; } - LOG(3) << "returning initial sync flag value of " << flag.get(); + LOGV2_DEBUG( + 21285, 3, "returning initial sync flag value of {flag_get}", "flag_get"_attr = flag.get()); return flag.get(); } void ReplicationConsistencyMarkersImpl::setInitialSyncFlag(OperationContext* opCtx) { - LOG(3) << "setting initial sync flag"; + LOGV2_DEBUG(21286, 3, "setting initial sync flag"); TimestampedBSONObj update; update.obj = BSON("$set" << kInitialSyncFlag); @@ -143,7 +147,7 @@ void ReplicationConsistencyMarkersImpl::setInitialSyncFlag(OperationContext* opC } void ReplicationConsistencyMarkersImpl::clearInitialSyncFlag(OperationContext* opCtx) { - LOG(3) << "clearing initial sync flag"; + LOGV2_DEBUG(21287, 3, "clearing initial sync flag"); auto replCoord = repl::ReplicationCoordinator::get(opCtx); OpTimeAndWallTime opTimeAndWallTime = replCoord->getMyLastAppliedOpTimeAndWallTime(); @@ -184,15 +188,22 @@ OpTime ReplicationConsistencyMarkersImpl::getMinValid(OperationContext* opCtx) c auto minValid = OpTime(doc->getMinValidTimestamp(), doc->getMinValidTerm()); - LOG(3) << "returning minvalid: " << minValid.toString() << "(" << minValid.toBSON() << ")"; + LOGV2_DEBUG(21288, + 3, + "returning minvalid: {minValid}({minValid2})", + "minValid"_attr = minValid.toString(), + "minValid2"_attr = minValid.toBSON()); return minValid; } void ReplicationConsistencyMarkersImpl::setMinValid(OperationContext* opCtx, const OpTime& minValid) { - LOG(3) << "setting minvalid to exactly: " << minValid.toString() << "(" << minValid.toBSON() - << ")"; + LOGV2_DEBUG(21289, + 3, + "setting minvalid to exactly: {minValid}({minValid2})", + "minValid"_attr = minValid.toString(), + "minValid2"_attr = minValid.toBSON()); TimestampedBSONObj update; update.obj = BSON("$set" << BSON(MinValidDocument::kMinValidTimestampFieldName @@ -209,8 +220,11 @@ void ReplicationConsistencyMarkersImpl::setMinValid(OperationContext* opCtx, void ReplicationConsistencyMarkersImpl::setMinValidToAtLeast(OperationContext* opCtx, const OpTime& minValid) { - LOG(3) << "setting minvalid to at least: " << minValid.toString() << "(" << minValid.toBSON() - << ")"; + LOGV2_DEBUG(21290, + 3, + "setting minvalid to at least: {minValid}({minValid2})", + "minValid"_attr = minValid.toString(), + "minValid2"_attr = minValid.toBSON()); auto& termField = MinValidDocument::kMinValidTermFieldName; auto& tsField = MinValidDocument::kMinValidTimestampFieldName; @@ -250,7 +264,11 @@ void ReplicationConsistencyMarkersImpl::setAppliedThrough(OperationContext* opCt const OpTime& optime, bool setTimestamp) { invariant(!optime.isNull()); - LOG(3) << "setting appliedThrough to: " << optime.toString() << "(" << optime.toBSON() << ")"; + LOGV2_DEBUG(21291, + 3, + "setting appliedThrough to: {optime}({optime2})", + "optime"_attr = optime.toString(), + "optime2"_attr = optime.toBSON()); // We set the 'appliedThrough' to the provided timestamp. The 'appliedThrough' is only valid // in checkpoints that contain all writes through this timestamp since it indicates the top of @@ -266,7 +284,10 @@ void ReplicationConsistencyMarkersImpl::setAppliedThrough(OperationContext* opCt void ReplicationConsistencyMarkersImpl::clearAppliedThrough(OperationContext* opCtx, const Timestamp& writeTimestamp) { - LOG(3) << "clearing appliedThrough at: " << writeTimestamp.toString(); + LOGV2_DEBUG(21292, + 3, + "clearing appliedThrough at: {writeTimestamp}", + "writeTimestamp"_attr = writeTimestamp.toString()); TimestampedBSONObj update; update.timestamp = writeTimestamp; @@ -281,11 +302,15 @@ OpTime ReplicationConsistencyMarkersImpl::getAppliedThrough(OperationContext* op auto appliedThrough = doc->getAppliedThrough(); if (!appliedThrough) { - LOG(3) << "No appliedThrough OpTime set, returning empty appliedThrough OpTime."; + LOGV2_DEBUG( + 21293, 3, "No appliedThrough OpTime set, returning empty appliedThrough OpTime."); return {}; } - LOG(3) << "returning appliedThrough: " << appliedThrough->toString() << "(" - << appliedThrough->toBSON() << ")"; + LOGV2_DEBUG(21294, + 3, + "returning appliedThrough: {appliedThrough}({appliedThrough2})", + "appliedThrough"_attr = appliedThrough->toString(), + "appliedThrough2"_attr = appliedThrough->toBSON()); return appliedThrough.get(); } @@ -313,8 +338,11 @@ ReplicationConsistencyMarkersImpl::_getOplogTruncateAfterPointDocument( void ReplicationConsistencyMarkersImpl::ensureFastCountOnOplogTruncateAfterPoint( OperationContext* opCtx) { - LOG(3) << "Updating cached fast-count on collection " << _oplogTruncateAfterPointNss - << " in case an unclean shutdown caused it to become incorrect."; + LOGV2_DEBUG(21295, + 3, + "Updating cached fast-count on collection {oplogTruncateAfterPointNss} in case an " + "unclean shutdown caused it to become incorrect.", + "oplogTruncateAfterPointNss"_attr = _oplogTruncateAfterPointNss); auto result = _storageInterface->findSingleton(opCtx, _oplogTruncateAfterPointNss); @@ -354,7 +382,10 @@ void ReplicationConsistencyMarkersImpl::_upsertOplogTruncateAfterPointDocument( void ReplicationConsistencyMarkersImpl::setOplogTruncateAfterPoint(OperationContext* opCtx, const Timestamp& timestamp) { - LOG(3) << "setting oplog truncate after point to: " << timestamp.toBSON(); + LOGV2_DEBUG(21296, + 3, + "setting oplog truncate after point to: {timestamp}", + "timestamp"_attr = timestamp.toBSON()); _upsertOplogTruncateAfterPointDocument( opCtx, BSON("$set" << BSON(OplogTruncateAfterPointDocument::kOplogTruncateAfterPointFieldName @@ -365,13 +396,14 @@ Timestamp ReplicationConsistencyMarkersImpl::getOplogTruncateAfterPoint( OperationContext* opCtx) const { auto doc = _getOplogTruncateAfterPointDocument(opCtx); if (!doc) { - LOG(3) << "Returning empty oplog truncate after point since document did not exist"; + LOGV2_DEBUG( + 21297, 3, "Returning empty oplog truncate after point since document did not exist"); return {}; } Timestamp out = doc->getOplogTruncateAfterPoint(); - LOG(3) << "returning oplog truncate after point: " << out; + LOGV2_DEBUG(21298, 3, "returning oplog truncate after point: {out}", "out"_attr = out); return out; } diff --git a/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp b/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp index 08bff69143a..29dfa758378 100644 --- a/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp +++ b/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp @@ -94,6 +94,7 @@ #include "mongo/executor/network_interface.h" #include "mongo/executor/network_interface_factory.h" #include "mongo/executor/thread_pool_task_executor.h" +#include "mongo/logv2/log.h" #include "mongo/rpc/metadata/egress_metadata_hook_list.h" #include "mongo/s/catalog/type_shard.h" #include "mongo/s/catalog_cache_loader.h" @@ -238,13 +239,13 @@ void ReplicationCoordinatorExternalStateImpl::startSteadyStateReplication( _bgSync = std::make_unique<BackgroundSync>(replCoord, this, _replicationProcess, _oplogApplier.get()); - log() << "Starting replication fetcher thread"; + LOGV2(21299, "Starting replication fetcher thread"); _bgSync->startup(opCtx); - log() << "Starting replication applier thread"; + LOGV2(21300, "Starting replication applier thread"); _oplogApplierShutdownFuture = _oplogApplier->startup(); - log() << "Starting replication reporter thread"; + LOGV2(21301, "Starting replication reporter thread"); invariant(!_syncSourceFeedbackThread); // Get the pointer while holding the lock so that _stopDataReplication_inlock() won't // leave the unique pointer empty if the _syncSourceFeedbackThread's function starts @@ -275,18 +276,18 @@ void ReplicationCoordinatorExternalStateImpl::_stopDataReplication_inlock( // _syncSourceFeedbackThread should be joined before _bgSync's shutdown because it has // a pointer of _bgSync. if (oldSSF) { - log() << "Stopping replication reporter thread"; + LOGV2(21302, "Stopping replication reporter thread"); _syncSourceFeedback.shutdown(); oldSSF->join(); } if (oldBgSync) { - log() << "Stopping replication fetcher thread"; + LOGV2(21303, "Stopping replication fetcher thread"); oldBgSync->shutdown(opCtx); } if (oldApplier) { - log() << "Stopping replication applier thread"; + LOGV2(21304, "Stopping replication applier thread"); oldApplier->shutdown(); } @@ -321,11 +322,12 @@ void ReplicationCoordinatorExternalStateImpl::startThreads(const ReplSettings& s return; } if (_inShutdown) { - log() << "Not starting replication storage threads because replication is shutting down."; + LOGV2(21305, + "Not starting replication storage threads because replication is shutting down."); return; } - log() << "Starting replication storage threads"; + LOGV2(21306, "Starting replication storage threads"); _service->getStorageEngine()->setJournalListener(this); _oplogApplierTaskExecutor = @@ -383,7 +385,7 @@ void ReplicationCoordinatorExternalStateImpl::shutdown(OperationContext* opCtx) _stopDataReplication_inlock(opCtx, lk); - log() << "Stopping replication storage threads"; + LOGV2(21307, "Stopping replication storage threads"); _taskExecutor->shutdown(); _oplogApplierTaskExecutor->shutdown(); @@ -396,7 +398,7 @@ void ReplicationCoordinatorExternalStateImpl::shutdown(OperationContext* opCtx) // itself can block on the ReplicationCoordinator mutex. It is safe to access _noopWriter // outside of _threadMutex because _noopWriter is protected by its own mutex. invariant(_noopWriter); - LOG(1) << "Stopping noop writer"; + LOGV2_DEBUG(21308, 1, "Stopping noop writer"); _noopWriter->stopWritingPeriodicNoops(); // We must wait for _taskExecutor outside of _threadMutex, since _taskExecutor is used to run @@ -877,7 +879,7 @@ void ReplicationCoordinatorExternalStateImpl::_dropAllTempCollections(OperationC // replica set members. if (*it == "local") continue; - LOG(2) << "Removing temporary collections from " << *it; + LOGV2_DEBUG(21309, 2, "Removing temporary collections from {it}", "it"_attr = *it); AutoGetDb autoDb(opCtx, *it, MODE_IX); invariant(autoDb.getDb(), str::stream() << "Unable to get reference to database " << *it); autoDb.getDb()->clearTmpCollections(opCtx); @@ -922,10 +924,11 @@ void ReplicationCoordinatorExternalStateImpl::notifyOplogMetadataWaiters( _taskExecutor.get(), [committedOpTime, reaper](const executor::TaskExecutor::CallbackArgs& args) { if (MONGO_unlikely(dropPendingCollectionReaperHang.shouldFail())) { - log() << "fail point dropPendingCollectionReaperHang enabled. " - "Blocking until fail point is disabled. " - "committedOpTime: " - << committedOpTime; + LOGV2(21310, + "fail point dropPendingCollectionReaperHang enabled. " + "Blocking until fail point is disabled. " + "committedOpTime: {committedOpTime}", + "committedOpTime"_attr = committedOpTime); dropPendingCollectionReaperHang.pauseWhileSet(); } auto opCtx = cc().makeOperationContext(); diff --git a/src/mongo/db/repl/replication_coordinator_impl.cpp b/src/mongo/db/repl/replication_coordinator_impl.cpp index f1db12180a2..43a9eab03a2 100644 --- a/src/mongo/db/repl/replication_coordinator_impl.cpp +++ b/src/mongo/db/repl/replication_coordinator_impl.cpp @@ -88,6 +88,7 @@ #include "mongo/db/write_concern_options.h" #include "mongo/executor/connection_pool_stats.h" #include "mongo/executor/network_interface.h" +#include "mongo/logv2/log.h" #include "mongo/platform/mutex.h" #include "mongo/rpc/metadata/oplog_query_metadata.h" #include "mongo/rpc/metadata/repl_set_metadata.h" @@ -443,12 +444,14 @@ bool ReplicationCoordinatorImpl::_startLoadLocalConfig(OperationContext* opCtx) StatusWith<LastVote> lastVote = _externalState->loadLocalLastVoteDocument(opCtx); if (!lastVote.isOK()) { - severe() << "Error loading local voted for document at startup; " << lastVote.getStatus(); + LOGV2_FATAL(21429, + "Error loading local voted for document at startup; {lastVote_getStatus}", + "lastVote_getStatus"_attr = lastVote.getStatus()); fassertFailedNoTrace(40367); } if (lastVote.getValue().getTerm() == OpTime::kInitialTerm) { // This log line is checked in unit tests. - log() << "Did not find local initialized voted for document at startup."; + LOGV2(21311, "Did not find local initialized voted for document at startup."); } { stdx::lock_guard<Latch> lk(_mutex); @@ -459,38 +462,45 @@ bool ReplicationCoordinatorImpl::_startLoadLocalConfig(OperationContext* opCtx) auto status = _replicationProcess->refreshRollbackID(opCtx); if (!status.isOK()) { if (status == ErrorCodes::NamespaceNotFound) { - log() << "Did not find local Rollback ID document at startup. Creating one."; + LOGV2(21312, "Did not find local Rollback ID document at startup. Creating one."); auto initializingStatus = _replicationProcess->initializeRollbackID(opCtx); fassert(40424, initializingStatus); } else { - severe() << "Error loading local Rollback ID document at startup; " << status; + LOGV2_FATAL(21430, + "Error loading local Rollback ID document at startup; {status}", + "status"_attr = status); fassertFailedNoTrace(40428); } } StatusWith<BSONObj> cfg = _externalState->loadLocalConfigDocument(opCtx); if (!cfg.isOK()) { - log() << "Did not find local replica set configuration document at startup; " - << cfg.getStatus(); + LOGV2(21313, + "Did not find local replica set configuration document at startup; {cfg_getStatus}", + "cfg_getStatus"_attr = cfg.getStatus()); return true; } ReplSetConfig localConfig; status = localConfig.initialize(cfg.getValue()); if (!status.isOK()) { if (status.code() == ErrorCodes::RepairedReplicaSetNode) { - severe() - << "This instance has been repaired and may contain modified replicated data that " - "would not match other replica set members. To see your repaired data, start " - "mongod without the --replSet option. When you are finished recovering your " - "data and would like to perform a complete re-sync, please refer to the " - "documentation here: " - "https://docs.mongodb.com/manual/tutorial/resync-replica-set-member/"; + LOGV2_FATAL( + 21431, + "This instance has been repaired and may contain modified replicated data that " + "would not match other replica set members. To see your repaired data, start " + "mongod without the --replSet option. When you are finished recovering your " + "data and would like to perform a complete re-sync, please refer to the " + "documentation here: " + "https://docs.mongodb.com/manual/tutorial/resync-replica-set-member/"); fassertFailedNoTrace(50923); } - error() << "Locally stored replica set configuration does not parse; See " - "http://www.mongodb.org/dochub/core/recover-replica-set-from-invalid-config " - "for information on how to recover from this. Got \"" - << status << "\" while parsing " << cfg.getValue(); + LOGV2_ERROR(21414, + "Locally stored replica set configuration does not parse; See " + "http://www.mongodb.org/dochub/core/recover-replica-set-from-invalid-config " + "for information on how to recover from this. Got \"{status}\" while parsing " + "{cfg_getValue}", + "status"_attr = status, + "cfg_getValue"_attr = cfg.getValue()); fassertFailedNoTrace(28545); } @@ -535,7 +545,10 @@ void ReplicationCoordinatorImpl::_finishLoadLocalConfig( const StatusWith<OpTimeAndWallTime>& lastOpTimeAndWallTimeStatus, const StatusWith<LastVote>& lastVoteStatus) { if (!cbData.status.isOK()) { - LOG(1) << "Loading local replica set configuration failed due to " << cbData.status; + LOGV2_DEBUG(21314, + 1, + "Loading local replica set configuration failed due to {cbData_status}", + "cbData_status"_attr = cbData.status); return; } @@ -544,37 +557,51 @@ void ReplicationCoordinatorImpl::_finishLoadLocalConfig( if (!myIndex.isOK()) { if (myIndex.getStatus() == ErrorCodes::NodeNotFound || myIndex.getStatus() == ErrorCodes::InvalidReplicaSetConfig) { - warning() << "Locally stored replica set configuration does not have a valid entry " - "for the current node; waiting for reconfig or remote heartbeat; Got \"" - << myIndex.getStatus() << "\" while validating " << localConfig.toBSON(); + LOGV2_WARNING(21405, + "Locally stored replica set configuration does not have a valid entry " + "for the current node; waiting for reconfig or remote heartbeat; Got " + "\"{myIndex_getStatus}\" while validating {localConfig}", + "myIndex_getStatus"_attr = myIndex.getStatus(), + "localConfig"_attr = localConfig.toBSON()); myIndex = StatusWith<int>(-1); } else { - error() << "Locally stored replica set configuration is invalid; See " - "http://www.mongodb.org/dochub/core/recover-replica-set-from-invalid-config" - " for information on how to recover from this. Got \"" - << myIndex.getStatus() << "\" while validating " << localConfig.toBSON(); + LOGV2_ERROR(21415, + "Locally stored replica set configuration is invalid; See " + "http://www.mongodb.org/dochub/core/recover-replica-set-from-invalid-config" + " for information on how to recover from this. Got \"{myIndex_getStatus}\" " + "while validating {localConfig}", + "myIndex_getStatus"_attr = myIndex.getStatus(), + "localConfig"_attr = localConfig.toBSON()); fassertFailedNoTrace(28544); } } if (localConfig.getReplSetName() != _settings.ourSetName()) { - warning() << "Local replica set configuration document reports set name of " - << localConfig.getReplSetName() << ", but command line reports " - << _settings.ourSetName() << "; waiting for reconfig or remote heartbeat"; + LOGV2_WARNING(21406, + "Local replica set configuration document reports set name of " + "{localConfig_getReplSetName}, but command line reports " + "{settings_ourSetName}; waiting for reconfig or remote heartbeat", + "localConfig_getReplSetName"_attr = localConfig.getReplSetName(), + "settings_ourSetName"_attr = _settings.ourSetName()); myIndex = StatusWith<int>(-1); } if (serverGlobalParams.enableMajorityReadConcern && localConfig.getNumMembers() == 3 && localConfig.getNumDataBearingMembers() == 2) { - log() << startupWarningsLog; - log() << "** WARNING: This replica set has a Primary-Secondary-Arbiter architecture, but " - "readConcern:majority is enabled " - << startupWarningsLog; - log() << "** for this node. This is not a recommended configuration. Please see " - << startupWarningsLog; - log() << "** https://dochub.mongodb.org/core/psa-disable-rc-majority" - << startupWarningsLog; - log() << startupWarningsLog; + LOGV2_OPTIONS(21315, {logv2::LogTag::kStartupWarnings}, ""); + LOGV2_OPTIONS( + 21316, + {logv2::LogTag::kStartupWarnings}, + "** WARNING: This replica set has a Primary-Secondary-Arbiter architecture, but " + "readConcern:majority is enabled "); + LOGV2_OPTIONS( + 21317, + {logv2::LogTag::kStartupWarnings}, + "** for this node. This is not a recommended configuration. Please see "); + LOGV2_OPTIONS(21318, + {logv2::LogTag::kStartupWarnings}, + "** https://dochub.mongodb.org/core/psa-disable-rc-majority"); + LOGV2_OPTIONS(21319, {logv2::LogTag::kStartupWarnings}, ""); } // Do not check optime, if this node is an arbiter. @@ -583,9 +610,12 @@ void ReplicationCoordinatorImpl::_finishLoadLocalConfig( OpTimeAndWallTime lastOpTimeAndWallTime = OpTimeAndWallTime(); if (!isArbiter) { if (!lastOpTimeAndWallTimeStatus.isOK()) { - warning() << "Failed to load timestamp and/or wall clock time of most recently applied " - "operation: " - << lastOpTimeAndWallTimeStatus.getStatus(); + LOGV2_WARNING( + 21407, + "Failed to load timestamp and/or wall clock time of most recently applied " + "operation: {lastOpTimeAndWallTimeStatus_getStatus}", + "lastOpTimeAndWallTimeStatus_getStatus"_attr = + lastOpTimeAndWallTimeStatus.getStatus()); } else { lastOpTimeAndWallTime = lastOpTimeAndWallTimeStatus.getValue(); } @@ -646,7 +676,7 @@ void ReplicationCoordinatorImpl::_finishLoadLocalConfig( // Step down is impossible, so we don't need to wait for the returned event. _updateTerm_inlock(term); } - LOG(1) << "Current term is now " << term; + LOGV2_DEBUG(21320, 1, "Current term is now {term}", "term"_attr = term); _performPostMemberStateUpdateAction(action); if (!isArbiter) { @@ -662,17 +692,21 @@ void ReplicationCoordinatorImpl::_stopDataReplication(OperationContext* opCtx) { _initialSyncer.swap(initialSyncerCopy); } if (initialSyncerCopy) { - LOG(1) - << "ReplicationCoordinatorImpl::_stopDataReplication calling InitialSyncer::shutdown."; + LOGV2_DEBUG( + 21321, + 1, + "ReplicationCoordinatorImpl::_stopDataReplication calling InitialSyncer::shutdown."); const auto status = initialSyncerCopy->shutdown(); if (!status.isOK()) { - warning() << "InitialSyncer shutdown failed: " << status; + LOGV2_WARNING(21408, "InitialSyncer shutdown failed: {status}", "status"_attr = status); } initialSyncerCopy.reset(); // Do not return here, fall through. } - LOG(1) << "ReplicationCoordinatorImpl::_stopDataReplication calling " - "ReplCoordExtState::stopDataReplication."; + LOGV2_DEBUG(21322, + 1, + "ReplicationCoordinatorImpl::_stopDataReplication calling " + "ReplCoordExtState::stopDataReplication."); _externalState->stopDataReplication(opCtx); } @@ -694,7 +728,7 @@ void ReplicationCoordinatorImpl::_startDataReplication(OperationContext* opCtx, // Do initial sync. if (!_externalState->getTaskExecutor()) { - log() << "not running initial sync during test."; + LOGV2(21323, "not running initial sync during test."); return; } @@ -702,16 +736,20 @@ void ReplicationCoordinatorImpl::_startDataReplication(OperationContext* opCtx, { stdx::lock_guard<Latch> lock(_mutex); if (opTimeStatus == ErrorCodes::CallbackCanceled) { - log() << "Initial Sync has been cancelled: " << opTimeStatus.getStatus(); + LOGV2(21324, + "Initial Sync has been cancelled: {opTimeStatus_getStatus}", + "opTimeStatus_getStatus"_attr = opTimeStatus.getStatus()); return; } else if (!opTimeStatus.isOK()) { if (_inShutdown) { - log() << "Initial Sync failed during shutdown due to " - << opTimeStatus.getStatus(); + LOGV2(21325, + "Initial Sync failed during shutdown due to {opTimeStatus_getStatus}", + "opTimeStatus_getStatus"_attr = opTimeStatus.getStatus()); return; } else { - error() << "Initial sync failed, shutting down now. Restart the server " - "to attempt a new initial sync."; + LOGV2_ERROR(21416, + "Initial sync failed, shutting down now. Restart the server " + "to attempt a new initial sync."); fassertFailedWithStatusNoTrace(40088, opTimeStatus.getStatus()); } } @@ -743,7 +781,7 @@ void ReplicationCoordinatorImpl::_startDataReplication(OperationContext* opCtx, // Must take the lock to set _initialSyncer, but not call it. stdx::lock_guard<Latch> lock(_mutex); if (_inShutdown) { - log() << "Initial Sync not starting because replication is shutting down."; + LOGV2(21326, "Initial Sync not starting because replication is shutting down."); return; } initialSyncerCopy = std::make_shared<InitialSyncer>( @@ -761,7 +799,7 @@ void ReplicationCoordinatorImpl::_startDataReplication(OperationContext* opCtx, uassertStatusOK(initialSyncerCopy->startup(opCtx, numInitialSyncAttempts.load())); } catch (const DBException& e) { auto status = e.toStatus(); - log() << "Initial Sync failed to start: " << status; + LOGV2(21327, "Initial Sync failed to start: {status}", "status"_attr = status); if (ErrorCodes::CallbackCanceled == status || ErrorCodes::isShutdownError(status.code())) { return; } @@ -853,7 +891,7 @@ void ReplicationCoordinatorImpl::shutdown(OperationContext* opCtx) { return; } - log() << "shutting down replication subsystems"; + LOGV2(21328, "shutting down replication subsystems"); // Used to shut down outside of the lock. std::shared_ptr<InitialSyncer> initialSyncerCopy; @@ -862,9 +900,10 @@ void ReplicationCoordinatorImpl::shutdown(OperationContext* opCtx) { fassert(28533, !_inShutdown); _inShutdown = true; if (_rsConfigState == kConfigPreStart) { - warning() << "ReplicationCoordinatorImpl::shutdown() called before " - "startup() finished. Shutting down without cleaning up the " - "replication system"; + LOGV2_WARNING(21409, + "ReplicationCoordinatorImpl::shutdown() called before " + "startup() finished. Shutting down without cleaning up the " + "replication system"); return; } if (_rsConfigState == kConfigStartingUp) { @@ -885,10 +924,11 @@ void ReplicationCoordinatorImpl::shutdown(OperationContext* opCtx) { // joining the replication executor is blocking so it must be run outside of the mutex if (initialSyncerCopy) { - LOG(1) << "ReplicationCoordinatorImpl::shutdown calling InitialSyncer::shutdown."; + LOGV2_DEBUG( + 21329, 1, "ReplicationCoordinatorImpl::shutdown calling InitialSyncer::shutdown."); const auto status = initialSyncerCopy->shutdown(); if (!status.isOK()) { - warning() << "InitialSyncer shutdown failed: " << status; + LOGV2_WARNING(21410, "InitialSyncer shutdown failed: {status}", "status"_attr = status); } initialSyncerCopy->join(); initialSyncerCopy.reset(); @@ -1068,7 +1108,9 @@ void ReplicationCoordinatorImpl::signalDrainComplete(OperationContext* opCtx, auto status = _topCoord->completeTransitionToPrimary(firstOpTime); if (status.code() == ErrorCodes::PrimarySteppedDown) { - log() << "Transition to primary failed" << causedBy(status); + LOGV2(21330, + "Transition to primary failed{causedBy_status}", + "causedBy_status"_attr = causedBy(status)); return; } invariant(status); @@ -1082,7 +1124,9 @@ void ReplicationCoordinatorImpl::signalDrainComplete(OperationContext* opCtx, // Update _canAcceptNonLocalWrites _updateMemberStateFromTopologyCoordinator(lk, opCtx); - log() << "transition to primary complete; database writes are now permitted" << rsLog; + LOGV2_OPTIONS(21331, + {logv2::LogTag::kRS}, + "transition to primary complete; database writes are now permitted"); _drainFinishedCond.notify_all(); _externalState->startNoopWriter(_getMyLastAppliedOpTime_inlock()); } @@ -1178,7 +1222,7 @@ void ReplicationCoordinatorImpl::resetMyLastOpTimes() { } void ReplicationCoordinatorImpl::_resetMyLastOpTimes(WithLock lk) { - LOG(1) << "resetting durable/applied optimes."; + LOGV2_DEBUG(21332, 1, "resetting durable/applied optimes."); // Reset to uninitialized OpTime bool isRollbackAllowed = true; _setMyLastAppliedOpTimeAndWallTime( @@ -1411,8 +1455,14 @@ Status ReplicationCoordinatorImpl::_waitUntilOpTime(OperationContext* opCtx, // We just need to wait for the opTime to catch up to what we need (not majority RC). auto future = _opTimeWaiterList.add_inlock(targetOpTime); - LOG(3) << "waitUntilOpTime: OpID " << opCtx->getOpID() << " is waiting for OpTime " - << targetOpTime << " until " << deadline.value_or(opCtx->getDeadline()); + LOGV2_DEBUG(21333, + 3, + "waitUntilOpTime: OpID {opCtx_getOpID} is waiting for OpTime " + "{targetOpTime} until {deadline_value_or_opCtx_getDeadline}", + "opCtx_getOpID"_attr = opCtx->getOpID(), + "targetOpTime"_attr = targetOpTime, + "deadline_value_or_opCtx_getDeadline"_attr = + deadline.value_or(opCtx->getDeadline())); lock.unlock(); auto waitStatus = futureGetNoThrowWithDeadline( @@ -1455,12 +1505,19 @@ Status ReplicationCoordinatorImpl::_waitUntilMajorityOpTime(mongo::OperationCont stdx::unique_lock lock(_mutex); - LOG(1) << "waitUntilOpTime: waiting for optime:" << targetOpTime - << " to be in a snapshot -- current snapshot: " - << _getCurrentCommittedSnapshotOpTime_inlock(); + LOGV2_DEBUG(21334, + 1, + "waitUntilOpTime: waiting for optime:{targetOpTime} to be in a snapshot -- current " + "snapshot: {getCurrentCommittedSnapshotOpTime_inlock}", + "targetOpTime"_attr = targetOpTime, + "getCurrentCommittedSnapshotOpTime_inlock"_attr = + _getCurrentCommittedSnapshotOpTime_inlock()); - LOG(3) << "waitUntilOpTime: waiting for a new snapshot until " - << deadline.value_or(opCtx->getDeadline()); + LOGV2_DEBUG( + 21335, + 3, + "waitUntilOpTime: waiting for a new snapshot until {deadline_value_or_opCtx_getDeadline}", + "deadline_value_or_opCtx_getDeadline"_attr = deadline.value_or(opCtx->getDeadline())); try { auto ok = opCtx->waitForConditionOrInterruptUntil( @@ -1484,7 +1541,10 @@ Status ReplicationCoordinatorImpl::_waitUntilMajorityOpTime(mongo::OperationCont // _currentCommittedSnapshot, _getCurrentCommittedSnapshotOpTime_inlock() also returns // default OpTime{}. Hence this branch only runs if _currentCommittedSnapshot actually // exists. - LOG(3) << "Got notified of new snapshot: " << _currentCommittedSnapshot->toString(); + LOGV2_DEBUG(21336, + 3, + "Got notified of new snapshot: {currentCommittedSnapshot}", + "currentCommittedSnapshot"_attr = _currentCommittedSnapshot->toString()); } return Status::OK(); } @@ -1638,8 +1698,13 @@ bool ReplicationCoordinatorImpl::_doneWaitingForReplication_inlock( // Wait for the "current" snapshot to advance to/past the opTime. const auto haveSnapshot = _currentCommittedSnapshot->opTime >= opTime; if (!haveSnapshot) { - LOG(1) << "Required snapshot optime: " << opTime << " is not yet part of the " - << "current 'committed' snapshot: " << _currentCommittedSnapshot->opTime; + LOGV2_DEBUG(21337, + 1, + "Required snapshot optime: {opTime} is not yet part of the current " + "'committed' snapshot: {currentCommittedSnapshot_opTime}", + "opTime"_attr = opTime, + "currentCommittedSnapshot_opTime"_attr = + _currentCommittedSnapshot->opTime); return false; } @@ -1650,14 +1715,16 @@ bool ReplicationCoordinatorImpl::_doneWaitingForReplication_inlock( // removed from storage. if (auto dropOpTime = _externalState->getEarliestDropPendingOpTime()) { if (*dropOpTime <= opTime) { - LOG(1) << "Unable to satisfy the requested majority write concern at " - "'committed' optime " - << opTime - << ". There are still drop pending collections (earliest drop optime: " - << *dropOpTime - << ") that have to be removed from storage before we can " - "satisfy the write concern " - << writeConcern.toBSON(); + LOGV2_DEBUG(21338, + 1, + "Unable to satisfy the requested majority write concern at " + "'committed' optime {opTime}. There are still drop pending collections " + "(earliest drop optime: {dropOpTime}) that have to be removed from " + "storage before we can " + "satisfy the write concern {writeConcern}", + "opTime"_attr = opTime, + "dropOpTime"_attr = *dropOpTime, + "writeConcern"_attr = writeConcern.toBSON()); return false; } } @@ -1717,9 +1784,13 @@ ReplicationCoordinator::StatusAndDuration ReplicationCoordinatorImpl::awaitRepli if (getTestCommandsEnabled() && !status.isOK()) { stdx::lock_guard lock(_mutex); - log() << "Replication failed for write concern: " << writeConcern.toBSON() - << ", waiting for optime: " << opTime << ", opID: " << opCtx->getOpID() - << ", progress: " << _getReplicationProgress(lock); + LOGV2(21339, + "Replication failed for write concern: {writeConcern}, waiting for optime: {opTime}, " + "opID: {opCtx_getOpID}, progress: {getReplicationProgress_lock}", + "writeConcern"_attr = writeConcern.toBSON(), + "opTime"_attr = opTime, + "opCtx_getOpID"_attr = opCtx->getOpID(), + "getReplicationProgress_lock"_attr = _getReplicationProgress(lock)); } return {std::move(status), duration_cast<Milliseconds>(timer.elapsed())}; } @@ -1862,7 +1933,7 @@ void ReplicationCoordinatorImpl::updateAndLogStateTransitionMetrics( bob.appendNumber("userOpsKilled", userOpsKilled.get()); bob.appendNumber("userOpsRunning", userOpsRunning.get()); - log() << "State transition ops metrics: " << bob.obj(); + LOGV2(21340, "State transition ops metrics: {bob_obj}", "bob_obj"_attr = bob.obj()); } std::shared_ptr<IsMasterResponse> ReplicationCoordinatorImpl::_makeIsMasterResponse( @@ -1954,12 +2025,16 @@ std::shared_ptr<const IsMasterResponse> ReplicationCoordinatorImpl::awaitIsMaste if (MONGO_unlikely(waitForIsMasterResponse.shouldFail())) { // Used in tests that wait for this failpoint to be entered before triggering a topology // change. - log() << "waitForIsMasterResponse failpoint enabled."; + LOGV2(21341, "waitForIsMasterResponse failpoint enabled."); } // Wait for a topology change with timeout set to deadline. - LOG(1) << "Waiting for an isMaster response from a topology change or until deadline: " - << deadline.get() << ". Current TopologyVersion counter is " << topologyVersionCounter; + LOGV2_DEBUG(21342, + 1, + "Waiting for an isMaster response from a topology change or until deadline: " + "{deadline_get}. Current TopologyVersion counter is {topologyVersionCounter}", + "deadline_get"_attr = deadline.get(), + "topologyVersionCounter"_attr = topologyVersionCounter); auto statusWithIsMaster = futureGetNoThrowWithDeadline(opCtx, future, deadline.get(), opCtx->getTimeoutError()); auto status = statusWithIsMaster.getStatus(); @@ -2050,7 +2125,7 @@ void ReplicationCoordinatorImpl::AutoGetRstlForStepUpStepDown::_killOpThreadFn() invariant(!cc().isFromUserConnection()); - log() << "Starting to kill user operations"; + LOGV2(21343, "Starting to kill user operations"); auto uniqueOpCtx = cc().makeOperationContext(); OperationContext* opCtx = uniqueOpCtx.get(); @@ -2078,7 +2153,7 @@ void ReplicationCoordinatorImpl::AutoGetRstlForStepUpStepDown::_killOpThreadFn() stdx::unique_lock<Latch> lock(_mutex); if (_stopKillingOps.wait_for( lock, Milliseconds(10).toSystemDuration(), [this] { return _killSignaled; })) { - log() << "Stopped killing user operations"; + LOGV2(21344, "Stopped killing user operations"); _replCord->updateAndLogStateTransitionMetrics( _stateTransition, getUserOpsKilled(), getUserOpsRunning()); _killSignaled = false; @@ -2189,9 +2264,10 @@ void ReplicationCoordinatorImpl::stepDown(OperationContext* opCtx, lk.unlock(); if (MONGO_unlikely(stepdownHangBeforePerformingPostMemberStateUpdateActions.shouldFail())) { - log() << "stepping down from primary - " - "stepdownHangBeforePerformingPostMemberStateUpdateActions fail point enabled. " - "Blocking until fail point is disabled."; + LOGV2(21345, + "stepping down from primary - " + "stepdownHangBeforePerformingPostMemberStateUpdateActions fail point enabled. " + "Blocking until fail point is disabled."); while (MONGO_unlikely( stepdownHangBeforePerformingPostMemberStateUpdateActions.shouldFail())) { mongo::sleepsecs(1); @@ -2313,31 +2389,41 @@ void ReplicationCoordinatorImpl::_performElectionHandoff() { auto candidateIndex = _topCoord->chooseElectionHandoffCandidate(); if (candidateIndex < 0) { - log() << "Could not find node to hand off election to."; + LOGV2(21346, "Could not find node to hand off election to."); return; } auto target = _rsConfig.getMemberAt(candidateIndex).getHostAndPort(); executor::RemoteCommandRequest request( target, "admin", BSON("replSetStepUp" << 1 << "skipDryRun" << true), nullptr); - log() << "Handing off election to " << target; + LOGV2(21347, "Handing off election to {target}", "target"_attr = target); auto callbackHandleSW = _replExecutor->scheduleRemoteCommand( request, [target](const executor::TaskExecutor::RemoteCommandCallbackArgs& callbackData) { auto status = callbackData.response.status; if (status.isOK()) { - LOG(1) << "replSetStepUp request to " << target << " succeeded with response -- " - << callbackData.response.data; + LOGV2_DEBUG(21348, + 1, + "replSetStepUp request to {target} succeeded with response -- " + "{callbackData_response_data}", + "target"_attr = target, + "callbackData_response_data"_attr = callbackData.response.data); } else { - log() << "replSetStepUp request to " << target << " failed due to " << status; + LOGV2(21349, + "replSetStepUp request to {target} failed due to {status}", + "target"_attr = target, + "status"_attr = status); } }); auto callbackHandleStatus = callbackHandleSW.getStatus(); if (!callbackHandleStatus.isOK()) { - error() << "Failed to schedule ReplSetStepUp request to " << target - << " for election handoff: " << callbackHandleStatus; + LOGV2_ERROR(21417, + "Failed to schedule ReplSetStepUp request to {target} for election handoff: " + "{callbackHandleStatus}", + "target"_attr = target, + "callbackHandleStatus"_attr = callbackHandleStatus); } } @@ -2643,18 +2729,24 @@ Status ReplicationCoordinatorImpl::setMaintenanceMode(bool activate) { int curMaintenanceCalls = _topCoord->getMaintenanceCount(); if (activate) { - log() << "going into maintenance mode with " << curMaintenanceCalls - << " other maintenance mode tasks in progress" << rsLog; + LOGV2_OPTIONS(21350, + {logv2::LogTag::kRS}, + "going into maintenance mode with {curMaintenanceCalls} other maintenance " + "mode tasks in progress", + "curMaintenanceCalls"_attr = curMaintenanceCalls); _topCoord->adjustMaintenanceCountBy(1); } else if (curMaintenanceCalls > 0) { invariant(_topCoord->getRole() == TopologyCoordinator::Role::kFollower); _topCoord->adjustMaintenanceCountBy(-1); - log() << "leaving maintenance mode (" << curMaintenanceCalls - 1 - << " other maintenance mode tasks ongoing)" << rsLog; + LOGV2_OPTIONS(21351, + {logv2::LogTag::kRS}, + "leaving maintenance mode ({curMaintenanceCalls_1} other maintenance mode " + "tasks ongoing)", + "curMaintenanceCalls_1"_attr = curMaintenanceCalls - 1); } else { - warning() << "Attempted to leave maintenance mode but it is not currently active"; + LOGV2_WARNING(21411, "Attempted to leave maintenance mode but it is not currently active"); return Status(ErrorCodes::OperationFailed, "already out of maintenance mode"); } @@ -2707,8 +2799,9 @@ Status ReplicationCoordinatorImpl::processReplSetFreeze(int secs, BSONObjBuilder Status ReplicationCoordinatorImpl::processReplSetReconfig(OperationContext* opCtx, const ReplSetReconfigArgs& args, BSONObjBuilder* resultObj) { - log() << "replSetReconfig admin command received from client; new config: " - << args.newConfigObj; + LOGV2(21352, + "replSetReconfig admin command received from client; new config: {args_newConfigObj}", + "args_newConfigObj"_attr = args.newConfigObj); stdx::unique_lock<Latch> lk(_mutex); @@ -2732,7 +2825,9 @@ Status ReplicationCoordinatorImpl::processReplSetReconfig(OperationContext* opCt "Cannot run replSetReconfig because the node is currently updating " "its configuration"); default: - severe() << "Unexpected _rsConfigState " << int(_rsConfigState); + LOGV2_FATAL(21432, + "Unexpected _rsConfigState {int_rsConfigState}", + "int_rsConfigState"_attr = int(_rsConfigState)); fassertFailed(18914); } @@ -2767,41 +2862,49 @@ Status ReplicationCoordinatorImpl::processReplSetReconfig(OperationContext* opCt // field passed in through its args. Instead, use this node's term. Status status = newConfig.initialize(newConfigObj, topCoordTerm, oldConfig.getReplicaSetId()); if (!status.isOK()) { - error() << "replSetReconfig got " << status << " while parsing " << newConfigObj; + LOGV2_ERROR(21418, + "replSetReconfig got {status} while parsing {newConfigObj}", + "status"_attr = status, + "newConfigObj"_attr = newConfigObj); return Status(ErrorCodes::InvalidReplicaSetConfig, status.reason()); } if (newConfig.getReplSetName() != _settings.ourSetName()) { str::stream errmsg; errmsg << "Attempting to reconfigure a replica set with name " << newConfig.getReplSetName() << ", but command line reports " << _settings.ourSetName() << "; rejecting"; - error() << std::string(errmsg); + LOGV2_ERROR(21419, "{std_string_errmsg}", "std_string_errmsg"_attr = std::string(errmsg)); return Status(ErrorCodes::InvalidReplicaSetConfig, errmsg); } StatusWith<int> myIndex = validateConfigForReconfig( _externalState.get(), oldConfig, newConfig, opCtx->getServiceContext(), args.force); if (!myIndex.isOK()) { - error() << "replSetReconfig got " << myIndex.getStatus() << " while validating " - << newConfigObj; + LOGV2_ERROR(21420, + "replSetReconfig got {myIndex_getStatus} while validating {newConfigObj}", + "myIndex_getStatus"_attr = myIndex.getStatus(), + "newConfigObj"_attr = newConfigObj); return Status(ErrorCodes::NewReplicaSetConfigurationIncompatible, myIndex.getStatus().reason()); } - log() << "replSetReconfig config object with " << newConfig.getNumMembers() - << " members parses ok"; + LOGV2(21353, + "replSetReconfig config object with {newConfig_getNumMembers} members parses ok", + "newConfig_getNumMembers"_attr = newConfig.getNumMembers()); if (!args.force) { status = checkQuorumForReconfig( _replExecutor.get(), newConfig, myIndex.getValue(), _topCoord->getTerm()); if (!status.isOK()) { - error() << "replSetReconfig failed; " << status; + LOGV2_ERROR(21421, "replSetReconfig failed; {status}", "status"_attr = status); return status; } } status = _externalState->storeLocalConfigDocument(opCtx, newConfig.toBSON()); if (!status.isOK()) { - error() << "replSetReconfig failed to store config document; " << status; + LOGV2_ERROR(21422, + "replSetReconfig failed to store config document; {status}", + "status"_attr = status); return status; } @@ -2825,8 +2928,11 @@ void ReplicationCoordinatorImpl::_finishReplSetReconfig(OperationContext* opCtx, // we have already set our ReplicationCoordinatorImpl::_rsConfigState state to // "kConfigReconfiguring" which prevents new elections from happening. if (electionFinishedEvent) { - LOG(2) << "Waiting for election to complete before finishing reconfig to config with " - << newConfig.getConfigVersionAndTerm(); + LOGV2_DEBUG(21354, + 2, + "Waiting for election to complete before finishing reconfig to config with " + "{newConfig_getConfigVersionAndTerm}", + "newConfig_getConfigVersionAndTerm"_attr = newConfig.getConfigVersionAndTerm()); // Wait for the election to complete and the node's Role to be set to follower. _replExecutor->waitForEvent(electionFinishedEvent); } @@ -2844,7 +2950,7 @@ void ReplicationCoordinatorImpl::_finishReplSetReconfig(OperationContext* opCtx, lk.lock(); if (_topCoord->isSteppingDownUnconditionally()) { invariant(opCtx->lockState()->isRSTLExclusive()); - log() << "stepping down from primary, because we received a new config"; + LOGV2(21355, "stepping down from primary, because we received a new config"); // We need to release the mutex before yielding locks for prepared transactions, which // might check out sessions, to avoid deadlocks with checked-out sessions accessing // this mutex. @@ -2892,7 +2998,7 @@ void ReplicationCoordinatorImpl::_finishReplSetReconfig(OperationContext* opCtx, Status ReplicationCoordinatorImpl::processReplSetInitiate(OperationContext* opCtx, const BSONObj& configObj, BSONObjBuilder* resultObj) { - log() << "replSetInitiate admin command received from client"; + LOGV2(21356, "replSetInitiate admin command received from client"); const auto replEnabled = _settings.usingReplSets(); stdx::unique_lock<Latch> lk(_mutex); @@ -2925,27 +3031,33 @@ Status ReplicationCoordinatorImpl::processReplSetInitiate(OperationContext* opCt ReplSetConfig newConfig; Status status = newConfig.initializeForInitiate(configObj); if (!status.isOK()) { - error() << "replSet initiate got " << status << " while parsing " << configObj; + LOGV2_ERROR(21423, + "replSet initiate got {status} while parsing {configObj}", + "status"_attr = status, + "configObj"_attr = configObj); return Status(ErrorCodes::InvalidReplicaSetConfig, status.reason()); } if (newConfig.getReplSetName() != _settings.ourSetName()) { str::stream errmsg; errmsg << "Attempting to initiate a replica set with name " << newConfig.getReplSetName() << ", but command line reports " << _settings.ourSetName() << "; rejecting"; - error() << std::string(errmsg); + LOGV2_ERROR(21424, "{std_string_errmsg}", "std_string_errmsg"_attr = std::string(errmsg)); return Status(ErrorCodes::InvalidReplicaSetConfig, errmsg); } StatusWith<int> myIndex = validateConfigForInitiate(_externalState.get(), newConfig, opCtx->getServiceContext()); if (!myIndex.isOK()) { - error() << "replSet initiate got " << myIndex.getStatus() << " while validating " - << configObj; + LOGV2_ERROR(21425, + "replSet initiate got {myIndex_getStatus} while validating {configObj}", + "myIndex_getStatus"_attr = myIndex.getStatus(), + "configObj"_attr = configObj); return Status(ErrorCodes::InvalidReplicaSetConfig, myIndex.getStatus().reason()); } - log() << "replSetInitiate config object with " << newConfig.getNumMembers() - << " members parses ok"; + LOGV2(21357, + "replSetInitiate config object with {newConfig_getNumMembers} members parses ok", + "newConfig_getNumMembers"_attr = newConfig.getNumMembers()); // In pv1, the TopologyCoordinator has not set the term yet. It will be set to kInitialTerm if // the initiate succeeds so we pass that here. @@ -2953,14 +3065,15 @@ Status ReplicationCoordinatorImpl::processReplSetInitiate(OperationContext* opCt _replExecutor.get(), newConfig, myIndex.getValue(), OpTime::kInitialTerm); if (!status.isOK()) { - error() << "replSetInitiate failed; " << status; + LOGV2_ERROR(21426, "replSetInitiate failed; {status}", "status"_attr = status); return status; } status = _externalState->initializeReplSetStorage(opCtx, newConfig.toBSON()); if (!status.isOK()) { - error() << "replSetInitiate failed to store config document or create the oplog; " - << status; + LOGV2_ERROR(21427, + "replSetInitiate failed to store config document or create the oplog; {status}", + "status"_attr = status); return status; } @@ -3177,7 +3290,11 @@ ReplicationCoordinatorImpl::_updateMemberStateFromTopologyCoordinator(WithLock l _cancelPriorityTakeover_inlock(); } - log() << "transition to " << newState << " from " << _memberState << rsLog; + LOGV2_OPTIONS(21358, + {logv2::LogTag::kRS}, + "transition to {newState} from {memberState}", + "newState"_attr = newState, + "memberState"_attr = _memberState); // Initializes the featureCompatibilityVersion to the latest value, because arbiters do not // receive the replicated version. This is to avoid bugs like SERVER-32639. if (newState.arbiter()) { @@ -3219,7 +3336,9 @@ void ReplicationCoordinatorImpl::_performPostMemberStateUpdateAction( _startElectSelfV1(StartElectionReasonEnum::kElectionTimeout); break; default: - severe() << "Unknown post member state update action " << static_cast<int>(action); + LOGV2_FATAL(21433, + "Unknown post member state update action {static_cast_int_action}", + "static_cast_int_action"_attr = static_cast<int>(action)); fassertFailed(26010); } } @@ -3249,7 +3368,7 @@ void ReplicationCoordinatorImpl::_onFollowerModeStateChange() { } void ReplicationCoordinatorImpl::CatchupState::start_inlock() { - log() << "Entering primary catch-up mode."; + LOGV2(21359, "Entering primary catch-up mode."); // Reset the number of catchup operations performed before starting catchup. _numCatchUpOps = 0; @@ -3264,7 +3383,7 @@ void ReplicationCoordinatorImpl::CatchupState::start_inlock() { // When catchUpTimeoutMillis is 0, we skip doing catchup entirely. if (catchupTimeout == ReplSetConfig::kCatchUpDisabled) { - log() << "Skipping primary catchup since the catchup timeout is 0."; + LOGV2(21360, "Skipping primary catchup since the catchup timeout is 0."); abort_inlock(PrimaryCatchUpConclusionReason::kSkipped); return; } @@ -3279,7 +3398,7 @@ void ReplicationCoordinatorImpl::CatchupState::start_inlock() { if (cbData.myHandle.isCanceled()) { return; } - log() << "Catchup timed out after becoming primary."; + LOGV2(21361, "Catchup timed out after becoming primary."); abort_inlock(PrimaryCatchUpConclusionReason::kTimedOut); }; @@ -3292,7 +3411,7 @@ void ReplicationCoordinatorImpl::CatchupState::start_inlock() { auto timeoutDate = _repl->_replExecutor->now() + catchupTimeout; auto status = _repl->_replExecutor->scheduleWorkAt(timeoutDate, std::move(timeoutCB)); if (!status.isOK()) { - log() << "Failed to schedule catchup timeout work."; + LOGV2(21362, "Failed to schedule catchup timeout work."); abort_inlock(PrimaryCatchUpConclusionReason::kFailedWithError); return; } @@ -3305,7 +3424,7 @@ void ReplicationCoordinatorImpl::CatchupState::abort_inlock(PrimaryCatchUpConclu ReplicationMetrics::get(getGlobalServiceContext()) .incrementNumCatchUpsConcludedForReason(reason); - log() << "Exited primary catch-up mode."; + LOGV2(21363, "Exited primary catch-up mode."); // Clean up its own members. if (_timeoutCbh) { _repl->_replExecutor->cancel(_timeoutCbh); @@ -3331,8 +3450,11 @@ void ReplicationCoordinatorImpl::CatchupState::signalHeartbeatUpdate_inlock() { // We've caught up. const auto myLastApplied = _repl->_getMyLastAppliedOpTime_inlock(); if (*targetOpTime <= myLastApplied) { - log() << "Caught up to the latest optime known via heartbeats after becoming primary. " - << "Target optime: " << *targetOpTime << ". My Last Applied: " << myLastApplied; + LOGV2(21364, + "Caught up to the latest optime known via heartbeats after becoming primary. Target " + "optime: {targetOpTime}. My Last Applied: {myLastApplied}", + "targetOpTime"_attr = *targetOpTime, + "myLastApplied"_attr = myLastApplied); // Report the number of ops applied during catchup in replSetGetStatus once the primary is // caught up. ReplicationMetrics::get(getGlobalServiceContext()).setNumCatchUpOps(_numCatchUpOps); @@ -3348,12 +3470,17 @@ void ReplicationCoordinatorImpl::CatchupState::signalHeartbeatUpdate_inlock() { ReplicationMetrics::get(getGlobalServiceContext()).setTargetCatchupOpTime(_targetOpTime); - log() << "Heartbeats updated catchup target optime to " << _targetOpTime; - log() << "Latest known optime per replica set member:"; + LOGV2(21365, + "Heartbeats updated catchup target optime to {targetOpTime}", + "targetOpTime"_attr = _targetOpTime); + LOGV2(21366, "Latest known optime per replica set member:"); auto opTimesPerMember = _repl->_topCoord->latestKnownOpTimeSinceHeartbeatRestartPerMember(); for (auto&& pair : opTimesPerMember) { - log() << "Member ID: " << pair.first - << ", latest known optime: " << (pair.second ? (*pair.second).toString() : "unknown"); + LOGV2(21367, + "Member ID: {pair_first}, latest known optime: {pair_second_pair_second_unknown}", + "pair_first"_attr = pair.first, + "pair_second_pair_second_unknown"_attr = + (pair.second ? (*pair.second).toString() : "unknown")); } if (_waiter) { @@ -3370,8 +3497,11 @@ void ReplicationCoordinatorImpl::CatchupState::signalHeartbeatUpdate_inlock() { // Double check the target time since stepdown may signal us too. const auto myLastApplied = _repl->_getMyLastAppliedOpTime_inlock(); if (_targetOpTime <= myLastApplied) { - log() << "Caught up to the latest known optime successfully after becoming primary. " - << "Target optime: " << _targetOpTime << ". My Last Applied: " << myLastApplied; + LOGV2(21368, + "Caught up to the latest known optime successfully after becoming primary. " + "Target optime: {targetOpTime}. My Last Applied: {myLastApplied}", + "targetOpTime"_attr = _targetOpTime, + "myLastApplied"_attr = myLastApplied); // Report the number of ops applied during catchup in replSetGetStatus once the primary // is caught up. ReplicationMetrics::get(getGlobalServiceContext()).setNumCatchUpOps(_numCatchUpOps); @@ -3442,21 +3572,32 @@ ReplicationCoordinatorImpl::_setCurrentRSConfig(WithLock lk, if (storageEngine && !storageEngine->isDurable() && (newConfig.getWriteConcernMajorityShouldJournal() && (!oldConfig.isInitialized() || !oldConfig.getWriteConcernMajorityShouldJournal()))) { - log() << startupWarningsLog; - log() << "** WARNING: This replica set node is running without journaling enabled but the " - << startupWarningsLog; - log() << "** writeConcernMajorityJournalDefault option to the replica set config " - << startupWarningsLog; - log() << "** is set to true. The writeConcernMajorityJournalDefault " - << startupWarningsLog; - log() << "** option to the replica set config must be set to false " - << startupWarningsLog; - log() << "** or w:majority write concerns will never complete." - << startupWarningsLog; - log() << "** In addition, this node's memory consumption may increase until all" - << startupWarningsLog; - log() << "** available free RAM is exhausted." << startupWarningsLog; - log() << startupWarningsLog; + LOGV2_OPTIONS(21369, {logv2::LogTag::kStartupWarnings}, ""); + LOGV2_OPTIONS( + 21370, + {logv2::LogTag::kStartupWarnings}, + "** WARNING: This replica set node is running without journaling enabled but the "); + LOGV2_OPTIONS( + 21371, + {logv2::LogTag::kStartupWarnings}, + "** writeConcernMajorityJournalDefault option to the replica set config "); + LOGV2_OPTIONS(21372, + {logv2::LogTag::kStartupWarnings}, + "** is set to true. The writeConcernMajorityJournalDefault "); + LOGV2_OPTIONS(21373, + {logv2::LogTag::kStartupWarnings}, + "** option to the replica set config must be set to false "); + LOGV2_OPTIONS(21374, + {logv2::LogTag::kStartupWarnings}, + "** or w:majority write concerns will never complete."); + LOGV2_OPTIONS( + 21375, + {logv2::LogTag::kStartupWarnings}, + "** In addition, this node's memory consumption may increase until all"); + LOGV2_OPTIONS(21376, + {logv2::LogTag::kStartupWarnings}, + "** available free RAM is exhausted."); + LOGV2_OPTIONS(21377, {logv2::LogTag::kStartupWarnings}, ""); } // Warn if using the in-memory (ephemeral) storage engine with @@ -3464,44 +3605,66 @@ ReplicationCoordinatorImpl::_setCurrentRSConfig(WithLock lk, if (storageEngine && storageEngine->isEphemeral() && (newConfig.getWriteConcernMajorityShouldJournal() && (!oldConfig.isInitialized() || !oldConfig.getWriteConcernMajorityShouldJournal()))) { - log() << startupWarningsLog; - log() << "** WARNING: This replica set node is using in-memory (ephemeral) storage with the" - << startupWarningsLog; - log() << "** writeConcernMajorityJournalDefault option to the replica set config " - << startupWarningsLog; - log() << "** set to true. The writeConcernMajorityJournalDefault option to the " - << startupWarningsLog; - log() << "** replica set config must be set to false " << startupWarningsLog; - log() << "** or w:majority write concerns will never complete." - << startupWarningsLog; - log() << "** In addition, this node's memory consumption may increase until all" - << startupWarningsLog; - log() << "** available free RAM is exhausted." << startupWarningsLog; - log() << startupWarningsLog; + LOGV2_OPTIONS(21378, {logv2::LogTag::kStartupWarnings}, ""); + LOGV2_OPTIONS( + 21379, + {logv2::LogTag::kStartupWarnings}, + "** WARNING: This replica set node is using in-memory (ephemeral) storage with the"); + LOGV2_OPTIONS( + 21380, + {logv2::LogTag::kStartupWarnings}, + "** writeConcernMajorityJournalDefault option to the replica set config "); + LOGV2_OPTIONS( + 21381, + {logv2::LogTag::kStartupWarnings}, + "** set to true. The writeConcernMajorityJournalDefault option to the "); + LOGV2_OPTIONS(21382, + {logv2::LogTag::kStartupWarnings}, + "** replica set config must be set to false "); + LOGV2_OPTIONS(21383, + {logv2::LogTag::kStartupWarnings}, + "** or w:majority write concerns will never complete."); + LOGV2_OPTIONS( + 21384, + {logv2::LogTag::kStartupWarnings}, + "** In addition, this node's memory consumption may increase until all"); + LOGV2_OPTIONS(21385, + {logv2::LogTag::kStartupWarnings}, + "** available free RAM is exhausted."); + LOGV2_OPTIONS(21386, {logv2::LogTag::kStartupWarnings}, ""); } // Since the ReplSetConfig always has a WriteConcernOptions, the only way to know if it has been // customized is if it's different to the implicit defaults of { w: 1, wtimeout: 0 }. if (const auto& wc = newConfig.getDefaultWriteConcern(); !(wc.wNumNodes == 1 && wc.wTimeout == 0)) { - log() << startupWarningsLog; - log() << "** WARNING: Replica set config contains customized getLastErrorDefaults," - << startupWarningsLog; - log() << "** which are deprecated. Use setDefaultRWConcern instead to set a" - << startupWarningsLog; - log() << "** cluster-wide default writeConcern." << startupWarningsLog; - log() << startupWarningsLog; + LOGV2_OPTIONS(21387, {logv2::LogTag::kStartupWarnings}, ""); + LOGV2_OPTIONS(21388, + {logv2::LogTag::kStartupWarnings}, + "** WARNING: Replica set config contains customized getLastErrorDefaults,"); + LOGV2_OPTIONS(21389, + {logv2::LogTag::kStartupWarnings}, + "** which are deprecated. Use setDefaultRWConcern instead to set a"); + LOGV2_OPTIONS(21390, + {logv2::LogTag::kStartupWarnings}, + "** cluster-wide default writeConcern."); + LOGV2_OPTIONS(21391, {logv2::LogTag::kStartupWarnings}, ""); } const bool horizonsChanged = _haveHorizonsChanged(oldConfig, newConfig, _selfIndex, myIndex); - log() << "New replica set config in use: " << _rsConfig.toBSON() << rsLog; + LOGV2_OPTIONS(21392, + {logv2::LogTag::kRS}, + "New replica set config in use: {rsConfig}", + "rsConfig"_attr = _rsConfig.toBSON()); _selfIndex = myIndex; if (_selfIndex >= 0) { - log() << "This node is " << _rsConfig.getMemberAt(_selfIndex).getHostAndPort() - << " in the config"; + LOGV2(21393, + "This node is {rsConfig_getMemberAt_selfIndex_getHostAndPort} in the config", + "rsConfig_getMemberAt_selfIndex_getHostAndPort"_attr = + _rsConfig.getMemberAt(_selfIndex).getHostAndPort()); } else { - log() << "This node is not a member of the config"; + LOGV2(21394, "This node is not a member of the config"); } if (horizonsChanged) { @@ -3736,9 +3899,11 @@ void ReplicationCoordinatorImpl::resetLastOpTimesFromOplog(OperationContext* opC auto lastOpTimeAndWallTimeStatus = _externalState->loadLastOpTimeAndWallTime(opCtx); OpTimeAndWallTime lastOpTimeAndWallTime = {OpTime(), Date_t()}; if (!lastOpTimeAndWallTimeStatus.getStatus().isOK()) { - warning() << "Failed to load timestamp and/or wall clock time of most recently applied " - "operation; " - << lastOpTimeAndWallTimeStatus.getStatus(); + LOGV2_WARNING(21412, + "Failed to load timestamp and/or wall clock time of most recently applied " + "operation; {lastOpTimeAndWallTimeStatus_getStatus}", + "lastOpTimeAndWallTimeStatus_getStatus"_attr = + lastOpTimeAndWallTimeStatus.getStatus()); } else { lastOpTimeAndWallTime = lastOpTimeAndWallTimeStatus.getValue(); } @@ -3899,7 +4064,7 @@ MONGO_FAIL_POINT_DEFINE(disableSnapshotting); void ReplicationCoordinatorImpl::_setStableTimestampForStorage(WithLock lk) { if (!_shouldSetStableTimestamp) { - LOG(2) << "Not setting stable timestamp for storage."; + LOGV2_DEBUG(21395, 2, "Not setting stable timestamp for storage."); return; } // Get the current stable optime. @@ -3908,7 +4073,10 @@ void ReplicationCoordinatorImpl::_setStableTimestampForStorage(WithLock lk) { // If there is a valid stable optime, set it for the storage engine, and then remove any // old, unneeded stable optime candidates. if (stableOpTime) { - LOG(2) << "Setting replication's stable optime to " << stableOpTime.value(); + LOGV2_DEBUG(21396, + 2, + "Setting replication's stable optime to {stableOpTime_value}", + "stableOpTime_value"_attr = stableOpTime.value()); if (!gTestingSnapshotBehaviorInIsolation) { // Update committed snapshot and wake up any threads waiting on read concern or @@ -3962,15 +4130,17 @@ void ReplicationCoordinatorImpl::finishRecoveryIfEligible(OperationContext* opCt // We can only transition to SECONDARY from RECOVERING state. MemberState state(getMemberState()); if (!state.recovering()) { - LOG(2) << "We cannot transition to SECONDARY state since we are not currently in " - "RECOVERING state. Current state: " - << state.toString(); + LOGV2_DEBUG(21397, + 2, + "We cannot transition to SECONDARY state since we are not currently in " + "RECOVERING state. Current state: {state}", + "state"_attr = state.toString()); return; } // Maintenance mode will force us to remain in RECOVERING state, no matter what. if (getMaintenanceMode()) { - LOG(1) << "We cannot transition to SECONDARY state while in maintenance mode."; + LOGV2_DEBUG(21398, 1, "We cannot transition to SECONDARY state while in maintenance mode."); return; } @@ -3980,17 +4150,26 @@ void ReplicationCoordinatorImpl::finishRecoveryIfEligible(OperationContext* opCt auto lastApplied = getMyLastAppliedOpTime(); auto minValid = _replicationProcess->getConsistencyMarkers()->getMinValid(opCtx); if (lastApplied < minValid) { - LOG(2) << "We cannot transition to SECONDARY state because our 'lastApplied' optime" - " is less than the 'minValid' optime. minValid optime: " - << minValid << ", lastApplied optime: " << lastApplied; + LOGV2_DEBUG(21399, + 2, + "We cannot transition to SECONDARY state because our 'lastApplied' optime" + " is less than the 'minValid' optime. minValid optime: {minValid}, lastApplied " + "optime: {lastApplied}", + "minValid"_attr = minValid, + "lastApplied"_attr = lastApplied); return; } // Execute the transition to SECONDARY. auto status = setFollowerMode(MemberState::RS_SECONDARY); if (!status.isOK()) { - warning() << "Failed to transition into " << MemberState(MemberState::RS_SECONDARY) - << ". Current state: " << getMemberState() << causedBy(status); + LOGV2_WARNING(21413, + "Failed to transition into {MemberState_MemberState_RS_SECONDARY}. Current " + "state: {getMemberState}{causedBy_status}", + "MemberState_MemberState_RS_SECONDARY"_attr = + MemberState(MemberState::RS_SECONDARY), + "getMemberState"_attr = getMemberState(), + "causedBy_status"_attr = causedBy(status)); } } @@ -4057,7 +4236,9 @@ Status ReplicationCoordinatorImpl::processReplSetRequestVotes( if (votedForCandidate) { Status status = _externalState->storeLocalLastVoteDocument(opCtx, lastVote); if (!status.isOK()) { - error() << "replSetRequestVotes failed to store LastVote document; " << status; + LOGV2_ERROR(21428, + "replSetRequestVotes failed to store LastVote document; {status}", + "status"_attr = status); return status; } } @@ -4160,8 +4341,10 @@ Status ReplicationCoordinatorImpl::processHeartbeatV1(const ReplSetHeartbeatArgs // a configuration that contains us. Chances are excellent that it will, since that // is the only reason for a remote node to send this node a heartbeat request. if (!senderHost.empty() && _seedList.insert(senderHost).second) { - log() << "Scheduling heartbeat to fetch a new config from: " << senderHost - << " since we are not a member of our current config."; + LOGV2(21400, + "Scheduling heartbeat to fetch a new config from: {senderHost} since we are not " + "a member of our current config.", + "senderHost"_attr = senderHost); _scheduleHeartbeatToTarget_inlock(senderHost, -1, now); } } else if (result.isOK() && @@ -4171,9 +4354,12 @@ Status ReplicationCoordinatorImpl::processHeartbeatV1(const ReplSetHeartbeatArgs // We cannot cancel the enqueued heartbeat, but either this one or the enqueued heartbeat // will trigger reconfig, which cancels and reschedules all heartbeats. if (args.hasSender()) { - log() << "Scheduling heartbeat to fetch a newer config with term " - << args.getConfigTerm() << " and version " << args.getConfigVersion() - << " from member: " << senderHost; + LOGV2(21401, + "Scheduling heartbeat to fetch a newer config with term {args_getConfigTerm} and " + "version {args_getConfigVersion} from member: {senderHost}", + "args_getConfigTerm"_attr = args.getConfigTerm(), + "args_getConfigVersion"_attr = args.getConfigVersion(), + "senderHost"_attr = senderHost); int senderIndex = _rsConfig.findMemberIndexByHostAndPort(senderHost); _scheduleHeartbeatToTarget_inlock(senderHost, senderIndex, now); } @@ -4251,12 +4437,16 @@ EventHandle ReplicationCoordinatorImpl::_updateTerm_inlock( _pendingTermUpdateDuringStepDown = term; } if (_topCoord->prepareForUnconditionalStepDown()) { - log() << "stepping down from primary, because a new term has begun: " << term; + LOGV2(21402, + "stepping down from primary, because a new term has begun: {term}", + "term"_attr = term); ReplicationMetrics::get(getServiceContext()).incrementNumStepDownsCausedByHigherTerm(); return _stepDownStart(); } else { - LOG(2) << "Updated term but not triggering stepdown because we are already in the " - "process of stepping down"; + LOGV2_DEBUG(21403, + 2, + "Updated term but not triggering stepdown because we are already in the " + "process of stepping down"); } } return EventHandle(); @@ -4314,7 +4504,7 @@ bool ReplicationCoordinatorImpl::_updateCommittedSnapshot( // If we are in ROLLBACK state, do not set any new _currentCommittedSnapshot, as it will be // cleared at the end of rollback anyway. if (_memberState.rollback()) { - log() << "Not updating committed snapshot because we are in rollback"; + LOGV2(21404, "Not updating committed snapshot because we are in rollback"); return false; } invariant(!newCommittedSnapshot.opTime.isNull()); diff --git a/src/mongo/db/repl/replication_coordinator_impl_elect_v1.cpp b/src/mongo/db/repl/replication_coordinator_impl_elect_v1.cpp index 6d44c96bc0e..61910c41c1e 100644 --- a/src/mongo/db/repl/replication_coordinator_impl_elect_v1.cpp +++ b/src/mongo/db/repl/replication_coordinator_impl_elect_v1.cpp @@ -37,6 +37,7 @@ #include "mongo/db/repl/replication_metrics.h" #include "mongo/db/repl/topology_coordinator.h" #include "mongo/db/repl/vote_requester.h" +#include "mongo/logv2/log.h" #include "mongo/platform/mutex.h" #include "mongo/util/log.h" #include "mongo/util/scopeguard.h" @@ -55,7 +56,9 @@ public: if (_dismissed) { return; } - log() << "Lost " << (_isDryRun ? "dry run " : "") << "election due to internal error"; + LOGV2(21434, + "Lost {isDryRun_dry_run}election due to internal error", + "isDryRun_dry_run"_attr = (_isDryRun ? "dry run " : "")); _replCoord->_topCoord->processLoseElection(); _replCoord->_voteRequester.reset(nullptr); if (_isDryRun && _replCoord->_electionDryRunFinishedEvent.isValid()) { @@ -105,13 +108,15 @@ void ReplicationCoordinatorImpl::_startElectSelfV1_inlock(StartElectionReasonEnu case kConfigInitiating: case kConfigReconfiguring: case kConfigHBReconfiguring: - LOG(2) << "Not standing for election; processing a configuration change"; + LOGV2_DEBUG(21435, 2, "Not standing for election; processing a configuration change"); // Transition out of candidate role. _topCoord->processLoseElection(); return; default: - severe() << "Entered replica set election code while in illegal config state " - << int(_rsConfigState); + LOGV2_FATAL(21452, + "Entered replica set election code while in illegal config state " + "{int_rsConfigState}", + "int_rsConfigState"_attr = int(_rsConfigState)); fassertFailed(28641); } @@ -134,8 +139,9 @@ void ReplicationCoordinatorImpl::_startElectSelfV1_inlock(StartElectionReasonEnu const auto lastOpTime = _getMyLastAppliedOpTime_inlock(); if (lastOpTime == OpTime()) { - log() << "not trying to elect self, " - "do not yet have a complete set of data from any point in time"; + LOGV2(21436, + "not trying to elect self, " + "do not yet have a complete set of data from any point in time"); return; } @@ -144,13 +150,17 @@ void ReplicationCoordinatorImpl::_startElectSelfV1_inlock(StartElectionReasonEnu if (reason == StartElectionReasonEnum::kStepUpRequestSkipDryRun) { long long newTerm = term + 1; - log() << "skipping dry run and running for election in term " << newTerm; + LOGV2(21437, + "skipping dry run and running for election in term {newTerm}", + "newTerm"_attr = newTerm); _startRealElection_inlock(newTerm, reason); lossGuard.dismiss(); return; } - log() << "conducting a dry run election to see if we could be elected. current term: " << term; + LOGV2(21438, + "conducting a dry run election to see if we could be elected. current term: {term}", + "term"_attr = term); _voteRequester.reset(new VoteRequester); // Only set primaryIndex if the primary's vote is required during the dry run. @@ -186,29 +196,34 @@ void ReplicationCoordinatorImpl::_processDryRunResult(long long originalTerm, invariant(_voteRequester); if (_topCoord->getTerm() != originalTerm) { - log() << "not running for primary, we have been superseded already during dry run. " - << "original term: " << originalTerm << ", current term: " << _topCoord->getTerm(); + LOGV2(21439, + "not running for primary, we have been superseded already during dry run. original " + "term: {originalTerm}, current term: {topCoord_getTerm}", + "originalTerm"_attr = originalTerm, + "topCoord_getTerm"_attr = _topCoord->getTerm()); return; } const VoteRequester::Result endResult = _voteRequester->getResult(); if (endResult == VoteRequester::Result::kInsufficientVotes) { - log() << "not running for primary, we received insufficient votes"; + LOGV2(21440, "not running for primary, we received insufficient votes"); return; } else if (endResult == VoteRequester::Result::kStaleTerm) { - log() << "not running for primary, we have been superseded already"; + LOGV2(21441, "not running for primary, we have been superseded already"); return; } else if (endResult == VoteRequester::Result::kPrimaryRespondedNo) { - log() << "not running for primary, the current primary responded no in the dry run"; + LOGV2(21442, "not running for primary, the current primary responded no in the dry run"); return; } else if (endResult != VoteRequester::Result::kSuccessfullyElected) { - log() << "not running for primary, we received an unexpected problem"; + LOGV2(21443, "not running for primary, we received an unexpected problem"); return; } long long newTerm = originalTerm + 1; - log() << "dry election run succeeded, running for election in term " << newTerm; + LOGV2(21444, + "dry election run succeeded, running for election in term {newTerm}", + "newTerm"_attr = newTerm); _startRealElection_inlock(newTerm, reason); lossGuard.dismiss(); @@ -291,14 +306,18 @@ void ReplicationCoordinatorImpl::_writeLastVoteForMyElection( } if (!status.isOK()) { - log() << "failed to store LastVote document when voting for myself: " << status; + LOGV2(21445, + "failed to store LastVote document when voting for myself: {status}", + "status"_attr = status); return; } if (_topCoord->getTerm() != lastVote.getTerm()) { - log() << "not running for primary, we have been superseded already while writing our last " - "vote. election term: " - << lastVote.getTerm() << ", current term: " << _topCoord->getTerm(); + LOGV2(21446, + "not running for primary, we have been superseded already while writing our last " + "vote. election term: {lastVote_getTerm}, current term: {topCoord_getTerm}", + "lastVote_getTerm"_attr = lastVote.getTerm(), + "topCoord_getTerm"_attr = _topCoord->getTerm()); return; } _startVoteRequester_inlock(lastVote.getTerm(), reason); @@ -336,8 +355,11 @@ void ReplicationCoordinatorImpl::_onVoteRequestComplete(long long newTerm, invariant(_voteRequester); if (_topCoord->getTerm() != newTerm) { - log() << "not becoming primary, we have been superseded already during election. " - << "election term: " << newTerm << ", current term: " << _topCoord->getTerm(); + LOGV2(21447, + "not becoming primary, we have been superseded already during election. election " + "term: {newTerm}, current term: {topCoord_getTerm}", + "newTerm"_attr = newTerm, + "topCoord_getTerm"_attr = _topCoord->getTerm()); return; } @@ -346,13 +368,15 @@ void ReplicationCoordinatorImpl::_onVoteRequestComplete(long long newTerm, switch (endResult) { case VoteRequester::Result::kInsufficientVotes: - log() << "not becoming primary, we received insufficient votes"; + LOGV2(21448, "not becoming primary, we received insufficient votes"); return; case VoteRequester::Result::kStaleTerm: - log() << "not becoming primary, we have been superseded already"; + LOGV2(21449, "not becoming primary, we have been superseded already"); return; case VoteRequester::Result::kSuccessfullyElected: - log() << "election succeeded, assuming primary role in term " << _topCoord->getTerm(); + LOGV2(21450, + "election succeeded, assuming primary role in term {topCoord_getTerm}", + "topCoord_getTerm"_attr = _topCoord->getTerm()); ReplicationMetrics::get(getServiceContext()) .incrementNumElectionsSuccessfulForReason(reason); break; @@ -372,9 +396,10 @@ void ReplicationCoordinatorImpl::_onVoteRequestComplete(long long newTerm, electionHangsBeforeUpdateMemberState.execute([&](const BSONObj& customWait) { auto waitForMillis = Milliseconds(customWait["waitForMillis"].numberInt()); - log() << "election succeeded - electionHangsBeforeUpdateMemberState fail point " - "enabled, sleeping " - << waitForMillis; + LOGV2(21451, + "election succeeded - electionHangsBeforeUpdateMemberState fail point " + "enabled, sleeping {waitForMillis}", + "waitForMillis"_attr = waitForMillis); sleepFor(waitForMillis); }); diff --git a/src/mongo/db/repl/replication_coordinator_impl_elect_v1_test.cpp b/src/mongo/db/repl/replication_coordinator_impl_elect_v1_test.cpp index bb44ed0bcd4..59db90faba0 100644 --- a/src/mongo/db/repl/replication_coordinator_impl_elect_v1_test.cpp +++ b/src/mongo/db/repl/replication_coordinator_impl_elect_v1_test.cpp @@ -44,6 +44,7 @@ #include "mongo/db/repl/replication_metrics.h" #include "mongo/db/repl/topology_coordinator.h" #include "mongo/executor/network_interface_mock.h" +#include "mongo/logv2/log.h" #include "mongo/unittest/unittest.h" #include "mongo/util/fail_point.h" #include "mongo/util/log.h" @@ -129,7 +130,9 @@ TEST_F(ReplCoordTest, ElectionSucceedsWhenNodeIsTheOnlyElectableNode) { auto electionTimeoutWhen = getReplCoord()->getElectionTimeout_forTest(); ASSERT_NOT_EQUALS(Date_t(), electionTimeoutWhen); - log() << "Election timeout scheduled at " << electionTimeoutWhen << " (simulator time)"; + LOGV2(21453, + "Election timeout scheduled at {electionTimeoutWhen} (simulator time)", + "electionTimeoutWhen"_attr = electionTimeoutWhen); NetworkInterfaceMock* net = getNet(); net->enterNetwork(); @@ -140,8 +143,10 @@ TEST_F(ReplCoordTest, ElectionSucceedsWhenNodeIsTheOnlyElectableNode) { } auto noi = net->getNextReadyRequest(); const auto& request = noi->getRequest(); - error() << "Black holing irrelevant request to " << request.target << ": " - << request.cmdObj; + LOGV2_ERROR(21473, + "Black holing irrelevant request to {request_target}: {request_cmdObj}", + "request_target"_attr = request.target, + "request_cmdObj"_attr = request.cmdObj); net->blackHole(noi); } net->exitNetwork(); @@ -354,7 +359,9 @@ TEST_F(ReplCoordTest, ElectionFailsWhenInsufficientVotesAreReceivedDuringDryRun) auto electionTimeoutWhen = getReplCoord()->getElectionTimeout_forTest(); ASSERT_NOT_EQUALS(Date_t(), electionTimeoutWhen); - log() << "Election timeout scheduled at " << electionTimeoutWhen << " (simulator time)"; + LOGV2(21454, + "Election timeout scheduled at {electionTimeoutWhen} (simulator time)", + "electionTimeoutWhen"_attr = electionTimeoutWhen); int voteRequests = 0; NetworkInterfaceMock* net = getNet(); @@ -366,7 +373,10 @@ TEST_F(ReplCoordTest, ElectionFailsWhenInsufficientVotesAreReceivedDuringDryRun) ASSERT_TRUE(net->hasReadyRequests()); const NetworkInterfaceMock::NetworkOperationIterator noi = net->getNextReadyRequest(); const RemoteCommandRequest& request = noi->getRequest(); - log() << request.target.toString() << " processing " << request.cmdObj; + LOGV2(21455, + "{request_target} processing {request_cmdObj}", + "request_target"_attr = request.target.toString(), + "request_cmdObj"_attr = request.cmdObj); if (consumeHeartbeatV1(noi)) { // The heartbeat has been consumed. } else if (request.cmdObj.firstElement().fieldNameStringData() == "replSetRequestVotes") { @@ -423,7 +433,9 @@ TEST_F(ReplCoordTest, ElectionFailsWhenDryRunResponseContainsANewerTerm) { auto electionTimeoutWhen = getReplCoord()->getElectionTimeout_forTest(); ASSERT_NOT_EQUALS(Date_t(), electionTimeoutWhen); - log() << "Election timeout scheduled at " << electionTimeoutWhen << " (simulator time)"; + LOGV2(21456, + "Election timeout scheduled at {electionTimeoutWhen} (simulator time)", + "electionTimeoutWhen"_attr = electionTimeoutWhen); int voteRequests = 0; NetworkInterfaceMock* net = getNet(); @@ -435,7 +447,10 @@ TEST_F(ReplCoordTest, ElectionFailsWhenDryRunResponseContainsANewerTerm) { ASSERT_TRUE(net->hasReadyRequests()); const NetworkInterfaceMock::NetworkOperationIterator noi = net->getNextReadyRequest(); const RemoteCommandRequest& request = noi->getRequest(); - log() << request.target.toString() << " processing " << request.cmdObj; + LOGV2(21457, + "{request_target} processing {request_cmdObj}", + "request_target"_attr = request.target.toString(), + "request_cmdObj"_attr = request.cmdObj); if (consumeHeartbeatV1(noi)) { // The heartbeat has been consumed. } else if (request.cmdObj.firstElement().fieldNameStringData() == "replSetRequestVotes") { @@ -529,7 +544,10 @@ TEST_F(ReplCoordTest, NodeWillNotStandForElectionDuringHeartbeatReconfig) { for (int i = 0; i < 2; ++i) { const NetworkInterfaceMock::NetworkOperationIterator noi = net->getNextReadyRequest(); const RemoteCommandRequest& request = noi->getRequest(); - log() << request.target.toString() << " processing " << request.cmdObj; + LOGV2(21458, + "{request_target} processing {request_cmdObj}", + "request_target"_attr = request.target.toString(), + "request_cmdObj"_attr = request.cmdObj); ReplSetHeartbeatArgsV1 hbArgs; if (hbArgs.initialize(request.cmdObj).isOK()) { ReplSetHeartbeatResponse hbResp; @@ -543,8 +561,10 @@ TEST_F(ReplCoordTest, NodeWillNotStandForElectionDuringHeartbeatReconfig) { BSONObjBuilder respObj; net->scheduleResponse(noi, net->now(), makeResponseStatus(hbResp.toBSON())); } else { - error() << "Black holing unexpected request to " << request.target << ": " - << request.cmdObj; + LOGV2_ERROR(21474, + "Black holing unexpected request to {request_target}: {request_cmdObj}", + "request_target"_attr = request.target, + "request_cmdObj"_attr = request.cmdObj); net->blackHole(noi); } net->runReadyNetworkOperations(); @@ -554,7 +574,9 @@ TEST_F(ReplCoordTest, NodeWillNotStandForElectionDuringHeartbeatReconfig) { // Advance the simulator clock sufficiently to trigger an election. auto electionTimeoutWhen = getReplCoord()->getElectionTimeout_forTest(); ASSERT_NOT_EQUALS(Date_t(), electionTimeoutWhen); - log() << "Election timeout scheduled at " << electionTimeoutWhen << " (simulator time)"; + LOGV2(21459, + "Election timeout scheduled at {electionTimeoutWhen} (simulator time)", + "electionTimeoutWhen"_attr = electionTimeoutWhen); net->enterNetwork(); while (net->now() < electionTimeoutWhen) { @@ -607,7 +629,10 @@ TEST_F(ReplCoordTest, ElectionFailsWhenInsufficientVotesAreReceivedDuringRequest while (net->hasReadyRequests()) { const NetworkInterfaceMock::NetworkOperationIterator noi = net->getNextReadyRequest(); const RemoteCommandRequest& request = noi->getRequest(); - log() << request.target.toString() << " processing " << request.cmdObj; + LOGV2(21460, + "{request_target} processing {request_cmdObj}", + "request_target"_attr = request.target.toString(), + "request_cmdObj"_attr = request.cmdObj); if (request.cmdObj.firstElement().fieldNameStringData() != "replSetRequestVotes") { net->blackHole(noi); } else { @@ -700,7 +725,10 @@ TEST_F(ReplCoordTest, ElectionFailsWhenVoteRequestResponseContainsANewerTerm) { while (net->hasReadyRequests()) { const NetworkInterfaceMock::NetworkOperationIterator noi = net->getNextReadyRequest(); const RemoteCommandRequest& request = noi->getRequest(); - log() << request.target.toString() << " processing " << request.cmdObj; + LOGV2(21461, + "{request_target} processing {request_cmdObj}", + "request_target"_attr = request.target.toString(), + "request_cmdObj"_attr = request.cmdObj); if (request.cmdObj.firstElement().fieldNameStringData() != "replSetRequestVotes") { net->blackHole(noi); } else { @@ -796,7 +824,10 @@ TEST_F(ReplCoordTest, ElectionFailsWhenTermChangesDuringActualElection) { while (net->hasReadyRequests()) { const NetworkInterfaceMock::NetworkOperationIterator noi = net->getNextReadyRequest(); const RemoteCommandRequest& request = noi->getRequest(); - log() << request.target.toString() << " processing " << request.cmdObj; + LOGV2(21462, + "{request_target} processing {request_cmdObj}", + "request_target"_attr = request.target.toString(), + "request_cmdObj"_attr = request.cmdObj); if (request.cmdObj.firstElement().fieldNameStringData() != "replSetRequestVotes") { net->blackHole(noi); } else { @@ -941,7 +972,11 @@ private: noi = net->getNextReadyRequest(); auto&& request = noi->getRequest(); - log() << request.target << " processing " << request.cmdObj << " at " << net->now(); + LOGV2(21463, + "{request_target} processing {request_cmdObj} at {net_now}", + "request_target"_attr = request.target, + "request_cmdObj"_attr = request.cmdObj, + "net_now"_attr = net->now()); // Make sure the heartbeat request is valid. ReplSetHeartbeatArgsV1 hbArgs; @@ -1549,10 +1584,16 @@ TEST_F(TakeoverTest, CatchupTakeoverDryRunFailsPrimarySaysNo) { NetworkInterfaceMock::NetworkOperationIterator noi_primary; Date_t until = net->now() + Seconds(1); while (voteRequests < votesExpected) { - unittest::log() << "request: " << voteRequests << " expected: " << votesExpected; + LOGV2(21464, + "request: {voteRequests} expected: {votesExpected}", + "voteRequests"_attr = voteRequests, + "votesExpected"_attr = votesExpected); const NetworkInterfaceMock::NetworkOperationIterator noi = net->getNextReadyRequest(); const RemoteCommandRequest& request = noi->getRequest(); - log() << request.target.toString() << " processing " << request.cmdObj; + LOGV2(21465, + "{request_target} processing {request_cmdObj}", + "request_target"_attr = request.target.toString(), + "request_cmdObj"_attr = request.cmdObj); if (request.cmdObj.firstElement().fieldNameStringData() != "replSetRequestVotes") { net->blackHole(noi); } else { @@ -2125,13 +2166,17 @@ protected: auto electionTimeoutWhen = replCoord->getElectionTimeout_forTest(); ASSERT_NOT_EQUALS(Date_t(), electionTimeoutWhen); - log() << "Election timeout scheduled at " << electionTimeoutWhen << " (simulator time)"; + LOGV2(21466, + "Election timeout scheduled at {electionTimeoutWhen} (simulator time)", + "electionTimeoutWhen"_attr = electionTimeoutWhen); ASSERT(replCoord->getMemberState().secondary()) << replCoord->getMemberState().toString(); // Process requests until we're primary but leave the heartbeats for the notification // of election win. Exit immediately on unexpected requests. while (!replCoord->getMemberState().primary()) { - log() << "Waiting on network in state " << replCoord->getMemberState(); + LOGV2(21467, + "Waiting on network in state {replCoord_getMemberState}", + "replCoord_getMemberState"_attr = replCoord->getMemberState()); net->enterNetwork(); if (net->now() < electionTimeoutWhen) { net->runUntil(electionTimeoutWhen); @@ -2139,7 +2184,10 @@ protected: // Peek the next request, don't consume it yet. const NetworkOpIter noi = net->getFrontOfUnscheduledQueue(); const RemoteCommandRequest& request = noi->getRequest(); - log() << request.target.toString() << " processing " << request.cmdObj; + LOGV2(21468, + "{request_target} processing {request_cmdObj}", + "request_target"_attr = request.target.toString(), + "request_cmdObj"_attr = request.cmdObj); if (ReplSetHeartbeatArgsV1().initialize(request.cmdObj).isOK()) { OpTime opTime(Timestamp(), getReplCoord()->getTerm()); net->scheduleResponse( @@ -2204,13 +2252,18 @@ protected: while (net->hasReadyRequests()) { const NetworkInterfaceMock::NetworkOperationIterator noi = net->getNextReadyRequest(); const RemoteCommandRequest& request = noi->getRequest(); - log() << request.target.toString() << " processing heartbeat " << request.cmdObj - << " at " << net->now(); + LOGV2(21469, + "{request_target} processing heartbeat {request_cmdObj} at {net_now}", + "request_target"_attr = request.target.toString(), + "request_cmdObj"_attr = request.cmdObj, + "net_now"_attr = net->now()); if (ReplSetHeartbeatArgsV1().initialize(request.cmdObj).isOK()) { onHeartbeatRequest(noi); } else { - log() << "Black holing unexpected request to " << request.target << ": " - << request.cmdObj; + LOGV2(21470, + "Black holing unexpected request to {request_target}: {request_cmdObj}", + "request_target"_attr = request.target, + "request_cmdObj"_attr = request.cmdObj); net->blackHole(noi); } net->runReadyNetworkOperations(); @@ -2227,7 +2280,11 @@ protected: // Peek the next request auto noi = net->getFrontOfUnscheduledQueue(); auto& request = noi->getRequest(); - log() << request.target << " at " << net->now() << " processing " << request.cmdObj; + LOGV2(21471, + "{request_target} at {net_now} processing {request_cmdObj}", + "request_target"_attr = request.target, + "net_now"_attr = net->now(), + "request_cmdObj"_attr = request.cmdObj); if (ReplSetHeartbeatArgsV1().initialize(request.cmdObj).isOK()) { // Consume the next request onHeartbeatRequest(net->getNextReadyRequest()); @@ -2434,7 +2491,9 @@ TEST_F(PrimaryCatchUpTest, HeartbeatTimeout) { replyHeartbeatsAndRunUntil(catchupTimeoutTime, [this, time1](const NetworkOpIter noi) { const RemoteCommandRequest& request = noi->getRequest(); if (request.target.host() == "node2") { - log() << "Black holing heartbeat from " << request.target.host(); + LOGV2(21472, + "Black holing heartbeat from {request_target_host}", + "request_target_host"_attr = request.target.host()); getNet()->blackHole(noi); } else { getNet()->scheduleResponse(noi, getNet()->now(), makeHeartbeatResponse(time1)); diff --git a/src/mongo/db/repl/replication_coordinator_impl_heartbeat.cpp b/src/mongo/db/repl/replication_coordinator_impl_heartbeat.cpp index db9d264aa16..3b928f80df0 100644 --- a/src/mongo/db/repl/replication_coordinator_impl_heartbeat.cpp +++ b/src/mongo/db/repl/replication_coordinator_impl_heartbeat.cpp @@ -54,6 +54,7 @@ #include "mongo/db/repl/topology_coordinator.h" #include "mongo/db/repl/vote_requester.h" #include "mongo/db/service_context.h" +#include "mongo/logv2/log.h" #include "mongo/platform/mutex.h" #include "mongo/rpc/get_status_from_command_result.h" #include "mongo/rpc/metadata/repl_set_metadata.h" @@ -274,11 +275,13 @@ stdx::unique_lock<Latch> ReplicationCoordinatorImpl::_handleHeartbeatResponseAct case HeartbeatResponseAction::StepDownSelf: invariant(action.getPrimaryConfigIndex() == _selfIndex); if (_topCoord->prepareForUnconditionalStepDown()) { - log() << "Stepping down from primary in response to heartbeat"; + LOGV2(21475, "Stepping down from primary in response to heartbeat"); _stepDownStart(); } else { - LOG(2) << "Heartbeat would have triggered a stepdown, but we're already in the " - "process of stepping down"; + LOGV2_DEBUG(21476, + 2, + "Heartbeat would have triggered a stepdown, but we're already in the " + "process of stepping down"); } break; case HeartbeatResponseAction::PriorityTakeover: { @@ -325,11 +328,18 @@ void remoteStepdownCallback(const executor::TaskExecutor::RemoteCommandCallbackA } if (status.isOK()) { - LOG(1) << "stepdown of primary(" << cbData.request.target << ") succeeded with response -- " - << cbData.response.data; + LOGV2_DEBUG(21477, + 1, + "stepdown of primary({cbData_request_target}) succeeded with response -- " + "{cbData_response_data}", + "cbData_request_target"_attr = cbData.request.target, + "cbData_response_data"_attr = cbData.response.data); } else { - warning() << "stepdown of primary(" << cbData.request.target << ") failed due to " - << cbData.response.status; + LOGV2_WARNING( + 21486, + "stepdown of primary({cbData_request_target}) failed due to {cbData_response_status}", + "cbData_request_target"_attr = cbData.request.target, + "cbData_response_status"_attr = cbData.response.status); } } } // namespace @@ -344,7 +354,7 @@ void ReplicationCoordinatorImpl::_requestRemotePrimaryStepdown(const HostAndPort 20LL)), nullptr); - log() << "Requesting " << target << " step down from primary"; + LOGV2(21478, "Requesting {target} step down from primary", "target"_attr = target); auto cbh = _replExecutor->scheduleRemoteCommand(request, remoteStepdownCallback); if (cbh.getStatus() != ErrorCodes::ShutdownInProgress) { fassert(18808, cbh.getStatus()); @@ -375,8 +385,9 @@ void ReplicationCoordinatorImpl::_stepDownFinish( if (MONGO_unlikely(blockHeartbeatStepdown.shouldFail())) { // This log output is used in js tests so please leave it. - log() << "stepDown - blockHeartbeatStepdown fail point enabled. " - "Blocking until fail point is disabled."; + LOGV2(21479, + "stepDown - blockHeartbeatStepdown fail point enabled. " + "Blocking until fail point is disabled."); auto inShutdown = [&] { stdx::lock_guard<Latch> lk(_mutex); @@ -458,8 +469,10 @@ void ReplicationCoordinatorImpl::_scheduleHeartbeatReconfig_inlock(const ReplSet case kConfigPreStart: case kConfigStartingUp: case kConfigReplicationDisabled: - severe() << "Reconfiguration request occurred while _rsConfigState == " - << int(_rsConfigState) << "; aborting."; + LOGV2_FATAL(21491, + "Reconfiguration request occurred while _rsConfigState == " + "{int_rsConfigState}; aborting.", + "int_rsConfigState"_attr = int(_rsConfigState)); fassertFailed(18807); } _setConfigState_inlock(kConfigHBReconfiguring); @@ -489,8 +502,10 @@ void ReplicationCoordinatorImpl::_heartbeatReconfigStore( const executor::TaskExecutor::CallbackArgs& cbd, const ReplSetConfig& newConfig) { if (cbd.status.code() == ErrorCodes::CallbackCanceled) { - log() << "The callback to persist the replica set configuration was canceled - " - << "the configuration was not persisted but was used: " << newConfig.toBSON(); + LOGV2(21480, + "The callback to persist the replica set configuration was canceled - the " + "configuration was not persisted but was used: {newConfig}", + "newConfig"_attr = newConfig.toBSON()); return; } @@ -513,9 +528,10 @@ void ReplicationCoordinatorImpl::_heartbeatReconfigStore( bool shouldStartDataReplication = false; if (!myIndex.getStatus().isOK() && myIndex.getStatus() != ErrorCodes::NodeNotFound) { - warning() << "Not persisting new configuration in heartbeat response to disk because " - "it is invalid: " - << myIndex.getStatus(); + LOGV2_WARNING(21487, + "Not persisting new configuration in heartbeat response to disk because " + "it is invalid: {myIndex_getStatus}", + "myIndex_getStatus"_attr = myIndex.getStatus()); } else { LOG_FOR_HEARTBEATS(2) << "Config with " << newConfig.getConfigVersionAndTerm() << " validated for reconfig; persisting to disk."; @@ -527,9 +543,10 @@ void ReplicationCoordinatorImpl::_heartbeatReconfigStore( stdx::lock_guard<Latch> lk(_mutex); isFirstConfig = !_rsConfig.isInitialized(); if (!status.isOK()) { - error() << "Ignoring new configuration in heartbeat response because we failed to" - " write it to stable storage; " - << status; + LOGV2_ERROR(21488, + "Ignoring new configuration in heartbeat response because we failed to" + " write it to stable storage; {status}", + "status"_attr = status); invariant(_rsConfigState == kConfigHBReconfiguring); if (isFirstConfig) { _setConfigState_inlock(kConfigUninitialized); @@ -626,7 +643,8 @@ void ReplicationCoordinatorImpl::_heartbeatReconfigFinish( lk.lock(); if (_topCoord->isSteppingDownUnconditionally()) { invariant(opCtx->lockState()->isRSTLExclusive()); - log() << "stepping down from primary, because we received a new config via heartbeat"; + LOGV2(21481, + "stepping down from primary, because we received a new config via heartbeat"); // We need to release the mutex before yielding locks for prepared transactions, which // might check out sessions, to avoid deadlocks with checked-out sessions accessing // this mutex. @@ -655,18 +673,24 @@ void ReplicationCoordinatorImpl::_heartbeatReconfigFinish( if (!myIndex.isOK()) { switch (myIndex.getStatus().code()) { case ErrorCodes::NodeNotFound: - log() << "Cannot find self in new replica set configuration; I must be removed; " - << myIndex.getStatus(); + LOGV2(21482, + "Cannot find self in new replica set configuration; I must be removed; " + "{myIndex_getStatus}", + "myIndex_getStatus"_attr = myIndex.getStatus()); break; case ErrorCodes::InvalidReplicaSetConfig: - error() << "Several entries in new config represent this node; " - "Removing self until an acceptable configuration arrives; " - << myIndex.getStatus(); + LOGV2_ERROR( + 21489, + "Several entries in new config represent this node; " + "Removing self until an acceptable configuration arrives; {myIndex_getStatus}", + "myIndex_getStatus"_attr = myIndex.getStatus()); break; default: - error() << "Could not validate configuration received from remote node; " - "Removing self until an acceptable configuration arrives; " - << myIndex.getStatus(); + LOGV2_ERROR( + 21490, + "Could not validate configuration received from remote node; " + "Removing self until an acceptable configuration arrives; {myIndex_getStatus}", + "myIndex_getStatus"_attr = myIndex.getStatus()); break; } myIndex = StatusWith<int>(-1); @@ -778,7 +802,8 @@ void ReplicationCoordinatorImpl::_scheduleNextLivenessUpdate_inlock() { } auto nextTimeout = earliestDate + _rsConfig.getElectionTimeoutPeriod(); - LOG(3) << "scheduling next check at " << nextTimeout; + LOGV2_DEBUG( + 21483, 3, "scheduling next check at {nextTimeout}", "nextTimeout"_attr = nextTimeout); // It is possible we will schedule the next timeout in the past. // ThreadPoolTaskExecutor::_scheduleWorkAt() schedules its work immediately if it's given a @@ -810,7 +835,7 @@ void ReplicationCoordinatorImpl::_cancelAndRescheduleLivenessUpdate_inlock(int u void ReplicationCoordinatorImpl::_cancelPriorityTakeover_inlock() { if (_priorityTakeoverCbh.isValid()) { - log() << "Canceling priority takeover callback"; + LOGV2(21484, "Canceling priority takeover callback"); _replExecutor->cancel(_priorityTakeoverCbh); _priorityTakeoverCbh = CallbackHandle(); _priorityTakeoverWhen = Date_t(); @@ -819,7 +844,7 @@ void ReplicationCoordinatorImpl::_cancelPriorityTakeover_inlock() { void ReplicationCoordinatorImpl::_cancelCatchupTakeover_inlock() { if (_catchupTakeoverCbh.isValid()) { - log() << "Canceling catchup takeover callback"; + LOGV2(21485, "Canceling catchup takeover callback"); _replExecutor->cancel(_catchupTakeoverCbh); _catchupTakeoverCbh = CallbackHandle(); _catchupTakeoverWhen = Date_t(); diff --git a/src/mongo/db/repl/replication_coordinator_impl_heartbeat_v1_test.cpp b/src/mongo/db/repl/replication_coordinator_impl_heartbeat_v1_test.cpp index 5532d72f701..27271617d88 100644 --- a/src/mongo/db/repl/replication_coordinator_impl_heartbeat_v1_test.cpp +++ b/src/mongo/db/repl/replication_coordinator_impl_heartbeat_v1_test.cpp @@ -41,6 +41,7 @@ #include "mongo/db/repl/replication_coordinator_test_fixture.h" #include "mongo/db/repl/topology_coordinator.h" #include "mongo/executor/network_interface_mock.h" +#include "mongo/logv2/log.h" #include "mongo/rpc/metadata/repl_set_metadata.h" #include "mongo/unittest/unittest.h" #include "mongo/util/log.h" @@ -718,7 +719,10 @@ TEST_F(ReplCoordHBV1Test, enterNetwork(); const NetworkInterfaceMock::NetworkOperationIterator noi = getNet()->getNextReadyRequest(); const RemoteCommandRequest& request = noi->getRequest(); - log() << request.target.toString() << " processing " << request.cmdObj; + LOGV2(21492, + "{request_target} processing {request_cmdObj}", + "request_target"_attr = request.target.toString(), + "request_cmdObj"_attr = request.cmdObj); getNet()->scheduleResponse( noi, getNet()->now(), @@ -728,8 +732,10 @@ TEST_F(ReplCoordHBV1Test, if (request.target != HostAndPort("node2", 12345) && request.cmdObj.firstElement().fieldNameStringData() != "replSetHeartbeat") { - error() << "Black holing unexpected request to " << request.target << ": " - << request.cmdObj; + LOGV2_ERROR(21496, + "Black holing unexpected request to {request_target}: {request_cmdObj}", + "request_target"_attr = request.target, + "request_cmdObj"_attr = request.cmdObj); getNet()->blackHole(noi); } getNet()->runReadyNetworkOperations(); @@ -792,11 +798,16 @@ TEST_F(ReplCoordHBV1Test, IgnoreTheContentsOfMetadataWhenItsReplicaSetIdDoesNotM const RemoteCommandRequest& request = noi->getRequest(); if (request.target == host2 && request.cmdObj.firstElement().fieldNameStringData() == "replSetHeartbeat") { - log() << request.target.toString() << " processing " << request.cmdObj; + LOGV2(21493, + "{request_target} processing {request_cmdObj}", + "request_target"_attr = request.target.toString(), + "request_cmdObj"_attr = request.cmdObj); net->scheduleResponse(noi, net->now(), heartbeatResponse); } else { - log() << "blackholing request to " << request.target.toString() << ": " - << request.cmdObj; + LOGV2(21494, + "blackholing request to {request_target}: {request_cmdObj}", + "request_target"_attr = request.target.toString(), + "request_cmdObj"_attr = request.cmdObj); net->blackHole(noi); } net->runReadyNetworkOperations(); @@ -810,7 +821,7 @@ TEST_F(ReplCoordHBV1Test, IgnoreTheContentsOfMetadataWhenItsReplicaSetIdDoesNotM ASSERT_OK(getReplCoord()->processReplSetGetStatus( &statusBuilder, ReplicationCoordinator::ReplSetGetStatusResponseStyle::kBasic)); auto statusObj = statusBuilder.obj(); - unittest::log() << "replica set status = " << statusObj; + LOGV2(21495, "replica set status = {statusObj}", "statusObj"_attr = statusObj); ASSERT_EQ(mongo::Array, statusObj["members"].type()); auto members = statusObj["members"].Array(); diff --git a/src/mongo/db/repl/replication_coordinator_impl_test.cpp b/src/mongo/db/repl/replication_coordinator_impl_test.cpp index aefefa5177e..1f1e6c32160 100644 --- a/src/mongo/db/repl/replication_coordinator_impl_test.cpp +++ b/src/mongo/db/repl/replication_coordinator_impl_test.cpp @@ -63,6 +63,7 @@ #include "mongo/db/service_context.h" #include "mongo/db/write_concern_options.h" #include "mongo/executor/network_interface_mock.h" +#include "mongo/logv2/log.h" #include "mongo/rpc/metadata/oplog_query_metadata.h" #include "mongo/rpc/metadata/repl_set_metadata.h" #include "mongo/stdx/future.h" @@ -1320,7 +1321,10 @@ protected: getNet()->runUntil(getNet()->now() + heartbeatInterval); NetworkInterfaceMock::NetworkOperationIterator noi = getNet()->getNextReadyRequest(); RemoteCommandRequest request = noi->getRequest(); - log() << request.target.toString() << " processing " << request.cmdObj; + LOGV2(21497, + "{request_target} processing {request_cmdObj}", + "request_target"_attr = request.target.toString(), + "request_cmdObj"_attr = request.cmdObj); ReplSetHeartbeatArgsV1 hbArgs; if (hbArgs.initialize(request.cmdObj).isOK()) { ReplSetHeartbeatResponse hbResp; @@ -1336,7 +1340,9 @@ protected: } while (getNet()->hasReadyRequests()) { auto noi = getNet()->getNextReadyRequest(); - log() << "Blackholing network request " << noi->getRequest().cmdObj; + LOGV2(21498, + "Blackholing network request {noi_getRequest_cmdObj}", + "noi_getRequest_cmdObj"_attr = noi->getRequest().cmdObj); getNet()->blackHole(noi); } @@ -1969,7 +1975,10 @@ protected: ReplSetHeartbeatArgsV1 hbArgs; ASSERT_OK(hbArgs.initialize(request.cmdObj)); - log() << request.target.toString() << " processing " << request.cmdObj; + LOGV2(21499, + "{request_target} processing {request_cmdObj}", + "request_target"_attr = request.target.toString(), + "request_cmdObj"_attr = request.cmdObj); // Catch up 'numNodesCaughtUp' nodes out of 5. OpTime optimeResponse = (hbNum <= numNodesCaughtUp) ? optimePrimary : optimeLagged; @@ -2365,7 +2374,10 @@ TEST_F(StepDownTest, getNet()->runUntil(getNet()->now() + Milliseconds(2000)); NetworkInterfaceMock::NetworkOperationIterator noi = getNet()->getNextReadyRequest(); RemoteCommandRequest request = noi->getRequest(); - log() << "HB1: " << request.target.toString() << " processing " << request.cmdObj; + LOGV2(21500, + "HB1: {request_target} processing {request_cmdObj}", + "request_target"_attr = request.target.toString(), + "request_cmdObj"_attr = request.cmdObj); ReplSetHeartbeatArgsV1 hbArgs; if (hbArgs.initialize(request.cmdObj).isOK()) { ReplSetHeartbeatResponse hbResp; @@ -2651,7 +2663,7 @@ TEST_F(ReplCoordTest, NodeIncludesOtherMembersProgressInUpdatePositionCommand) { long long memberId = entry[UpdatePositionArgs::kMemberIdFieldName].Number(); memberIds.insert(memberId); if (memberId == 0) { - log() << 0; + LOGV2(21501, "{_0}", "_0"_attr = 0); ASSERT_OK(bsonExtractOpTimeField( entry, UpdatePositionArgs::kAppliedOpTimeFieldName, &appliedOpTime)); ASSERT_OK(bsonExtractOpTimeField( @@ -2659,7 +2671,7 @@ TEST_F(ReplCoordTest, NodeIncludesOtherMembersProgressInUpdatePositionCommand) { ASSERT_EQUALS(optime1, appliedOpTime); ASSERT_EQUALS(optime1, durableOpTime); } else if (memberId == 1) { - log() << 1; + LOGV2(21502, "{_1}", "_1"_attr = 1); ASSERT_OK(bsonExtractOpTimeField( entry, UpdatePositionArgs::kAppliedOpTimeFieldName, &appliedOpTime)); ASSERT_OK(bsonExtractOpTimeField( @@ -2667,7 +2679,7 @@ TEST_F(ReplCoordTest, NodeIncludesOtherMembersProgressInUpdatePositionCommand) { ASSERT_EQUALS(optime2, appliedOpTime); ASSERT_EQUALS(OpTime(), durableOpTime); } else if (memberId == 2) { - log() << 2; + LOGV2(21503, "{_2}", "_2"_attr = 2); ASSERT_OK(bsonExtractOpTimeField( entry, UpdatePositionArgs::kAppliedOpTimeFieldName, &appliedOpTime)); ASSERT_OK(bsonExtractOpTimeField( @@ -2675,7 +2687,7 @@ TEST_F(ReplCoordTest, NodeIncludesOtherMembersProgressInUpdatePositionCommand) { ASSERT_EQUALS(optime3, appliedOpTime); ASSERT_EQUALS(optime3, durableOpTime); } else { - log() << 3; + LOGV2(21504, "{_3}", "_3"_attr = 3); ASSERT_EQUALS(3, memberId); ASSERT_OK(bsonExtractOpTimeField( entry, UpdatePositionArgs::kAppliedOpTimeFieldName, &appliedOpTime)); @@ -3778,8 +3790,9 @@ TEST_F(ReplCoordTest, AwaitIsMasterResponseReturnsOnElectionWin) { waitForIsMasterFailPoint->waitForTimesEntered(timesEnteredFailPoint + 1); auto electionTimeoutWhen = getReplCoord()->getElectionTimeout_forTest(); ASSERT_NOT_EQUALS(Date_t(), electionTimeoutWhen); - unittest::log() << "Election timeout scheduled at " << electionTimeoutWhen - << " (simulator time)"; + LOGV2(21505, + "Election timeout scheduled at {electionTimeoutWhen} (simulator time)", + "electionTimeoutWhen"_attr = electionTimeoutWhen); simulateSuccessfulV1ElectionWithoutExitingDrainMode(electionTimeoutWhen, opCtx.get()); waitForIsMasterFailPoint->waitForTimesEntered(timesEnteredFailPoint + 2); @@ -5682,7 +5695,7 @@ TEST_F(ReplCoordTest, PrepareOplogQueryMetadata) { &metadataBob); BSONObj metadata = metadataBob.done(); - log() << metadata; + LOGV2(21506, "{metadata}", "metadata"_attr = metadata); auto oqMetadata = rpc::OplogQueryMetadata::readFromMetadata(metadata); ASSERT_OK(oqMetadata.getStatus()); @@ -5788,12 +5801,16 @@ TEST_F(ReplCoordTest, const auto& request = noi->getRequest(); ASSERT_EQUALS(HostAndPort("node2", 12345), request.target); ASSERT_EQUALS("replSetHeartbeat", request.cmdObj.firstElement().fieldNameStringData()); - log() << "black holing " << noi->getRequest().cmdObj; + LOGV2(21507, + "black holing {noi_getRequest_cmdObj}", + "noi_getRequest_cmdObj"_attr = noi->getRequest().cmdObj); net->blackHole(noi); // Advance simulator clock to some time before the first scheduled election. auto electionTimeoutWhen = replCoord->getElectionTimeout_forTest(); - log() << "Election initially scheduled at " << electionTimeoutWhen << " (simulator time)"; + LOGV2(21508, + "Election initially scheduled at {electionTimeoutWhen} (simulator time)", + "electionTimeoutWhen"_attr = electionTimeoutWhen); ASSERT_GREATER_THAN(electionTimeoutWhen, net->now()); auto until = net->now() + (electionTimeoutWhen - net->now()) / 2; net->runUntil(until); @@ -5877,7 +5894,7 @@ TEST_F(ReplCoordTest, ASSERT_TRUE(net->hasReadyRequests()); auto noi = net->getNextReadyRequest(); auto&& request = noi->getRequest(); - log() << "processing " << request.cmdObj; + LOGV2(21509, "processing {request_cmdObj}", "request_cmdObj"_attr = request.cmdObj); ASSERT_EQUALS(HostAndPort("node2", 12345), request.target); ASSERT_EQUALS("replSetHeartbeat", request.cmdObj.firstElement().fieldNameStringData()); @@ -5934,7 +5951,7 @@ TEST_F(ReplCoordTest, ASSERT_TRUE(net->hasReadyRequests()); auto noi = net->getNextReadyRequest(); auto&& request = noi->getRequest(); - log() << "processing " << request.cmdObj; + LOGV2(21510, "processing {request_cmdObj}", "request_cmdObj"_attr = request.cmdObj); ASSERT_EQUALS(HostAndPort("node2", 12345), request.target); ASSERT_EQUALS("replSetHeartbeat", request.cmdObj.firstElement().fieldNameStringData()); @@ -5987,7 +6004,7 @@ TEST_F(ReplCoordTest, ASSERT_TRUE(net->hasReadyRequests()); auto noi = net->getNextReadyRequest(); auto&& request = noi->getRequest(); - log() << "processing " << request.cmdObj; + LOGV2(21511, "processing {request_cmdObj}", "request_cmdObj"_attr = request.cmdObj); ASSERT_EQUALS(HostAndPort("node2", 12345), request.target); ASSERT_EQUALS("replSetHeartbeat", request.cmdObj.firstElement().fieldNameStringData()); @@ -6035,7 +6052,7 @@ TEST_F(ReplCoordTest, ASSERT_TRUE(net->hasReadyRequests()); auto noi = net->getNextReadyRequest(); auto&& request = noi->getRequest(); - log() << "processing " << request.cmdObj; + LOGV2(21512, "processing {request_cmdObj}", "request_cmdObj"_attr = request.cmdObj); ASSERT_EQUALS(HostAndPort("node2", 12345), request.target); ASSERT_EQUALS("replSetHeartbeat", request.cmdObj.firstElement().fieldNameStringData()); @@ -6097,12 +6114,16 @@ TEST_F(ReplCoordTest, CancelAndRescheduleElectionTimeoutLogging) { const auto& request = noi->getRequest(); ASSERT_EQUALS(HostAndPort("node2", 12345), request.target); ASSERT_EQUALS("replSetHeartbeat", request.cmdObj.firstElement().fieldNameStringData()); - log() << "black holing " << noi->getRequest().cmdObj; + LOGV2(21513, + "black holing {noi_getRequest_cmdObj}", + "noi_getRequest_cmdObj"_attr = noi->getRequest().cmdObj); net->blackHole(noi); // Advance simulator clock to some time after the first scheduled election. auto electionTimeoutWhen = replCoord->getElectionTimeout_forTest(); - log() << "Election initially scheduled at " << electionTimeoutWhen << " (simulator time)"; + LOGV2(21514, + "Election initially scheduled at {electionTimeoutWhen} (simulator time)", + "electionTimeoutWhen"_attr = electionTimeoutWhen); ASSERT_GREATER_THAN(electionTimeoutWhen, net->now()); auto until = electionTimeoutWhen + Milliseconds(1); net->runUntil(until); diff --git a/src/mongo/db/repl/replication_coordinator_test_fixture.cpp b/src/mongo/db/repl/replication_coordinator_test_fixture.cpp index f773b0d03ff..4d2c4c82dd4 100644 --- a/src/mongo/db/repl/replication_coordinator_test_fixture.cpp +++ b/src/mongo/db/repl/replication_coordinator_test_fixture.cpp @@ -52,6 +52,7 @@ #include "mongo/executor/network_interface_mock.h" #include "mongo/executor/thread_pool_mock.h" #include "mongo/executor/thread_pool_task_executor.h" +#include "mongo/logv2/log.h" #include "mongo/unittest/unittest.h" #include "mongo/util/fail_point.h" #include "mongo/util/log.h" @@ -217,7 +218,10 @@ void ReplCoordTest::assertStartSuccess(const BSONObj& configDoc, const HostAndPo executor::RemoteCommandResponse ReplCoordTest::makeResponseStatus(const BSONObj& doc, Milliseconds millis) { - log() << "Responding with " << doc << " (elapsed: " << millis << ")"; + LOGV2(21515, + "Responding with {doc} (elapsed: {millis})", + "doc"_attr = doc, + "millis"_attr = millis); return RemoteCommandResponse(doc, millis); } @@ -229,7 +233,10 @@ void ReplCoordTest::simulateEnoughHeartbeatsForAllNodesUp() { for (int i = 0; i < rsConfig.getNumMembers() - 1; ++i) { const NetworkInterfaceMock::NetworkOperationIterator noi = net->getNextReadyRequest(); const RemoteCommandRequest& request = noi->getRequest(); - log() << request.target.toString() << " processing " << request.cmdObj; + LOGV2(21516, + "{request_target} processing {request_cmdObj}", + "request_target"_attr = request.target.toString(), + "request_cmdObj"_attr = request.cmdObj); ReplSetHeartbeatArgsV1 hbArgs; if (hbArgs.initialize(request.cmdObj).isOK()) { ReplSetHeartbeatResponse hbResp; @@ -243,8 +250,10 @@ void ReplCoordTest::simulateEnoughHeartbeatsForAllNodesUp() { BSONObjBuilder respObj; net->scheduleResponse(noi, net->now(), makeResponseStatus(hbResp.toBSON())); } else { - error() << "Black holing unexpected request to " << request.target << ": " - << request.cmdObj; + LOGV2_ERROR(21526, + "Black holing unexpected request to {request_target}: {request_cmdObj}", + "request_target"_attr = request.target, + "request_cmdObj"_attr = request.cmdObj); net->blackHole(noi); } net->runReadyNetworkOperations(); @@ -260,12 +269,15 @@ void ReplCoordTest::simulateSuccessfulDryRun( auto electionTimeoutWhen = replCoord->getElectionTimeout_forTest(); ASSERT_NOT_EQUALS(Date_t(), electionTimeoutWhen); - log() << "Election timeout scheduled at " << electionTimeoutWhen << " (simulator time)"; + LOGV2(21517, + "Election timeout scheduled at {electionTimeoutWhen} (simulator time)", + "electionTimeoutWhen"_attr = electionTimeoutWhen); int voteRequests = 0; int votesExpected = rsConfig.getNumMembers() / 2; - log() << "Simulating dry run responses - expecting " << votesExpected - << " replSetRequestVotes requests"; + LOGV2(21518, + "Simulating dry run responses - expecting {votesExpected} replSetRequestVotes requests", + "votesExpected"_attr = votesExpected); net->enterNetwork(); while (voteRequests < votesExpected) { if (net->now() < electionTimeoutWhen) { @@ -273,7 +285,10 @@ void ReplCoordTest::simulateSuccessfulDryRun( } const NetworkInterfaceMock::NetworkOperationIterator noi = net->getNextReadyRequest(); const RemoteCommandRequest& request = noi->getRequest(); - log() << request.target.toString() << " processing " << request.cmdObj; + LOGV2(21519, + "{request_target} processing {request_cmdObj}", + "request_target"_attr = request.target.toString(), + "request_cmdObj"_attr = request.cmdObj); if (request.cmdObj.firstElement().fieldNameStringData() == "replSetRequestVotes") { ASSERT_TRUE(request.cmdObj.getBoolField("dryRun")); onDryRunRequest(request); @@ -288,17 +303,20 @@ void ReplCoordTest::simulateSuccessfulDryRun( } else if (consumeHeartbeatV1(noi)) { // The heartbeat has been consumed. } else { - error() << "Black holing unexpected request to " << request.target << ": " - << request.cmdObj; + LOGV2_ERROR(21527, + "Black holing unexpected request to {request_target}: {request_cmdObj}", + "request_target"_attr = request.target, + "request_cmdObj"_attr = request.cmdObj); net->blackHole(noi); } net->runReadyNetworkOperations(); } net->exitNetwork(); - log() << "Simulating dry run responses - scheduled " << voteRequests - << " replSetRequestVotes responses"; + LOGV2(21520, + "Simulating dry run responses - scheduled {voteRequests} replSetRequestVotes responses", + "voteRequests"_attr = voteRequests); getReplCoord()->waitForElectionDryRunFinish_forTest(); - log() << "Simulating dry run responses - dry run completed"; + LOGV2(21521, "Simulating dry run responses - dry run completed"); } void ReplCoordTest::simulateSuccessfulDryRun() { @@ -309,7 +327,9 @@ void ReplCoordTest::simulateSuccessfulDryRun() { void ReplCoordTest::simulateSuccessfulV1Election() { auto electionTimeoutWhen = getReplCoord()->getElectionTimeout_forTest(); ASSERT_NOT_EQUALS(Date_t(), electionTimeoutWhen); - log() << "Election timeout scheduled at " << electionTimeoutWhen << " (simulator time)"; + LOGV2(21522, + "Election timeout scheduled at {electionTimeoutWhen} (simulator time)", + "electionTimeoutWhen"_attr = electionTimeoutWhen); simulateSuccessfulV1ElectionAt(electionTimeoutWhen); } @@ -325,14 +345,19 @@ void ReplCoordTest::simulateSuccessfulV1ElectionWithoutExitingDrainMode(Date_t e // Process requests until we're primary and consume the heartbeats for the notification // of election win. while (!replCoord->getMemberState().primary() || hasReadyRequests) { - log() << "Waiting on network in state " << replCoord->getMemberState(); + LOGV2(21523, + "Waiting on network in state {replCoord_getMemberState}", + "replCoord_getMemberState"_attr = replCoord->getMemberState()); getNet()->enterNetwork(); if (net->now() < electionTime) { net->runUntil(electionTime); } const NetworkInterfaceMock::NetworkOperationIterator noi = net->getNextReadyRequest(); const RemoteCommandRequest& request = noi->getRequest(); - log() << request.target.toString() << " processing " << request.cmdObj; + LOGV2(21524, + "{request_target} processing {request_cmdObj}", + "request_target"_attr = request.target.toString(), + "request_cmdObj"_attr = request.cmdObj); ReplSetHeartbeatArgsV1 hbArgs; Status status = hbArgs.initialize(request.cmdObj); if (status.isOK()) { @@ -354,8 +379,10 @@ void ReplCoordTest::simulateSuccessfulV1ElectionWithoutExitingDrainMode(Date_t e << "term" << request.cmdObj["term"].Long() << "voteGranted" << true))); } else { - error() << "Black holing unexpected request to " << request.target << ": " - << request.cmdObj; + LOGV2_ERROR(21528, + "Black holing unexpected request to {request_target}: {request_cmdObj}", + "request_target"_attr = request.target, + "request_cmdObj"_attr = request.cmdObj); net->blackHole(noi); } net->runReadyNetworkOperations(); @@ -464,7 +491,10 @@ void ReplCoordTest::simulateCatchUpAbort() { auto noi = net->getNextReadyRequest(); auto request = noi->getRequest(); // Black hole heartbeat requests caused by time advance. - log() << "Black holing request to " << request.target.toString() << " : " << request.cmdObj; + LOGV2(21525, + "Black holing request to {request_target} : {request_cmdObj}", + "request_target"_attr = request.target.toString(), + "request_cmdObj"_attr = request.cmdObj); net->blackHole(noi); if (net->now() < heartbeatTimeoutWhen) { net->runUntil(heartbeatTimeoutWhen); diff --git a/src/mongo/db/repl/replication_process.cpp b/src/mongo/db/repl/replication_process.cpp index 117972289af..194c0194d12 100644 --- a/src/mongo/db/repl/replication_process.cpp +++ b/src/mongo/db/repl/replication_process.cpp @@ -41,6 +41,7 @@ #include "mongo/db/repl/rollback_gen.h" #include "mongo/db/repl/storage_interface.h" #include "mongo/db/service_context.h" +#include "mongo/logv2/log.h" #include "mongo/util/assert_util.h" #include "mongo/util/log.h" #include "mongo/util/str.h" @@ -92,9 +93,14 @@ Status ReplicationProcess::refreshRollbackID(OperationContext* opCtx) { } if (kUninitializedRollbackId == _rbid) { - log() << "Rollback ID is " << rbidResult.getValue(); + LOGV2(21529, + "Rollback ID is {rbidResult_getValue}", + "rbidResult_getValue"_attr = rbidResult.getValue()); } else { - log() << "Rollback ID is " << rbidResult.getValue() << " (previously " << _rbid << ")"; + LOGV2(21530, + "Rollback ID is {rbidResult_getValue} (previously {rbid})", + "rbidResult_getValue"_attr = rbidResult.getValue(), + "rbid"_attr = _rbid); } _rbid = rbidResult.getValue(); @@ -106,7 +112,7 @@ int ReplicationProcess::getRollbackID() const { if (kUninitializedRollbackId == _rbid) { // This may happen when serverStatus is called by an internal client before we have a chance // to read the rollback ID from storage. - warning() << "Rollback ID is not initialized yet."; + LOGV2_WARNING(21533, "Rollback ID is not initialized yet."); } return _rbid; } @@ -122,11 +128,15 @@ Status ReplicationProcess::initializeRollbackID(OperationContext* opCtx) { auto initRbidSW = _storageInterface->initializeRollbackID(opCtx); if (initRbidSW.isOK()) { - log() << "Initialized the rollback ID to " << initRbidSW.getValue(); + LOGV2(21531, + "Initialized the rollback ID to {initRbidSW_getValue}", + "initRbidSW_getValue"_attr = initRbidSW.getValue()); _rbid = initRbidSW.getValue(); invariant(kUninitializedRollbackId != _rbid); } else { - warning() << "Failed to initialize the rollback ID: " << initRbidSW.getStatus().reason(); + LOGV2_WARNING(21534, + "Failed to initialize the rollback ID: {initRbidSW_getStatus_reason}", + "initRbidSW_getStatus_reason"_attr = initRbidSW.getStatus().reason()); } return initRbidSW.getStatus(); } @@ -139,11 +149,15 @@ Status ReplicationProcess::incrementRollbackID(OperationContext* opCtx) { // If the rollback ID was incremented successfully, cache the new value in _rbid to be returned // the next time getRollbackID() is called. if (status.isOK()) { - log() << "Incremented the rollback ID to " << status.getValue(); + LOGV2(21532, + "Incremented the rollback ID to {status_getValue}", + "status_getValue"_attr = status.getValue()); _rbid = status.getValue(); invariant(kUninitializedRollbackId != _rbid); } else { - warning() << "Failed to increment the rollback ID: " << status.getStatus().reason(); + LOGV2_WARNING(21535, + "Failed to increment the rollback ID: {status_getStatus_reason}", + "status_getStatus_reason"_attr = status.getStatus().reason()); } return status.getStatus(); diff --git a/src/mongo/db/repl/replication_recovery.cpp b/src/mongo/db/repl/replication_recovery.cpp index 3c7b47d3285..f28c9aaa2d1 100644 --- a/src/mongo/db/repl/replication_recovery.cpp +++ b/src/mongo/db/repl/replication_recovery.cpp @@ -50,6 +50,7 @@ #include "mongo/db/storage/storage_parameters_gen.h" #include "mongo/db/transaction_history_iterator.h" #include "mongo/db/transaction_participant.h" +#include "mongo/logv2/log.h" #include "mongo/util/log.h" #include "mongo/util/timer.h" @@ -90,8 +91,12 @@ public: void onBatchEnd(const StatusWith<OpTime>&, const std::vector<OplogEntry>&) final {} void complete(const OpTime& applyThroughOpTime) const { - log() << "Applied " << _numOpsApplied << " operations in " << _numBatches - << " batches. Last operation applied with optime: " << applyThroughOpTime; + LOGV2(21536, + "Applied {numOpsApplied} operations in {numBatches} batches. Last operation applied " + "with optime: {applyThroughOpTime}", + "numOpsApplied"_attr = _numOpsApplied, + "numBatches"_attr = _numBatches, + "applyThroughOpTime"_attr = applyThroughOpTime); } private: @@ -134,16 +139,21 @@ public: ss << " and <= " << _oplogApplicationEndPoint->toBSON(); } - severe() << "Couldn't find any entries in the oplog" << ss.str() - << ", which should be impossible."; + LOGV2_FATAL( + 21559, + "Couldn't find any entries in the oplog{ss_str}, which should be impossible.", + "ss_str"_attr = ss.str()); fassertFailedNoTrace(40293); } auto firstTimestampFound = fassert(40291, OpTime::parseFromOplogEntry(_cursor->nextSafe())).getTimestamp(); if (firstTimestampFound != _oplogApplicationStartPoint) { - severe() << "Oplog entry at " << _oplogApplicationStartPoint.toBSON() - << " is missing; actual entry found is " << firstTimestampFound.toBSON(); + LOGV2_FATAL(21560, + "Oplog entry at {oplogApplicationStartPoint} is missing; actual entry " + "found is {firstTimestampFound}", + "oplogApplicationStartPoint"_attr = _oplogApplicationStartPoint.toBSON(), + "firstTimestampFound"_attr = firstTimestampFound.toBSON()); fassertFailedNoTrace(40292); } } @@ -210,8 +220,9 @@ private: boost::optional<Timestamp> recoverFromOplogPrecursor(OperationContext* opCtx, StorageInterface* storageInterface) { if (!storageInterface->supportsRecoveryTimestamp(opCtx->getServiceContext())) { - severe() << "Cannot recover from the oplog with a storage engine that does not support " - << "recover to stable timestamp."; + LOGV2_FATAL(21561, + "Cannot recover from the oplog with a storage engine that does not support " + "recover to stable timestamp."); fassertFailedNoTrace(50805); } @@ -220,7 +231,8 @@ boost::optional<Timestamp> recoverFromOplogPrecursor(OperationContext* opCtx, // happen. auto recoveryTS = storageInterface->getRecoveryTimestamp(opCtx->getServiceContext()); if (recoveryTS && recoveryTS->isNull()) { - severe() << "Cannot recover from the oplog with stable checkpoint at null timestamp."; + LOGV2_FATAL(21562, + "Cannot recover from the oplog with stable checkpoint at null timestamp."); fassertFailedNoTrace(50806); } @@ -238,36 +250,44 @@ void ReplicationRecoveryImpl::_assertNoRecoveryNeededOnUnstableCheckpoint(Operat invariant(!_storageInterface->getRecoveryTimestamp(opCtx->getServiceContext())); if (_consistencyMarkers->getInitialSyncFlag(opCtx)) { - severe() << "Unexpected recovery needed, initial sync flag set."; + LOGV2_FATAL(21563, "Unexpected recovery needed, initial sync flag set."); fassertFailedNoTrace(31362); } const auto truncateAfterPoint = _consistencyMarkers->getOplogTruncateAfterPoint(opCtx); if (!truncateAfterPoint.isNull()) { - severe() << "Unexpected recovery needed, oplog requires truncation. Truncate after point: " - << truncateAfterPoint.toString(); + LOGV2_FATAL(21564, + "Unexpected recovery needed, oplog requires truncation. Truncate after point: " + "{truncateAfterPoint}", + "truncateAfterPoint"_attr = truncateAfterPoint.toString()); fassertFailedNoTrace(31363); } auto topOfOplogSW = _getTopOfOplog(opCtx); if (!topOfOplogSW.isOK()) { - severe() << "Recovery not possible, no oplog found: " << topOfOplogSW.getStatus(); + LOGV2_FATAL(21565, + "Recovery not possible, no oplog found: {topOfOplogSW_getStatus}", + "topOfOplogSW_getStatus"_attr = topOfOplogSW.getStatus()); fassertFailedNoTrace(31364); } const auto topOfOplog = topOfOplogSW.getValue(); const auto appliedThrough = _consistencyMarkers->getAppliedThrough(opCtx); if (!appliedThrough.isNull() && appliedThrough != topOfOplog) { - severe() << "Unexpected recovery needed, appliedThrough is not at top of oplog, indicating " - "oplog has not been fully applied. appliedThrough: " - << appliedThrough.toString(); + LOGV2_FATAL(21566, + "Unexpected recovery needed, appliedThrough is not at top of oplog, indicating " + "oplog has not been fully applied. appliedThrough: {appliedThrough}", + "appliedThrough"_attr = appliedThrough.toString()); fassertFailedNoTrace(31365); } const auto minValid = _consistencyMarkers->getMinValid(opCtx); if (minValid > topOfOplog) { - severe() << "Unexpected recovery needed, top of oplog is not consistent. topOfOplog: " - << topOfOplog << ", minValid: " << minValid; + LOGV2_FATAL(21567, + "Unexpected recovery needed, top of oplog is not consistent. topOfOplog: " + "{topOfOplog}, minValid: {minValid}", + "topOfOplog"_attr = topOfOplog, + "minValid"_attr = minValid); fassertFailedNoTrace(31366); } } @@ -287,21 +307,25 @@ void ReplicationRecoveryImpl::recoverFromOplogAsStandalone(OperationContext* opC if (gTakeUnstableCheckpointOnShutdown) { // Ensure 'recoverFromOplogAsStandalone' with 'takeUnstableCheckpointOnShutdown' // is safely idempotent when it succeeds. - log() << "Recovering from unstable checkpoint with 'takeUnstableCheckpointOnShutdown'." - << " Confirming that no oplog recovery is needed."; + LOGV2(21537, + "Recovering from unstable checkpoint with 'takeUnstableCheckpointOnShutdown'. " + "Confirming that no oplog recovery is needed."); _assertNoRecoveryNeededOnUnstableCheckpoint(opCtx); - log() << "Not doing any oplog recovery since there is an unstable checkpoint that is " - << "up to date."; + LOGV2(21538, + "Not doing any oplog recovery since there is an unstable checkpoint that is up " + "to date."); } else { - severe() << "Cannot use 'recoverFromOplogAsStandalone' without a stable checkpoint."; + LOGV2_FATAL(21568, + "Cannot use 'recoverFromOplogAsStandalone' without a stable checkpoint."); fassertFailedNoTrace(31229); } } reconstructPreparedTransactions(opCtx, OplogApplication::Mode::kRecovering); - warning() << "Setting mongod to readOnly mode as a result of specifying " - "'recoverFromOplogAsStandalone'."; + LOGV2_WARNING(21558, + "Setting mongod to readOnly mode as a result of specifying " + "'recoverFromOplogAsStandalone'."); storageGlobalParams.readOnly = true; } @@ -313,7 +337,7 @@ void ReplicationRecoveryImpl::recoverFromOplogUpTo(OperationContext* opCtx, Time auto recoveryTS = recoverFromOplogPrecursor(opCtx, _storageInterface); if (!recoveryTS) { - severe() << "Cannot use 'recoverToOplogTimestamp' without a stable checkpoint."; + LOGV2_FATAL(21569, "Cannot use 'recoverToOplogTimestamp' without a stable checkpoint."); fassertFailedNoTrace(31399); } @@ -322,15 +346,18 @@ void ReplicationRecoveryImpl::recoverFromOplogUpTo(OperationContext* opCtx, Time Timestamp startPoint = _consistencyMarkers->getAppliedThrough(opCtx).getTimestamp(); if (startPoint.isNull()) { - log() << "No stored oplog entries to apply for recovery."; + LOGV2(21539, "No stored oplog entries to apply for recovery."); return; } invariant(!endPoint.isNull()); if (startPoint == endPoint) { - log() << "No oplog entries to apply for recovery. Start point '" << startPoint - << "' is at the end point '" << endPoint << "' in the oplog."; + LOGV2(21540, + "No oplog entries to apply for recovery. Start point '{startPoint}' is at the end " + "point '{endPoint}' in the oplog.", + "startPoint"_attr = startPoint, + "endPoint"_attr = endPoint); return; } else if (startPoint > endPoint) { uasserted(ErrorCodes::BadValue, @@ -341,8 +368,11 @@ void ReplicationRecoveryImpl::recoverFromOplogUpTo(OperationContext* opCtx, Time Timestamp appliedUpTo = _applyOplogOperations(opCtx, startPoint, endPoint); if (appliedUpTo.isNull()) { - log() << "No stored oplog entries to apply for recovery between " << startPoint.toString() - << " (inclusive) and " << endPoint.toString() << " (inclusive)."; + LOGV2(21541, + "No stored oplog entries to apply for recovery between {startPoint} (inclusive) and " + "{endPoint} (inclusive).", + "startPoint"_attr = startPoint.toString(), + "endPoint"_attr = endPoint.toString()); } else { invariant(appliedUpTo <= endPoint); } @@ -353,7 +383,7 @@ void ReplicationRecoveryImpl::recoverFromOplogUpTo(OperationContext* opCtx, Time void ReplicationRecoveryImpl::recoverFromOplog(OperationContext* opCtx, boost::optional<Timestamp> stableTimestamp) try { if (_consistencyMarkers->getInitialSyncFlag(opCtx)) { - log() << "No recovery needed. Initial sync flag set."; + LOGV2(21542, "No recovery needed. Initial sync flag set."); return; // Initial Sync will take over so no cleanup is needed. } @@ -393,7 +423,7 @@ void ReplicationRecoveryImpl::recoverFromOplog(OperationContext* opCtx, topOfOplogSW.getStatus() == ErrorCodes::NamespaceNotFound) { // Oplog is empty. There are no oplog entries to apply, so we exit recovery and go into // initial sync. - log() << "No oplog entries to apply for recovery. Oplog is empty."; + LOGV2(21543, "No oplog entries to apply for recovery. Oplog is empty."); return; } fassert(40290, topOfOplogSW); @@ -406,7 +436,9 @@ void ReplicationRecoveryImpl::recoverFromOplog(OperationContext* opCtx, _recoverFromUnstableCheckpoint(opCtx, appliedThrough, topOfOplog); } } catch (...) { - severe() << "Caught exception during replication recovery: " << exceptionToStatus(); + LOGV2_FATAL(21570, + "Caught exception during replication recovery: {exceptionToStatus}", + "exceptionToStatus"_attr = exceptionToStatus()); std::terminate(); } @@ -419,11 +451,17 @@ void ReplicationRecoveryImpl::_recoverFromStableTimestamp(OperationContext* opCt const auto truncateAfterPoint = _consistencyMarkers->getOplogTruncateAfterPoint(opCtx); - log() << "Recovering from stable timestamp: " << stableTimestamp - << " (top of oplog: " << topOfOplog << ", appliedThrough: " << appliedThrough - << ", TruncateAfter: " << truncateAfterPoint << ")"; - - log() << "Starting recovery oplog application at the stable timestamp: " << stableTimestamp; + LOGV2(21544, + "Recovering from stable timestamp: {stableTimestamp} (top of oplog: {topOfOplog}, " + "appliedThrough: {appliedThrough}, TruncateAfter: {truncateAfterPoint})", + "stableTimestamp"_attr = stableTimestamp, + "topOfOplog"_attr = topOfOplog, + "appliedThrough"_attr = appliedThrough, + "truncateAfterPoint"_attr = truncateAfterPoint); + + LOGV2(21545, + "Starting recovery oplog application at the stable timestamp: {stableTimestamp}", + "stableTimestamp"_attr = stableTimestamp); _applyToEndOfOplog(opCtx, stableTimestamp, topOfOplog.getTimestamp()); } @@ -431,18 +469,24 @@ void ReplicationRecoveryImpl::_recoverFromUnstableCheckpoint(OperationContext* o OpTime appliedThrough, OpTime topOfOplog) { invariant(!topOfOplog.isNull()); - log() << "Recovering from an unstable checkpoint (top of oplog: " << topOfOplog - << ", appliedThrough: " << appliedThrough << ")"; + LOGV2(21546, + "Recovering from an unstable checkpoint (top of oplog: {topOfOplog}, appliedThrough: " + "{appliedThrough})", + "topOfOplog"_attr = topOfOplog, + "appliedThrough"_attr = appliedThrough); if (appliedThrough.isNull()) { // The appliedThrough would be null if we shut down cleanly or crashed as a primary. Either // way we are consistent at the top of the oplog. - log() << "No oplog entries to apply for recovery. appliedThrough is null."; + LOGV2(21547, "No oplog entries to apply for recovery. appliedThrough is null."); } else { // If the appliedThrough is not null, then we shut down uncleanly during secondary oplog // application and must apply from the appliedThrough to the top of the oplog. - log() << "Starting recovery oplog application at the appliedThrough: " << appliedThrough - << ", through the top of the oplog: " << topOfOplog; + LOGV2(21548, + "Starting recovery oplog application at the appliedThrough: {appliedThrough}, " + "through the top of the oplog: {topOfOplog}", + "appliedThrough"_attr = appliedThrough, + "topOfOplog"_attr = topOfOplog); // When `recoverFromOplog` truncates the oplog, that also happens to set the "oldest // timestamp" to the truncation point[1]. `_applyToEndOfOplog` will then perform writes @@ -492,11 +536,15 @@ void ReplicationRecoveryImpl::_applyToEndOfOplog(OperationContext* opCtx, // Check if we have any unapplied ops in our oplog. It is important that this is done after // deleting the ragged end of the oplog. if (oplogApplicationStartPoint == topOfOplog) { - log() << "No oplog entries to apply for recovery. Start point is at the top of the oplog."; + LOGV2(21549, + "No oplog entries to apply for recovery. Start point is at the top of the oplog."); return; // We've applied all the valid oplog we have. } else if (oplogApplicationStartPoint > topOfOplog) { - severe() << "Applied op " << oplogApplicationStartPoint.toBSON() - << " not found. Top of oplog is " << topOfOplog.toBSON() << '.'; + LOGV2_FATAL( + 21571, + "Applied op {oplogApplicationStartPoint} not found. Top of oplog is {topOfOplog}.", + "oplogApplicationStartPoint"_attr = oplogApplicationStartPoint.toBSON(), + "topOfOplog"_attr = topOfOplog.toBSON()); fassertFailedNoTrace(40313); } @@ -511,8 +559,10 @@ void ReplicationRecoveryImpl::_applyToEndOfOplog(OperationContext* opCtx, Timestamp ReplicationRecoveryImpl::_applyOplogOperations(OperationContext* opCtx, const Timestamp& startPoint, const Timestamp& endPoint) { - log() << "Replaying stored operations from " << startPoint << " (inclusive) to " << endPoint - << " (inclusive)."; + LOGV2(21550, + "Replaying stored operations from {startPoint} (inclusive) to {endPoint} (inclusive).", + "startPoint"_attr = startPoint, + "endPoint"_attr = endPoint); OplogBufferLocalOplog oplogBuffer(startPoint, endPoint); oplogBuffer.startup(opCtx); @@ -616,9 +666,10 @@ void ReplicationRecoveryImpl::_truncateOplogTo(OperationContext* opCtx, const auto tsElem = entry["ts"]; if (count == 1) { if (tsElem.eoo()) { - LOG(2) << "Oplog tail entry: " << redact(entry); + LOGV2_DEBUG(21551, 2, "Oplog tail entry: {entry}", "entry"_attr = redact(entry)); } else { - LOG(2) << "Oplog tail entry ts field: " << tsElem; + LOGV2_DEBUG( + 21552, 2, "Oplog tail entry ts field: {tsElem}", "tsElem"_attr = tsElem); topOfOplog = tsElem.timestamp(); } } @@ -627,24 +678,35 @@ void ReplicationRecoveryImpl::_truncateOplogTo(OperationContext* opCtx, // If count == 1, that means that we have nothing to delete because everything in the // oplog is <= truncateTimestamp. if (count != 1) { - log() << "Truncating oplog from [" << Timestamp(previousRecordId.repr()) << " to " - << topOfOplog << "]. Truncate after point is " << truncateTimestamp; + LOGV2(21553, + "Truncating oplog from [{Timestamp_previousRecordId_repr} to {topOfOplog}]. " + "Truncate after point is {truncateTimestamp}", + "Timestamp_previousRecordId_repr"_attr = Timestamp(previousRecordId.repr()), + "topOfOplog"_attr = topOfOplog, + "truncateTimestamp"_attr = truncateTimestamp); invariant(!previousRecordId.isNull()); oplogCollection->cappedTruncateAfter(opCtx, previousRecordId, /*inclusive*/ true); } else { - log() << "There is no oplog after " << truncateTimestamp - << " to truncate. The top of the oplog is " << topOfOplog; + LOGV2(21554, + "There is no oplog after {truncateTimestamp} to truncate. The top of the " + "oplog is {topOfOplog}", + "truncateTimestamp"_attr = truncateTimestamp, + "topOfOplog"_attr = topOfOplog); } - log() << "Replication recovery oplog truncation finished in: " << timer.millis() - << "ms"; + LOGV2(21555, + "Replication recovery oplog truncation finished in: {timer_millis}ms", + "timer_millis"_attr = timer.millis()); return; } previousRecordId = id; } - severe() << "Reached end of oplog looking for oplog entry before " << truncateTimestamp.toBSON() - << " but couldn't find any after looking through " << count << " entries."; + LOGV2_FATAL(21572, + "Reached end of oplog looking for oplog entry before {truncateTimestamp} but " + "couldn't find any after looking through {count} entries.", + "truncateTimestamp"_attr = truncateTimestamp.toBSON(), + "count"_attr = count); fassertFailedNoTrace(40296); } @@ -661,14 +723,18 @@ void ReplicationRecoveryImpl::_truncateOplogIfNeededAndThenClearOplogTruncateAft AutoGetCollectionForRead oplog(opCtx, NamespaceString::kRsOplogNamespace); invariant(oplog.getCollection()); - log() << "The oplog truncation point (" << truncatePoint - << ") is equal to or earlier than the stable timestamp (" << stableTimestamp.get() - << "), so truncating after the stable timestamp instead"; + LOGV2(21556, + "The oplog truncation point ({truncatePoint}) is equal to or earlier than the stable " + "timestamp ({stableTimestamp_get}), so truncating after the stable timestamp instead", + "truncatePoint"_attr = truncatePoint, + "stableTimestamp_get"_attr = stableTimestamp.get()); truncatePoint = stableTimestamp.get(); } - log() << "Removing unapplied oplog entries starting after: " << truncatePoint.toBSON(); + LOGV2(21557, + "Removing unapplied oplog entries starting after: {truncatePoint}", + "truncatePoint"_attr = truncatePoint.toBSON()); _truncateOplogTo(opCtx, truncatePoint); // Clear the oplogTruncateAfterPoint now that we have removed any holes that might exist in the diff --git a/src/mongo/db/repl/reporter.cpp b/src/mongo/db/repl/reporter.cpp index f66eb930893..2d23ef4e256 100644 --- a/src/mongo/db/repl/reporter.cpp +++ b/src/mongo/db/repl/reporter.cpp @@ -37,6 +37,7 @@ #include "mongo/bson/util/bson_extract.h" #include "mongo/db/commands/server_status_metric.h" #include "mongo/db/repl/update_position_args.h" +#include "mongo/logv2/log.h" #include "mongo/rpc/get_status_from_command_result.h" #include "mongo/util/assert_util.h" #include "mongo/util/destructor_guard.h" @@ -190,8 +191,11 @@ Status Reporter::trigger() { _status = scheduleResult.getStatus(); if (!_status.isOK()) { - LOG(2) << "Reporter failed to schedule callback to prepare and send update command: " - << _status; + LOGV2_DEBUG( + 21585, + 2, + "Reporter failed to schedule callback to prepare and send update command: {status}", + "status"_attr = _status); return _status; } @@ -212,8 +216,11 @@ StatusWith<BSONObj> Reporter::_prepareCommand() { // If there was an error in preparing the command, abort and return that error. if (!prepareResult.isOK()) { - LOG(2) << "Reporter failed to prepare update command with status: " - << prepareResult.getStatus(); + LOGV2_DEBUG( + 21586, + 2, + "Reporter failed to prepare update command with status: {prepareResult_getStatus}", + "prepareResult_getStatus"_attr = prepareResult.getStatus()); _status = prepareResult.getStatus(); return _status; } @@ -222,8 +229,12 @@ StatusWith<BSONObj> Reporter::_prepareCommand() { } void Reporter::_sendCommand_inlock(BSONObj commandRequest, Milliseconds netTimeout) { - LOG(2) << "Reporter sending slave oplog progress to upstream updater " << _target << ": " - << commandRequest; + LOGV2_DEBUG( + 21587, + 2, + "Reporter sending slave oplog progress to upstream updater {target}: {commandRequest}", + "target"_attr = _target, + "commandRequest"_attr = commandRequest); auto scheduleResult = _executor->scheduleRemoteCommand( executor::RemoteCommandRequest(_target, "admin", commandRequest, nullptr, netTimeout), @@ -233,7 +244,8 @@ void Reporter::_sendCommand_inlock(BSONObj commandRequest, Milliseconds netTimeo _status = scheduleResult.getStatus(); if (!_status.isOK()) { - LOG(2) << "Reporter failed to schedule with status: " << _status; + LOGV2_DEBUG( + 21588, 2, "Reporter failed to schedule with status: {status}", "status"_attr = _status); if (_status != ErrorCodes::ShutdownInProgress) { fassert(34434, _status); } @@ -272,8 +284,10 @@ void Reporter::_processResponseCallback( // sync target. if (_status == ErrorCodes::InvalidReplicaSetConfig && _isTargetConfigNewerThanRequest(commandResult, rcbd.request.cmdObj)) { - LOG(1) << "Reporter found newer configuration on sync source: " << _target - << ". Retrying."; + LOGV2_DEBUG(21589, + 1, + "Reporter found newer configuration on sync source: {target}. Retrying.", + "target"_attr = _target); _status = Status::OK(); // Do not resend update command immediately. _isWaitingToSendReporter = false; diff --git a/src/mongo/db/repl/roll_back_local_operations.cpp b/src/mongo/db/repl/roll_back_local_operations.cpp index 1e5b102a595..04ea36b725d 100644 --- a/src/mongo/db/repl/roll_back_local_operations.cpp +++ b/src/mongo/db/repl/roll_back_local_operations.cpp @@ -33,6 +33,7 @@ #include "mongo/db/repl/roll_back_local_operations.h" +#include "mongo/logv2/log.h" #include "mongo/util/assert_util.h" #include "mongo/util/log.h" #include "mongo/util/str.h" @@ -114,7 +115,10 @@ StatusWith<RollBackLocalOperations::RollbackCommonPoint> RollBackLocalOperations while (getTimestamp(_localOplogValue) > getTimestamp(operation)) { _scanned++; - LOG(2) << "Local oplog entry to roll back: " << redact(_localOplogValue.first); + LOGV2_DEBUG(21656, + 2, + "Local oplog entry to roll back: {localOplogValue_first}", + "localOplogValue_first"_attr = redact(_localOplogValue.first)); auto status = _rollbackOperation(_localOplogValue.first); if (!status.isOK()) { invariant(ErrorCodes::NoSuchKey != status.code()); diff --git a/src/mongo/db/repl/roll_back_local_operations_test.cpp b/src/mongo/db/repl/roll_back_local_operations_test.cpp index 6d4261938ea..659ac806d09 100644 --- a/src/mongo/db/repl/roll_back_local_operations_test.cpp +++ b/src/mongo/db/repl/roll_back_local_operations_test.cpp @@ -27,6 +27,8 @@ * it in the license file. */ +#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kDefault + #include "mongo/platform/basic.h" #include <iterator> @@ -39,7 +41,9 @@ #include "mongo/db/repl/oplog_interface_mock.h" #include "mongo/db/repl/oplog_interface_remote.h" #include "mongo/db/repl/roll_back_local_operations.h" +#include "mongo/logv2/log.h" #include "mongo/unittest/unittest.h" +#include "mongo/util/log.h" namespace { @@ -326,14 +330,15 @@ public: int batchSize) override { if (_initFailuresLeft > 0) { _initFailuresLeft--; - unittest::log() - << "Throwing DBException on DBClientCursorForTest::query(). Failures left: " - << _initFailuresLeft; + LOGV2(21657, + "Throwing DBException on DBClientCursorForTest::query(). Failures left: " + "{initFailuresLeft}", + "initFailuresLeft"_attr = _initFailuresLeft); uasserted(50852, "Simulated network error"); MONGO_UNREACHABLE; } - unittest::log() << "Returning success on DBClientCursorForTest::query()"; + LOGV2(21658, "Returning success on DBClientCursorForTest::query()"); BSONArrayBuilder builder; builder.append(makeOp(1)); diff --git a/src/mongo/db/repl/rollback_impl.cpp b/src/mongo/db/repl/rollback_impl.cpp index b61005e8878..aa5d60beb85 100644 --- a/src/mongo/db/repl/rollback_impl.cpp +++ b/src/mongo/db/repl/rollback_impl.cpp @@ -62,6 +62,7 @@ #include "mongo/db/session_txn_record_gen.h" #include "mongo/db/storage/remove_saver.h" #include "mongo/db/transaction_history_iterator.h" +#include "mongo/logv2/log.h" #include "mongo/s/catalog/type_config_version.h" #include "mongo/util/log.h" #include "mongo/util/scopeguard.h" @@ -99,10 +100,11 @@ boost::optional<long long> _parseDroppedCollectionCount(const OplogEntry& oplogE auto obj2 = oplogEntry.getObject2(); if (!obj2) { - warning() << "Unable to get collection count from " << desc - << " without the o2 " - "field. oplog op: " - << redact(oplogEntry.toBSON()); + LOGV2_WARNING(21634, + "Unable to get collection count from {desc} without the o2 " + "field. oplog op: {oplogEntry}", + "desc"_attr = desc, + "oplogEntry"_attr = redact(oplogEntry.toBSON())); return boost::none; } @@ -110,19 +112,30 @@ boost::optional<long long> _parseDroppedCollectionCount(const OplogEntry& oplogE // TODO: Use IDL to parse o2 object. See txn_cmds.idl for example. auto status = bsonExtractIntegerField(*obj2, kNumRecordsFieldName, &count); if (!status.isOK()) { - warning() << "Failed to parse " << desc << " for collection count: " << status - << ". oplog op: " << redact(oplogEntry.toBSON()); + LOGV2_WARNING( + 21635, + "Failed to parse {desc} for collection count: {status}. oplog op: {oplogEntry}", + "desc"_attr = desc, + "status"_attr = status, + "oplogEntry"_attr = redact(oplogEntry.toBSON())); return boost::none; } if (count < 0) { - warning() << "Invalid collection count found in " << desc << ": " << count - << ". oplog op: " << redact(oplogEntry.toBSON()); + LOGV2_WARNING(21636, + "Invalid collection count found in {desc}: {count}. oplog op: {oplogEntry}", + "desc"_attr = desc, + "count"_attr = count, + "oplogEntry"_attr = redact(oplogEntry.toBSON())); return boost::none; } - LOG(2) << "Parsed collection count of " << count << " from " << desc - << ". oplog op: " << redact(oplogEntry.toBSON()); + LOGV2_DEBUG(21590, + 2, + "Parsed collection count of {count} from {desc}. oplog op: {oplogEntry}", + "count"_attr = count, + "desc"_attr = desc, + "oplogEntry"_attr = redact(oplogEntry.toBSON())); return count; } @@ -182,8 +195,9 @@ Status RollbackImpl::runRollback(OperationContext* opCtx) { _listener->onTransitionToRollback(); if (MONGO_unlikely(rollbackHangAfterTransitionToRollback.shouldFail())) { - log() << "rollbackHangAfterTransitionToRollback fail point enabled. Blocking until fail " - "point is disabled (rollback_impl)."; + LOGV2(21591, + "rollbackHangAfterTransitionToRollback fail point enabled. Blocking until fail " + "point is disabled (rollback_impl)."); rollbackHangAfterTransitionToRollback.pauseWhileSet(opCtx); } @@ -251,7 +265,7 @@ Status RollbackImpl::runRollback(OperationContext* opCtx) { } _listener->onRollbackOpObserver(_observerInfo); - log() << "Rollback complete"; + LOGV2(21592, "Rollback complete"); return Status::OK(); } @@ -308,7 +322,7 @@ Status RollbackImpl::_transitionToRollback(OperationContext* opCtx) { return Status(ErrorCodes::ShutdownInProgress, "rollback shutting down"); } - log() << "transition to ROLLBACK"; + LOGV2(21593, "transition to ROLLBACK"); { ReplicationStateTransitionLockGuard rstlLock( opCtx, MODE_X, ReplicationStateTransitionLockGuard::EnqueueOnly()); @@ -327,7 +341,7 @@ Status RollbackImpl::_transitionToRollback(OperationContext* opCtx) { << "Cannot transition from " << _replicationCoordinator->getMemberState().toString() << " to " << MemberState(MemberState::RS_ROLLBACK).toString()); - log() << status; + LOGV2(21594, "{status}", "status"_attr = status); return status; } } @@ -350,14 +364,19 @@ Status RollbackImpl::_awaitBgIndexCompletion(OperationContext* opCtx) { // Wait for all background operations to complete by waiting on each database. std::vector<StringData> dbNames(dbs.begin(), dbs.end()); - log() << "Waiting for all background operations to complete before starting rollback"; + LOGV2(21595, "Waiting for all background operations to complete before starting rollback"); for (auto db : dbNames) { auto numInProg = BackgroundOperation::numInProgForDb(db); auto numInProgInCoordinator = IndexBuildsCoordinator::get(opCtx)->numInProgForDb(db); if (numInProg > 0 || numInProgInCoordinator > 0) { - LOG(1) << "Waiting for " - << (numInProg > numInProgInCoordinator ? numInProg : numInProgInCoordinator) - << " background operations to complete on database '" << db << "'"; + LOGV2_DEBUG( + 21596, + 1, + "Waiting for {numInProg_numInProgInCoordinator_numInProg_numInProgInCoordinator} " + "background operations to complete on database '{db}'", + "numInProg_numInProgInCoordinator_numInProg_numInProgInCoordinator"_attr = + (numInProg > numInProgInCoordinator ? numInProg : numInProgInCoordinator), + "db"_attr = db); BackgroundOperation::awaitNoBgOpInProgForDb(db); IndexBuildsCoordinator::get(opCtx)->awaitNoBgOpInProgForDb(db); } @@ -368,7 +387,7 @@ Status RollbackImpl::_awaitBgIndexCompletion(OperationContext* opCtx) { } } - log() << "Finished waiting for background operations to complete before rollback"; + LOGV2(21597, "Finished waiting for background operations to complete before rollback"); return Status::OK(); } @@ -478,7 +497,7 @@ void RollbackImpl::_runPhaseFromAbortToReconstructPreparedTxns( status = _writeRollbackFiles(opCtx); fassert(31228, status); } else { - log() << "Not writing rollback files. 'createRollbackDataFiles' set to false."; + LOGV2(21598, "Not writing rollback files. 'createRollbackDataFiles' set to false."); } // If there were rolled back operations on any session, invalidate all sessions. @@ -497,16 +516,24 @@ void RollbackImpl::_runPhaseFromAbortToReconstructPreparedTxns( // Log the total number of insert and update operations that have been rolled back as a // result of recovering to the stable timestamp. - log() << "Rollback reverted " << _observerInfo.rollbackCommandCounts[kInsertCmdName] - << " insert operations, " << _observerInfo.rollbackCommandCounts[kUpdateCmdName] - << " update operations and " << _observerInfo.rollbackCommandCounts[kDeleteCmdName] - << " delete operations."; + LOGV2(21599, + "Rollback reverted {observerInfo_rollbackCommandCounts_kInsertCmdName} insert " + "operations, {observerInfo_rollbackCommandCounts_kUpdateCmdName} update operations and " + "{observerInfo_rollbackCommandCounts_kDeleteCmdName} delete operations.", + "observerInfo_rollbackCommandCounts_kInsertCmdName"_attr = + _observerInfo.rollbackCommandCounts[kInsertCmdName], + "observerInfo_rollbackCommandCounts_kUpdateCmdName"_attr = + _observerInfo.rollbackCommandCounts[kUpdateCmdName], + "observerInfo_rollbackCommandCounts_kDeleteCmdName"_attr = + _observerInfo.rollbackCommandCounts[kDeleteCmdName]); // During replication recovery, we truncate all oplog entries with timestamps greater than the // oplog truncate after point. If we entered rollback, we are guaranteed to have at least one // oplog entry after the common point. - log() << "Marking to truncate all oplog entries with timestamps greater than " - << commonPoint.getOpTime().getTimestamp(); + LOGV2(21600, + "Marking to truncate all oplog entries with timestamps greater than " + "{commonPoint_getOpTime_getTimestamp}", + "commonPoint_getOpTime_getTimestamp"_attr = commonPoint.getOpTime().getTimestamp()); Timestamp truncatePoint = commonPoint.getOpTime().getTimestamp(); // Persist the truncate point to the 'oplogTruncateAfterPoint' document. We save this value so @@ -568,9 +595,14 @@ void RollbackImpl::_correctRecordStoreCounts(OperationContext* opCtx) { // if we just set the collection count here. if (sizeRecoveryState(opCtx->getServiceContext()) .collectionAlwaysNeedsSizeAdjustment(ident)) { - LOG(2) << "Not setting collection count to " << newCount << " for " << nss.ns() << " (" - << uuid.toString() << ") [" << ident - << "] because it is marked for size adjustment."; + LOGV2_DEBUG(21601, + 2, + "Not setting collection count to {newCount} for {nss_ns} ({uuid}) " + "[{ident}] because it is marked for size adjustment.", + "newCount"_attr = newCount, + "nss_ns"_attr = nss.ns(), + "uuid"_attr = uuid.toString(), + "ident"_attr = ident); continue; } @@ -578,8 +610,10 @@ void RollbackImpl::_correctRecordStoreCounts(OperationContext* opCtx) { // (most likely due to a 4.0 drop oplog entry without the count information), we will // determine the correct count here post-recovery using a collection scan. if (kCollectionScanRequired == newCount) { - log() << "Scanning collection " << nss.ns() << " (" << uuid.toString() - << ") to fix collection count."; + LOGV2(21602, + "Scanning collection {nss_ns} ({uuid}) to fix collection count.", + "nss_ns"_attr = nss.ns(), + "uuid"_attr = uuid.toString()); AutoGetCollectionForRead autoCollToScan(opCtx, nss); auto collToScan = autoCollToScan.getCollection(); invariant(coll == collToScan, @@ -596,9 +630,13 @@ void RollbackImpl::_correctRecordStoreCounts(OperationContext* opCtx) { if (PlanExecutor::IS_EOF != state) { // We ignore errors here because crashing or leaving rollback would only leave // collection counts more inaccurate. - warning() << "Failed to set count of " << nss.ns() << " (" << uuid.toString() - << ") [" << ident - << "] due to failed collection scan: " << exec->statestr(state); + LOGV2_WARNING(21637, + "Failed to set count of {nss_ns} ({uuid}) [{ident}] due to failed " + "collection scan: {exec_statestr_state}", + "nss_ns"_attr = nss.ns(), + "uuid"_attr = uuid.toString(), + "ident"_attr = ident, + "exec_statestr_state"_attr = exec->statestr(state)); continue; } newCount = countFromScan; @@ -609,11 +647,22 @@ void RollbackImpl::_correctRecordStoreCounts(OperationContext* opCtx) { if (!status.isOK()) { // We ignore errors here because crashing or leaving rollback would only leave // collection counts more inaccurate. - warning() << "Failed to set count of " << nss.ns() << " (" << uuid.toString() << ") [" - << ident << "] to " << newCount << ". Received: " << status; + LOGV2_WARNING(21638, + "Failed to set count of {nss_ns} ({uuid}) [{ident}] to {newCount}. " + "Received: {status}", + "nss_ns"_attr = nss.ns(), + "uuid"_attr = uuid.toString(), + "ident"_attr = ident, + "newCount"_attr = newCount, + "status"_attr = status); } else { - LOG(2) << "Set collection count of " << nss.ns() << " (" << uuid.toString() << ") [" - << ident << "] to " << newCount << "."; + LOGV2_DEBUG(21603, + 2, + "Set collection count of {nss_ns} ({uuid}) [{ident}] to {newCount}.", + "nss_ns"_attr = nss.ns(), + "uuid"_attr = uuid.toString(), + "ident"_attr = ident, + "newCount"_attr = newCount); } } } @@ -622,7 +671,7 @@ Status RollbackImpl::_findRecordStoreCounts(OperationContext* opCtx) { const auto& catalog = CollectionCatalog::get(opCtx); auto storageEngine = opCtx->getServiceContext()->getStorageEngine(); - log() << "finding record store counts"; + LOGV2(21604, "finding record store counts"); for (const auto& uiCount : _countDiffs) { auto uuid = uiCount.first; auto countDiff = uiCount.second; @@ -662,10 +711,13 @@ Status RollbackImpl::_findRecordStoreCounts(OperationContext* opCtx) { } if (oldCount > static_cast<uint64_t>(std::numeric_limits<long long>::max())) { - warning() << "Count for " << nss->ns() << " (" << uuid.toString() << ") was " - << oldCount - << " which is larger than the maximum int64_t value. Not attempting to fix " - "count during rollback."; + LOGV2_WARNING(21639, + "Count for {nss_ns} ({uuid}) was {oldCount} which is larger than the " + "maximum int64_t value. Not attempting to fix " + "count during rollback.", + "nss_ns"_attr = nss->ns(), + "uuid"_attr = uuid.toString(), + "oldCount"_attr = oldCount); continue; } @@ -673,17 +725,28 @@ Status RollbackImpl::_findRecordStoreCounts(OperationContext* opCtx) { auto newCount = oldCountSigned + countDiff; if (newCount < 0) { - warning() << "Attempted to set count for " << nss->ns() << " (" << uuid.toString() - << ") to " << newCount - << " but set it to 0 instead. This is likely due to the count previously " - "becoming inconsistent from an unclean shutdown or a rollback that could " - "not fix the count correctly. Old count: " - << oldCount << ". Count change: " << countDiff; + LOGV2_WARNING( + 21640, + "Attempted to set count for {nss_ns} ({uuid}) to {newCount} but set it to 0 " + "instead. This is likely due to the count previously " + "becoming inconsistent from an unclean shutdown or a rollback that could " + "not fix the count correctly. Old count: {oldCount}. Count change: {countDiff}", + "nss_ns"_attr = nss->ns(), + "uuid"_attr = uuid.toString(), + "newCount"_attr = newCount, + "oldCount"_attr = oldCount, + "countDiff"_attr = countDiff); newCount = 0; } - LOG(2) << "Record count of " << nss->ns() << " (" << uuid.toString() - << ") before rollback is " << oldCount << ". Setting it to " << newCount - << ", due to change of " << countDiff; + LOGV2_DEBUG(21605, + 2, + "Record count of {nss_ns} ({uuid}) before rollback is {oldCount}. Setting it " + "to {newCount}, due to change of {countDiff}", + "nss_ns"_attr = nss->ns(), + "uuid"_attr = uuid.toString(), + "oldCount"_attr = oldCount, + "newCount"_attr = newCount, + "countDiff"_attr = countDiff); _newCounts[uuid] = newCount; } @@ -774,15 +837,17 @@ Status RollbackImpl::_processRollbackOp(OperationContext* opCtx, const OplogEntr idVal == ShardIdentityType::IdName) { // Check if the creation of the shard identity document is being rolled back. _observerInfo.shardIdentityRolledBack = true; - warning() << "Shard identity document rollback detected. oplog op: " - << redact(oplogEntry.toBSON()); + LOGV2_WARNING(21641, + "Shard identity document rollback detected. oplog op: {oplogEntry}", + "oplogEntry"_attr = redact(oplogEntry.toBSON())); } else if (serverGlobalParams.clusterRole == ClusterRole::ConfigServer && opNss == VersionType::ConfigNS) { // Check if the creation of the config server config version document is being rolled // back. _observerInfo.configServerConfigVersionRolledBack = true; - warning() << "Config version document rollback detected. oplog op: " - << redact(oplogEntry.toBSON()); + LOGV2_WARNING(21642, + "Config version document rollback detected. oplog op: {oplogEntry}", + "oplogEntry"_attr = redact(oplogEntry.toBSON())); } // Rolling back an insert must decrement the count by 1. @@ -902,7 +967,7 @@ StatusWith<RollBackLocalOperations::RollbackCommonPoint> RollbackImpl::_findComm return Status(ErrorCodes::ShutdownInProgress, "rollback shutting down"); } - log() << "finding common point"; + LOGV2(21606, "finding common point"); // We save some aggregate information about all operations that are rolled back, so that we can // pass this information to the rollback op observer. In most cases, other subsystems do not @@ -934,7 +999,9 @@ StatusWith<RollBackLocalOperations::RollbackCommonPoint> RollbackImpl::_findComm auto stableTimestamp = _storageInterface->getLastStableRecoveryTimestamp(opCtx->getServiceContext()); - log() << "Rollback common point is " << commonPointOpTime; + LOGV2(21607, + "Rollback common point is {commonPointOpTime}", + "commonPointOpTime"_attr = commonPointOpTime); // Rollback common point should be >= the replication commit point. invariant(commonPointOpTime.getTimestamp() >= lastCommittedOpTime.getTimestamp()); @@ -949,8 +1016,11 @@ StatusWith<RollBackLocalOperations::RollbackCommonPoint> RollbackImpl::_findComm if (commonPointOpTime.getTimestamp() < *stableTimestamp) { // This is an fassert rather than an invariant, since it can happen if the server was // recently upgraded to enableMajorityReadConcern=true. - severe() << "Common point must be at least stable timestamp, common point: " - << commonPointOpTime.getTimestamp() << ", stable timestamp: " << *stableTimestamp; + LOGV2_FATAL(21644, + "Common point must be at least stable timestamp, common point: " + "{commonPointOpTime_getTimestamp}, stable timestamp: {stableTimestamp}", + "commonPointOpTime_getTimestamp"_attr = commonPointOpTime.getTimestamp(), + "stableTimestamp"_attr = *stableTimestamp); fassertFailedNoTrace(51121); } @@ -996,11 +1066,13 @@ Status RollbackImpl::_checkAgainstTimeLimit( } } else { - warning() - << "Wall clock times on oplog entries not monotonically increasing. This " - "might indicate a backward clock skew. Time at first oplog after common point: " - << firstOpWallClockTimeAfterCommonPoint - << ". Time at top of oplog: " << topOfOplogWallTime; + LOGV2_WARNING( + 21643, + "Wall clock times on oplog entries not monotonically increasing. This " + "might indicate a backward clock skew. Time at first oplog after common point: " + "{firstOpWallClockTimeAfterCommonPoint}. Time at top of oplog: {topOfOplogWallTime}", + "firstOpWallClockTimeAfterCommonPoint"_attr = firstOpWallClockTimeAfterCommonPoint, + "topOfOplogWallTime"_attr = topOfOplogWallTime); } return Status::OK(); @@ -1016,8 +1088,13 @@ boost::optional<BSONObj> RollbackImpl::_findDocumentById(OperationContext* opCtx } else if (document.getStatus().code() == ErrorCodes::NoSuchKey) { return boost::none; } else { - severe() << "Rollback failed to read document with " << redact(id) << " in namespace " - << nss.ns() << " with uuid " << uuid.toString() << causedBy(document.getStatus()); + LOGV2_FATAL(21645, + "Rollback failed to read document with {id} in namespace {nss_ns} with uuid " + "{uuid}{causedBy_document_getStatus}", + "id"_attr = redact(id), + "nss_ns"_attr = nss.ns(), + "uuid"_attr = uuid.toString(), + "causedBy_document_getStatus"_attr = causedBy(document.getStatus())); fassert(50751, document.getStatus()); } @@ -1034,10 +1111,12 @@ Status RollbackImpl::_writeRollbackFiles(OperationContext* opCtx) { // Drop-pending collections are not visible to rollback via the catalog when they are // managed by the storage engine. See StorageEngine::supportsPendingDrops(). if (!nss && storageEngine->supportsPendingDrops()) { - log() << "The collection with UUID " << uuid - << " is missing in the CollectionCatalog. This could be due to a dropped " - " collection. Not writing rollback file for uuid " - << uuid; + LOGV2(21608, + "The collection with UUID {uuid} is missing in the CollectionCatalog. This could " + "be due to a dropped " + " collection. Not writing rollback file for uuid {uuid2}", + "uuid"_attr = uuid, + "uuid2"_attr = uuid); continue; } @@ -1056,8 +1135,12 @@ void RollbackImpl::_writeRollbackFileForNamespace(OperationContext* opCtx, NamespaceString nss, const SimpleBSONObjUnorderedSet& idSet) { RemoveSaver removeSaver(kRollbackRemoveSaverType, uuid.toString(), kRollbackRemoveSaverWhy); - log() << "Preparing to write deleted documents to a rollback file for collection " << nss.ns() - << " with uuid " << uuid.toString() << " to " << removeSaver.file().generic_string(); + LOGV2(21609, + "Preparing to write deleted documents to a rollback file for collection {nss_ns} with " + "uuid {uuid} to {removeSaver_file_generic_string}", + "nss_ns"_attr = nss.ns(), + "uuid"_attr = uuid.toString(), + "removeSaver_file_generic_string"_attr = removeSaver.file().generic_string()); // The RemoveSaver will save the data files in a directory structure similar to the following: // @@ -1106,7 +1189,7 @@ Status RollbackImpl::_triggerOpObserver(OperationContext* opCtx) { if (_isInShutdown()) { return Status(ErrorCodes::ShutdownInProgress, "rollback shutting down"); } - log() << "Triggering the rollback op observer"; + LOGV2(21610, "Triggering the rollback op observer"); opCtx->getServiceContext()->getOpObserver()->onReplicationRollback(opCtx, _observerInfo); return Status::OK(); } @@ -1115,16 +1198,22 @@ void RollbackImpl::_transitionFromRollbackToSecondary(OperationContext* opCtx) { invariant(opCtx); invariant(_replicationCoordinator->getMemberState() == MemberState(MemberState::RS_ROLLBACK)); - log() << "transition to SECONDARY"; + LOGV2(21611, "transition to SECONDARY"); ReplicationStateTransitionLockGuard transitionGuard(opCtx, MODE_X); auto status = _replicationCoordinator->setFollowerMode(MemberState::RS_SECONDARY); if (!status.isOK()) { - severe() << "Failed to transition into " << MemberState(MemberState::RS_SECONDARY) - << "; expected to be in state " << MemberState(MemberState::RS_ROLLBACK) - << "; found self in " << _replicationCoordinator->getMemberState() - << causedBy(status); + LOGV2_FATAL( + 21646, + "Failed to transition into {MemberState_MemberState_RS_SECONDARY}; expected to be in " + "state {MemberState_MemberState_RS_ROLLBACK}; found self in " + "{replicationCoordinator_getMemberState}{causedBy_status}", + "MemberState_MemberState_RS_SECONDARY"_attr = MemberState(MemberState::RS_SECONDARY), + "MemberState_MemberState_RS_ROLLBACK"_attr = MemberState(MemberState::RS_ROLLBACK), + "replicationCoordinator_getMemberState"_attr = + _replicationCoordinator->getMemberState(), + "causedBy_status"_attr = causedBy(status)); fassertFailedNoTrace(40408); } } @@ -1150,21 +1239,36 @@ void RollbackImpl::_resetDropPendingState(OperationContext* opCtx) { } void RollbackImpl::_summarizeRollback(OperationContext* opCtx) const { - log() << "Rollback summary:"; - log() << "\tstart time: " << _rollbackStats.startTime; - log() << "\tend time: " << opCtx->getServiceContext()->getFastClockSource()->now(); - log() << "\tsync source: " << _remoteOplog->hostAndPort().toString(); - log() << "\trollback data file directory: " - << _rollbackStats.rollbackDataFileDirectory.value_or("none; no files written"); + LOGV2(21612, "Rollback summary:"); + LOGV2(21613, + "\tstart time: {rollbackStats_startTime}", + "rollbackStats_startTime"_attr = _rollbackStats.startTime); + LOGV2(21614, + "\tend time: {opCtx_getServiceContext_getFastClockSource_now}", + "opCtx_getServiceContext_getFastClockSource_now"_attr = + opCtx->getServiceContext()->getFastClockSource()->now()); + LOGV2(21615, + "\tsync source: {remoteOplog_hostAndPort}", + "remoteOplog_hostAndPort"_attr = _remoteOplog->hostAndPort().toString()); + LOGV2(21616, + "\trollback data file directory: " + "{rollbackStats_rollbackDataFileDirectory_value_or_none_no_files_written}", + "rollbackStats_rollbackDataFileDirectory_value_or_none_no_files_written"_attr = + _rollbackStats.rollbackDataFileDirectory.value_or("none; no files written")); if (_rollbackStats.rollbackId) { - log() << "\trollback id: " << *_rollbackStats.rollbackId; + LOGV2(21617, + "\trollback id: {rollbackStats_rollbackId}", + "rollbackStats_rollbackId"_attr = *_rollbackStats.rollbackId); } if (_rollbackStats.lastLocalOptime) { - log() << "\tlast optime on branch of history rolled back: " - << *_rollbackStats.lastLocalOptime; + LOGV2(21618, + "\tlast optime on branch of history rolled back: {rollbackStats_lastLocalOptime}", + "rollbackStats_lastLocalOptime"_attr = *_rollbackStats.lastLocalOptime); } if (_rollbackStats.commonPoint) { - log() << "\tcommon point optime: " << *_rollbackStats.commonPoint; + LOGV2(21619, + "\tcommon point optime: {rollbackStats_commonPoint}", + "rollbackStats_commonPoint"_attr = *_rollbackStats.commonPoint); } if (_rollbackStats.lastLocalWallClockTime && _rollbackStats.firstOpWallClockTimeAfterCommonPoint) { @@ -1175,36 +1279,62 @@ void RollbackImpl::_summarizeRollback(OperationContext* opCtx) const { unsigned long long diff = durationCount<Seconds>(Milliseconds(lastWall - firstOpWallClockTimeAfterCommonPoint)); - log() << "\tlast wall clock time on the branch of history rolled back: " << lastWall; - log() << "\twall clock time of the first operation after the common point: " - << firstOpWallClockTimeAfterCommonPoint; - log() << "\tdifference in wall clock times: " << diff << " second(s)"; + LOGV2(21620, + "\tlast wall clock time on the branch of history rolled back: {lastWall}", + "lastWall"_attr = lastWall); + LOGV2(21621, + "\twall clock time of the first operation after the common point: " + "{firstOpWallClockTimeAfterCommonPoint}", + "firstOpWallClockTimeAfterCommonPoint"_attr = firstOpWallClockTimeAfterCommonPoint); + LOGV2(21622, "\tdifference in wall clock times: {diff} second(s)", "diff"_attr = diff); } if (_rollbackStats.truncateTimestamp) { - log() << "\ttruncate timestamp: " << *_rollbackStats.truncateTimestamp; + LOGV2(21623, + "\ttruncate timestamp: {rollbackStats_truncateTimestamp}", + "rollbackStats_truncateTimestamp"_attr = *_rollbackStats.truncateTimestamp); } if (_rollbackStats.stableTimestamp) { - log() << "\tstable timestamp: " << *_rollbackStats.stableTimestamp; + LOGV2(21624, + "\tstable timestamp: {rollbackStats_stableTimestamp}", + "rollbackStats_stableTimestamp"_attr = *_rollbackStats.stableTimestamp); } - log() << "\tshard identity document rolled back: " << std::boolalpha - << _observerInfo.shardIdentityRolledBack; - log() << "\tconfig server config version document rolled back: " << std::boolalpha - << _observerInfo.configServerConfigVersionRolledBack; - log() << "\taffected sessions: " << (_observerInfo.rollbackSessionIds.empty() ? "none" : ""); + LOGV2(21625, + "\tshard identity document rolled back: {observerInfo_shardIdentityRolledBack}", + "observerInfo_shardIdentityRolledBack"_attr = _observerInfo.shardIdentityRolledBack); + LOGV2(21626, + "\tconfig server config version document rolled back: " + "{observerInfo_configServerConfigVersionRolledBack}", + "observerInfo_configServerConfigVersionRolledBack"_attr = + _observerInfo.configServerConfigVersionRolledBack); + LOGV2(21627, + "\taffected sessions: {observerInfo_rollbackSessionIds_empty_none}", + "observerInfo_rollbackSessionIds_empty_none"_attr = + (_observerInfo.rollbackSessionIds.empty() ? "none" : "")); for (const auto& sessionId : _observerInfo.rollbackSessionIds) { - log() << "\t\t" << sessionId; + LOGV2(21628, "\t\t{sessionId}", "sessionId"_attr = sessionId); } - log() << "\taffected namespaces: " << (_observerInfo.rollbackNamespaces.empty() ? "none" : ""); + LOGV2(21629, + "\taffected namespaces: {observerInfo_rollbackNamespaces_empty_none}", + "observerInfo_rollbackNamespaces_empty_none"_attr = + (_observerInfo.rollbackNamespaces.empty() ? "none" : "")); for (const auto& nss : _observerInfo.rollbackNamespaces) { - log() << "\t\t" << nss.ns(); + LOGV2(21630, "\t\t{nss_ns}", "nss_ns"_attr = nss.ns()); } - log() << "\tcounts of interesting commands rolled back: " - << (_observerInfo.rollbackCommandCounts.empty() ? "none" : ""); + LOGV2(21631, + "\tcounts of interesting commands rolled back: " + "{observerInfo_rollbackCommandCounts_empty_none}", + "observerInfo_rollbackCommandCounts_empty_none"_attr = + (_observerInfo.rollbackCommandCounts.empty() ? "none" : "")); for (const auto& entry : _observerInfo.rollbackCommandCounts) { - log() << "\t\t" << entry.first << ": " << entry.second; + LOGV2(21632, + "\t\t{entry_first}: {entry_second}", + "entry_first"_attr = entry.first, + "entry_second"_attr = entry.second); } - log() << "\ttotal number of entries rolled back (including no-ops): " - << _observerInfo.numberOfEntriesObserved; + LOGV2(21633, + "\ttotal number of entries rolled back (including no-ops): " + "{observerInfo_numberOfEntriesObserved}", + "observerInfo_numberOfEntriesObserved"_attr = _observerInfo.numberOfEntriesObserved); } } // namespace repl diff --git a/src/mongo/db/repl/rollback_impl_test.cpp b/src/mongo/db/repl/rollback_impl_test.cpp index e33b10f60cc..fa5b5af0a96 100644 --- a/src/mongo/db/repl/rollback_impl_test.cpp +++ b/src/mongo/db/repl/rollback_impl_test.cpp @@ -48,6 +48,7 @@ #include "mongo/db/s/shard_identity_rollback_notifier.h" #include "mongo/db/s/type_shard_identity.h" #include "mongo/db/service_context.h" +#include "mongo/logv2/log.h" #include "mongo/s/catalog/type_config_version.h" #include "mongo/stdx/thread.h" #include "mongo/transport/session.h" @@ -113,10 +114,15 @@ protected: UUID uuid, NamespaceString nss, const SimpleBSONObjUnorderedSet& idSet) final { - log() << "Simulating writing a rollback file for namespace " << nss.ns() << " with uuid " - << uuid; + LOGV2(21647, + "Simulating writing a rollback file for namespace {nss_ns} with uuid {uuid}", + "nss_ns"_attr = nss.ns(), + "uuid"_attr = uuid); for (auto&& id : idSet) { - log() << "Looking up " << id.jsonString(JsonStringFormat::LegacyStrict); + LOGV2(21648, + "Looking up {id_jsonString_JsonStringFormat_LegacyStrict}", + "id_jsonString_JsonStringFormat_LegacyStrict"_attr = + id.jsonString(JsonStringFormat::LegacyStrict)); auto document = _findDocumentById(opCtx, uuid, nss, id.firstElement()); if (document) { _uuidToObjsMap[uuid].push_back(*document); @@ -531,12 +537,12 @@ TEST_F(RollbackImplTest, RollbackKillsNecessaryOperations) { } // We assume that an interrupted opCtx would release its locks. - unittest::log() << "Both opCtx's marked for kill"; + LOGV2(21649, "Both opCtx's marked for kill"); ASSERT_EQ(ErrorCodes::InterruptedDueToReplStateChange, writeOpCtx->checkForInterruptNoAssert()); globalWrite = boost::none; ASSERT_EQ(ErrorCodes::InterruptedDueToReplStateChange, readOpCtx->checkForInterruptNoAssert()); globalRead = boost::none; - unittest::log() << "Both opCtx's were interrupted"; + LOGV2(21650, "Both opCtx's were interrupted"); rollbackThread.join(); ASSERT_OK(status); @@ -879,7 +885,7 @@ DEATH_TEST_F(RollbackImplTest, _storageInterface->setStableTimestamp(nullptr, Timestamp(1, 1)); auto status = _rollback->runRollback(_opCtx.get()); - unittest::log() << "Mongod did not crash. Status: " << status; + LOGV2(21651, "Mongod did not crash. Status: {status}", "status"_attr = status); MONGO_UNREACHABLE; } @@ -1234,7 +1240,7 @@ DEATH_TEST_F(RollbackImplTest, } auto status = _rollback->runRollback(_opCtx.get()); - unittest::log() << "mongod did not crash when expected; status: " << status; + LOGV2(21652, "mongod did not crash when expected; status: {status}", "status"_attr = status); } DEATH_TEST_F(RollbackImplTest, @@ -1248,7 +1254,7 @@ DEATH_TEST_F(RollbackImplTest, _storageInterface->setStableTimestamp(nullptr, Timestamp(1, 1)); auto status = _rollback->runRollback(_opCtx.get()); - unittest::log() << "mongod did not crash when expected; status: " << status; + LOGV2(21653, "mongod did not crash when expected; status: {status}", "status"_attr = status); } TEST_F(RollbackImplTest, RollbackSetsMultipleCollectionCounts) { @@ -1753,7 +1759,9 @@ DEATH_TEST_F(RollbackImplObserverInfoTest, Timestamp(2, 2), boost::none, "admin.$cmd", BSON("applyOps" << subops.arr()), 2); auto status = _rollback->_namespacesForOp_forTest(OplogEntry(applyOpsCmdOp.first)); - unittest::log() << "Mongod did not crash. Status: " << status.getStatus(); + LOGV2(21654, + "Mongod did not crash. Status: {status_getStatus}", + "status_getStatus"_attr = status.getStatus()); MONGO_UNREACHABLE; } @@ -1855,7 +1863,7 @@ DEATH_TEST_F(RollbackImplObserverInfoTest, ASSERT_OK(_insertOplogEntry(unknownCmdOp.first)); auto status = _rollback->runRollback(_opCtx.get()); - unittest::log() << "Mongod did not crash. Status: " << status; + LOGV2(21655, "Mongod did not crash. Status: {status}", "status"_attr = status); MONGO_UNREACHABLE; } diff --git a/src/mongo/db/repl/rs_rollback.cpp b/src/mongo/db/repl/rs_rollback.cpp index 647247a228c..aad64b6db3c 100644 --- a/src/mongo/db/repl/rs_rollback.cpp +++ b/src/mongo/db/repl/rs_rollback.cpp @@ -77,6 +77,7 @@ #include "mongo/db/storage/durable_catalog.h" #include "mongo/db/storage/remove_saver.h" #include "mongo/db/transaction_participant.h" +#include "mongo/logv2/log.h" #include "mongo/s/client/shard_registry.h" #include "mongo/s/grid.h" #include "mongo/util/exit.h" @@ -134,9 +135,12 @@ void FixUpInfo::removeRedundantOperations() { } bool FixUpInfo::removeRedundantIndexCommands(UUID uuid, std::string indexName) { - LOG(2) << "Attempting to remove redundant index operations from the set of indexes to create " - "for collection " - << uuid << ", for index '" << indexName << "'"; + LOGV2_DEBUG(21659, + 2, + "Attempting to remove redundant index operations from the set of indexes to create " + "for collection {uuid}, for index '{indexName}'", + "uuid"_attr = uuid, + "indexName"_attr = indexName); // See if there are any indexes to create for this collection. auto indexes = indexesToCreate.find(uuid); @@ -144,10 +148,12 @@ bool FixUpInfo::removeRedundantIndexCommands(UUID uuid, std::string indexName) { // There are no indexes to create for this collection UUID, so there are no index creation // operations to remove. if (indexes == indexesToCreate.end()) { - LOG(2) - << "Collection " << uuid - << " has no indexes to create. Not removing any index creation operations for index '" - << indexName << "'."; + LOGV2_DEBUG(21660, + 2, + "Collection {uuid} has no indexes to create. Not removing any index creation " + "operations for index '{indexName}'.", + "uuid"_attr = uuid, + "indexName"_attr = indexName); return false; } @@ -158,15 +164,23 @@ bool FixUpInfo::removeRedundantIndexCommands(UUID uuid, std::string indexName) { // If this index was not previously added to the set of indexes that need to be created for this // collection, then we do nothing. if (indexesToCreateForColl->find(indexName) == indexesToCreateForColl->end()) { - LOG(2) << "Index '" << indexName << "' was not previously set to be created for collection " - << uuid << ". Not removing any index creation operations."; + LOGV2_DEBUG(21661, + 2, + "Index '{indexName}' was not previously set to be created for collection " + "{uuid}. Not removing any index creation operations.", + "indexName"_attr = indexName, + "uuid"_attr = uuid); return false; } // This index was previously added to the set of indexes to create for this collection, so we // remove it from that set. - LOG(2) << "Index '" << indexName << "' was previously set to be created for collection " << uuid - << ". Removing this redundant index creation operation."; + LOGV2_DEBUG(21662, + 2, + "Index '{indexName}' was previously set to be created for collection {uuid}. " + "Removing this redundant index creation operation.", + "indexName"_attr = indexName, + "uuid"_attr = uuid); indexesToCreateForColl->erase(indexName); // If there are now no remaining indexes to create for this collection, remove it from // the set of collections that we need to create indexes for. @@ -206,7 +220,7 @@ Status FixUpInfo::recordDropTargetInfo(const BSONElement& dropTarget, << "Unable to roll back renameCollection. Cannot parse " "dropTarget UUID. Returned status: " << redact(dropTargetUUIDStatus.getStatus()) << ", oplog entry: " << redact(obj); - error() << message; + LOGV2_ERROR(21729, "{message}", "message"_attr = message); return dropTargetUUIDStatus.getStatus(); } UUID dropTargetUUID = dropTargetUUIDStatus.getValue(); @@ -253,8 +267,10 @@ Status rollback_internal::updateFixUpInfoFromLocalOplogEntry(OperationContext* o const OplogEntry oplogEntry(fixedObj); if (isNestedApplyOpsCommand) { - LOG(2) << "Updating rollback FixUpInfo for nested applyOps oplog entry: " - << redact(oplogEntry.toBSON()); + LOGV2_DEBUG(21663, + 2, + "Updating rollback FixUpInfo for nested applyOps oplog entry: {oplogEntry}", + "oplogEntry"_attr = redact(oplogEntry.toBSON())); } // Extract the op's collection namespace and UUID. @@ -371,9 +387,10 @@ Status rollback_internal::updateFixUpInfoFromLocalOplogEntry(OperationContext* o string indexName; auto status = bsonExtractStringField(obj, "index", &indexName); if (!status.isOK()) { - severe() - << "Missing index name in dropIndexes operation on rollback, document: " - << redact(oplogEntry.toBSON()); + LOGV2_FATAL(21731, + "Missing index name in dropIndexes operation on rollback, " + "document: {oplogEntry}", + "oplogEntry"_attr = redact(oplogEntry.toBSON())); throw RSFatalException( "Missing index name in dropIndexes operation on rollback."); } @@ -409,9 +426,10 @@ Status rollback_internal::updateFixUpInfoFromLocalOplogEntry(OperationContext* o string indexName; auto status = bsonExtractStringField(obj, "name", &indexName); if (!status.isOK()) { - severe() - << "Missing index name in createIndexes operation on rollback, document: " - << redact(oplogEntry.toBSON()); + LOGV2_FATAL(21732, + "Missing index name in createIndexes operation on rollback, " + "document: {oplogEntry}", + "oplogEntry"_attr = redact(oplogEntry.toBSON())); throw RSFatalException( "Missing index name in createIndexes operation on rollback."); } @@ -457,9 +475,12 @@ Status rollback_internal::updateFixUpInfoFromLocalOplogEntry(OperationContext* o auto buildUUID = indexBuildOplogEntry.buildUUID; auto existingIt = buildsToRestart.find(buildUUID); if (existingIt != buildsToRestart.end()) { - LOG(2) << "Index build that was previously marked to be restarted will now be " - "dropped due to a rolled-back 'startIndexBuild' oplog entry: " - << buildUUID; + LOGV2_DEBUG( + 21664, + 2, + "Index build that was previously marked to be restarted will now be " + "dropped due to a rolled-back 'startIndexBuild' oplog entry: {buildUUID}", + "buildUUID"_attr = buildUUID); buildsToRestart.erase(existingIt); // If the index build was committed or aborted, we must mark the index as @@ -500,8 +521,11 @@ Status rollback_internal::updateFixUpInfoFromLocalOplogEntry(OperationContext* o "UUID is already marked to be restarted: " << buildUUID); - LOG(2) << "Index build will be restarted after a rolled-back 'abortIndexBuild': " - << buildUUID; + LOGV2_DEBUG(21665, + 2, + "Index build will be restarted after a rolled-back 'abortIndexBuild': " + "{buildUUID}", + "buildUUID"_attr = buildUUID); IndexBuildDetails details{*uuid}; for (auto& spec : indexBuildOplogEntry.indexSpecs) { invariant(spec.isOwned()); @@ -542,8 +566,11 @@ Status rollback_internal::updateFixUpInfoFromLocalOplogEntry(OperationContext* o "UUID is already marked to be restarted: " << buildUUID); - LOG(2) << "Index build will be restarted after a rolled-back 'commitIndexBuild': " - << buildUUID; + LOGV2_DEBUG(21666, + 2, + "Index build will be restarted after a rolled-back 'commitIndexBuild': " + "{buildUUID}", + "buildUUID"_attr = buildUUID); IndexBuildDetails details{*uuid}; for (auto& spec : indexBuildOplogEntry.indexSpecs) { @@ -571,7 +598,7 @@ Status rollback_internal::updateFixUpInfoFromLocalOplogEntry(OperationContext* o if (ns.empty()) { std::string message = str::stream() << "Collection name missing from oplog entry: " << redact(obj); - log() << message; + LOGV2(21667, "{message}", "message"_attr = message); return Status(ErrorCodes::UnrecoverableRollbackError, message); } @@ -650,7 +677,10 @@ Status rollback_internal::updateFixUpInfoFromLocalOplogEntry(OperationContext* o } // Some collMod fields cannot be rolled back, such as the index field. string message = "Cannot roll back a collMod command: "; - severe() << message << redact(obj); + LOGV2_FATAL(21733, + "{message}{obj}", + "message"_attr = message, + "obj"_attr = redact(obj)); throw RSFatalException(message); } return Status::OK(); @@ -687,7 +717,7 @@ Status rollback_internal::updateFixUpInfoFromLocalOplogEntry(OperationContext* o std::string message = str::stream() << "Expected applyOps argument to be an array; found " << redact(operations); - severe() << message; + LOGV2_FATAL(21734, "{message}", "message"_attr = message); return Status(ErrorCodes::UnrecoverableRollbackError, message); } for (const auto& subopElement : operations.Array()) { @@ -695,7 +725,7 @@ Status rollback_internal::updateFixUpInfoFromLocalOplogEntry(OperationContext* o std::string message = str::stream() << "Expected applyOps operations to be of Object type, but found " << redact(subopElement); - severe() << message; + LOGV2_FATAL(21735, "{message}", "message"_attr = message); return Status(ErrorCodes::UnrecoverableRollbackError, message); } // In applyOps, the object contains an array of different oplog entries, we @@ -729,7 +759,10 @@ Status rollback_internal::updateFixUpInfoFromLocalOplogEntry(OperationContext* o default: { std::string message = str::stream() << "Can't roll back this command yet: " << " cmdname = " << first.fieldName(); - severe() << message << " document: " << redact(obj); + LOGV2_FATAL(21736, + "{message} document: {obj}", + "message"_attr = message, + "obj"_attr = redact(obj)); throw RSFatalException(message); } } @@ -742,7 +775,10 @@ Status rollback_internal::updateFixUpInfoFromLocalOplogEntry(OperationContext* o doc._id = oplogEntry.getIdElement(); if (doc._id.eoo()) { std::string message = str::stream() << "Cannot roll back op with no _id. ns: " << nss.ns(); - severe() << message << ", document: " << redact(oplogEntry.toBSON()); + LOGV2_FATAL(21737, + "{message}, document: {oplogEntry}", + "message"_attr = message, + "oplogEntry"_attr = redact(oplogEntry.toBSON())); throw RSFatalException(message); } fixUpInfo.docsToRefetch.insert(doc); @@ -779,7 +815,7 @@ void checkRbidAndUpdateMinValid(OperationContext* opCtx, // RECOVERING state to SECONDARY state until we have reached the minValid oplog entry. OpTime minValid = fassert(40492, OpTime::parseFromOplogEntry(newMinValidDoc)); - log() << "Setting minvalid to " << minValid; + LOGV2(21668, "Setting minvalid to {minValid}", "minValid"_attr = minValid); // This method is only used with storage engines that do not support recover to stable // timestamp. As a result, the timestamp on the 'appliedThrough' update does not matter. @@ -790,8 +826,9 @@ void checkRbidAndUpdateMinValid(OperationContext* opCtx, if (MONGO_unlikely(rollbackHangThenFailAfterWritingMinValid.shouldFail())) { // This log output is used in jstests so please leave it. - log() << "rollback - rollbackHangThenFailAfterWritingMinValid fail point " - "enabled. Blocking until fail point is disabled."; + LOGV2(21669, + "rollback - rollbackHangThenFailAfterWritingMinValid fail point " + "enabled. Blocking until fail point is disabled."); while (MONGO_unlikely(rollbackHangThenFailAfterWritingMinValid.shouldFail())) { invariant(!globalInShutdownDeprecated()); // It is an error to shutdown while enabled. mongo::sleepsecs(1); @@ -813,8 +850,10 @@ void dropIndex(OperationContext* opCtx, auto indexDescriptor = indexCatalog->findIndexByName(opCtx, indexName, includeUnfinishedIndexes); if (!indexDescriptor) { - warning() << "Rollback failed to drop index " << indexName << " in " << nss.toString() - << ": index not found."; + LOGV2_WARNING(21725, + "Rollback failed to drop index {indexName} in {nss}: index not found.", + "indexName"_attr = indexName, + "nss"_attr = nss.toString()); return; } WriteUnitOfWork wunit(opCtx); @@ -822,14 +861,20 @@ void dropIndex(OperationContext* opCtx, if (entry->isReady(opCtx)) { auto status = indexCatalog->dropIndex(opCtx, indexDescriptor); if (!status.isOK()) { - severe() << "Rollback failed to drop index " << indexName << " in " << nss.toString() - << ": " << redact(status); + LOGV2_FATAL(21738, + "Rollback failed to drop index {indexName} in {nss}: {status}", + "indexName"_attr = indexName, + "nss"_attr = nss.toString(), + "status"_attr = redact(status)); } } else { auto status = indexCatalog->dropUnfinishedIndex(opCtx, indexDescriptor); if (!status.isOK()) { - severe() << "Rollback failed to drop unfinished index " << indexName << " in " - << nss.toString() << ": " << redact(status); + LOGV2_FATAL(21739, + "Rollback failed to drop unfinished index {indexName} in {nss}: {status}", + "indexName"_attr = indexName, + "nss"_attr = nss.toString(), + "status"_attr = redact(status)); } } wunit.commit(); @@ -849,29 +894,43 @@ void rollbackCreateIndexes(OperationContext* opCtx, UUID uuid, std::set<std::str // If we cannot find the collection, we skip over dropping the index. if (!collection) { - LOG(2) << "Cannot find the collection with uuid: " << uuid.toString() - << " in CollectionCatalog during roll back of a createIndexes command."; + LOGV2_DEBUG(21670, + 2, + "Cannot find the collection with uuid: {uuid} in CollectionCatalog during roll " + "back of a createIndexes command.", + "uuid"_attr = uuid.toString()); return; } // If we cannot find the index catalog, we skip over dropping the index. auto indexCatalog = collection->getIndexCatalog(); if (!indexCatalog) { - LOG(2) << "Cannot find the index catalog in collection with uuid: " << uuid.toString() - << " during roll back of a createIndexes command."; + LOGV2_DEBUG(21671, + 2, + "Cannot find the index catalog in collection with uuid: {uuid} during roll " + "back of a createIndexes command.", + "uuid"_attr = uuid.toString()); return; } for (auto itIndex = indexNames.begin(); itIndex != indexNames.end(); itIndex++) { const string& indexName = *itIndex; - log() << "Dropping index in rollback for collection: " << *nss << ", UUID: " << uuid - << ", index: " << indexName; + LOGV2(21672, + "Dropping index in rollback for collection: {nss}, UUID: {uuid}, index: {indexName}", + "nss"_attr = *nss, + "uuid"_attr = uuid, + "indexName"_attr = indexName); dropIndex(opCtx, indexCatalog, indexName, *nss); - LOG(1) << "Dropped index in rollback for collection: " << *nss << ", UUID: " << uuid - << ", index: " << indexName; + LOGV2_DEBUG( + 21673, + 1, + "Dropped index in rollback for collection: {nss}, UUID: {uuid}, index: {indexName}", + "nss"_attr = *nss, + "uuid"_attr = uuid, + "indexName"_attr = indexName); } } @@ -891,8 +950,11 @@ void rollbackDropIndexes(OperationContext* opCtx, // If we cannot find the collection, we skip over dropping the index. if (!collection) { - LOG(2) << "Cannot find the collection with uuid: " << uuid.toString() - << "in CollectionCatalog during roll back of a dropIndexes command."; + LOGV2_DEBUG(21674, + 2, + "Cannot find the collection with uuid: {uuid}in CollectionCatalog during roll " + "back of a dropIndexes command.", + "uuid"_attr = uuid.toString()); return; } @@ -900,13 +962,21 @@ void rollbackDropIndexes(OperationContext* opCtx, const string indexName = itIndex->first; BSONObj indexSpec = itIndex->second; - log() << "Creating index in rollback for collection: " << *nss << ", UUID: " << uuid - << ", index: " << indexName; + LOGV2(21675, + "Creating index in rollback for collection: {nss}, UUID: {uuid}, index: {indexName}", + "nss"_attr = *nss, + "uuid"_attr = uuid, + "indexName"_attr = indexName); createIndexForApplyOps(opCtx, indexSpec, *nss, OplogApplication::Mode::kRecovering); - LOG(1) << "Created index in rollback for collection: " << *nss << ", UUID: " << uuid - << ", index: " << indexName; + LOGV2_DEBUG( + 21676, + 1, + "Created index in rollback for collection: {nss}, UUID: {uuid}, index: {indexName}", + "nss"_attr = *nss, + "uuid"_attr = uuid, + "indexName"_attr = indexName); } } @@ -919,10 +989,14 @@ void dropCollection(OperationContext* opCtx, Database* db) { if (RollbackImpl::shouldCreateDataFiles()) { RemoveSaver removeSaver("rollback", "", collection->uuid().toString()); - log() << "Rolling back createCollection on " << nss - << ": Preparing to write documents to a rollback file for a collection " << nss - << " with uuid " << collection->uuid() << " to " - << removeSaver.file().generic_string(); + LOGV2(21677, + "Rolling back createCollection on {nss}: Preparing to write documents to a rollback " + "file for a collection {nss2} with uuid {collection_uuid} to " + "{removeSaver_file_generic_string}", + "nss"_attr = nss, + "nss2"_attr = nss, + "collection_uuid"_attr = collection->uuid(), + "removeSaver_file_generic_string"_attr = removeSaver.file().generic_string()); // Performs a collection scan and writes all documents in the collection to disk // in order to keep an archive of items that were rolled back. @@ -933,8 +1007,11 @@ void dropCollection(OperationContext* opCtx, while (PlanExecutor::ADVANCED == (execState = exec->getNext(&curObj, nullptr))) { auto status = removeSaver.goingToDelete(curObj); if (!status.isOK()) { - severe() << "Rolling back createCollection on " << nss - << " failed to write document to remove saver file: " << redact(status); + LOGV2_FATAL(21740, + "Rolling back createCollection on {nss} failed to write document to " + "remove saver file: {status}", + "nss"_attr = nss, + "status"_attr = redact(status)); throw RSFatalException( "Rolling back createCollection. Failed to write document to remove saver " "file."); @@ -951,13 +1028,18 @@ void dropCollection(OperationContext* opCtx, if (execState == PlanExecutor::FAILURE && WorkingSetCommon::isValidStatusMemberObject(curObj)) { Status errorStatus = WorkingSetCommon::getMemberObjectStatus(curObj); - severe() << "Rolling back createCollection on " << nss << " failed with " - << redact(errorStatus) << ". A full resync is necessary."; + LOGV2_FATAL(21741, + "Rolling back createCollection on {nss} failed with {errorStatus}. A " + "full resync is necessary.", + "nss"_attr = nss, + "errorStatus"_attr = redact(errorStatus)); throw RSFatalException( "Rolling back createCollection failed. A full resync is necessary."); } else { - severe() << "Rolling back createCollection on " << nss - << " failed. A full resync is necessary."; + LOGV2_FATAL( + 21742, + "Rolling back createCollection on {nss} failed. A full resync is necessary.", + "nss"_attr = nss); throw RSFatalException( "Rolling back createCollection failed. A full resync is necessary."); } @@ -992,17 +1074,27 @@ void renameOutOfTheWay(OperationContext* opCtx, RenameCollectionInfo info, Datab // namespace. auto tmpNameResult = db->makeUniqueCollectionNamespace(opCtx, "rollback.tmp%%%%%"); if (!tmpNameResult.isOK()) { - severe() << "Unable to generate temporary namespace to rename collection " << info.renameTo - << " out of the way. " << tmpNameResult.getStatus().reason(); + LOGV2_FATAL(21743, + "Unable to generate temporary namespace to rename collection {info_renameTo} " + "out of the way. {tmpNameResult_getStatus_reason}", + "info_renameTo"_attr = info.renameTo, + "tmpNameResult_getStatus_reason"_attr = tmpNameResult.getStatus().reason()); throw RSFatalException( "Unable to generate temporary namespace to rename collection out of the way."); } const auto& tempNss = tmpNameResult.getValue(); - LOG(2) << "Attempted to rename collection from " << info.renameFrom << " to " << info.renameTo - << " but " << info.renameTo << " exists already. Temporarily renaming collection " - << info.renameTo << " with UUID " << collection->uuid() << " out of the way to " - << tempNss; + LOGV2_DEBUG(21678, + 2, + "Attempted to rename collection from {info_renameFrom} to {info_renameTo} but " + "{info_renameTo2} exists already. Temporarily renaming collection {info_renameTo3} " + "with UUID {collection_uuid} out of the way to {tempNss}", + "info_renameFrom"_attr = info.renameFrom, + "info_renameTo"_attr = info.renameTo, + "info_renameTo2"_attr = info.renameTo, + "info_renameTo3"_attr = info.renameTo, + "collection_uuid"_attr = collection->uuid(), + "tempNss"_attr = tempNss); // Renaming the collection that was clashing with the attempted rename // operation to a different collection name. @@ -1010,8 +1102,10 @@ void renameOutOfTheWay(OperationContext* opCtx, RenameCollectionInfo info, Datab auto renameStatus = renameCollectionForRollback(opCtx, tempNss, uuid); if (!renameStatus.isOK()) { - severe() << "Unable to rename collection " << info.renameTo << " out of the way to " - << tempNss; + LOGV2_FATAL(21744, + "Unable to rename collection {info_renameTo} out of the way to {tempNss}", + "info_renameTo"_attr = info.renameTo, + "tempNss"_attr = tempNss); throw RSFatalException("Unable to rename collection out of the way"); } } @@ -1023,8 +1117,12 @@ void rollbackRenameCollection(OperationContext* opCtx, UUID uuid, RenameCollecti auto dbName = info.renameFrom.db(); - log() << "Attempting to rename collection with UUID: " << uuid << ", from: " << info.renameFrom - << ", to: " << info.renameTo; + LOGV2(21679, + "Attempting to rename collection with UUID: {uuid}, from: {info_renameFrom}, to: " + "{info_renameTo}", + "uuid"_attr = uuid, + "info_renameFrom"_attr = info.renameFrom, + "info_renameTo"_attr = info.renameTo); Lock::DBLock dbLock(opCtx, dbName, MODE_X); auto databaseHolder = DatabaseHolder::get(opCtx); auto db = databaseHolder->openDb(opCtx, dbName); @@ -1044,20 +1142,30 @@ void rollbackRenameCollection(OperationContext* opCtx, UUID uuid, RenameCollecti status = renameCollectionForRollback(opCtx, info.renameTo, uuid); if (!status.isOK()) { - severe() << "Rename collection failed to roll back twice. We were unable to rename " - << "collection " << info.renameFrom << " to " << info.renameTo << ". " - << status.toString(); + LOGV2_FATAL(21745, + "Rename collection failed to roll back twice. We were unable to rename " + "collection {info_renameFrom} to {info_renameTo}. {status}", + "info_renameFrom"_attr = info.renameFrom, + "info_renameTo"_attr = info.renameTo, + "status"_attr = status.toString()); throw RSFatalException( "Rename collection failed to roll back twice. We were unable to rename " "the collection."); } } else if (!status.isOK()) { - severe() << "Unable to roll back renameCollection command: " << status.toString(); + LOGV2_FATAL(21746, + "Unable to roll back renameCollection command: {status}", + "status"_attr = status.toString()); throw RSFatalException("Unable to rollback renameCollection command"); } - LOG(1) << "Renamed collection with UUID: " << uuid << ", from: " << info.renameFrom - << ", to: " << info.renameTo; + LOGV2_DEBUG( + 21680, + 1, + "Renamed collection with UUID: {uuid}, from: {info_renameFrom}, to: {info_renameTo}", + "uuid"_attr = uuid, + "info_renameFrom"_attr = info.renameFrom, + "info_renameTo"_attr = info.renameTo); } Status _syncRollback(OperationContext* opCtx, @@ -1071,7 +1179,10 @@ Status _syncRollback(OperationContext* opCtx, FixUpInfo how; how.localTopOfOplog = replCoord->getMyLastAppliedOpTime(); - log() << "Starting rollback. Sync source: " << rollbackSource.getSource() << rsLog; + LOGV2_OPTIONS(21681, + {logv2::LogTag::kRS}, + "Starting rollback. Sync source: {rollbackSource_getSource}", + "rollbackSource_getSource"_attr = rollbackSource.getSource()); how.rbid = rollbackSource.getRollbackId(); uassert( 40506, "Upstream node rolled back. Need to retry our rollback.", how.rbid == requiredRBID); @@ -1085,7 +1196,7 @@ Status _syncRollback(OperationContext* opCtx, // they may be made redundant by a rolled-back startIndexBuild oplog entry. how.indexBuildsToRestart.insert(abortedIndexBuilds.begin(), abortedIndexBuilds.end()); - log() << "Finding the Common Point"; + LOGV2(21682, "Finding the Common Point"); try { auto processOperationForFixUp = [&how, &opCtx, &localOplog](const BSONObj& operation) { @@ -1122,7 +1233,7 @@ Status _syncRollback(OperationContext* opCtx, OpTime lastCommittedOpTime = replCoord->getLastCommittedOpTime(); OpTime committedSnapshot = replCoord->getCurrentCommittedSnapshotOpTime(); - log() << "Rollback common point is " << commonPoint; + LOGV2(21683, "Rollback common point is {commonPoint}", "commonPoint"_attr = commonPoint); // Rollback common point should be >= the replication commit point. invariant(commonPoint.getTimestamp() >= lastCommittedOpTime.getTimestamp()); @@ -1140,8 +1251,9 @@ Status _syncRollback(OperationContext* opCtx, syncFixUp(opCtx, how, rollbackSource, replCoord, replicationProcess); if (MONGO_unlikely(rollbackExitEarlyAfterCollectionDrop.shouldFail())) { - log() << "rollbackExitEarlyAfterCollectionDrop fail point enabled. Returning early " - "until fail point is disabled."; + LOGV2(21684, + "rollbackExitEarlyAfterCollectionDrop fail point enabled. Returning early " + "until fail point is disabled."); return Status(ErrorCodes::NamespaceNotFound, str::stream() << "Failing rollback because " "rollbackExitEarlyAfterCollectionDrop fail point " @@ -1164,8 +1276,9 @@ Status _syncRollback(OperationContext* opCtx, if (MONGO_unlikely(rollbackHangBeforeFinish.shouldFail())) { // This log output is used in js tests so please leave it. - log() << "rollback - rollbackHangBeforeFinish fail point " - "enabled. Blocking until fail point is disabled."; + LOGV2(21685, + "rollback - rollbackHangBeforeFinish fail point " + "enabled. Blocking until fail point is disabled."); while (MONGO_unlikely(rollbackHangBeforeFinish.shouldFail())) { invariant(!globalInShutdownDeprecated()); // It is an error to shutdown while enabled. mongo::sleepsecs(1); @@ -1191,7 +1304,7 @@ void rollback_internal::syncFixUp(OperationContext* opCtx, // Fetches all the goodVersions of each document from the current sync source. unsigned long long numFetched = 0; - log() << "Starting refetching documents"; + LOGV2(21686, "Starting refetching documents"); for (auto&& doc : fixUpInfo.docsToRefetch) { invariant(!doc._id.eoo()); // This is checked when we insert to the set. @@ -1201,10 +1314,18 @@ void rollback_internal::syncFixUp(OperationContext* opCtx, try { if (nss) { - LOG(2) << "Refetching document, collection: " << *nss << ", UUID: " << uuid << ", " - << redact(doc._id); + LOGV2_DEBUG(21687, + 2, + "Refetching document, collection: {nss}, UUID: {uuid}, {doc_id}", + "nss"_attr = *nss, + "uuid"_attr = uuid, + "doc_id"_attr = redact(doc._id)); } else { - LOG(2) << "Refetching document, UUID: " << uuid << ", " << redact(doc._id); + LOGV2_DEBUG(21688, + 2, + "Refetching document, UUID: {uuid}, {doc_id}", + "uuid"_attr = uuid, + "doc_id"_attr = redact(doc._id)); } // TODO : Slow. Lots of round trips. numFetched++; @@ -1253,27 +1374,34 @@ void rollback_internal::syncFixUp(OperationContext* opCtx, ex.code() == ErrorCodes::NamespaceNotFound) continue; - log() << "Rollback couldn't re-fetch from uuid: " << uuid << " _id: " << redact(doc._id) - << ' ' << numFetched << '/' << fixUpInfo.docsToRefetch.size() << ": " - << redact(ex); + LOGV2(21689, + "Rollback couldn't re-fetch from uuid: {uuid} _id: {doc_id} " + "{numFetched}/{fixUpInfo_docsToRefetch_size}: {ex}", + "uuid"_attr = uuid, + "doc_id"_attr = redact(doc._id), + "numFetched"_attr = numFetched, + "fixUpInfo_docsToRefetch_size"_attr = fixUpInfo.docsToRefetch.size(), + "ex"_attr = redact(ex)); throw; } } - log() << "Finished refetching documents. Total size of documents refetched: " - << goodVersions.size(); + LOGV2(21690, + "Finished refetching documents. Total size of documents refetched: {goodVersions_size}", + "goodVersions_size"_attr = goodVersions.size()); // We must start taking unstable checkpoints before rolling back oplog entries. Otherwise, a // stable checkpoint could include the fixup write (since it is untimestamped) but not the write // being rolled back (if it is after the stable timestamp), leading to inconsistent state. An // unstable checkpoint will include both writes. if (!serverGlobalParams.enableMajorityReadConcern) { - log() << "Setting initialDataTimestamp to 0 so that we start taking unstable checkpoints."; + LOGV2(21691, + "Setting initialDataTimestamp to 0 so that we start taking unstable checkpoints."); opCtx->getServiceContext()->getStorageEngine()->setInitialDataTimestamp( Timestamp::kAllowUnstableCheckpointsSentinel); } - log() << "Checking the RollbackID and updating the MinValid if necessary"; + LOGV2(21692, "Checking the RollbackID and updating the MinValid if necessary"); checkRbidAndUpdateMinValid(opCtx, fixUpInfo.rbid, rollbackSource, replicationProcess); @@ -1288,7 +1416,7 @@ void rollback_internal::syncFixUp(OperationContext* opCtx, // indexes. // We drop indexes before renaming collections so that if a collection name gets longer, // any indexes with names that are now too long will already be dropped. - log() << "Rolling back createIndexes and startIndexBuild operations"; + LOGV2(21693, "Rolling back createIndexes and startIndexBuild operations"); for (auto it = fixUpInfo.indexesToDrop.begin(); it != fixUpInfo.indexesToDrop.end(); it++) { UUID uuid = it->first; @@ -1301,7 +1429,7 @@ void rollback_internal::syncFixUp(OperationContext* opCtx, // rolled-back, but the unfinished index still exists in the catalog. Drop these before any // collection drops, because one of the preconditions of dropping a collection is that there are // no unfinished indxes. - log() << "Rolling back unfinished startIndexBuild operations"; + LOGV2(21694, "Rolling back unfinished startIndexBuild operations"); for (auto index : fixUpInfo.unfinishedIndexesToDrop) { UUID uuid = index.first; std::set<std::string> indexNames = index.second; @@ -1309,7 +1437,7 @@ void rollback_internal::syncFixUp(OperationContext* opCtx, rollbackCreateIndexes(opCtx, uuid, indexNames); } - log() << "Dropping collections to roll back create operations"; + LOGV2(21695, "Dropping collections to roll back create operations"); // Drops collections before updating individual documents. We drop these collections before // rolling back any other commands to prevent namespace collisions that may occur @@ -1332,9 +1460,12 @@ void rollback_internal::syncFixUp(OperationContext* opCtx, // Do not attempt to acquire the database lock with an empty namespace. We should survive // an attempt to drop a non-existent collection. if (!nss) { - log() << "This collection does not exist, UUID: " << uuid; + LOGV2(21696, "This collection does not exist, UUID: {uuid}", "uuid"_attr = uuid); } else { - log() << "Dropping collection: " << *nss << ", UUID: " << uuid; + LOGV2(21697, + "Dropping collection: {nss}, UUID: {uuid}", + "nss"_attr = *nss, + "uuid"_attr = uuid); AutoGetDb dbLock(opCtx, nss->db(), MODE_X); Database* db = dbLock.getDb(); @@ -1342,7 +1473,11 @@ void rollback_internal::syncFixUp(OperationContext* opCtx, Collection* collection = CollectionCatalog::get(opCtx).lookupCollectionByUUID(opCtx, uuid); dropCollection(opCtx, *nss, collection, db); - LOG(1) << "Dropped collection: " << *nss << ", UUID: " << uuid; + LOGV2_DEBUG(21698, + 1, + "Dropped collection: {nss}, UUID: {uuid}", + "nss"_attr = *nss, + "uuid"_attr = uuid); } } } @@ -1352,7 +1487,7 @@ void rollback_internal::syncFixUp(OperationContext* opCtx, } // Rolling back renameCollection commands. - log() << "Rolling back renameCollection commands and collection drop commands."; + LOGV2(21699, "Rolling back renameCollection commands and collection drop commands."); for (auto it = fixUpInfo.collectionsToRename.begin(); it != fixUpInfo.collectionsToRename.end(); it++) { @@ -1363,16 +1498,21 @@ void rollback_internal::syncFixUp(OperationContext* opCtx, rollbackRenameCollection(opCtx, uuid, info); } - log() << "Rolling back collections pending being dropped: Removing them from the list of " - "drop-pending collections in the DropPendingCollectionReaper."; + LOGV2(21700, + "Rolling back collections pending being dropped: Removing them from the list of " + "drop-pending collections in the DropPendingCollectionReaper."); // Roll back any drop-pending collections. This must be done first so that the collection // exists when we attempt to resync its metadata or insert documents into it. for (const auto& collPair : fixUpInfo.collectionsToRemoveFromDropPendingCollections) { const auto& optime = collPair.second.first; const auto& collectionNamespace = collPair.second.second; - LOG(1) << "Rolling back collection pending being dropped for OpTime: " << optime - << ", collection: " << collectionNamespace; + LOGV2_DEBUG(21701, + 1, + "Rolling back collection pending being dropped for OpTime: {optime}, " + "collection: {collectionNamespace}", + "optime"_attr = optime, + "collectionNamespace"_attr = collectionNamespace); DropPendingCollectionReaper::get(opCtx)->rollBackDropPendingCollection( opCtx, optime, collectionNamespace); } @@ -1391,7 +1531,10 @@ void rollback_internal::syncFixUp(OperationContext* opCtx, CollectionCatalog::get(opCtx).lookupNSSByUUID(opCtx, uuid); invariant(nss); - log() << "Resyncing collection metadata for collection: " << *nss << ", UUID: " << uuid; + LOGV2(21702, + "Resyncing collection metadata for collection: {nss}, UUID: {uuid}", + "nss"_attr = *nss, + "uuid"_attr = uuid); Lock::DBLock dbLock(opCtx, nss->db(), MODE_X); @@ -1411,9 +1554,10 @@ void rollback_internal::syncFixUp(OperationContext* opCtx, // is rolled back upstream and we restart, we expect to still have the // collection. - log() << nss->ns() - << " not found on remote host, so we do not roll back collmod " - "operation. Instead, we will drop the collection soon."; + LOGV2(21703, + "{nss_ns} not found on remote host, so we do not roll back collmod " + "operation. Instead, we will drop the collection soon.", + "nss_ns"_attr = nss->ns()); continue; } @@ -1460,22 +1604,29 @@ void rollback_internal::syncFixUp(OperationContext* opCtx, wuow.commit(); - LOG(1) << "Resynced collection metadata for collection: " << *nss << ", UUID: " << uuid - << ", with: " << redact(info) << ", to: " - << redact(DurableCatalog::get(opCtx) - ->getCollectionOptions(opCtx, collection->getCatalogId()) - .toBSON()); + LOGV2_DEBUG( + 21704, + 1, + "Resynced collection metadata for collection: {nss}, UUID: {uuid}, with: {info}, " + "to: {DurableCatalog_get_opCtx_getCollectionOptions_opCtx_collection_getCatalogId}", + "nss"_attr = *nss, + "uuid"_attr = uuid, + "info"_attr = redact(info), + "DurableCatalog_get_opCtx_getCollectionOptions_opCtx_collection_getCatalogId"_attr = + redact(DurableCatalog::get(opCtx) + ->getCollectionOptions(opCtx, collection->getCatalogId()) + .toBSON())); } // Since we read from the sync source to retrieve the metadata of the // collection, we must check if the sync source rolled back as well as update // minValid if necessary. - log() << "Rechecking the Rollback ID and minValid"; + LOGV2(21705, "Rechecking the Rollback ID and minValid"); checkRbidAndUpdateMinValid(opCtx, fixUpInfo.rbid, rollbackSource, replicationProcess); } // Rolls back dropIndexes commands by re-creating the indexes that were dropped. - log() << "Rolling back dropIndexes commands."; + LOGV2(21706, "Rolling back dropIndexes commands."); for (auto it = fixUpInfo.indexesToCreate.begin(); it != fixUpInfo.indexesToCreate.end(); it++) { UUID uuid = it->first; @@ -1484,12 +1635,13 @@ void rollback_internal::syncFixUp(OperationContext* opCtx, rollbackDropIndexes(opCtx, uuid, indexNames); } - log() << "Restarting rolled-back committed or aborted index builds."; + LOGV2(21707, "Restarting rolled-back committed or aborted index builds."); IndexBuildsCoordinator::get(opCtx)->restartIndexBuildsForRecovery( opCtx, fixUpInfo.indexBuildsToRestart); - log() << "Deleting and updating documents to roll back insert, update and remove " - "operations"; + LOGV2(21708, + "Deleting and updating documents to roll back insert, update and remove " + "operations"); unsigned deletes = 0, updates = 0; time_t lastProgressUpdate = time(nullptr); time_t progressUpdateGap = 10; @@ -1510,18 +1662,24 @@ void rollback_internal::syncFixUp(OperationContext* opCtx, if (RollbackImpl::shouldCreateDataFiles()) { removeSaver = std::make_unique<RemoveSaver>("rollback", "", uuid.toString()); - log() << "Preparing to write deleted documents to a rollback file for collection " - << *nss << " with uuid " << uuid.toString() << " to " - << removeSaver->file().generic_string(); + LOGV2(21709, + "Preparing to write deleted documents to a rollback file for collection {nss} " + "with uuid {uuid} to {removeSaver_file_generic_string}", + "nss"_attr = *nss, + "uuid"_attr = uuid.toString(), + "removeSaver_file_generic_string"_attr = removeSaver->file().generic_string()); } const auto& goodVersionsByDocID = nsAndGoodVersionsByDocID.second; for (const auto& idAndDoc : goodVersionsByDocID) { time_t now = time(nullptr); if (now - lastProgressUpdate > progressUpdateGap) { - log() << deletes << " delete and " << updates - << " update operations processed out of " << goodVersions.size() - << " total operations."; + LOGV2(21710, + "{deletes} delete and {updates} update operations processed out of " + "{goodVersions_size} total operations.", + "deletes"_attr = deletes, + "updates"_attr = updates, + "goodVersions_size"_attr = goodVersions.size()); lastProgressUpdate = now; } const DocID& doc = idAndDoc.first; @@ -1546,21 +1704,31 @@ void rollback_internal::syncFixUp(OperationContext* opCtx, if (found) { auto status = removeSaver->goingToDelete(obj); if (!status.isOK()) { - severe() << "Rollback cannot write document in namespace " << nss->ns() - << " to archive file: " << redact(status); + LOGV2_FATAL(21747, + "Rollback cannot write document in namespace {nss_ns} to " + "archive file: {status}", + "nss_ns"_attr = nss->ns(), + "status"_attr = redact(status)); throw RSFatalException(str::stream() << "Rollback cannot write document in namespace " << nss->ns() << " to archive file."); } } else { - error() << "Rollback cannot find object: " << pattern << " in namespace " - << nss->ns(); + LOGV2_ERROR(21730, + "Rollback cannot find object: {pattern} in namespace {nss_ns}", + "pattern"_attr = pattern, + "nss_ns"_attr = nss->ns()); } } if (idAndDoc.second.isEmpty()) { - LOG(2) << "Deleting document with: " << redact(doc._id) - << ", from collection: " << doc.ns << ", with UUID: " << uuid; + LOGV2_DEBUG(21711, + 2, + "Deleting document with: {doc_id}, from collection: {doc_ns}, with " + "UUID: {uuid}", + "doc_id"_attr = redact(doc._id), + "doc_ns"_attr = doc.ns, + "uuid"_attr = uuid); // If the document could not be found on the primary, deletes the document. // TODO 1.6 : can't delete from a capped collection. Need to handle that // here. @@ -1580,8 +1748,10 @@ void rollback_internal::syncFixUp(OperationContext* opCtx, const auto findOneStart = clock->now(); RecordId loc = Helpers::findOne(opCtx, collection, pattern, false); if (clock->now() - findOneStart > Milliseconds(200)) - warning() << "Roll back slow no _id index for " << nss->ns() - << " perhaps?"; + LOGV2_WARNING( + 21726, + "Roll back slow no _id index for {nss_ns} perhaps?", + "nss_ns"_attr = nss->ns()); // Would be faster but requires index: // RecordId loc = Helpers::findById(nsd, pattern); if (!loc.isNull()) { @@ -1614,11 +1784,15 @@ void rollback_internal::syncFixUp(OperationContext* opCtx, // inconsistent. We rely on age-out to make these problems go away // eventually. - warning() << "Ignoring failure to roll back change to capped " - << "collection " << nss->ns() << " with _id " - << redact(idAndDoc.first._id.toString( - /*includeFieldName*/ false)) - << ": " << redact(e); + LOGV2_WARNING(21727, + "Ignoring failure to roll back change to capped " + "collection {nss_ns} with _id " + "{idAndDoc_first_id_includeFieldName_false}: {e}", + "nss_ns"_attr = nss->ns(), + "idAndDoc_first_id_includeFieldName_false"_attr = + redact(idAndDoc.first._id.toString( + /*includeFieldName*/ false)), + "e"_attr = redact(e)); } } else { deleteObjects(opCtx, @@ -1630,9 +1804,14 @@ void rollback_internal::syncFixUp(OperationContext* opCtx, } } } else { - LOG(2) << "Updating document with: " << redact(doc._id) - << ", from collection: " << doc.ns << ", UUID: " << uuid - << ", to: " << redact(idAndDoc.second); + LOGV2_DEBUG(21712, + 2, + "Updating document with: {doc_id}, from collection: {doc_ns}, " + "UUID: {uuid}, to: {idAndDoc_second}", + "doc_id"_attr = redact(doc._id), + "doc_ns"_attr = doc.ns, + "uuid"_attr = uuid, + "idAndDoc_second"_attr = redact(idAndDoc.second)); // TODO faster... updates++; @@ -1646,23 +1825,31 @@ void rollback_internal::syncFixUp(OperationContext* opCtx, update(opCtx, ctx.db(), request); } } catch (const DBException& e) { - log() << "Exception in rollback ns:" << nss->ns() << ' ' << pattern.toString() - << ' ' << redact(e) << " ndeletes:" << deletes; + LOGV2(21713, + "Exception in rollback ns:{nss_ns} {pattern} {e} ndeletes:{deletes}", + "nss_ns"_attr = nss->ns(), + "pattern"_attr = pattern.toString(), + "e"_attr = redact(e), + "deletes"_attr = deletes); throw; } } } - log() << "Rollback deleted " << deletes << " documents and updated " << updates - << " documents."; + LOGV2(21714, + "Rollback deleted {deletes} documents and updated {updates} documents.", + "deletes"_attr = deletes, + "updates"_attr = updates); if (!serverGlobalParams.enableMajorityReadConcern) { // When majority read concern is disabled, the stable timestamp may be ahead of the common // point. Force the stable timestamp back to the common point, to allow writes after the // common point. const bool force = true; - log() << "Forcing the stable timestamp to the common point: " - << fixUpInfo.commonPoint.getTimestamp(); + LOGV2(21715, + "Forcing the stable timestamp to the common point: " + "{fixUpInfo_commonPoint_getTimestamp}", + "fixUpInfo_commonPoint_getTimestamp"_attr = fixUpInfo.commonPoint.getTimestamp()); opCtx->getServiceContext()->getStorageEngine()->setStableTimestamp( fixUpInfo.commonPoint.getTimestamp(), force); @@ -1676,22 +1863,28 @@ void rollback_internal::syncFixUp(OperationContext* opCtx, auto syncSourceTopOfOplog = OpTime::parseFromOplogEntry(rollbackSource.getLastOperation()) .getValue() .getTimestamp(); - log() << "Setting initialDataTimestamp to the max of local top of oplog and sync source " - "top of oplog. Local top of oplog: " - << fixUpInfo.localTopOfOplog.getTimestamp() - << ", sync source top of oplog: " << syncSourceTopOfOplog; + LOGV2(21716, + "Setting initialDataTimestamp to the max of local top of oplog and sync source " + "top of oplog. Local top of oplog: {fixUpInfo_localTopOfOplog_getTimestamp}, sync " + "source top of oplog: {syncSourceTopOfOplog}", + "fixUpInfo_localTopOfOplog_getTimestamp"_attr = + fixUpInfo.localTopOfOplog.getTimestamp(), + "syncSourceTopOfOplog"_attr = syncSourceTopOfOplog); opCtx->getServiceContext()->getStorageEngine()->setInitialDataTimestamp( std::max(fixUpInfo.localTopOfOplog.getTimestamp(), syncSourceTopOfOplog)); // Take an unstable checkpoint to ensure that all of the writes performed during rollback // are persisted to disk before truncating oplog. - log() << "Waiting for an unstable checkpoint"; + LOGV2(21717, "Waiting for an unstable checkpoint"); const bool stableCheckpoint = false; opCtx->recoveryUnit()->waitUntilUnjournaledWritesDurable(opCtx, stableCheckpoint); } - log() << "Truncating the oplog at " << fixUpInfo.commonPoint.toString() << " (" - << fixUpInfo.commonPointOurDiskloc << "), non-inclusive"; + LOGV2(21718, + "Truncating the oplog at {fixUpInfo_commonPoint} ({fixUpInfo_commonPointOurDiskloc}), " + "non-inclusive", + "fixUpInfo_commonPoint"_attr = fixUpInfo.commonPoint.toString(), + "fixUpInfo_commonPointOurDiskloc"_attr = fixUpInfo.commonPointOurDiskloc); // Cleans up the oplog. { @@ -1718,24 +1911,28 @@ void rollback_internal::syncFixUp(OperationContext* opCtx, // This is done using an untimestamped write, since timestamping the write with the common // point TS would be incorrect (since this is equal to the stable timestamp), and this write // will be included in the unstable checkpoint regardless of its timestamp. - log() << "Setting appliedThrough to the common point: " << fixUpInfo.commonPoint; + LOGV2(21719, + "Setting appliedThrough to the common point: {fixUpInfo_commonPoint}", + "fixUpInfo_commonPoint"_attr = fixUpInfo.commonPoint); const bool setTimestamp = false; replicationProcess->getConsistencyMarkers()->setAppliedThrough( opCtx, fixUpInfo.commonPoint, setTimestamp); // Take an unstable checkpoint to ensure the appliedThrough write is persisted to disk. - log() << "Waiting for an unstable checkpoint"; + LOGV2(21720, "Waiting for an unstable checkpoint"); const bool stableCheckpoint = false; opCtx->recoveryUnit()->waitUntilUnjournaledWritesDurable(opCtx, stableCheckpoint); // Ensure that appliedThrough is unset in the next stable checkpoint. - log() << "Clearing appliedThrough"; + LOGV2(21721, "Clearing appliedThrough"); replicationProcess->getConsistencyMarkers()->clearAppliedThrough(opCtx, Timestamp()); } Status status = AuthorizationManager::get(opCtx->getServiceContext())->initialize(opCtx); if (!status.isOK()) { - severe() << "Failed to reinitialize auth data after rollback: " << redact(status); + LOGV2_FATAL(21748, + "Failed to reinitialize auth data after rollback: {status}", + "status"_attr = redact(status)); fassertFailedNoTrace(40496); } @@ -1790,8 +1987,12 @@ Status syncRollback(OperationContext* opCtx, replCoord, replicationProcess); - log() << "Rollback finished. The final minValid is: " - << replicationProcess->getConsistencyMarkers()->getMinValid(opCtx) << rsLog; + LOGV2_OPTIONS(21722, + {logv2::LogTag::kRS}, + "Rollback finished. The final minValid is: " + "{replicationProcess_getConsistencyMarkers_getMinValid_opCtx}", + "replicationProcess_getConsistencyMarkers_getMinValid_opCtx"_attr = + replicationProcess->getConsistencyMarkers()->getMinValid(opCtx)); return status; } @@ -1821,15 +2022,21 @@ void rollback(OperationContext* opCtx, auto status = replCoord->setFollowerModeStrict(opCtx, MemberState::RS_ROLLBACK); if (!status.isOK()) { - log() << "Cannot transition from " << replCoord->getMemberState().toString() << " to " - << MemberState(MemberState::RS_ROLLBACK).toString() << causedBy(status); + LOGV2(21723, + "Cannot transition from {replCoord_getMemberState} to " + "{MemberState_MemberState_RS_ROLLBACK}{causedBy_status}", + "replCoord_getMemberState"_attr = replCoord->getMemberState().toString(), + "MemberState_MemberState_RS_ROLLBACK"_attr = + MemberState(MemberState::RS_ROLLBACK).toString(), + "causedBy_status"_attr = causedBy(status)); return; } } if (MONGO_unlikely(rollbackHangAfterTransitionToRollback.shouldFail())) { - log() << "rollbackHangAfterTransitionToRollback fail point enabled. Blocking until fail " - "point is disabled (rs_rollback)."; + LOGV2(21724, + "rollbackHangAfterTransitionToRollback fail point enabled. Blocking until fail " + "point is disabled (rs_rollback)."); rollbackHangAfterTransitionToRollback.pauseWhileSet(opCtx); } @@ -1846,8 +2053,9 @@ void rollback(OperationContext* opCtx, // WARNING: these statuses sometimes have location codes which are lost with uassertStatusOK // so we need to check here first. if (ErrorCodes::UnrecoverableRollbackError == status.code()) { - severe() << "Unable to complete rollback. A full resync may be needed: " - << redact(status); + LOGV2_FATAL(21749, + "Unable to complete rollback. A full resync may be needed: {status}", + "status"_attr = redact(status)); fassertFailedNoTrace(40507); } @@ -1858,9 +2066,14 @@ void rollback(OperationContext* opCtx, // above. invariant(ex.code() != ErrorCodes::UnrecoverableRollbackError); - warning() << "Rollback cannot complete at this time (retrying later): " << redact(ex) - << " appliedThrough= " << replCoord->getMyLastAppliedOpTime() << " minvalid= " - << replicationProcess->getConsistencyMarkers()->getMinValid(opCtx); + LOGV2_WARNING(21728, + "Rollback cannot complete at this time (retrying later): {ex} " + "appliedThrough= {replCoord_getMyLastAppliedOpTime} minvalid= " + "{replicationProcess_getConsistencyMarkers_getMinValid_opCtx}", + "ex"_attr = redact(ex), + "replCoord_getMyLastAppliedOpTime"_attr = replCoord->getMyLastAppliedOpTime(), + "replicationProcess_getConsistencyMarkers_getMinValid_opCtx"_attr = + replicationProcess->getConsistencyMarkers()->getMinValid(opCtx)); // Sleep a bit to allow upstream node to coalesce, if that was the cause of the failure. If // we failed in a way that will keep failing, but wasn't flagged as a fatal failure, this @@ -1882,17 +2095,24 @@ void rollback(OperationContext* opCtx, // then we must shut down to clear the in-memory ShardingState associated with the // shardIdentity document. if (ShardIdentityRollbackNotifier::get(opCtx)->didRollbackHappen()) { - severe() << "shardIdentity document rollback detected. Shutting down to clear " + LOGV2_FATAL(21750, + "shardIdentity document rollback detected. Shutting down to clear " "in-memory sharding state. Restarting this process should safely return it " - "to a healthy state"; + "to a healthy state"); fassertFailedNoTrace(40498); } auto status = replCoord->setFollowerMode(MemberState::RS_RECOVERING); if (!status.isOK()) { - severe() << "Failed to transition into " << MemberState(MemberState::RS_RECOVERING) - << "; expected to be in state " << MemberState(MemberState::RS_ROLLBACK) - << "; found self in " << replCoord->getMemberState() << causedBy(status); + LOGV2_FATAL( + 21751, + "Failed to transition into {MemberState_MemberState_RS_RECOVERING}; expected to be in " + "state {MemberState_MemberState_RS_ROLLBACK}; found self in " + "{replCoord_getMemberState}{causedBy_status}", + "MemberState_MemberState_RS_RECOVERING"_attr = MemberState(MemberState::RS_RECOVERING), + "MemberState_MemberState_RS_ROLLBACK"_attr = MemberState(MemberState::RS_ROLLBACK), + "replCoord_getMemberState"_attr = replCoord->getMemberState(), + "causedBy_status"_attr = causedBy(status)); fassertFailedNoTrace(40499); } } diff --git a/src/mongo/db/repl/scatter_gather_runner.cpp b/src/mongo/db/repl/scatter_gather_runner.cpp index 18e3bc761b9..9986987b3f4 100644 --- a/src/mongo/db/repl/scatter_gather_runner.cpp +++ b/src/mongo/db/repl/scatter_gather_runner.cpp @@ -38,6 +38,7 @@ #include "mongo/base/status_with.h" #include "mongo/db/repl/scatter_gather_algorithm.h" +#include "mongo/logv2/log.h" #include "mongo/util/assert_util.h" #include "mongo/util/log.h" #include "mongo/util/scopeguard.h" @@ -105,8 +106,10 @@ StatusWith<EventHandle> ScatterGatherRunner::RunnerImpl::start( std::vector<RemoteCommandRequest> requests = _algorithm->getRequests(); for (size_t i = 0; i < requests.size(); ++i) { - log() << "Scheduling remote command request for " << _logMessage << ": " - << requests[i].toString(); + LOGV2(21752, + "Scheduling remote command request for {logMessage}: {requests_i}", + "logMessage"_attr = _logMessage, + "requests_i"_attr = requests[i].toString()); const StatusWith<CallbackHandle> cbh = _executor->scheduleRemoteCommand(requests[i], processResponseCB); if (cbh.getStatus() == ErrorCodes::ShutdownInProgress) { diff --git a/src/mongo/db/repl/storage_interface_impl.cpp b/src/mongo/db/repl/storage_interface_impl.cpp index 17af180cd45..92841eeacae 100644 --- a/src/mongo/db/repl/storage_interface_impl.cpp +++ b/src/mongo/db/repl/storage_interface_impl.cpp @@ -76,6 +76,7 @@ #include "mongo/db/service_context.h" #include "mongo/db/storage/durable_catalog.h" #include "mongo/db/storage/oplog_cap_maintainer_thread.h" +#include "mongo/logv2/log.h" #include "mongo/util/assert_util.h" #include "mongo/util/background.h" #include "mongo/util/log.h" @@ -190,7 +191,10 @@ StorageInterfaceImpl::createCollectionForBulkLoading( const BSONObj idIndexSpec, const std::vector<BSONObj>& secondaryIndexSpecs) { - LOG(2) << "StorageInterfaceImpl::createCollectionForBulkLoading called for ns: " << nss.ns(); + LOGV2_DEBUG(21753, + 2, + "StorageInterfaceImpl::createCollectionForBulkLoading called for ns: {nss_ns}", + "nss_ns"_attr = nss.ns()); class StashClient { public: @@ -384,7 +388,9 @@ Status StorageInterfaceImpl::dropReplicatedDatabases(OperationContext* opCtx) { std::vector<std::string> dbNames = opCtx->getServiceContext()->getStorageEngine()->listDatabases(); invariant(!dbNames.empty()); - log() << "dropReplicatedDatabases - dropping " << dbNames.size() << " databases"; + LOGV2(21754, + "dropReplicatedDatabases - dropping {dbNames_size} databases", + "dbNames_size"_attr = dbNames.size()); ReplicationCoordinator::get(opCtx)->dropAllSnapshots(); @@ -401,14 +407,17 @@ Status StorageInterfaceImpl::dropReplicatedDatabases(OperationContext* opCtx) { } else { // This is needed since dropDatabase can't be rolled back. // This is safe be replaced by "invariant(db);dropDatabase(opCtx, db);" once fixed. - log() << "dropReplicatedDatabases - database disappeared after retrieving list of " - "database names but before drop: " - << dbName; + LOGV2(21755, + "dropReplicatedDatabases - database disappeared after retrieving list of " + "database names but before drop: {dbName}", + "dbName"_attr = dbName); } }); } invariant(hasLocalDatabase, "local database missing"); - log() << "dropReplicatedDatabases - dropped " << dbNames.size() << " databases"; + LOGV2(21756, + "dropReplicatedDatabases - dropped {dbNames_size} databases", + "dbNames_size"_attr = dbNames.size()); return Status::OK(); } diff --git a/src/mongo/db/repl/storage_interface_mock.cpp b/src/mongo/db/repl/storage_interface_mock.cpp index e9fa17504be..30d60626641 100644 --- a/src/mongo/db/repl/storage_interface_mock.cpp +++ b/src/mongo/db/repl/storage_interface_mock.cpp @@ -34,6 +34,7 @@ #include "mongo/db/repl/storage_interface_mock.h" +#include "mongo/logv2/log.h" #include "mongo/util/log.h" #include "mongo/util/str.h" @@ -103,14 +104,14 @@ bool StorageInterfaceMock::supportsDocLocking(ServiceContext* serviceCtx) const } Status CollectionBulkLoaderMock::init(const std::vector<BSONObj>& secondaryIndexSpecs) { - LOG(1) << "CollectionBulkLoaderMock::init called"; + LOGV2_DEBUG(21757, 1, "CollectionBulkLoaderMock::init called"); stats->initCalled = true; return Status::OK(); }; Status CollectionBulkLoaderMock::insertDocuments(const std::vector<BSONObj>::const_iterator begin, const std::vector<BSONObj>::const_iterator end) { - LOG(1) << "CollectionBulkLoaderMock::insertDocuments called"; + LOGV2_DEBUG(21758, 1, "CollectionBulkLoaderMock::insertDocuments called"); const auto status = insertDocsFn(begin, end); // Only count if it succeeds. @@ -121,7 +122,7 @@ Status CollectionBulkLoaderMock::insertDocuments(const std::vector<BSONObj>::con }; Status CollectionBulkLoaderMock::commit() { - LOG(1) << "CollectionBulkLoaderMock::commit called"; + LOGV2_DEBUG(21759, 1, "CollectionBulkLoaderMock::commit called"); stats->commitCalled = true; return commitFn(); }; diff --git a/src/mongo/db/repl/sync_source_feedback.cpp b/src/mongo/db/repl/sync_source_feedback.cpp index 03b5af98376..0fc2f3f16ac 100644 --- a/src/mongo/db/repl/sync_source_feedback.cpp +++ b/src/mongo/db/repl/sync_source_feedback.cpp @@ -39,6 +39,7 @@ #include "mongo/db/repl/replication_coordinator.h" #include "mongo/db/repl/reporter.h" #include "mongo/executor/task_executor.h" +#include "mongo/logv2/log.h" #include "mongo/util/concurrency/idle_thread_block.h" #include "mongo/util/log.h" #include "mongo/util/net/hostandport.h" @@ -104,8 +105,11 @@ void SyncSourceFeedback::forwardSlaveProgress() { if (_reporter) { auto triggerStatus = _reporter->trigger(); if (!triggerStatus.isOK()) { - warning() << "unable to forward slave progress to " << _reporter->getTarget() - << ": " << triggerStatus; + LOGV2_WARNING( + 21764, + "unable to forward slave progress to {reporter_getTarget}: {triggerStatus}", + "reporter_getTarget"_attr = _reporter->getTarget(), + "triggerStatus"_attr = triggerStatus); } } } @@ -116,15 +120,21 @@ Status SyncSourceFeedback::_updateUpstream(Reporter* reporter) { auto triggerStatus = reporter->trigger(); if (!triggerStatus.isOK()) { - warning() << "unable to schedule reporter to update replication progress on " << syncTarget - << ": " << triggerStatus; + LOGV2_WARNING(21765, + "unable to schedule reporter to update replication progress on {syncTarget}: " + "{triggerStatus}", + "syncTarget"_attr = syncTarget, + "triggerStatus"_attr = triggerStatus); return triggerStatus; } auto status = reporter->join(); if (!status.isOK()) { - log() << "SyncSourceFeedback error sending update to " << syncTarget << ": " << status; + LOGV2(21760, + "SyncSourceFeedback error sending update to {syncTarget}: {status}", + "syncTarget"_attr = syncTarget, + "status"_attr = status); } // Sync source blacklisting will be done in BackgroundSync and SyncSourceResolver. @@ -202,15 +212,19 @@ void SyncSourceFeedback::run(executor::TaskExecutor* executor, } if (syncTarget != target) { - LOG(1) << "setting syncSourceFeedback to " << target; + LOGV2_DEBUG(21761, 1, "setting syncSourceFeedback to {target}", "target"_attr = target); syncTarget = target; // Update keepalive value from config. auto oldKeepAliveInterval = keepAliveInterval; keepAliveInterval = calculateKeepAliveInterval(replCoord->getConfig()); if (oldKeepAliveInterval != keepAliveInterval) { - LOG(1) << "new syncSourceFeedback keep alive duration = " << keepAliveInterval - << " (previously " << oldKeepAliveInterval << ")"; + LOGV2_DEBUG(21762, + 1, + "new syncSourceFeedback keep alive duration = {keepAliveInterval} " + "(previously {oldKeepAliveInterval})", + "keepAliveInterval"_attr = keepAliveInterval, + "oldKeepAliveInterval"_attr = oldKeepAliveInterval); } } @@ -233,9 +247,12 @@ void SyncSourceFeedback::run(executor::TaskExecutor* executor, auto status = _updateUpstream(&reporter); if (!status.isOK()) { - LOG(1) << "The replication progress command (replSetUpdatePosition) failed and will be " - "retried: " - << status; + LOGV2_DEBUG( + 21763, + 1, + "The replication progress command (replSetUpdatePosition) failed and will be " + "retried: {status}", + "status"_attr = status); } } } diff --git a/src/mongo/db/repl/sync_source_resolver.cpp b/src/mongo/db/repl/sync_source_resolver.cpp index a6c87de8731..8ecdad2ebe4 100644 --- a/src/mongo/db/repl/sync_source_resolver.cpp +++ b/src/mongo/db/repl/sync_source_resolver.cpp @@ -39,6 +39,7 @@ #include "mongo/db/repl/oplog_entry.h" #include "mongo/db/repl/replication_process.h" #include "mongo/db/repl/sync_source_selector.h" +#include "mongo/logv2/log.h" #include "mongo/rpc/get_status_from_command_result.h" #include "mongo/util/assert_util.h" #include "mongo/util/destructor_guard.h" @@ -224,8 +225,11 @@ Status SyncSourceResolver::_scheduleFetcher(std::unique_ptr<Fetcher> fetcher) { _shuttingDownFetcher = std::move(_fetcher); _fetcher = std::move(fetcher); } else { - error() << "Error scheduling fetcher to evaluate host as sync source, host:" - << fetcher->getSource() << ", error: " << status; + LOGV2_ERROR(21776, + "Error scheduling fetcher to evaluate host as sync source, " + "host:{fetcher_getSource}, error: {status}", + "fetcher_getSource"_attr = fetcher->getSource(), + "status"_attr = status); } return status; } @@ -235,8 +239,12 @@ OpTime SyncSourceResolver::_parseRemoteEarliestOpTime(const HostAndPort& candida if (queryResponse.documents.empty()) { // Remote oplog is empty. const auto until = _taskExecutor->now() + kOplogEmptyBlacklistDuration; - log() << "Blacklisting " << candidate << " due to empty oplog for " - << kOplogEmptyBlacklistDuration << " until: " << until; + LOGV2(21766, + "Blacklisting {candidate} due to empty oplog for {kOplogEmptyBlacklistDuration} " + "until: {until}", + "candidate"_attr = candidate, + "kOplogEmptyBlacklistDuration"_attr = kOplogEmptyBlacklistDuration, + "until"_attr = until); _syncSourceSelector->blacklistSyncSource(candidate, until); return OpTime(); } @@ -245,8 +253,13 @@ OpTime SyncSourceResolver::_parseRemoteEarliestOpTime(const HostAndPort& candida if (firstObjFound.isEmpty()) { // First document in remote oplog is empty. const auto until = _taskExecutor->now() + kFirstOplogEntryEmptyBlacklistDuration; - log() << "Blacklisting " << candidate << " due to empty first document for " - << kFirstOplogEntryEmptyBlacklistDuration << " until: " << until; + LOGV2(21767, + "Blacklisting {candidate} due to empty first document for " + "{kFirstOplogEntryEmptyBlacklistDuration} until: {until}", + "candidate"_attr = candidate, + "kFirstOplogEntryEmptyBlacklistDuration"_attr = + kFirstOplogEntryEmptyBlacklistDuration, + "until"_attr = until); _syncSourceSelector->blacklistSyncSource(candidate, until); return OpTime(); } @@ -254,10 +267,16 @@ OpTime SyncSourceResolver::_parseRemoteEarliestOpTime(const HostAndPort& candida const auto remoteEarliestOpTime = OpTime::parseFromOplogEntry(firstObjFound); if (!remoteEarliestOpTime.isOK()) { const auto until = _taskExecutor->now() + kFirstOplogEntryNullTimestampBlacklistDuration; - log() << "Blacklisting " << candidate << " due to error parsing OpTime from the oldest" - << " oplog entry for " << kFirstOplogEntryNullTimestampBlacklistDuration - << " until: " << until << ". Error: " << remoteEarliestOpTime.getStatus() - << ", Entry: " << redact(firstObjFound); + LOGV2(21768, + "Blacklisting {candidate} due to error parsing OpTime from the oldest oplog entry " + "for {kFirstOplogEntryNullTimestampBlacklistDuration} until: {until}. Error: " + "{remoteEarliestOpTime_getStatus}, Entry: {firstObjFound}", + "candidate"_attr = candidate, + "kFirstOplogEntryNullTimestampBlacklistDuration"_attr = + kFirstOplogEntryNullTimestampBlacklistDuration, + "until"_attr = until, + "remoteEarliestOpTime_getStatus"_attr = remoteEarliestOpTime.getStatus(), + "firstObjFound"_attr = redact(firstObjFound)); _syncSourceSelector->blacklistSyncSource(candidate, until); return OpTime(); } @@ -265,8 +284,13 @@ OpTime SyncSourceResolver::_parseRemoteEarliestOpTime(const HostAndPort& candida if (remoteEarliestOpTime.getValue().isNull()) { // First document in remote oplog is empty. const auto until = _taskExecutor->now() + kFirstOplogEntryNullTimestampBlacklistDuration; - log() << "Blacklisting " << candidate << " due to null timestamp in first document for " - << kFirstOplogEntryNullTimestampBlacklistDuration << " until: " << until; + LOGV2(21769, + "Blacklisting {candidate} due to null timestamp in first document for " + "{kFirstOplogEntryNullTimestampBlacklistDuration} until: {until}", + "candidate"_attr = candidate, + "kFirstOplogEntryNullTimestampBlacklistDuration"_attr = + kFirstOplogEntryNullTimestampBlacklistDuration, + "until"_attr = until); _syncSourceSelector->blacklistSyncSource(candidate, until); return OpTime(); } @@ -295,8 +319,13 @@ void SyncSourceResolver::_firstOplogEntryFetcherCallback( if (!queryResult.isOK()) { // We got an error. const auto until = _taskExecutor->now() + kFetcherErrorBlacklistDuration; - log() << "Blacklisting " << candidate << " due to error: '" << queryResult.getStatus() - << "' for " << kFetcherErrorBlacklistDuration << " until: " << until; + LOGV2(21770, + "Blacklisting {candidate} due to error: '{queryResult_getStatus}' for " + "{kFetcherErrorBlacklistDuration} until: {until}", + "candidate"_attr = candidate, + "queryResult_getStatus"_attr = queryResult.getStatus(), + "kFetcherErrorBlacklistDuration"_attr = kFetcherErrorBlacklistDuration, + "until"_attr = until); _syncSourceSelector->blacklistSyncSource(candidate, until); _chooseAndProbeNextSyncSource(earliestOpTimeSeen).transitional_ignore(); @@ -316,11 +345,16 @@ void SyncSourceResolver::_firstOplogEntryFetcherCallback( const auto blacklistDuration = kTooStaleBlacklistDuration; const auto until = _taskExecutor->now() + Minutes(1); - log() << "We are too stale to use " << candidate << " as a sync source. " - << "Blacklisting this sync source" - << " because our last fetched timestamp: " << _lastOpTimeFetched.getTimestamp() - << " is before their earliest timestamp: " << remoteEarliestOpTime.getTimestamp() - << " for " << blacklistDuration << " until: " << until; + LOGV2(21771, + "We are too stale to use {candidate} as a sync source. Blacklisting this sync source " + "because our last fetched timestamp: {lastOpTimeFetched_getTimestamp} is before " + "their earliest timestamp: {remoteEarliestOpTime_getTimestamp} for " + "{blacklistDuration} until: {until}", + "candidate"_attr = candidate, + "lastOpTimeFetched_getTimestamp"_attr = _lastOpTimeFetched.getTimestamp(), + "remoteEarliestOpTime_getTimestamp"_attr = remoteEarliestOpTime.getTimestamp(), + "blacklistDuration"_attr = blacklistDuration, + "until"_attr = until); _syncSourceSelector->blacklistSyncSource(candidate, until); @@ -386,8 +420,13 @@ void SyncSourceResolver::_rbidRequestCallback( rbid = rbidReply.response.data["rbid"].Int(); } catch (const DBException& ex) { const auto until = _taskExecutor->now() + kFetcherErrorBlacklistDuration; - log() << "Blacklisting " << candidate << " due to error: '" << ex << "' for " - << kFetcherErrorBlacklistDuration << " until: " << until; + LOGV2(21772, + "Blacklisting {candidate} due to error: '{ex}' for {kFetcherErrorBlacklistDuration} " + "until: {until}", + "candidate"_attr = candidate, + "ex"_attr = ex, + "kFetcherErrorBlacklistDuration"_attr = kFetcherErrorBlacklistDuration, + "until"_attr = until); _syncSourceSelector->blacklistSyncSource(candidate, until); _chooseAndProbeNextSyncSource(earliestOpTimeSeen).transitional_ignore(); return; @@ -456,9 +495,15 @@ void SyncSourceResolver::_requiredOpTimeFetcherCallback( if (!queryResult.isOK()) { // We got an error. const auto until = _taskExecutor->now() + kFetcherErrorBlacklistDuration; - log() << "Blacklisting " << candidate << " due to required optime fetcher error: '" - << queryResult.getStatus() << "' for " << kFetcherErrorBlacklistDuration - << " until: " << until << ". required optime: " << _requiredOpTime; + LOGV2(21773, + "Blacklisting {candidate} due to required optime fetcher error: " + "'{queryResult_getStatus}' for {kFetcherErrorBlacklistDuration} until: {until}. " + "required optime: {requiredOpTime}", + "candidate"_attr = candidate, + "queryResult_getStatus"_attr = queryResult.getStatus(), + "kFetcherErrorBlacklistDuration"_attr = kFetcherErrorBlacklistDuration, + "until"_attr = until, + "requiredOpTime"_attr = _requiredOpTime); _syncSourceSelector->blacklistSyncSource(candidate, until); _chooseAndProbeNextSyncSource(earliestOpTimeSeen).transitional_ignore(); @@ -469,13 +514,18 @@ void SyncSourceResolver::_requiredOpTimeFetcherCallback( auto status = _compareRequiredOpTimeWithQueryResponse(queryResponse); if (!status.isOK()) { const auto until = _taskExecutor->now() + kNoRequiredOpTimeBlacklistDuration; - warning() << "We cannot use " << candidate.toString() - << " as a sync source because it does not contain the necessary " - "operations for us to reach a consistent state: " - << status << " last fetched optime: " << _lastOpTimeFetched - << ". required optime: " << _requiredOpTime - << ". Blacklisting this sync source for " << kNoRequiredOpTimeBlacklistDuration - << " until: " << until; + LOGV2_WARNING( + 21774, + "We cannot use {candidate} as a sync source because it does not contain the necessary " + "operations for us to reach a consistent state: {status} last fetched optime: " + "{lastOpTimeFetched}. required optime: {requiredOpTime}. Blacklisting this sync source " + "for {kNoRequiredOpTimeBlacklistDuration} until: {until}", + "candidate"_attr = candidate.toString(), + "status"_attr = status, + "lastOpTimeFetched"_attr = _lastOpTimeFetched, + "requiredOpTime"_attr = _requiredOpTime, + "kNoRequiredOpTimeBlacklistDuration"_attr = kNoRequiredOpTimeBlacklistDuration, + "until"_attr = until); _syncSourceSelector->blacklistSyncSource(candidate, until); _chooseAndProbeNextSyncSource(earliestOpTimeSeen).transitional_ignore(); @@ -532,8 +582,9 @@ Status SyncSourceResolver::_finishCallback(const SyncSourceResolverResponse& res try { _onCompletion(response); } catch (...) { - warning() << "sync source resolver finish callback threw exception: " - << exceptionToStatus(); + LOGV2_WARNING(21775, + "sync source resolver finish callback threw exception: {exceptionToStatus}", + "exceptionToStatus"_attr = exceptionToStatus()); } stdx::lock_guard<Latch> lock(_mutex); diff --git a/src/mongo/db/repl/task_runner.cpp b/src/mongo/db/repl/task_runner.cpp index 86edc6da9c5..5889b149393 100644 --- a/src/mongo/db/repl/task_runner.cpp +++ b/src/mongo/db/repl/task_runner.cpp @@ -40,6 +40,7 @@ #include "mongo/db/client.h" #include "mongo/db/operation_context.h" #include "mongo/db/service_context.h" +#include "mongo/logv2/log.h" #include "mongo/util/assert_util.h" #include "mongo/util/concurrency/thread_name.h" #include "mongo/util/destructor_guard.h" @@ -65,7 +66,9 @@ TaskRunner::NextAction runSingleTask(const TaskRunner::Task& task, try { return task(opCtx, status); } catch (...) { - log() << "Unhandled exception in task runner: " << redact(exceptionToStatus()); + LOGV2(21777, + "Unhandled exception in task runner: {exceptionToStatus}", + "exceptionToStatus"_attr = redact(exceptionToStatus())); } return TaskRunner::NextAction::kCancel; } diff --git a/src/mongo/db/repl/topology_coordinator.cpp b/src/mongo/db/repl/topology_coordinator.cpp index a4e4ab8f9e9..fb7d8c65bd3 100644 --- a/src/mongo/db/repl/topology_coordinator.cpp +++ b/src/mongo/db/repl/topology_coordinator.cpp @@ -53,6 +53,7 @@ #include "mongo/db/repl/isself.h" #include "mongo/db/repl/member_data.h" #include "mongo/db/repl/rslog.h" +#include "mongo/logv2/log.h" #include "mongo/rpc/metadata/oplog_query_metadata.h" #include "mongo/rpc/metadata/repl_set_metadata.h" #include "mongo/util/assert_util.h" @@ -201,7 +202,8 @@ HostAndPort TopologyCoordinator::chooseNewSyncSource(Date_t now, ChainingPreference chainingPreference) { // If we are not a member of the current replica set configuration, no sync source is valid. if (_selfIndex == -1) { - LOG(1) << "Cannot sync from any members because we are not in the replica set config"; + LOGV2_DEBUG( + 21778, 1, "Cannot sync from any members because we are not in the replica set config"); return HostAndPort(); } @@ -209,31 +211,38 @@ HostAndPort TopologyCoordinator::chooseNewSyncSource(Date_t now, const auto& data = sfp.getData(); const auto hostAndPortElem = data["hostAndPort"]; if (!hostAndPortElem) { - severe() << "'forceSyncSoureCandidate' parameter set with invalid host and port: " - << data; + LOGV2_FATAL( + 21839, + "'forceSyncSoureCandidate' parameter set with invalid host and port: {data}", + "data"_attr = data); fassertFailed(50835); } const auto hostAndPort = HostAndPort(hostAndPortElem.checkAndGetStringData()); const int syncSourceIndex = _rsConfig.findMemberIndexByHostAndPort(hostAndPort); if (syncSourceIndex < 0) { - log() << "'forceSyncSourceCandidate' failed due to host and port not in " - "replica set config: " - << hostAndPort.toString(); + LOGV2(21779, + "'forceSyncSourceCandidate' failed due to host and port not in " + "replica set config: {hostAndPort}", + "hostAndPort"_attr = hostAndPort.toString()); fassertFailed(50836); } if (_memberIsBlacklisted(_rsConfig.getMemberAt(syncSourceIndex), now)) { - log() << "Cannot select a sync source because forced candidate is blacklisted: " - << hostAndPort.toString(); + LOGV2(21780, + "Cannot select a sync source because forced candidate is blacklisted: " + "{hostAndPort}", + "hostAndPort"_attr = hostAndPort.toString()); _syncSource = HostAndPort(); return _syncSource; } _syncSource = _rsConfig.getMemberAt(syncSourceIndex).getHostAndPort(); - log() << "choosing sync source candidate due to 'forceSyncSourceCandidate' parameter: " - << _syncSource; + LOGV2(21781, + "choosing sync source candidate due to 'forceSyncSourceCandidate' parameter: " + "{syncSource}", + "syncSource"_attr = _syncSource); std::string msg(str::stream() << "syncing from: " << _syncSource.toString() << " by 'forceSyncSourceCandidate' parameter"); setMyHeartbeatMessage(now, msg); @@ -245,7 +254,9 @@ HostAndPort TopologyCoordinator::chooseNewSyncSource(Date_t now, invariant(_forceSyncSourceIndex < _rsConfig.getNumMembers()); _syncSource = _rsConfig.getMemberAt(_forceSyncSourceIndex).getHostAndPort(); _forceSyncSourceIndex = -1; - log() << "choosing sync source candidate by request: " << _syncSource; + LOGV2(21782, + "choosing sync source candidate by request: {syncSource}", + "syncSource"_attr = _syncSource); std::string msg(str::stream() << "syncing from: " << _syncSource.toString() << " by request"); setMyHeartbeatMessage(now, msg); @@ -258,7 +269,9 @@ HostAndPort TopologyCoordinator::chooseNewSyncSource(Date_t now, if (needMorePings > 0) { static Occasionally sampler; if (sampler.tick()) { - log() << "waiting for " << needMorePings << " pings from other members before syncing"; + LOGV2(21783, + "waiting for {needMorePings} pings from other members before syncing", + "needMorePings"_attr = needMorePings); } _syncSource = HostAndPort(); return _syncSource; @@ -268,19 +281,26 @@ HostAndPort TopologyCoordinator::chooseNewSyncSource(Date_t now, if (chainingPreference == ChainingPreference::kUseConfiguration && !_rsConfig.isChainingAllowed()) { if (_currentPrimaryIndex == -1) { - LOG(1) << "Cannot select a sync source because chaining is" - " not allowed and primary is unknown/down"; + LOGV2_DEBUG(21784, + 1, + "Cannot select a sync source because chaining is" + " not allowed and primary is unknown/down"); _syncSource = HostAndPort(); return _syncSource; } else if (_memberIsBlacklisted(*_currentPrimaryMember(), now)) { - LOG(1) << "Cannot select a sync source because chaining is not allowed and primary " - "member is blacklisted: " - << _currentPrimaryMember()->getHostAndPort(); + LOGV2_DEBUG(21785, + 1, + "Cannot select a sync source because chaining is not allowed and primary " + "member is blacklisted: {currentPrimaryMember_getHostAndPort}", + "currentPrimaryMember_getHostAndPort"_attr = + _currentPrimaryMember()->getHostAndPort()); _syncSource = HostAndPort(); return _syncSource; } else if (_currentPrimaryIndex == _selfIndex) { - LOG(1) - << "Cannot select a sync source because chaining is not allowed and we are primary"; + LOGV2_DEBUG( + 21786, + 1, + "Cannot select a sync source because chaining is not allowed and we are primary"); _syncSource = HostAndPort(); return _syncSource; } else if (_memberData.at(_currentPrimaryIndex).getLastAppliedOpTime() < @@ -295,8 +315,9 @@ HostAndPort TopologyCoordinator::chooseNewSyncSource(Date_t now, return _syncSource; } else { _syncSource = _currentPrimaryMember()->getHostAndPort(); - log() << "chaining not allowed, choosing primary as sync source candidate: " - << _syncSource; + LOGV2(21787, + "chaining not allowed, choosing primary as sync source candidate: {syncSource}", + "syncSource"_attr = _syncSource); std::string msg(str::stream() << "syncing from primary: " << _syncSource.toString()); setMyHeartbeatMessage(now, msg); return _syncSource; @@ -347,14 +368,20 @@ HostAndPort TopologyCoordinator::chooseNewSyncSource(Date_t now, // Candidate must be up to be considered. if (!it->up()) { - LOG(2) << "Cannot select sync source because it is not up: " - << itMemberConfig.getHostAndPort(); + LOGV2_DEBUG(21788, + 2, + "Cannot select sync source because it is not up: " + "{itMemberConfig_getHostAndPort}", + "itMemberConfig_getHostAndPort"_attr = itMemberConfig.getHostAndPort()); continue; } // Candidate must be PRIMARY or SECONDARY state to be considered. if (!it->getState().readable()) { - LOG(2) << "Cannot select sync source because it is not readable: " - << itMemberConfig.getHostAndPort(); + LOGV2_DEBUG(21789, + 2, + "Cannot select sync source because it is not readable: " + "{itMemberConfig_getHostAndPort}", + "itMemberConfig_getHostAndPort"_attr = itMemberConfig.getHostAndPort()); continue; } @@ -362,61 +389,94 @@ HostAndPort TopologyCoordinator::chooseNewSyncSource(Date_t now, if (attempts == 0) { // Candidate must be a voter if we are a voter. if (_selfConfig().isVoter() && !itMemberConfig.isVoter()) { - LOG(2) << "Cannot select sync source because we are a voter and it is not: " - << itMemberConfig.getHostAndPort(); + LOGV2_DEBUG(21790, + 2, + "Cannot select sync source because we are a voter and it is not: " + "{itMemberConfig_getHostAndPort}", + "itMemberConfig_getHostAndPort"_attr = + itMemberConfig.getHostAndPort()); continue; } // Candidates must not be hidden. if (itMemberConfig.isHidden()) { - LOG(2) << "Cannot select sync source because it is hidden: " - << itMemberConfig.getHostAndPort(); + LOGV2_DEBUG(21791, + 2, + "Cannot select sync source because it is hidden: " + "{itMemberConfig_getHostAndPort}", + "itMemberConfig_getHostAndPort"_attr = + itMemberConfig.getHostAndPort()); continue; } // Candidates cannot be excessively behind. if (it->getHeartbeatAppliedOpTime() < oldestSyncOpTime) { - LOG(2) << "Cannot select sync source because it is too far behind." - << "Latest optime of sync candidate " << itMemberConfig.getHostAndPort() - << ": " << it->getHeartbeatAppliedOpTime() - << ", oldest acceptable optime: " << oldestSyncOpTime; + LOGV2_DEBUG( + 21792, + 2, + "Cannot select sync source because it is too far behind.Latest optime of " + "sync candidate {itMemberConfig_getHostAndPort}: " + "{it_getHeartbeatAppliedOpTime}, oldest acceptable optime: " + "{oldestSyncOpTime}", + "itMemberConfig_getHostAndPort"_attr = itMemberConfig.getHostAndPort(), + "it_getHeartbeatAppliedOpTime"_attr = it->getHeartbeatAppliedOpTime(), + "oldestSyncOpTime"_attr = oldestSyncOpTime); continue; } // Candidate must not have a configured delay larger than ours. if (_selfConfig().getSlaveDelay() < itMemberConfig.getSlaveDelay()) { - LOG(2) << "Cannot select sync source with larger slaveDelay than ours: " - << itMemberConfig.getHostAndPort(); + LOGV2_DEBUG(21793, + 2, + "Cannot select sync source with larger slaveDelay than ours: " + "{itMemberConfig_getHostAndPort}", + "itMemberConfig_getHostAndPort"_attr = + itMemberConfig.getHostAndPort()); continue; } } // Candidate must build indexes if we build indexes, to be considered. if (_selfConfig().shouldBuildIndexes()) { if (!itMemberConfig.shouldBuildIndexes()) { - LOG(2) << "Cannot select sync source with shouldBuildIndex differences: " - << itMemberConfig.getHostAndPort(); + LOGV2_DEBUG(21794, + 2, + "Cannot select sync source with shouldBuildIndex differences: " + "{itMemberConfig_getHostAndPort}", + "itMemberConfig_getHostAndPort"_attr = + itMemberConfig.getHostAndPort()); continue; } } // Only select a candidate that is ahead of me. if (it->getHeartbeatAppliedOpTime() <= lastOpTimeFetched) { - LOG(1) << "Cannot select this sync source. Sync source must be ahead of me. " - << "Sync candidate: " << itMemberConfig.getHostAndPort() - << ", my last fetched oplog optime: " << lastOpTimeFetched.toBSON() - << ", latest oplog optime of sync candidate: " - << it->getHeartbeatAppliedOpTime().toBSON(); + LOGV2_DEBUG(21795, + 1, + "Cannot select this sync source. Sync source must be ahead of me. Sync " + "candidate: {itMemberConfig_getHostAndPort}, my last fetched oplog " + "optime: {lastOpTimeFetched}, latest oplog optime of sync candidate: " + "{it_getHeartbeatAppliedOpTime}", + "itMemberConfig_getHostAndPort"_attr = itMemberConfig.getHostAndPort(), + "lastOpTimeFetched"_attr = lastOpTimeFetched.toBSON(), + "it_getHeartbeatAppliedOpTime"_attr = + it->getHeartbeatAppliedOpTime().toBSON()); continue; } // Candidate cannot be more latent than anything we've already considered. if ((closestIndex != -1) && (_getPing(itMemberConfig.getHostAndPort()) > _getPing(_rsConfig.getMemberAt(closestIndex).getHostAndPort()))) { - LOG(2) << "Cannot select sync source with higher latency than the best candidate: " - << itMemberConfig.getHostAndPort(); + LOGV2_DEBUG(21796, + 2, + "Cannot select sync source with higher latency than the best " + "candidate: {itMemberConfig_getHostAndPort}", + "itMemberConfig_getHostAndPort"_attr = itMemberConfig.getHostAndPort()); continue; } // Candidate cannot be blacklisted. if (_memberIsBlacklisted(itMemberConfig, now)) { - LOG(1) << "Cannot select sync source which is blacklisted: " - << itMemberConfig.getHostAndPort(); + LOGV2_DEBUG(21797, + 1, + "Cannot select sync source which is blacklisted: " + "{itMemberConfig_getHostAndPort}", + "itMemberConfig_getHostAndPort"_attr = itMemberConfig.getHostAndPort()); continue; } @@ -432,7 +492,7 @@ HostAndPort TopologyCoordinator::chooseNewSyncSource(Date_t now, std::string msg("could not find member to sync from"); // Only log when we had a valid sync source before if (!_syncSource.empty()) { - log() << msg << rsLog; + LOGV2_OPTIONS(21798, {logv2::LogTag::kRS}, "{msg}", "msg"_attr = msg); } setMyHeartbeatMessage(now, msg); @@ -440,7 +500,7 @@ HostAndPort TopologyCoordinator::chooseNewSyncSource(Date_t now, return _syncSource; } _syncSource = _rsConfig.getMemberAt(closestIndex).getHostAndPort(); - log() << "sync source candidate: " << _syncSource; + LOGV2(21799, "sync source candidate: {syncSource}", "syncSource"_attr = _syncSource); std::string msg(str::stream() << "syncing from: " << _syncSource.toString(), 0); setMyHeartbeatMessage(now, msg); return _syncSource; @@ -458,14 +518,18 @@ bool TopologyCoordinator::_memberIsBlacklisted(const MemberConfig& memberConfig, } void TopologyCoordinator::blacklistSyncSource(const HostAndPort& host, Date_t until) { - LOG(2) << "blacklisting " << host << " until " << until.toString(); + LOGV2_DEBUG(21800, + 2, + "blacklisting {host} until {until}", + "host"_attr = host, + "until"_attr = until.toString()); _syncSourceBlacklist[host] = until; } void TopologyCoordinator::unblacklistSyncSource(const HostAndPort& host, Date_t now) { std::map<HostAndPort, Date_t>::iterator hostItr = _syncSourceBlacklist.find(host); if (hostItr != _syncSourceBlacklist.end() && now >= hostItr->second) { - LOG(2) << "unblacklisting " << host; + LOGV2_DEBUG(21801, 2, "unblacklisting {host}", "host"_attr = host); _syncSourceBlacklist.erase(hostItr); } } @@ -549,9 +613,14 @@ void TopologyCoordinator::prepareSyncFromResponse(const HostAndPort& target, } const OpTime lastOpApplied = getMyLastAppliedOpTime(); if (hbdata.getHeartbeatAppliedOpTime().getSecs() + 10 < lastOpApplied.getSecs()) { - warning() << "attempting to sync from " << target << ", but its latest opTime is " - << hbdata.getHeartbeatAppliedOpTime().getSecs() << " and ours is " - << lastOpApplied.getSecs() << " so this may not work"; + LOGV2_WARNING(21837, + "attempting to sync from {target}, but its latest opTime is " + "{hbdata_getHeartbeatAppliedOpTime_getSecs} and ours is " + "{lastOpApplied_getSecs} so this may not work", + "target"_attr = target, + "hbdata_getHeartbeatAppliedOpTime_getSecs"_attr = + hbdata.getHeartbeatAppliedOpTime().getSecs(), + "lastOpApplied_getSecs"_attr = lastOpApplied.getSecs()); response->append("warning", str::stream() << "requested member \"" << target.toString() << "\" is more than 10 seconds behind us"); @@ -575,8 +644,10 @@ Status TopologyCoordinator::prepareHeartbeatResponseV1(Date_t now, // Verify that replica set names match const std::string rshb = args.getSetName(); if (ourSetName != rshb) { - log() << "replSet set names do not match, ours: " << ourSetName - << "; remote node's: " << rshb; + LOGV2(21802, + "replSet set names do not match, ours: {ourSetName}; remote node's: {rshb}", + "ourSetName"_attr = ourSetName, + "rshb"_attr = rshb); return Status(ErrorCodes::InconsistentReplicaSetNames, str::stream() << "Our set name of " << ourSetName << " does not match name " << rshb << " reported by remote node"); @@ -761,9 +832,9 @@ HeartbeatResponseAction TopologyCoordinator::processHeartbeatResponse( // Could be we got the newer version before we got the response, or the // target erroneously sent us one, even though it isn't newer. if (newConfig.getConfigVersionAndTerm() < currentConfigVersionAndTerm) { - LOG(1) << "Config version from heartbeat was older than ours."; + LOGV2_DEBUG(21803, 1, "Config version from heartbeat was older than ours."); } else { - LOG(2) << "Config from heartbeat response was same as ours."; + LOGV2_DEBUG(21804, 2, "Config from heartbeat response was same as ours."); } if (shouldLog(MongoLogDefaultComponent_component, ::mongo::LogstreamBuilder::severityCast(2))) { @@ -785,18 +856,24 @@ HeartbeatResponseAction TopologyCoordinator::processHeartbeatResponse( } // If we're not in the config, we don't need to respond to heartbeats. if (_selfIndex == -1) { - LOG(1) << "Could not find ourself in current config so ignoring heartbeat from " << target - << " -- current config: " << _rsConfig.toBSON(); + LOGV2_DEBUG(21805, + 1, + "Could not find ourself in current config so ignoring heartbeat from {target} " + "-- current config: {rsConfig}", + "target"_attr = target, + "rsConfig"_attr = _rsConfig.toBSON()); HeartbeatResponseAction nextAction = HeartbeatResponseAction::makeNoAction(); nextAction.setNextHeartbeatStartDate(nextHeartbeatStartDate); return nextAction; } const int memberIndex = _rsConfig.findMemberIndexByHostAndPort(target); if (memberIndex == -1) { - LOG(1) << "Could not find " << target - << " in current config so ignoring --" - " current config: " - << _rsConfig.toBSON(); + LOGV2_DEBUG(21806, + 1, + "Could not find {target} in current config so ignoring --" + " current config: {rsConfig}", + "target"_attr = target, + "rsConfig"_attr = _rsConfig.toBSON()); HeartbeatResponseAction nextAction = HeartbeatResponseAction::makeNoAction(); nextAction.setNextHeartbeatStartDate(nextHeartbeatStartDate); return nextAction; @@ -816,13 +893,20 @@ HeartbeatResponseAction TopologyCoordinator::processHeartbeatResponse( else if (hbStats.failed() || (alreadyElapsed >= _rsConfig.getHeartbeatTimeoutPeriod())) { hbData.setDownValues(now, hbResponse.getStatus().reason()); } else { - LOG(3) << "Bad heartbeat response from " << target - << "; trying again; Retries left: " << (hbStats.retriesLeft()) << "; " - << alreadyElapsed << " have already elapsed"; + LOGV2_DEBUG(21807, + 3, + "Bad heartbeat response from {target}; trying again; Retries left: " + "{hbStats_retriesLeft}; {alreadyElapsed} have already elapsed", + "target"_attr = target, + "hbStats_retriesLeft"_attr = (hbStats.retriesLeft()), + "alreadyElapsed"_attr = alreadyElapsed); } } else { ReplSetHeartbeatResponse hbr = std::move(hbResponse.getValue()); - LOG(3) << "setUpValues: heartbeat response good for member _id:" << member.getId(); + LOGV2_DEBUG(21808, + 3, + "setUpValues: heartbeat response good for member _id:{member_getId}", + "member_getId"_attr = member.getId()); advancedOpTime = hbData.setUpValues(now, std::move(hbr)); } @@ -932,7 +1016,7 @@ HeartbeatResponseAction TopologyCoordinator::checkMemberTimeouts(Date_t now) { } } if (stepdown) { - log() << "can't see a majority of the set, relinquishing primary"; + LOGV2(21809, "can't see a majority of the set, relinquishing primary"); return HeartbeatResponseAction::makeStepDownSelfAction(_selfIndex); } return HeartbeatResponseAction::makeNoAction(); @@ -981,13 +1065,20 @@ std::pair<MemberId, Date_t> TopologyCoordinator::getStalestLiveMember() const { // Already stale. continue; } - LOG(3) << "memberData lastupdate is: " << memberData.getLastUpdate(); + LOGV2_DEBUG(21810, + 3, + "memberData lastupdate is: {memberData_getLastUpdate}", + "memberData_getLastUpdate"_attr = memberData.getLastUpdate()); if (earliestDate > memberData.getLastUpdate()) { earliestDate = memberData.getLastUpdate(); earliestMemberId = memberData.getMemberId(); } } - LOG(3) << "stalest member " << earliestMemberId << " date: " << earliestDate; + LOGV2_DEBUG(21811, + 3, + "stalest member {earliestMemberId} date: {earliestDate}", + "earliestMemberId"_attr = earliestMemberId, + "earliestDate"_attr = earliestDate); return std::make_pair(earliestMemberId, earliestDate); } @@ -1071,10 +1162,15 @@ StatusWith<bool> TopologyCoordinator::setLastOptime(const UpdatePositionArgs::Up return false; } - LOG(2) << "received notification that node with memberID " << memberId - << " in config with version " << args.cfgver - << " has reached optime: " << args.appliedOpTime - << " and is durable through: " << args.durableOpTime; + LOGV2_DEBUG(21812, + 2, + "received notification that node with memberID {memberId} in config with version " + "{args_cfgver} has reached optime: {args_appliedOpTime} and is durable through: " + "{args_durableOpTime}", + "memberId"_attr = memberId, + "args_cfgver"_attr = args.cfgver, + "args_appliedOpTime"_attr = args.appliedOpTime, + "args_durableOpTime"_attr = args.durableOpTime); // If we're in FCV 4.4, allow replSetUpdatePosition commands between config versions. if (!serverGlobalParams.featureCompatibility.isVersion( @@ -1084,7 +1180,7 @@ StatusWith<bool> TopologyCoordinator::setLastOptime(const UpdatePositionArgs::Up << "Received replSetUpdatePosition for node with memberId " << memberId << " whose config version of " << args.cfgver << " doesn't match our config version of " << _rsConfig.getConfigVersion(); - LOG(1) << errmsg; + LOGV2_DEBUG(21813, 1, "{errmsg}", "errmsg"_attr = errmsg); *configVersion = _rsConfig.getConfigVersion(); return Status(ErrorCodes::InvalidReplicaSetConfig, errmsg); } @@ -1099,16 +1195,23 @@ StatusWith<bool> TopologyCoordinator::setLastOptime(const UpdatePositionArgs::Up std::string errmsg = str::stream() << "Received replSetUpdatePosition for node with memberId " << memberId << " which doesn't exist in our config"; - LOG(1) << errmsg; + LOGV2_DEBUG(21814, 1, "{errmsg}", "errmsg"_attr = errmsg); return Status(ErrorCodes::NodeNotFound, errmsg); } invariant(memberId == memberData->getMemberId()); - LOG(3) << "Node with memberID " << memberId << " currently has optime " - << memberData->getLastAppliedOpTime() << " durable through " - << memberData->getLastDurableOpTime() << "; updating to optime " << args.appliedOpTime - << " and durable through " << args.durableOpTime; + LOGV2_DEBUG( + 21815, + 3, + "Node with memberID {memberId} currently has optime {memberData_getLastAppliedOpTime} " + "durable through {memberData_getLastDurableOpTime}; updating to optime " + "{args_appliedOpTime} and durable through {args_durableOpTime}", + "memberId"_attr = memberId, + "memberData_getLastAppliedOpTime"_attr = memberData->getLastAppliedOpTime(), + "memberData_getLastDurableOpTime"_attr = memberData->getLastDurableOpTime(), + "args_appliedOpTime"_attr = args.appliedOpTime, + "args_durableOpTime"_attr = args.durableOpTime); bool advancedOpTime = memberData->advanceLastAppliedOpTimeAndWallTime( {args.appliedOpTime, args.appliedWallTime}, now); @@ -1401,15 +1504,19 @@ void TopologyCoordinator::changeMemberState_forTest(const MemberState& newMember updateConfig(ReplSetConfig(), -1, Date_t()); break; default: - severe() << "Cannot switch to state " << newMemberState; + LOGV2_FATAL(21840, + "Cannot switch to state {newMemberState}", + "newMemberState"_attr = newMemberState); MONGO_UNREACHABLE; } if (getMemberState() != newMemberState.s) { - severe() << "Expected to enter state " << newMemberState << " but am now in " - << getMemberState(); + LOGV2_FATAL(21841, + "Expected to enter state {newMemberState} but am now in {getMemberState}", + "newMemberState"_attr = newMemberState, + "getMemberState"_attr = getMemberState()); MONGO_UNREACHABLE; } - log() << newMemberState; + LOGV2(21816, "{newMemberState}", "newMemberState"_attr = newMemberState); } void TopologyCoordinator::setCurrentPrimary_forTest(int primaryIndex, @@ -1823,13 +1930,13 @@ TopologyCoordinator::prepareFreezeResponse(Date_t now, int secs, BSONObjBuilder* std::string msg = str::stream() << "cannot freeze node when primary or running for election. state: " << (_role == TopologyCoordinator::Role::kLeader ? "Primary" : "Running-Election"); - log() << msg; + LOGV2(21817, "{msg}", "msg"_attr = msg); return Status(ErrorCodes::NotSecondary, msg); } if (secs == 0) { _stepDownUntil = now; - log() << "'unfreezing'"; + LOGV2(21818, "'unfreezing'"); response->append("info", "unfreezing"); return PrepareFreezeResponseResult::kSingleNodeSelfElect; } else { @@ -1837,7 +1944,7 @@ TopologyCoordinator::prepareFreezeResponse(Date_t now, int secs, BSONObjBuilder* response->append("warning", "you really want to freeze for only 1 second?"); _stepDownUntil = std::max(_stepDownUntil, now + Seconds(secs)); - log() << "'freezing' for " << secs << " seconds"; + LOGV2(21819, "'freezing' for {secs} seconds", "secs"_attr = secs); } return PrepareFreezeResponseResult::kNoAction; @@ -1914,7 +2021,10 @@ void TopologyCoordinator::updateConfig(const ReplSetConfig& newConfig, int selfI // Reset term on startup. if (!_rsConfig.isInitialized()) { _term = OpTime::kInitialTerm; - LOG(1) << "Updated term in topology coordinator to " << _term << " due to new config"; + LOGV2_DEBUG(21820, + 1, + "Updated term in topology coordinator to {term} due to new config", + "term"_attr = _term); } _updateHeartbeatDataForReconfig(newConfig, selfIndex, now); @@ -1924,9 +2034,9 @@ void TopologyCoordinator::updateConfig(const ReplSetConfig& newConfig, int selfI if (_role == Role::kLeader) { if (_selfIndex == -1) { - log() << "Could not remain primary because no longer a member of the replica set"; + LOGV2(21821, "Could not remain primary because no longer a member of the replica set"); } else if (!_selfConfig().isElectable()) { - log() << " Could not remain primary because no longer electable"; + LOGV2(21822, " Could not remain primary because no longer electable"); } else { // Don't stepdown if you don't have to. _currentPrimaryIndex = _selfIndex; @@ -2109,7 +2219,9 @@ std::string TopologyCoordinator::_getUnelectableReasonString(const UnelectableRe ss << "node is not a member of a valid replica set configuration"; } if (!hasWrittenToStream) { - severe() << "Invalid UnelectableReasonMask value 0x" << integerToHex(ur); + LOGV2_FATAL(21842, + "Invalid UnelectableReasonMask value 0x{integerToHex_ur}", + "integerToHex_ur"_attr = integerToHex(ur)); fassertFailed(26011); } ss << " (mask 0x" << integerToHex(ur) << ")"; @@ -2422,7 +2534,7 @@ void TopologyCoordinator::finishUnconditionalStepDown() { // two other nodes think they are primary (asynchronously polled) // -- wait for things to settle down. remotePrimaryIndex = -1; - warning() << "two remote primaries (transiently)"; + LOGV2_WARNING(21838, "two remote primaries (transiently)"); break; } remotePrimaryIndex = itIndex; @@ -2494,8 +2606,12 @@ bool TopologyCoordinator::advanceLastCommittedOpTimeAndWallTime(OpTimeAndWallTim // This check is performed to ensure primaries do not commit an OpTime from a previous term. if (_iAmPrimary() && committedOpTime.opTime < _firstOpTimeOfMyTerm) { - LOG(1) << "Ignoring older committed snapshot from before I became primary, optime: " - << committedOpTime.opTime << ", firstOpTimeOfMyTerm: " << _firstOpTimeOfMyTerm; + LOGV2_DEBUG(21823, + 1, + "Ignoring older committed snapshot from before I became primary, optime: " + "{committedOpTime_opTime}, firstOpTimeOfMyTerm: {firstOpTimeOfMyTerm}", + "committedOpTime_opTime"_attr = committedOpTime.opTime, + "firstOpTimeOfMyTerm"_attr = _firstOpTimeOfMyTerm); return false; } @@ -2505,11 +2621,15 @@ bool TopologyCoordinator::advanceLastCommittedOpTimeAndWallTime(OpTimeAndWallTim if (fromSyncSource) { committedOpTime = std::min(committedOpTime, getMyLastAppliedOpTimeAndWallTime()); } else { - LOG(1) << "Ignoring commit point with different term than my lastApplied, since it " - "may " - "not be on the same oplog branch as mine. optime: " - << committedOpTime - << ", my last applied: " << getMyLastAppliedOpTimeAndWallTime(); + LOGV2_DEBUG(21824, + 1, + "Ignoring commit point with different term than my lastApplied, since it " + "may " + "not be on the same oplog branch as mine. optime: {committedOpTime}, my " + "last applied: {getMyLastAppliedOpTimeAndWallTime}", + "committedOpTime"_attr = committedOpTime, + "getMyLastAppliedOpTimeAndWallTime"_attr = + getMyLastAppliedOpTimeAndWallTime()); return false; } } @@ -2519,12 +2639,19 @@ bool TopologyCoordinator::advanceLastCommittedOpTimeAndWallTime(OpTimeAndWallTim } if (committedOpTime.opTime < _lastCommittedOpTimeAndWallTime.opTime) { - LOG(1) << "Ignoring older committed snapshot optime: " << committedOpTime - << ", currentCommittedOpTime: " << _lastCommittedOpTimeAndWallTime; + LOGV2_DEBUG(21825, + 1, + "Ignoring older committed snapshot optime: {committedOpTime}, " + "currentCommittedOpTime: {lastCommittedOpTimeAndWallTime}", + "committedOpTime"_attr = committedOpTime, + "lastCommittedOpTimeAndWallTime"_attr = _lastCommittedOpTimeAndWallTime); return false; } - LOG(2) << "Updating _lastCommittedOpTimeAndWallTime to " << committedOpTime; + LOGV2_DEBUG(21826, + 2, + "Updating _lastCommittedOpTimeAndWallTime to {committedOpTime}", + "committedOpTime"_attr = committedOpTime); _lastCommittedOpTimeAndWallTime = committedOpTime; return true; } @@ -2586,7 +2713,8 @@ TopologyCoordinator::UpdateTermResult TopologyCoordinator::updateTerm(long long if (_iAmPrimary()) { return TopologyCoordinator::UpdateTermResult::kTriggerStepDown; } - LOG(1) << "Updating term from " << _term << " to " << term; + LOGV2_DEBUG( + 21827, 1, "Updating term from {term} to {term2}", "term"_attr = _term, "term2"_attr = term); _term = term; return TopologyCoordinator::UpdateTermResult::kUpdatedTerm; } @@ -2611,15 +2739,17 @@ bool TopologyCoordinator::shouldChangeSyncSource( // progress, return true. if (_selfIndex == -1) { - log() << "Not choosing new sync source because we are not in the config."; + LOGV2(21828, "Not choosing new sync source because we are not in the config."); return false; } // If the user requested a sync source change, return true. if (_forceSyncSourceIndex != -1) { - log() << "Choosing new sync source because the user has requested to use " - << _rsConfig.getMemberAt(_forceSyncSourceIndex).getHostAndPort() - << " as a sync source"; + LOGV2(21829, + "Choosing new sync source because the user has requested to use " + "{rsConfig_getMemberAt_forceSyncSourceIndex_getHostAndPort} as a sync source", + "rsConfig_getMemberAt_forceSyncSourceIndex_getHostAndPort"_attr = + _rsConfig.getMemberAt(_forceSyncSourceIndex).getHostAndPort()); return true; } @@ -2628,9 +2758,13 @@ bool TopologyCoordinator::shouldChangeSyncSource( if (!serverGlobalParams.featureCompatibility.isVersion( ServerGlobalParams::FeatureCompatibility::Version::kFullyUpgradedTo44)) { if (replMetadata.getConfigVersion() != _rsConfig.getConfigVersion()) { - log() << "Choosing new sync source because the config version supplied by " - << currentSource << ", " << replMetadata.getConfigVersion() - << ", does not match ours, " << _rsConfig.getConfigVersion(); + LOGV2( + 21830, + "Choosing new sync source because the config version supplied by {currentSource}, " + "{replMetadata_getConfigVersion}, does not match ours, {rsConfig_getConfigVersion}", + "currentSource"_attr = currentSource, + "replMetadata_getConfigVersion"_attr = replMetadata.getConfigVersion(), + "rsConfig_getConfigVersion"_attr = _rsConfig.getConfigVersion()); return true; } } @@ -2639,8 +2773,9 @@ bool TopologyCoordinator::shouldChangeSyncSource( // from a node that is not in our config. const int currentSourceIndex = _rsConfig.findMemberIndexByHostAndPort(currentSource); if (currentSourceIndex == -1) { - log() << "Choosing new sync source because " << currentSource.toString() - << " is not in our config"; + LOGV2(21831, + "Choosing new sync source because {currentSource} is not in our config", + "currentSource"_attr = currentSource.toString()); return true; } @@ -2689,15 +2824,17 @@ bool TopologyCoordinator::shouldChangeSyncSource( } else { logMessage << " (sync source does not know the primary)"; } - log() << logMessage.str(); + LOGV2(21832, "{logMessage_str}", "logMessage_str"_attr = logMessage.str()); return true; } if (MONGO_unlikely(disableMaxSyncSourceLagSecs.shouldFail())) { - log() << "disableMaxSyncSourceLagSecs fail point enabled - not checking the most recent " - "OpTime, " - << currentSourceOpTime.toString() << ", of our current sync source, " << currentSource - << ", against the OpTimes of the other nodes in this replica set."; + LOGV2(21833, + "disableMaxSyncSourceLagSecs fail point enabled - not checking the most recent " + "OpTime, {currentSourceOpTime}, of our current sync source, {currentSource}, against " + "the OpTimes of the other nodes in this replica set.", + "currentSourceOpTime"_attr = currentSourceOpTime.toString(), + "currentSource"_attr = currentSource); } else { unsigned int currentSecs = currentSourceOpTime.getSecs(); unsigned int goalSecs = currentSecs + durationCount<Seconds>(_options.maxSyncSourceLagSecs); @@ -2711,13 +2848,19 @@ bool TopologyCoordinator::shouldChangeSyncSource( (candidateConfig.shouldBuildIndexes() || !_selfConfig().shouldBuildIndexes()) && it->getState().readable() && !_memberIsBlacklisted(candidateConfig, now) && goalSecs < it->getHeartbeatAppliedOpTime().getSecs()) { - log() << "Choosing new sync source because the most recent OpTime of our sync " - "source, " - << currentSource << ", is " << currentSourceOpTime.toString() - << " which is more than " << _options.maxSyncSourceLagSecs - << " behind member " << candidateConfig.getHostAndPort().toString() - << " whose most recent OpTime is " - << it->getHeartbeatAppliedOpTime().toString(); + LOGV2( + 21834, + "Choosing new sync source because the most recent OpTime of our sync " + "source, {currentSource}, is {currentSourceOpTime} which is more than " + "{options_maxSyncSourceLagSecs} behind member {candidateConfig_getHostAndPort} " + "whose most recent OpTime is {it_getHeartbeatAppliedOpTime}", + "currentSource"_attr = currentSource, + "currentSourceOpTime"_attr = currentSourceOpTime.toString(), + "options_maxSyncSourceLagSecs"_attr = _options.maxSyncSourceLagSecs, + "candidateConfig_getHostAndPort"_attr = + candidateConfig.getHostAndPort().toString(), + "it_getHeartbeatAppliedOpTime"_attr = + it->getHeartbeatAppliedOpTime().toString()); invariant(itIndex != _selfIndex); return true; } @@ -2751,7 +2894,7 @@ void TopologyCoordinator::processReplSetRequestVotes(const ReplSetRequestVotesAr response->setTerm(_term); if (MONGO_unlikely(voteNoInElection.shouldFail())) { - log() << "failpoint voteNoInElection enabled"; + LOGV2(21835, "failpoint voteNoInElection enabled"); response->setVoteGranted(false); response->setReason(str::stream() << "forced to vote no during dry run election due to " "failpoint voteNoInElection set"); @@ -2759,7 +2902,7 @@ void TopologyCoordinator::processReplSetRequestVotes(const ReplSetRequestVotesAr } if (MONGO_unlikely(voteYesInDryRunButNoInRealElection.shouldFail())) { - log() << "failpoint voteYesInDryRunButNoInRealElection enabled"; + LOGV2(21836, "failpoint voteYesInDryRunButNoInRealElection enabled"); if (args.isADryRun()) { response->setVoteGranted(true); response->setReason(str::stream() << "forced to vote yes in dry run due to failpoint " diff --git a/src/mongo/db/repl/topology_coordinator_v1_test.cpp b/src/mongo/db/repl/topology_coordinator_v1_test.cpp index 9fc313a6042..cf41b85b78d 100644 --- a/src/mongo/db/repl/topology_coordinator_v1_test.cpp +++ b/src/mongo/db/repl/topology_coordinator_v1_test.cpp @@ -27,6 +27,8 @@ * it in the license file. */ +#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kDefault + #include "mongo/platform/basic.h" #include <iostream> @@ -42,10 +44,12 @@ #include "mongo/db/server_options.h" #include "mongo/executor/task_executor.h" #include "mongo/logger/logger.h" +#include "mongo/logv2/log.h" #include "mongo/rpc/metadata/oplog_query_metadata.h" #include "mongo/rpc/metadata/repl_set_metadata.h" #include "mongo/unittest/unittest.h" #include "mongo/util/assert_util.h" +#include "mongo/util/log.h" #include "mongo/util/log_global_settings.h" #include "mongo/util/net/hostandport.h" #include "mongo/util/scopeguard.h" @@ -1626,7 +1630,7 @@ TEST_F(TopoCoordTest, ReplSetGetStatus) { &resultStatus); ASSERT_OK(resultStatus); BSONObj rsStatus = statusBuilder.obj(); - unittest::log() << rsStatus; + LOGV2(21843, "{rsStatus}", "rsStatus"_attr = rsStatus); // Test results for all non-self members ASSERT_EQUALS(setName, rsStatus["set"].String()); @@ -1742,7 +1746,7 @@ TEST_F(TopoCoordTest, ReplSetGetStatus) { &resultStatus); ASSERT_OK(resultStatus); rsStatus = statusBuilder2.obj(); - unittest::log() << rsStatus; + LOGV2(21844, "{rsStatus}", "rsStatus"_attr = rsStatus); ASSERT_EQUALS(setName, rsStatus["set"].String()); ASSERT_FALSE(rsStatus.hasField("lastStableRecoveryTimestamp")); ASSERT_FALSE(rsStatus.hasField("electionCandidateMetrics")); diff --git a/src/mongo/db/repl/transaction_oplog_application.cpp b/src/mongo/db/repl/transaction_oplog_application.cpp index 83bbe54deb9..3d2f77905dc 100644 --- a/src/mongo/db/repl/transaction_oplog_application.cpp +++ b/src/mongo/db/repl/transaction_oplog_application.cpp @@ -45,6 +45,7 @@ #include "mongo/db/session_catalog_mongod.h" #include "mongo/db/transaction_history_iterator.h" #include "mongo/db/transaction_participant.h" +#include "mongo/logv2/log.h" #include "mongo/util/log.h" namespace mongo { @@ -78,14 +79,20 @@ Status _applyOperationsForTransaction(OperationContext* opCtx, oplogApplicationMode == repl::OplogApplication::Mode::kRecovering); if (!ignoreException) { - LOG(1) << "Error applying operation in transaction. " << redact(ex) - << "- oplog entry: " << redact(op.toBSON()); + LOGV2_DEBUG(21845, + 1, + "Error applying operation in transaction. {ex}- oplog entry: {op}", + "ex"_attr = redact(ex), + "op"_attr = redact(op.toBSON())); return exceptionToStatus(); } - LOG(1) << "Encountered but ignoring error: " << redact(ex) - << " while applying operations for transaction because we are either in initial " - "sync or recovering mode - oplog entry: " - << redact(op.toBSON()); + LOGV2_DEBUG(21846, + 1, + "Encountered but ignoring error: {ex} while applying operations for " + "transaction because we are either in initial " + "sync or recovering mode - oplog entry: {op}", + "ex"_attr = redact(ex), + "op"_attr = redact(op.toBSON())); } } return Status::OK(); @@ -379,8 +386,10 @@ Status _applyPrepareTransaction(OperationContext* opCtx, auto ns = op.getNss(); auto uuid = *op.getUuid(); if (BackgroundOperation::inProgForNs(ns)) { - warning() << "blocking replication until index builds are finished on " - << redact(ns.toString()) << ", due to prepared transaction"; + LOGV2_WARNING(21849, + "blocking replication until index builds are finished on {ns}, due " + "to prepared transaction", + "ns"_attr = redact(ns.toString())); BackgroundOperation::awaitNoBgOpInProgForNs(ns); IndexBuildsCoordinator::get(opCtx)->awaitNoIndexBuildInProgressForCollection(uuid); } @@ -411,7 +420,7 @@ Status _applyPrepareTransaction(OperationContext* opCtx, fassert(31137, status); if (MONGO_unlikely(applyOpsHangBeforePreparingTransaction.shouldFail())) { - LOG(0) << "Hit applyOpsHangBeforePreparingTransaction failpoint"; + LOGV2(21847, "Hit applyOpsHangBeforePreparingTransaction failpoint"); applyOpsHangBeforePreparingTransaction.pauseWhileSet(opCtx); } @@ -460,10 +469,11 @@ Status applyPrepareTransaction(OperationContext* opCtx, switch (mode) { case repl::OplogApplication::Mode::kRecovering: { if (!serverGlobalParams.enableMajorityReadConcern) { - error() - << "Cannot replay a prepared transaction when 'enableMajorityReadConcern' is " - "set to false. Restart the server with --enableMajorityReadConcern=true " - "to complete recovery."; + LOGV2_ERROR( + 21850, + "Cannot replay a prepared transaction when 'enableMajorityReadConcern' is " + "set to false. Restart the server with --enableMajorityReadConcern=true " + "to complete recovery."); fassertFailed(51146); } @@ -490,7 +500,7 @@ Status applyPrepareTransaction(OperationContext* opCtx, void reconstructPreparedTransactions(OperationContext* opCtx, repl::OplogApplication::Mode mode) { if (MONGO_unlikely(skipReconstructPreparedTransactions.shouldFail())) { - log() << "Hit skipReconstructPreparedTransactions failpoint"; + LOGV2(21848, "Hit skipReconstructPreparedTransactions failpoint"); return; } // Read the transactions table and the oplog collection without a timestamp. |