diff options
| -rw-r--r-- | jstests/multiVersion/index_signature_fcv.js | 35 | ||||
| -rw-r--r-- | src/mongo/SConscript | 4 | ||||
| -rw-r--r-- | src/mongo/db/SConscript | 10 | ||||
| -rw-r--r-- | src/mongo/db/commands/feature_compatibility_version.cpp | 118 | ||||
| -rw-r--r-- | src/mongo/db/commands/feature_compatibility_version.h | 15 | ||||
| -rw-r--r-- | src/mongo/db/index/index_descriptor.cpp | 4 | ||||
| -rw-r--r-- | src/mongo/db/mongod_main.cpp | 34 | ||||
| -rw-r--r-- | src/mongo/db/repair.cpp (renamed from src/mongo/db/repair_database.cpp) | 155 | ||||
| -rw-r--r-- | src/mongo/db/repair.h (renamed from src/mongo/db/repair_database.h) | 20 | ||||
| -rw-r--r-- | src/mongo/db/repair_database_and_check_version.cpp | 653 | ||||
| -rw-r--r-- | src/mongo/db/startup_recovery.cpp | 588 | ||||
| -rw-r--r-- | src/mongo/db/startup_recovery.h (renamed from src/mongo/db/repair_database_and_check_version.h) | 14 | ||||
| -rw-r--r-- | src/mongo/db/storage/storage_engine_impl.cpp | 1 | ||||
| -rw-r--r-- | src/mongo/embedded/SConscript | 2 | ||||
| -rw-r--r-- | src/mongo/embedded/embedded.cpp | 15 |
15 files changed, 873 insertions, 795 deletions
diff --git a/jstests/multiVersion/index_signature_fcv.js b/jstests/multiVersion/index_signature_fcv.js index db0c4fdc329..a97dfb91cff 100644 --- a/jstests/multiVersion/index_signature_fcv.js +++ b/jstests/multiVersion/index_signature_fcv.js @@ -13,8 +13,9 @@ (function() { "use strict"; -load("jstests/libs/analyze_plan.js"); // For isIxscan and hasRejectedPlans. -load("jstests/multiVersion/libs/multi_rs.js"); // For upgradeSet. +load("jstests/libs/analyze_plan.js"); // For isIxscan and hasRejectedPlans. +load("jstests/multiVersion/libs/multi_rs.js"); // For upgradeSet. +load('jstests/noPassthrough/libs/index_build.js'); // For IndexBuildTest const rst = new ReplSetTest({ nodes: 2, @@ -23,13 +24,14 @@ const rst = new ReplSetTest({ rst.startSet(); rst.initiate(); -let testDB = rst.getPrimary().getDB(jsTestName()); +let primary = rst.getPrimary(); +let testDB = primary.getDB(jsTestName()); let coll = testDB.test; +coll.insert({a: 100}); // Verifies that the given query is indexed, and that 'numAlternativePlans' were generated. function assertIndexedQuery(query, numAlternativePlans) { const explainOut = coll.explain().find(query).finish(); - const results = coll.find(query).toArray(); assert(isIxscan(testDB, explainOut), explainOut); assert.eq(getRejectedPlans(explainOut).length, numAlternativePlans, explainOut); } @@ -48,6 +50,31 @@ assertIndexedQuery({a: 1}, 0); assertIndexedQuery({a: 11}, 1); assertIndexedQuery({a: 101}, 2); +// Test that an index build restarted during startup recovery in FCV 4.6 does not revert to FCV 4.4 +// behavior. +jsTestLog("Starting index build on primary and pausing before completion"); +IndexBuildTest.pauseIndexBuilds(primary); +IndexBuildTest.startIndexBuild( + primary, coll.getFullName(), {a: 1}, {name: "index4", partialFilterExpression: {a: {$lt: 0}}}); + +jsTestLog("Waiting for secondary to start the index build"); +let secondary = rst.getSecondary(); +let secondaryDB = secondary.getDB(jsTestName()); +IndexBuildTest.waitForIndexBuildToStart(secondaryDB); +rst.restart(secondary.nodeId); + +jsTestLog("Waiting for all nodes to finish building the index"); +IndexBuildTest.resumeIndexBuilds(primary); +IndexBuildTest.waitForIndexBuildToStop(testDB, coll.getName(), "index4"); +rst.awaitReplication(); + +// Reset connection in case leadership has changed. +primary = rst.getPrimary(); +testDB = primary.getDB(jsTestName()); +coll = testDB.test; + +assertIndexedQuery({a: -1}, 0); + // Test that these indexes are retained and can be used by the planner when we downgrade to FCV 4.4. testDB.adminCommand({setFeatureCompatibilityVersion: lastStableFCV}); assertIndexedQuery({a: 1}, 0); diff --git a/src/mongo/SConscript b/src/mongo/SConscript index 54e0f1239c7..4f2c249820d 100644 --- a/src/mongo/SConscript +++ b/src/mongo/SConscript @@ -458,7 +458,6 @@ env.Library( 'db/query_exec', 'db/read_concern_d_impl', 'db/read_write_concern_defaults', - 'db/repair_database_and_check_version', 'db/repl/bgsync', 'db/repl/oplog_application', 'db/repl/oplog_buffer_blocking_queue', @@ -481,6 +480,7 @@ env.Library( 'db/service_liaison_mongod', 'db/sessions_collection_rs', 'db/sessions_collection_standalone', + 'db/startup_recovery', 'db/startup_warnings_mongod', 'db/stats/counters', 'db/stats/serveronly_stats', @@ -569,7 +569,6 @@ env.Library( 'db/op_observer', 'db/periodic_runner_job_abort_expired_transactions', 'db/pipeline/process_interface/mongod_process_interface_factory', - 'db/repair_database_and_check_version', 'db/repl/drop_pending_collection_reaper', 'db/repl/repl_coordinator_impl', 'db/repl/replication_recovery', @@ -584,6 +583,7 @@ env.Library( 'db/service_liaison_mongod', 'db/sessions_collection_rs', 'db/sessions_collection_standalone', + 'db/startup_recovery', 'db/startup_warnings_mongod', 'db/storage/backup_cursor_hooks', 'db/storage/flow_control', diff --git a/src/mongo/db/SConscript b/src/mongo/db/SConscript index a649e04031d..4997bdea383 100644 --- a/src/mongo/db/SConscript +++ b/src/mongo/db/SConscript @@ -1012,9 +1012,9 @@ env.Library( ) env.Library( - target="repair_database", + target="repair", source=[ - "repair_database.cpp", + "repair.cpp", ], LIBDEPS=[ '$BUILD_DIR/mongo/db/catalog/collection', @@ -1191,9 +1191,9 @@ env.Library( ) env.Library( - target="repair_database_and_check_version", + target="startup_recovery", source=[ - "repair_database_and_check_version.cpp", + "startup_recovery.cpp", ], LIBDEPS_PRIVATE=[ 'rebuild_indexes', @@ -1202,7 +1202,7 @@ env.Library( 'commands/mongod_fcv', 'dbdirectclient', 'dbhelpers', - 'repair_database', + 'repair', 'repl/repl_settings', 'storage/storage_repair_observer', ], diff --git a/src/mongo/db/commands/feature_compatibility_version.cpp b/src/mongo/db/commands/feature_compatibility_version.cpp index 82acfd8f676..c82dbcdc468 100644 --- a/src/mongo/db/commands/feature_compatibility_version.cpp +++ b/src/mongo/db/commands/feature_compatibility_version.cpp @@ -45,6 +45,7 @@ #include "mongo/db/operation_context.h" #include "mongo/db/repl/optime.h" #include "mongo/db/repl/replication_coordinator.h" +#include "mongo/db/repl/replication_process.h" #include "mongo/db/repl/storage_interface.h" #include "mongo/db/s/collection_sharding_state.h" #include "mongo/db/s/sharding_state.h" @@ -67,6 +68,25 @@ Lock::ResourceMutex FeatureCompatibilityVersion::fcvLock("featureCompatibilityVe MONGO_FAIL_POINT_DEFINE(hangBeforeAbortingRunningTransactionsOnFCVDowngrade); +namespace { +bool isWriteableStorageEngine() { + return !storageGlobalParams.readOnly && (storageGlobalParams.engine != "devnull"); +} + +// Returns the featureCompatibilityVersion document if it exists. +boost::optional<BSONObj> findFcvDocument(OperationContext* opCtx) { + // Ensure database is opened and exists. + AutoGetOrCreateDb autoDb(opCtx, NamespaceString::kServerConfigurationNamespace.db(), MODE_IX); + + const auto query = BSON("_id" << FeatureCompatibilityVersionParser::kParameterName); + const auto swFcv = repl::StorageInterface::get(opCtx)->findById( + opCtx, NamespaceString::kServerConfigurationNamespace, query["_id"]); + if (!swFcv.isOK()) { + return boost::none; + } + return swFcv.getValue(); +} + /** * Build update command for featureCompatibilityVersion document updates. */ @@ -103,7 +123,7 @@ void runUpdateCommand(OperationContext* opCtx, const FeatureCompatibilityVersion client.runCommand(nss.db().toString(), updateCmd.obj(), updateResult); uassertStatusOK(getStatusFromWriteCommandReply(updateResult)); } - +} // namespace void FeatureCompatibilityVersion::setTargetUpgradeFrom( OperationContext* opCtx, ServerGlobalParams::FeatureCompatibility::Version fromVersion) { @@ -230,6 +250,102 @@ void FeatureCompatibilityVersion::updateMinWireVersion() { } } +void FeatureCompatibilityVersion::initializeForStartup(OperationContext* opCtx) { + // Global write lock must be held. + invariant(opCtx->lockState()->isW()); + auto featureCompatibilityVersion = findFcvDocument(opCtx); + if (!featureCompatibilityVersion) { + return; + } + + // If the server configuration collection already contains a valid featureCompatibilityVersion + // document, cache it in-memory as a server parameter. + auto swVersion = FeatureCompatibilityVersionParser::parse(*featureCompatibilityVersion); + + // Note this error path captures all cases of an FCV document existing, but with any + // unacceptable value. This includes unexpected cases with no path forward such as the FCV value + // not being a string. + if (!swVersion.isOK()) { + uassertStatusOK({ErrorCodes::MustDowngrade, + str::stream() + << "UPGRADE PROBLEM: Found an invalid featureCompatibilityVersion " + "document (ERROR: " + << swVersion.getStatus() + << "). If the current featureCompatibilityVersion is below 4.4, see " + "the documentation on upgrading at " + << feature_compatibility_version_documentation::kUpgradeLink << "."}); + } + + auto version = swVersion.getValue(); + serverGlobalParams.featureCompatibility.setVersion(version); + FeatureCompatibilityVersion::updateMinWireVersion(); + + // On startup, if the version is in an upgrading or downgrading state, print a warning. + if (version == ServerGlobalParams::FeatureCompatibility::Version::kUpgradingFrom44To451) { + LOGV2_WARNING_OPTIONS( + 21011, + {logv2::LogTag::kStartupWarnings}, + "A featureCompatibilityVersion upgrade did not complete. The current " + "featureCompatibilityVersion is {currentfeatureCompatibilityVersion}. To fix this, " + "use the setFeatureCompatibilityVersion command to resume upgrade to 4.5.1", + "A featureCompatibilityVersion upgrade did not complete. To fix this, use the " + "setFeatureCompatibilityVersion command to resume upgrade to 4.5.1", + "currentfeatureCompatibilityVersion"_attr = + FeatureCompatibilityVersionParser::toString(version)); + } else if (version == + ServerGlobalParams::FeatureCompatibility::Version::kDowngradingFrom451To44) { + LOGV2_WARNING_OPTIONS( + 21014, + {logv2::LogTag::kStartupWarnings}, + "A featureCompatibilityVersion downgrade did not complete. The current " + "featureCompatibilityVersion is {currentfeatureCompatibilityVersion}. To fix this, " + "use the setFeatureCompatibilityVersion command to resume downgrade to 4.4.", + "A featureCompatibilityVersion downgrade did not complete. To fix this, use the " + "setFeatureCompatibilityVersion command to resume downgrade to 4.5.1", + "currentfeatureCompatibilityVersion"_attr = + FeatureCompatibilityVersionParser::toString(version)); + } +} + +// Fatally asserts if the featureCompatibilityVersion document is not initialized, when required. +void FeatureCompatibilityVersion::fassertInitializedAfterStartup(OperationContext* opCtx) { + Lock::GlobalWrite lk(opCtx); + const auto replProcess = repl::ReplicationProcess::get(opCtx); + const auto& replSettings = repl::ReplicationCoordinator::get(opCtx)->getSettings(); + + // The node did not complete the last initial sync. If the initial sync flag is set and we are + // part of a replica set, we expect the version to be initialized as part of initial sync after + // startup. + bool needInitialSync = replSettings.usingReplSets() && replProcess && + replProcess->getConsistencyMarkers()->getInitialSyncFlag(opCtx); + if (needInitialSync) { + return; + } + + auto fcvDocument = findFcvDocument(opCtx); + + auto const storageEngine = opCtx->getServiceContext()->getStorageEngine(); + auto dbNames = storageEngine->listDatabases(); + bool nonLocalDatabases = std::any_of(dbNames.begin(), dbNames.end(), [](auto name) { + return name != NamespaceString::kLocalDb; + }); + + // Fail to start up if there is no featureCompatibilityVersion document and there are non-local + // databases present. + if (!fcvDocument && nonLocalDatabases) { + LOGV2_FATAL_NOTRACE(40652, + "Unable to start up mongod due to missing featureCompatibilityVersion " + "document. Please run with --repair to restore the document."); + } + + // If we are part of a replica set and are started up with no data files, we do not set the + // featureCompatibilityVersion until a primary is chosen. For this case, we expect the in-memory + // featureCompatibilityVersion parameter to still be uninitialized until after startup. + if (isWriteableStorageEngine() && (!replSettings.usingReplSets() || nonLocalDatabases)) { + invariant(serverGlobalParams.featureCompatibility.isVersionInitialized()); + } +} + void FeatureCompatibilityVersion::_setVersion( OperationContext* opCtx, ServerGlobalParams::FeatureCompatibility::Version newVersion) { serverGlobalParams.featureCompatibility.setVersion(newVersion); diff --git a/src/mongo/db/commands/feature_compatibility_version.h b/src/mongo/db/commands/feature_compatibility_version.h index b2076f6de7b..9ac8f0073f5 100644 --- a/src/mongo/db/commands/feature_compatibility_version.h +++ b/src/mongo/db/commands/feature_compatibility_version.h @@ -52,6 +52,21 @@ public: static Lock::ResourceMutex fcvLock; /** + * Reads the featureCompatibilityVersion (FCV) document in admin.system.version and initializes + * the FCV global state. Returns an error if the FCV document exists and is invalid. Does not + * return an error if it is missing. This should be checked after startup with + * fassertInitializedAfterStartup. + * + * Throws a MustDowngrade error if an existing FCV document contains an invalid version. + */ + static void initializeForStartup(OperationContext* opCtx); + + /** + * Fatally asserts if the featureCompatibilityVersion is not properly initialized after startup. + */ + static void fassertInitializedAfterStartup(OperationContext* opCtx); + + /** * Records intent to perform a currentVersion -> kLatest upgrade by updating the on-disk * feature compatibility version document to have 'version'=currentVersion, * 'targetVersion'=kLatest. Should be called before schemas are modified. diff --git a/src/mongo/db/index/index_descriptor.cpp b/src/mongo/db/index/index_descriptor.cpp index 55e8ec0eb8c..6dddcf4c0d2 100644 --- a/src/mongo/db/index/index_descriptor.cpp +++ b/src/mongo/db/index/index_descriptor.cpp @@ -179,8 +179,8 @@ IndexDescriptor::Comparison IndexDescriptor::compareIndexOptions( // The partialFilterExpression is only part of the index signature if FCV has been set to 4.6. // TODO SERVER-47766: remove these FCV checks after we branch for 4.7. - auto isFCV46 = serverGlobalParams.featureCompatibility.isVersion( - ServerGlobalParams::FeatureCompatibility::Version::kVersion451); + auto isFCV46 = serverGlobalParams.featureCompatibility.getVersion() == + ServerGlobalParams::FeatureCompatibility::Version::kVersion451; // If we have a partial filter expression and the other index doesn't, or vice-versa, then the // two indexes are not equivalent. We therefore return Comparison::kDifferent immediately. diff --git a/src/mongo/db/mongod_main.cpp b/src/mongo/db/mongod_main.cpp index 368c7605491..9c3de78b56d 100644 --- a/src/mongo/db/mongod_main.cpp +++ b/src/mongo/db/mongod_main.cpp @@ -109,7 +109,6 @@ #include "mongo/db/pipeline/process_interface/replica_set_node_process_interface.h" #include "mongo/db/query/internal_plans.h" #include "mongo/db/read_write_concern_defaults_cache_lookup_mongod.h" -#include "mongo/db/repair_database_and_check_version.h" #include "mongo/db/repl/drop_pending_collection_reaper.h" #include "mongo/db/repl/oplog.h" #include "mongo/db/repl/repl_settings.h" @@ -140,6 +139,7 @@ #include "mongo/db/service_context.h" #include "mongo/db/service_entry_point_mongod.h" #include "mongo/db/session_killer.h" +#include "mongo/db/startup_recovery.h" #include "mongo/db/startup_warnings_mongod.h" #include "mongo/db/stats/counters.h" #include "mongo/db/storage/backup_cursor_hooks.h" @@ -445,17 +445,8 @@ ExitCode _initAndListen(ServiceContext* serviceContext, int listenPort) { auto startupOpCtx = serviceContext->makeOperationContext(&cc()); - bool canCallFCVSetIfCleanStartup = - !storageGlobalParams.readOnly && (storageGlobalParams.engine != "devnull"); - if (canCallFCVSetIfCleanStartup && !replSettings.usingReplSets()) { - Lock::GlobalWrite lk(startupOpCtx.get()); - FeatureCompatibilityVersion::setIfCleanStartup(startupOpCtx.get(), - repl::StorageInterface::get(serviceContext)); - } - - bool nonLocalDatabases; try { - nonLocalDatabases = repairDatabasesAndCheckVersion(startupOpCtx.get()); + startup_recovery::repairAndRecoverDatabases(startupOpCtx.get()); } catch (const ExceptionFor<ErrorCodes::MustDowngrade>& error) { LOGV2_FATAL_OPTIONS( 20573, @@ -466,6 +457,10 @@ ExitCode _initAndListen(ServiceContext* serviceContext, int listenPort) { exitCleanly(EXIT_NEED_DOWNGRADE); } + // Ensure FCV document exists and is initialized in-memory. Fatally asserts if there is an + // error. + FeatureCompatibilityVersion::fassertInitializedAfterStartup(startupOpCtx.get()); + // This flag is used during storage engine initialization to perform behavior that is specific // to recovering from an unclean shutdown. It is also used to determine whether temporary files // should be removed. The last of these uses is done by repairDatabasesAndCheckVersion() above, @@ -474,23 +469,6 @@ ExitCode _initAndListen(ServiceContext* serviceContext, int listenPort) { // shutdown. startingAfterUncleanShutdown(serviceContext) = false; - auto replProcess = repl::ReplicationProcess::get(serviceContext); - invariant(replProcess); - const bool initialSyncFlag = - replProcess->getConsistencyMarkers()->getInitialSyncFlag(startupOpCtx.get()); - - // Assert that the in-memory featureCompatibilityVersion parameter has been explicitly set. If - // we are part of a replica set and are started up with no data files, we do not set the - // featureCompatibilityVersion until a primary is chosen. For this case, we expect the in-memory - // featureCompatibilityVersion parameter to still be uninitialized until after startup. If the - // initial sync flag is set and we are part of a replica set, we expect the version to be - // initialized as part of initial sync after startup. - const bool initializeFCVAtInitialSync = replSettings.usingReplSets() && initialSyncFlag; - if (canCallFCVSetIfCleanStartup && (!replSettings.usingReplSets() || nonLocalDatabases) && - !initializeFCVAtInitialSync) { - invariant(serverGlobalParams.featureCompatibility.isVersionInitialized()); - } - if (gFlowControlEnabled.load()) { LOGV2(20536, "Flow Control is enabled on this deployment"); } diff --git a/src/mongo/db/repair_database.cpp b/src/mongo/db/repair.cpp index 171f1efd2ad..89f7e28e244 100644 --- a/src/mongo/db/repair_database.cpp +++ b/src/mongo/db/repair.cpp @@ -1,5 +1,5 @@ /** - * Copyright (C) 2018-present MongoDB, Inc. + * Copyright (C) 2020-present MongoDB, Inc. * * This program is free software: you can redistribute it and/or modify * it under the terms of the Server Side Public License, version 1, @@ -34,7 +34,7 @@ #include <algorithm> #include <fmt/format.h> -#include "mongo/db/repair_database.h" +#include "mongo/db/repair.h" #include "mongo/base/status.h" #include "mongo/base/string_data.h" @@ -118,87 +118,16 @@ Status repairCollections(OperationContext* opCtx, auto colls = CollectionCatalog::get(opCtx).getAllCollectionNamesFromDb(opCtx, dbName); for (const auto& nss : colls) { - opCtx->checkForInterrupt(); - - LOGV2(21027, "Repairing collection", "namespace"_attr = nss); - - auto collection = CollectionCatalog::get(opCtx).lookupCollectionByNamespace(opCtx, nss); - Status status = engine->repairRecordStore(opCtx, collection->getCatalogId(), nss); - - // Need to lookup from catalog again because the old collection object was invalidated by - // repairRecordStore. - collection = CollectionCatalog::get(opCtx).lookupCollectionByNamespace(opCtx, nss); - - // If data was modified during repairRecordStore, we know to rebuild indexes without needing - // to run an expensive collection validation. - if (status.code() == ErrorCodes::DataModifiedByRepair) { - invariant(StorageRepairObserver::get(opCtx->getServiceContext())->isDataInvalidated(), - "Collection '{}' ({})"_format(collection->ns().toString(), - collection->uuid().toString())); - - // If we are a replica set member in standalone mode and we have unfinished indexes, - // drop them before rebuilding any completed indexes. Since we have already made - // invalidating modifications to our data, it is safe to just drop the indexes entirely - // to avoid the risk of the index rebuild failing. - if (getReplSetMemberInStandaloneMode(opCtx->getServiceContext())) { - if (auto status = dropUnfinishedIndexes(opCtx, collection); !status.isOK()) { - return status; - } - } - - Status status = rebuildIndexesForNamespace(opCtx, nss, engine); - if (!status.isOK()) { - return status; - } - continue; - } else if (!status.isOK()) { - return status; - } - - // Run collection validation to avoid unecessarily rebuilding indexes on valid collections - // with consistent indexes. Initialize the collection prior to validation. - collection->init(opCtx); - - ValidateResults validateResults; - BSONObjBuilder output; - - // Exclude full record store validation because we have already validated the underlying - // record store in the call to repairRecordStore above. - status = CollectionValidation::validate( - opCtx, - nss, - CollectionValidation::ValidateMode::kForegroundFullIndexOnly, - CollectionValidation::RepairMode::kRepair, - &validateResults, - &output); + auto status = repair::repairCollection(opCtx, engine, nss); if (!status.isOK()) { return status; } - - LOGV2(21028, "Collection validation", "results"_attr = output.done()); - - if (validateResults.repaired) { - if (validateResults.valid) { - LOGV2(4934000, "Validate successfully repaired all data", "collection"_attr = nss); - } else { - LOGV2(4934001, "Validate was unable to repair all data", "collection"_attr = nss); - } - } else { - LOGV2(4934002, "Validate did not make any repairs", "collection"_attr = nss); - } - - // If not valid, whether repair ran or not, indexes will need to be rebuilt. - if (!validateResults.valid) { - status = rebuildIndexesForNamespace(opCtx, nss, engine); - if (!status.isOK()) { - return status; - } - } } return Status::OK(); } } // namespace +namespace repair { Status repairDatabase(OperationContext* opCtx, StorageEngine* engine, const std::string& dbName) { DisableDocumentValidation validationDisabler(opCtx); @@ -258,4 +187,80 @@ Status repairDatabase(OperationContext* opCtx, StorageEngine* engine, const std: return status; } +Status repairCollection(OperationContext* opCtx, + StorageEngine* engine, + const NamespaceString& nss) { + opCtx->checkForInterrupt(); + + LOGV2(21027, "Repairing collection", "namespace"_attr = nss); + + auto collection = CollectionCatalog::get(opCtx).lookupCollectionByNamespace(opCtx, nss); + Status status = engine->repairRecordStore(opCtx, collection->getCatalogId(), nss); + + // Need to lookup from catalog again because the old collection object was invalidated by + // repairRecordStore. + collection = CollectionCatalog::get(opCtx).lookupCollectionByNamespace(opCtx, nss); + + // If data was modified during repairRecordStore, we know to rebuild indexes without needing + // to run an expensive collection validation. + if (status.code() == ErrorCodes::DataModifiedByRepair) { + invariant(StorageRepairObserver::get(opCtx->getServiceContext())->isDataInvalidated(), + "Collection '{}' ({})"_format(collection->ns().toString(), + collection->uuid().toString())); + + // If we are a replica set member in standalone mode and we have unfinished indexes, + // drop them before rebuilding any completed indexes. Since we have already made + // invalidating modifications to our data, it is safe to just drop the indexes entirely + // to avoid the risk of the index rebuild failing. + if (getReplSetMemberInStandaloneMode(opCtx->getServiceContext())) { + if (auto status = dropUnfinishedIndexes(opCtx, collection); !status.isOK()) { + return status; + } + } + + return rebuildIndexesForNamespace(opCtx, nss, engine); + } else if (!status.isOK()) { + return status; + } + + // Run collection validation to avoid unecessarily rebuilding indexes on valid collections + // with consistent indexes. Initialize the collection prior to validation. + collection->init(opCtx); + + ValidateResults validateResults; + BSONObjBuilder output; + + // Exclude full record store validation because we have already validated the underlying + // record store in the call to repairRecordStore above. + status = + CollectionValidation::validate(opCtx, + nss, + CollectionValidation::ValidateMode::kForegroundFullIndexOnly, + CollectionValidation::RepairMode::kRepair, + &validateResults, + &output); + if (!status.isOK()) { + return status; + } + + LOGV2(21028, "Collection validation", "results"_attr = output.done()); + + if (validateResults.repaired) { + if (validateResults.valid) { + LOGV2(4934000, "Validate successfully repaired all data", "collection"_attr = nss); + } else { + LOGV2(4934001, "Validate was unable to repair all data", "collection"_attr = nss); + } + } else { + LOGV2(4934002, "Validate did not make any repairs", "collection"_attr = nss); + } + + // If not valid, whether repair ran or not, indexes will need to be rebuilt. + if (!validateResults.valid) { + return rebuildIndexesForNamespace(opCtx, nss, engine); + } + return Status::OK(); +} +} // namespace repair + } // namespace mongo diff --git a/src/mongo/db/repair_database.h b/src/mongo/db/repair.h index db4e110bcf1..719b11c57a3 100644 --- a/src/mongo/db/repair_database.h +++ b/src/mongo/db/repair.h @@ -1,5 +1,5 @@ /** - * Copyright (C) 2018-present MongoDB, Inc. + * Copyright (C) 2020-present MongoDB, Inc. * * This program is free software: you can redistribute it and/or modify * it under the terms of the Server Side Public License, version 1, @@ -36,20 +36,28 @@ #include "mongo/db/record_id.h" namespace mongo { -class Collection; class StorageEngine; class NamespaceString; class OperationContext; class Status; -class StringData; + +namespace repair { /** - * Repairs a database using a storage engine-specific, best-effort process. - * Some data may be lost or modified in the process but the output will - * be structurally valid on successful return. + * Repairs a database using a storage engine-specific, best-effort process. Some data may be lost or + * modified in the process but the result will be readable collections consistent with their indexes + * on a successful return. * * It is expected that the local database will be repaired first when running in repair mode. */ Status repairDatabase(OperationContext* opCtx, StorageEngine* engine, const std::string& dbName); +/** + * Repairs a collection using a storage engine-specific, best-effort process. + * Some data may be lost or modified in the process but the result will be a readable collection + * consistent with its indexes on a successful return. + */ +Status repairCollection(OperationContext* opCtx, StorageEngine* engine, const NamespaceString& nss); + +} // namespace repair } // namespace mongo diff --git a/src/mongo/db/repair_database_and_check_version.cpp b/src/mongo/db/repair_database_and_check_version.cpp deleted file mode 100644 index 3624ce8b6f3..00000000000 --- a/src/mongo/db/repair_database_and_check_version.cpp +++ /dev/null @@ -1,653 +0,0 @@ -/** - * Copyright (C) 2018-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kStorage - -#include "mongo/platform/basic.h" - -#include "repair_database_and_check_version.h" - -#include <functional> - -#include "mongo/db/catalog/collection.h" -#include "mongo/db/catalog/create_collection.h" -#include "mongo/db/catalog/database.h" -#include "mongo/db/catalog/database_holder.h" -#include "mongo/db/catalog/multi_index_block.h" -#include "mongo/db/commands/feature_compatibility_version.h" -#include "mongo/db/commands/feature_compatibility_version_document_gen.h" -#include "mongo/db/commands/feature_compatibility_version_documentation.h" -#include "mongo/db/commands/feature_compatibility_version_parser.h" -#include "mongo/db/concurrency/write_conflict_exception.h" -#include "mongo/db/db_raii.h" -#include "mongo/db/dbhelpers.h" -#include "mongo/db/index_builds_coordinator.h" -#include "mongo/db/namespace_string.h" -#include "mongo/db/operation_context.h" -#include "mongo/db/rebuild_indexes.h" -#include "mongo/db/repair_database.h" -#include "mongo/db/repl/replication_coordinator.h" -#include "mongo/db/repl/replication_process.h" -#include "mongo/db/repl_set_member_in_standalone_mode.h" -#include "mongo/db/server_options.h" -#include "mongo/db/storage/durable_catalog.h" -#include "mongo/db/storage/storage_repair_observer.h" -#include "mongo/logv2/log.h" -#include "mongo/util/exit.h" -#include "mongo/util/fail_point.h" -#include "mongo/util/quick_exit.h" - -#if !defined(_WIN32) -#include <sys/file.h> -#endif - -namespace mongo { - -// Exit after repair has started, but before data is repaired. -MONGO_FAIL_POINT_DEFINE(exitBeforeDataRepair); -// Exit after repairing data, but before the replica set configuration is invalidated. -MONGO_FAIL_POINT_DEFINE(exitBeforeRepairInvalidatesConfig); - -namespace { - -const std::string mustDowngradeErrorMsg = str::stream() - << "UPGRADE PROBLEM: The data files need to be fully upgraded to version 4.4 before attempting " - "an upgrade to 4.5.1; see " - << feature_compatibility_version_documentation::kUpgradeLink << " for more details."; - -Status restoreMissingFeatureCompatibilityVersionDocument(OperationContext* opCtx, - const std::vector<std::string>& dbNames) { - NamespaceString fcvNss(NamespaceString::kServerConfigurationNamespace); - - // If the admin database, which contains the server configuration collection with the - // featureCompatibilityVersion document, does not exist, create it. - auto databaseHolder = DatabaseHolder::get(opCtx); - auto db = databaseHolder->getDb(opCtx, fcvNss.db()); - if (!db) { - LOGV2(20998, "Re-creating admin database that was dropped."); - } - db = databaseHolder->openDb(opCtx, fcvNss.db()); - invariant(db); - - // If the server configuration collection, which contains the FCV document, does not exist, then - // create it. - if (!CollectionCatalog::get(opCtx).lookupCollectionByNamespace( - opCtx, NamespaceString::kServerConfigurationNamespace)) { - LOGV2(20999, - "Re-creating the server configuration collection (admin.system.version) that was " - "dropped."); - uassertStatusOK( - createCollection(opCtx, fcvNss.db().toString(), BSON("create" << fcvNss.coll()))); - } - - Collection* fcvColl = CollectionCatalog::get(opCtx).lookupCollectionByNamespace( - opCtx, NamespaceString::kServerConfigurationNamespace); - invariant(fcvColl); - - // Restore the featureCompatibilityVersion document if it is missing. - BSONObj featureCompatibilityVersion; - if (!Helpers::findOne(opCtx, - fcvColl, - BSON("_id" << FeatureCompatibilityVersionParser::kParameterName), - featureCompatibilityVersion)) { - LOGV2(4926905, - "Re-creating featureCompatibilityVersion document that was deleted. Creating new " - "document with last LTS version.", - "version"_attr = FeatureCompatibilityVersionParser::kLastLTS); - - FeatureCompatibilityVersionDocument fcvDoc; - fcvDoc.setVersion(ServerGlobalParams::FeatureCompatibility::kLastLTS); - - writeConflictRetry(opCtx, "insertFCVDocument", fcvNss.ns(), [&] { - WriteUnitOfWork wunit(opCtx); - OpDebug* const nullOpDebug = nullptr; - uassertStatusOK(fcvColl->insertDocument( - opCtx, InsertStatement(fcvDoc.toBSON()), nullOpDebug, false)); - wunit.commit(); - }); - } - - invariant(Helpers::findOne(opCtx, - fcvColl, - BSON("_id" << FeatureCompatibilityVersionParser::kParameterName), - featureCompatibilityVersion)); - - return Status::OK(); -} - -/** - * Returns true if the collection associated with the given CollectionCatalogEntry has an index on - * the _id field - */ -bool checkIdIndexExists(OperationContext* opCtx, RecordId catalogId) { - auto durableCatalog = DurableCatalog::get(opCtx); - auto indexCount = durableCatalog->getTotalIndexCount(opCtx, catalogId); - auto indexNames = std::vector<std::string>(indexCount); - durableCatalog->getAllIndexes(opCtx, catalogId, &indexNames); - - for (auto name : indexNames) { - if (name == "_id_") { - return true; - } - } - return false; -} - -Status buildMissingIdIndex(OperationContext* opCtx, Collection* collection) { - MultiIndexBlock indexer; - auto abortOnExit = makeGuard( - [&] { indexer.abortIndexBuild(opCtx, collection, MultiIndexBlock::kNoopOnCleanUpFn); }); - - const auto indexCatalog = collection->getIndexCatalog(); - const auto idIndexSpec = indexCatalog->getDefaultIdIndexSpec(); - - auto swSpecs = indexer.init(opCtx, collection, idIndexSpec, MultiIndexBlock::kNoopOnInitFn); - if (!swSpecs.isOK()) { - return swSpecs.getStatus(); - } - - auto status = indexer.insertAllDocumentsInCollection(opCtx, collection); - if (!status.isOK()) { - return status; - } - - status = indexer.checkConstraints(opCtx); - if (!status.isOK()) { - return status; - } - - WriteUnitOfWork wuow(opCtx); - status = indexer.commit( - opCtx, collection, MultiIndexBlock::kNoopOnCreateEachFn, MultiIndexBlock::kNoopOnCommitFn); - wuow.commit(); - abortOnExit.dismiss(); - return status; -} - - -/** - * Checks that all collections in the given list of 'dbNames' have valid properties for this version - * of MongoDB. - * - * This validates that all collections have UUIDs and an _id index. If a collection is missing an - * _id index, this function will build it. - * - * Returns a MustDowngrade error if any collections are missing UUIDs. - * Returns a MustDowngrade error if any index builds on the required _id field fail. - */ -Status ensureCollectionProperties(OperationContext* opCtx, - const std::vector<std::string>& dbNames) { - auto databaseHolder = DatabaseHolder::get(opCtx); - auto downgradeError = Status{ErrorCodes::MustDowngrade, mustDowngradeErrorMsg}; - invariant(opCtx->lockState()->isW()); - - for (const auto& dbName : dbNames) { - auto db = databaseHolder->openDb(opCtx, dbName); - invariant(db); - - for (auto collIt = db->begin(opCtx); collIt != db->end(opCtx); ++collIt) { - auto coll = *collIt; - if (!coll) { - break; - } - - // All user-created replicated collections created since MongoDB 4.0 have _id indexes. - auto requiresIndex = coll->requiresIdIndex() && coll->ns().isReplicated(); - auto collOptions = - DurableCatalog::get(opCtx)->getCollectionOptions(opCtx, coll->getCatalogId()); - auto hasAutoIndexIdField = collOptions.autoIndexId == CollectionOptions::YES; - - // Even if the autoIndexId field is not YES, the collection may still have an _id index - // that was created manually by the user. Check the list of indexes to confirm index - // does not exist before attempting to build it or returning an error. - if (requiresIndex && !hasAutoIndexIdField && - !checkIdIndexExists(opCtx, coll->getCatalogId())) { - LOGV2(21001, - "collection {coll_ns} is missing an _id index; building it now", - "Collection is missing an _id index; building it now", - logAttrs(*coll)); - auto status = buildMissingIdIndex(opCtx, coll); - if (!status.isOK()) { - LOGV2_ERROR(21021, - "could not build an _id index on collection {coll_ns}: {error}", - "Could not build an _id index on collection", - logAttrs(*coll), - "error"_attr = status); - return downgradeError; - } - } - } - } - return Status::OK(); -} - -const NamespaceString startupLogCollectionName("local.startup_log"); - -/** - * Returns 'true' if this server has a configuration document in local.system.replset. - */ -bool hasReplSetConfigDoc(OperationContext* opCtx) { - Lock::GlobalWrite lk(opCtx); - BSONObj config; - return Helpers::getSingleton( - opCtx, NamespaceString::kSystemReplSetNamespace.ns().c_str(), config); -} - -/** - * Check that the oplog is capped, and abort the process if it is not. - * Caller must lock DB before calling this function. - */ -void checkForCappedOplog(OperationContext* opCtx, Database* db) { - const NamespaceString oplogNss(NamespaceString::kRsOplogNamespace); - invariant(opCtx->lockState()->isDbLockedForMode(oplogNss.db(), MODE_IS)); - Collection* oplogCollection = - CollectionCatalog::get(opCtx).lookupCollectionByNamespace(opCtx, oplogNss); - if (oplogCollection && !oplogCollection->isCapped()) { - LOGV2_FATAL_NOTRACE( - 40115, - "The oplog collection {oplogNamespace} is not capped; a capped oplog is a " - "requirement for replication to function.", - "The oplog collection is not capped; a capped oplog is a " - "requirement for replication to function.", - "oplogNamespace"_attr = oplogNss); - } -} - -void rebuildIndexes(OperationContext* opCtx, StorageEngine* storageEngine) { - auto reconcileResult = fassert(40593, storageEngine->reconcileCatalogAndIdents(opCtx)); - - // Determine which indexes need to be rebuilt. rebuildIndexesOnCollection() requires that all - // indexes on that collection are done at once, so we use a map to group them together. - StringMap<IndexNameObjs> nsToIndexNameObjMap; - for (auto&& idxIdentifier : reconcileResult.indexesToRebuild) { - NamespaceString collNss = idxIdentifier.nss; - const std::string& indexName = idxIdentifier.indexName; - auto swIndexSpecs = - getIndexNameObjs(opCtx, idxIdentifier.catalogId, [&indexName](const std::string& name) { - return name == indexName; - }); - if (!swIndexSpecs.isOK() || swIndexSpecs.getValue().first.empty()) { - fassert(40590, - {ErrorCodes::InternalError, - str::stream() << "failed to get index spec for index " << indexName - << " in collection " << collNss.toString()}); - } - - auto& indexesToRebuild = swIndexSpecs.getValue(); - invariant(indexesToRebuild.first.size() == 1 && indexesToRebuild.second.size() == 1, - str::stream() << "Num Index Names: " << indexesToRebuild.first.size() - << " Num Index Objects: " << indexesToRebuild.second.size()); - auto& ino = nsToIndexNameObjMap[collNss.ns()]; - ino.first.emplace_back(std::move(indexesToRebuild.first.back())); - ino.second.emplace_back(std::move(indexesToRebuild.second.back())); - } - - for (const auto& entry : nsToIndexNameObjMap) { - NamespaceString collNss(entry.first); - - auto collection = CollectionCatalog::get(opCtx).lookupCollectionByNamespace(opCtx, collNss); - for (const auto& indexName : entry.second.first) { - LOGV2(21004, - "Rebuilding index. Collection: {collNss} Index: {indexName}", - "Rebuilding index", - "namespace"_attr = collNss, - "index"_attr = indexName); - } - - std::vector<BSONObj> indexSpecs = entry.second.second; - fassert(40592, rebuildIndexesOnCollection(opCtx, collection, indexSpecs, RepairData::kNo)); - } - - - // Two-phase index builds depend on a replicated 'commitIndexBuild' oplog entry to commit. - // Therefore, when a replica set member is started in standalone mode, we cannot restart the - // index build. - if (getReplSetMemberInStandaloneMode(opCtx->getServiceContext())) { - LOGV2(21005, "Not restarting unfinished index builds because we are in standalone mode"); - return; - } - - // Once all unfinished indexes have been rebuilt, restart any unfinished index builds. This will - // not build any indexes to completion, but rather start the background thread to build the - // index, and wait for a replicated commit or abort oplog entry. - IndexBuildsCoordinator::get(opCtx)->restartIndexBuildsForRecovery( - opCtx, reconcileResult.indexBuildsToRestart); -} - -/** - * Sets the appropriate flag on the service context decorable 'replSetMemberInStandaloneMode' to - * 'true' if this is a replica set node running in standalone mode, otherwise 'false'. - */ -void setReplSetMemberInStandaloneMode(OperationContext* opCtx) { - const repl::ReplSettings& replSettings = - repl::ReplicationCoordinator::get(opCtx)->getSettings(); - - if (replSettings.usingReplSets()) { - // Not in standalone mode. - setReplSetMemberInStandaloneMode(opCtx->getServiceContext(), false); - return; - } - - invariant(opCtx->lockState()->isW()); - Collection* collection = CollectionCatalog::get(opCtx).lookupCollectionByNamespace( - opCtx, NamespaceString::kSystemReplSetNamespace); - if (collection && collection->numRecords(opCtx) > 0) { - setReplSetMemberInStandaloneMode(opCtx->getServiceContext(), true); - return; - } - - setReplSetMemberInStandaloneMode(opCtx->getServiceContext(), false); -} - -} // namespace - -/** - * Return whether there are non-local databases. If there was an error becauses the wrong mongod - * version was used for these datafiles, a DBException with status ErrorCodes::MustDowngrade is - * thrown. - */ -bool repairDatabasesAndCheckVersion(OperationContext* opCtx) { - auto const storageEngine = opCtx->getServiceContext()->getStorageEngine(); - Lock::GlobalWrite lk(opCtx); - - std::vector<std::string> dbNames = storageEngine->listDatabases(); - - // Rebuilding indexes must be done before a database can be opened, except when using repair, - // which rebuilds all indexes when it is done. - if (!storageGlobalParams.readOnly && !storageGlobalParams.repair) { - // Determine whether this is a replica set node running in standalone mode. If we're in - // repair mode, we cannot set the flag yet as it needs to open a database and look through a - // collection. Rebuild the necessary indexes after setting the flag. - setReplSetMemberInStandaloneMode(opCtx); - rebuildIndexes(opCtx, storageEngine); - } - - bool ensuredCollectionProperties = false; - - // Repair all databases first, so that we do not try to open them if they are in bad shape - auto databaseHolder = DatabaseHolder::get(opCtx); - if (storageGlobalParams.repair) { - invariant(!storageGlobalParams.readOnly); - - if (MONGO_unlikely(exitBeforeDataRepair.shouldFail())) { - LOGV2(21006, "Exiting because 'exitBeforeDataRepair' fail point was set."); - quickExit(EXIT_ABRUPT); - } - - // Ensure that the local database is repaired first, if it exists, so that we can open it - // before any other database to be able to determine if this is a replica set node running - // in standalone mode before rebuilding any indexes. - auto dbNamesIt = std::find(dbNames.begin(), dbNames.end(), NamespaceString::kLocalDb); - if (dbNamesIt != dbNames.end()) { - std::swap(dbNames.front(), *dbNamesIt); - invariant(dbNames.front() == NamespaceString::kLocalDb); - } - - for (const auto& dbName : dbNames) { - LOGV2_DEBUG(21007, 1, " Repairing database: {dbName}", "dbName"_attr = dbName); - fassertNoTrace(18506, repairDatabase(opCtx, storageEngine, dbName)); - - if (dbName == NamespaceString::kLocalDb) { - setReplSetMemberInStandaloneMode(opCtx); - } - } - - // All collections must have UUIDs before restoring the FCV document to a version that - // requires UUIDs. - uassertStatusOK(ensureCollectionProperties(opCtx, dbNames)); - ensuredCollectionProperties = true; - - // Attempt to restore the featureCompatibilityVersion document if it is missing. - NamespaceString fcvNSS(NamespaceString::kServerConfigurationNamespace); - - auto db = databaseHolder->getDb(opCtx, fcvNSS.db()); - Collection* versionColl; - BSONObj featureCompatibilityVersion; - if (!db || - !(versionColl = - CollectionCatalog::get(opCtx).lookupCollectionByNamespace(opCtx, fcvNSS)) || - !Helpers::findOne(opCtx, - versionColl, - BSON("_id" << FeatureCompatibilityVersionParser::kParameterName), - featureCompatibilityVersion)) { - uassertStatusOK(restoreMissingFeatureCompatibilityVersionDocument(opCtx, dbNames)); - } - } - - if (!ensuredCollectionProperties) { - uassertStatusOK(ensureCollectionProperties(opCtx, dbNames)); - } - - if (!storageGlobalParams.readOnly) { - // We open the "local" database before calling hasReplSetConfigDoc() to ensure the in-memory - // catalog entries for the 'kSystemReplSetNamespace' collection have been populated if the - // collection exists. If the "local" database didn't exist at this point yet, then it will - // be created. If the mongod is running in a read-only mode, then it is fine to not open the - // "local" database and populate the catalog entries because we won't attempt to drop the - // temporary collections anyway. - Lock::DBLock dbLock(opCtx, NamespaceString::kSystemReplSetNamespace.db(), MODE_X); - databaseHolder->openDb(opCtx, NamespaceString::kSystemReplSetNamespace.db()); - } - - if (storageGlobalParams.repair) { - if (MONGO_unlikely(exitBeforeRepairInvalidatesConfig.shouldFail())) { - LOGV2(21008, "Exiting because 'exitBeforeRepairInvalidatesConfig' fail point was set."); - quickExit(EXIT_ABRUPT); - } - // This must be done after opening the "local" database as it modifies the replica set - // config. - auto repairObserver = StorageRepairObserver::get(opCtx->getServiceContext()); - repairObserver->onRepairDone(opCtx); - if (repairObserver->getModifications().size() > 0) { - const auto& mods = repairObserver->getModifications(); - for (const auto& mod : mods) { - LOGV2_WARNING( - 21019, "repairModification", "description"_attr = mod.getDescription()); - } - } - if (repairObserver->isDataInvalidated()) { - if (hasReplSetConfigDoc(opCtx)) { - LOGV2_WARNING( - 21020, - "WARNING: Repair may have modified replicated data. This node will no " - "longer be able to join a replica set without a full re-sync"); - } - } - - // There were modifications, but only benign ones. - if (repairObserver->getModifications().size() > 0 && !repairObserver->isDataInvalidated()) { - LOGV2(21009, - "Repair has made modifications to unreplicated data. The data is healthy and " - "the node is eligible to be returned to the replica set."); - } - } - - const repl::ReplSettings& replSettings = - repl::ReplicationCoordinator::get(opCtx)->getSettings(); - - // On replica set members we only clear temp collections on DBs other than "local" during - // promotion to primary. On pure slaves, they are only cleared when the oplog tells them - // to. The local DB is special because it is not replicated. See SERVER-10927 for more - // details. - const bool shouldClearNonLocalTmpCollections = - !(hasReplSetConfigDoc(opCtx) || replSettings.usingReplSets()); - - // To check whether a featureCompatibilityVersion document exists. - bool fcvDocumentExists = false; - - // To check whether we have databases other than local. - bool nonLocalDatabases = false; - - // Refresh list of database names to include newly-created admin, if it exists. - dbNames = storageEngine->listDatabases(); - - // We want to recover the admin database first so we can load the FCV early since - // some collection validation may depend on the FCV being set. - if (auto it = std::find(dbNames.begin(), dbNames.end(), "admin"); it != dbNames.end()) { - std::swap(*it, dbNames.front()); - } - - for (const auto& dbName : dbNames) { - if (dbName != "local") { - nonLocalDatabases = true; - } - LOGV2_DEBUG(21010, 1, " Recovering database: {dbName}", "dbName"_attr = dbName); - - auto db = databaseHolder->openDb(opCtx, dbName); - invariant(db); - - // First thing after opening the database is to check for file compatibility, - // otherwise we might crash if this is a deprecated format. - auto status = storageEngine->currentFilesCompatible(opCtx); - if (!status.isOK()) { - if (status.code() == ErrorCodes::CanRepairToDowngrade) { - // Convert CanRepairToDowngrade statuses to MustUpgrade statuses to avoid logging a - // potentially confusing and inaccurate message. - // - // TODO SERVER-24097: Log a message informing the user that they can start the - // current version of mongod with --repair and then proceed with normal startup. - status = {ErrorCodes::MustUpgrade, status.reason()}; - } - LOGV2_FATAL_CONTINUE( - 21023, - "Unable to start mongod due to an incompatibility with the data files and" - " this version of mongod: {error}. Please consult our documentation when trying " - "to downgrade to a previous major release", - "Unable to start mongod due to an incompatibility with the data files and" - " this version of mongod. Please consult our documentation when trying " - "to downgrade to a previous major release", - "error"_attr = redact(status)); - quickExit(EXIT_NEED_UPGRADE); - MONGO_UNREACHABLE; - } - - - // If the server configuration collection already contains a valid - // featureCompatibilityVersion document, cache it in-memory as a server parameter. - if (dbName == "admin") { - if (Collection* versionColl = CollectionCatalog::get(opCtx).lookupCollectionByNamespace( - opCtx, NamespaceString::kServerConfigurationNamespace)) { - BSONObj featureCompatibilityVersion; - if (Helpers::findOne( - opCtx, - versionColl, - BSON("_id" << FeatureCompatibilityVersionParser::kParameterName), - featureCompatibilityVersion)) { - auto swVersion = - FeatureCompatibilityVersionParser::parse(featureCompatibilityVersion); - // Note this error path captures all cases of an FCV document existing, - // but with any value other than "4.4" or "4.6". This includes unexpected - // cases with no path forward such as the FCV value not being a string. - uassert(ErrorCodes::MustDowngrade, - str::stream() - << "UPGRADE PROBLEM: Found an invalid " - "featureCompatibilityVersion document (ERROR: " - << swVersion.getStatus() - << "). If the current featureCompatibilityVersion is below " - "4.4, see the documentation on upgrading at " - << feature_compatibility_version_documentation::kUpgradeLink << ".", - swVersion.isOK()); - - fcvDocumentExists = true; - auto version = swVersion.getValue(); - serverGlobalParams.featureCompatibility.setVersion(version); - FeatureCompatibilityVersion::updateMinWireVersion(); - - // On startup, if the version is in an upgrading or downgrading state, print a - // warning. - if (version == - ServerGlobalParams::FeatureCompatibility::Version::kUpgradingFrom44To451) { - LOGV2_WARNING_OPTIONS( - 21011, - {logv2::LogTag::kStartupWarnings}, - "A featureCompatibilityVersion upgrade did not complete. The current " - "featureCompatibilityVersion is " - "{currentfeatureCompatibilityVersion}. To fix this, use the " - "setFeatureCompatibilityVersion command to resume upgrade to 4.5.1.", - "A featureCompatibilityVersion upgrade did not complete. To fix this, " - "use the " - "setFeatureCompatibilityVersion command to resume upgrade to 4.5.1", - "currentfeatureCompatibilityVersion"_attr = - FeatureCompatibilityVersionParser::toString(version)); - } else if (version == - ServerGlobalParams::FeatureCompatibility::Version:: - kDowngradingFrom451To44) { - LOGV2_WARNING_OPTIONS( - 21014, - {logv2::LogTag::kStartupWarnings}, - "A featureCompatibilityVersion downgrade did not complete. The current " - "featureCompatibilityVersion is " - "{currentfeatureCompatibilityVersion}. To fix this, use the " - "setFeatureCompatibilityVersion command to resume downgrade to 4.4.", - "A featureCompatibilityVersion downgrade did not complete. To fix " - "this, use the setFeatureCompatibilityVersion command to resume " - "downgrade to 4.4", - "currentfeatureCompatibilityVersion"_attr = - FeatureCompatibilityVersionParser::toString(version)); - } - } - } - } - - if (replSettings.usingReplSets()) { - // We only care about _id indexes and drop-pending collections if we are in a replset. - db->checkForIdIndexesAndDropPendingCollections(opCtx); - // Ensure oplog is capped (mongodb does not guarantee order of inserts on noncapped - // collections) - if (db->name() == "local") { - checkForCappedOplog(opCtx, db); - } - } - - if (!storageGlobalParams.readOnly && - (shouldClearNonLocalTmpCollections || dbName == "local")) { - db->clearTmpCollections(opCtx); - } - } - - auto replProcess = repl::ReplicationProcess::get(opCtx); - auto needInitialSync = false; - if (auto initialSyncFlag = false; replProcess) { - initialSyncFlag = replProcess->getConsistencyMarkers()->getInitialSyncFlag(opCtx); - // The node did not complete the last initial sync. We should attempt initial sync again. - needInitialSync = initialSyncFlag && replSettings.usingReplSets(); - } - // Fail to start up if there is no featureCompatibilityVersion document and there are non-local - // databases present and we do not need to start up via initial sync. - if (!fcvDocumentExists && nonLocalDatabases && !needInitialSync) { - LOGV2_FATAL_NOTRACE(40652, - "Unable to start up mongod due to missing featureCompatibilityVersion " - "document. Please run with --repair to restore the document."); - } - - LOGV2_DEBUG(21017, 1, "done repairDatabases"); - return nonLocalDatabases; -} - -} // namespace mongo diff --git a/src/mongo/db/startup_recovery.cpp b/src/mongo/db/startup_recovery.cpp new file mode 100644 index 00000000000..cbc10c7be7a --- /dev/null +++ b/src/mongo/db/startup_recovery.cpp @@ -0,0 +1,588 @@ +/** + * Copyright (C) 2020-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kStorage + +#include "mongo/platform/basic.h" + +#include "startup_recovery.h" + +#include "mongo/db/catalog/collection.h" +#include "mongo/db/catalog/create_collection.h" +#include "mongo/db/catalog/database.h" +#include "mongo/db/catalog/database_holder.h" +#include "mongo/db/catalog/multi_index_block.h" +#include "mongo/db/commands/feature_compatibility_version.h" +#include "mongo/db/commands/feature_compatibility_version_document_gen.h" +#include "mongo/db/commands/feature_compatibility_version_documentation.h" +#include "mongo/db/commands/feature_compatibility_version_parser.h" +#include "mongo/db/concurrency/write_conflict_exception.h" +#include "mongo/db/db_raii.h" +#include "mongo/db/dbhelpers.h" +#include "mongo/db/index_builds_coordinator.h" +#include "mongo/db/namespace_string.h" +#include "mongo/db/operation_context.h" +#include "mongo/db/rebuild_indexes.h" +#include "mongo/db/repair.h" +#include "mongo/db/repl/replication_coordinator.h" +#include "mongo/db/repl_set_member_in_standalone_mode.h" +#include "mongo/db/server_options.h" +#include "mongo/db/storage/durable_catalog.h" +#include "mongo/db/storage/storage_repair_observer.h" +#include "mongo/logv2/log.h" +#include "mongo/util/exit.h" +#include "mongo/util/fail_point.h" +#include "mongo/util/quick_exit.h" + +#if !defined(_WIN32) +#include <sys/file.h> +#endif + +namespace mongo { + +// Exit after repair has started, but before data is repaired. +MONGO_FAIL_POINT_DEFINE(exitBeforeDataRepair); +// Exit after repairing data, but before the replica set configuration is invalidated. +MONGO_FAIL_POINT_DEFINE(exitBeforeRepairInvalidatesConfig); + +namespace { + +// Returns true if storage engine is writable. +bool isWriteableStorageEngine() { + return !storageGlobalParams.readOnly && (storageGlobalParams.engine != "devnull"); +} + +// Attempt to restore the featureCompatibilityVersion document if it is missing. +Status restoreMissingFeatureCompatibilityVersionDocument(OperationContext* opCtx) { + NamespaceString fcvNss(NamespaceString::kServerConfigurationNamespace); + + // If the admin database, which contains the server configuration collection with the + // featureCompatibilityVersion document, does not exist, create it. + auto databaseHolder = DatabaseHolder::get(opCtx); + auto db = databaseHolder->getDb(opCtx, fcvNss.db()); + if (!db) { + LOGV2(20998, "Re-creating admin database that was dropped."); + } + db = databaseHolder->openDb(opCtx, fcvNss.db()); + invariant(db); + + // If the server configuration collection, which contains the FCV document, does not exist, then + // create it. + if (!CollectionCatalog::get(opCtx).lookupCollectionByNamespace( + opCtx, NamespaceString::kServerConfigurationNamespace)) { + LOGV2(4926905, + "Re-creating featureCompatibilityVersion document that was deleted. Creating new " + "document with last LTS version.", + "version"_attr = FeatureCompatibilityVersionParser::kLastLTS); + uassertStatusOK( + createCollection(opCtx, fcvNss.db().toString(), BSON("create" << fcvNss.coll()))); + } + + Collection* fcvColl = CollectionCatalog::get(opCtx).lookupCollectionByNamespace( + opCtx, NamespaceString::kServerConfigurationNamespace); + invariant(fcvColl); + + // Restore the featureCompatibilityVersion document if it is missing. + BSONObj featureCompatibilityVersion; + if (!Helpers::findOne(opCtx, + fcvColl, + BSON("_id" << FeatureCompatibilityVersionParser::kParameterName), + featureCompatibilityVersion)) { + LOGV2(21000, + "Re-creating featureCompatibilityVersion document that was deleted. Creating new " + "document with version " + "{FeatureCompatibilityVersionParser_kVersion44}.", + "Re-creating featureCompatibilityVersion document that was deleted", + "version"_attr = FeatureCompatibilityVersionParser::kVersion44); + + FeatureCompatibilityVersionDocument fcvDoc; + fcvDoc.setVersion(ServerGlobalParams::FeatureCompatibility::kLastLTS); + + writeConflictRetry(opCtx, "insertFCVDocument", fcvNss.ns(), [&] { + WriteUnitOfWork wunit(opCtx); + OpDebug* const nullOpDebug = nullptr; + uassertStatusOK(fcvColl->insertDocument( + opCtx, InsertStatement(fcvDoc.toBSON()), nullOpDebug, false)); + wunit.commit(); + }); + } + + invariant(Helpers::findOne(opCtx, + fcvColl, + BSON("_id" << FeatureCompatibilityVersionParser::kParameterName), + featureCompatibilityVersion)); + + return Status::OK(); +} + +/** + * Returns true if the collection associated with the given CollectionCatalogEntry has an index on + * the _id field + */ +bool checkIdIndexExists(OperationContext* opCtx, RecordId catalogId) { + auto durableCatalog = DurableCatalog::get(opCtx); + auto indexCount = durableCatalog->getTotalIndexCount(opCtx, catalogId); + auto indexNames = std::vector<std::string>(indexCount); + durableCatalog->getAllIndexes(opCtx, catalogId, &indexNames); + + for (auto name : indexNames) { + if (name == "_id_") { + return true; + } + } + return false; +} + +Status buildMissingIdIndex(OperationContext* opCtx, Collection* collection) { + LOGV2(4805002, "Building missing _id index", logAttrs(*collection)); + MultiIndexBlock indexer; + auto abortOnExit = makeGuard( + [&] { indexer.abortIndexBuild(opCtx, collection, MultiIndexBlock::kNoopOnCleanUpFn); }); + + const auto indexCatalog = collection->getIndexCatalog(); + const auto idIndexSpec = indexCatalog->getDefaultIdIndexSpec(); + + auto swSpecs = indexer.init(opCtx, collection, idIndexSpec, MultiIndexBlock::kNoopOnInitFn); + if (!swSpecs.isOK()) { + return swSpecs.getStatus(); + } + + auto status = indexer.insertAllDocumentsInCollection(opCtx, collection); + if (!status.isOK()) { + return status; + } + + status = indexer.checkConstraints(opCtx); + if (!status.isOK()) { + return status; + } + + WriteUnitOfWork wuow(opCtx); + status = indexer.commit( + opCtx, collection, MultiIndexBlock::kNoopOnCreateEachFn, MultiIndexBlock::kNoopOnCommitFn); + wuow.commit(); + abortOnExit.dismiss(); + return status; +} + +auto downgradeError = + Status{ErrorCodes::MustDowngrade, + str::stream() << "UPGRADE PROBLEM: The data files need to be fully upgraded to version " + "4.4 before attempting " + "an upgrade to 4.5.1; see " + << feature_compatibility_version_documentation::kUpgradeLink + << " for more details."}; + +/** + * Checks that all collections on a database have valid properties for this version of MongoDB. + * + * This validates that required collections have an _id index. If a collection is missing an _id + * index, this function will build it if EnsureIndexPolicy is kBuildMissing. + * + * Returns a MustDowngrade error if any index builds on the required _id field fail. + */ +enum class EnsureIndexPolicy { kBuildMissing, kError }; +Status ensureCollectionProperties(OperationContext* opCtx, + Database* db, + EnsureIndexPolicy ensureIndexPolicy) { + for (auto collIt = db->begin(opCtx); collIt != db->end(opCtx); ++collIt) { + auto coll = *collIt; + if (!coll) { + break; + } + + // All user-created replicated collections created since MongoDB 4.0 have _id indexes. + auto requiresIndex = coll->requiresIdIndex() && coll->ns().isReplicated(); + auto collOptions = + DurableCatalog::get(opCtx)->getCollectionOptions(opCtx, coll->getCatalogId()); + auto hasAutoIndexIdField = collOptions.autoIndexId == CollectionOptions::YES; + + // Even if the autoIndexId field is not YES, the collection may still have an _id index + // that was created manually by the user. Check the list of indexes to confirm index + // does not exist before attempting to build it or returning an error. + if (requiresIndex && !hasAutoIndexIdField && + !checkIdIndexExists(opCtx, coll->getCatalogId())) { + LOGV2(21001, + "collection {coll_ns} is missing an _id index", + "Collection is missing an _id index", + logAttrs(*coll)); + if (EnsureIndexPolicy::kBuildMissing == ensureIndexPolicy) { + auto status = buildMissingIdIndex(opCtx, coll); + if (!status.isOK()) { + LOGV2_ERROR(21021, + "could not build an _id index on collection {coll_ns}: {error}", + "Could not build an _id index on collection", + logAttrs(*coll), + "error"_attr = status); + return downgradeError; + } + } else { + return downgradeError; + } + } + } + return Status::OK(); +} + +/** + * Opens each database and provides a callback on each one. + */ +template <typename Func> +void openDatabases(OperationContext* opCtx, const StorageEngine* storageEngine, Func&& onDatabase) { + invariant(opCtx->lockState()->isW()); + + auto databaseHolder = DatabaseHolder::get(opCtx); + auto dbNames = storageEngine->listDatabases(); + for (const auto& dbName : dbNames) { + LOGV2_DEBUG(21010, 1, " Opening database: {dbName}", "dbName"_attr = dbName); + auto db = databaseHolder->openDb(opCtx, dbName); + invariant(db); + + onDatabase(db); + } +} + +// Check for storage engine file compatibility. Exits the process if there is an incompatibility. +void assertFilesCompatible(OperationContext* opCtx, StorageEngine* storageEngine) { + auto status = storageEngine->currentFilesCompatible(opCtx); + if (status.isOK()) { + return; + } + + if (status.code() == ErrorCodes::CanRepairToDowngrade) { + // Convert CanRepairToDowngrade statuses to MustUpgrade statuses to avoid logging a + // potentially confusing and inaccurate message. + // + // TODO SERVER-24097: Log a message informing the user that they can start the current + // version of mongod with --repair and then proceed with normal startup. + status = {ErrorCodes::MustUpgrade, status.reason()}; + } + LOGV2_FATAL_CONTINUE( + 21023, + "Unable to start mongod due to an incompatibility with the data files and this version " + "of mongod: {error}. Please consult our documentation when trying to downgrade to a " + "previous major release", + "Unable to start mongod due to an incompatibility with the data files and this version " + "of mongod. Please consult our documentation when trying to downgrade to a previous " + "major release", + "error"_attr = redact(status)); + quickExit(EXIT_NEED_UPGRADE); + MONGO_UNREACHABLE; +} + + +/** + * Returns 'true' if this server has a configuration document in local.system.replset. + */ +bool hasReplSetConfigDoc(OperationContext* opCtx) { + auto databaseHolder = DatabaseHolder::get(opCtx); + + // We open the "local" database before reading to ensure the in-memory catalog entries for the + // 'kSystemReplSetNamespace' collection have been populated if the collection exists. If the + // "local" database doesn't exist at this point yet, then it will be created. + const auto nss = NamespaceString::kSystemReplSetNamespace; + databaseHolder->openDb(opCtx, nss.db()); + BSONObj config; + return Helpers::getSingleton(opCtx, nss.ns().c_str(), config); +} + +/** + * Check that the oplog is capped, and abort the process if it is not. + * Caller must lock DB before calling this function. + */ +void assertCappedOplog(OperationContext* opCtx, Database* db) { + const NamespaceString oplogNss(NamespaceString::kRsOplogNamespace); + invariant(opCtx->lockState()->isDbLockedForMode(oplogNss.db(), MODE_IS)); + Collection* oplogCollection = + CollectionCatalog::get(opCtx).lookupCollectionByNamespace(opCtx, oplogNss); + if (oplogCollection && !oplogCollection->isCapped()) { + LOGV2_FATAL_NOTRACE( + 40115, + "The oplog collection {oplogNamespace} is not capped; a capped oplog is a " + "requirement for replication to function.", + "The oplog collection is not capped; a capped oplog is a " + "requirement for replication to function.", + "oplogNamespace"_attr = oplogNss); + } +} + +void reconcileCatalogAndRebuildUnfinishedIndexes(OperationContext* opCtx, + StorageEngine* storageEngine) { + auto reconcileResult = fassert(40593, storageEngine->reconcileCatalogAndIdents(opCtx)); + + // Determine which indexes need to be rebuilt. rebuildIndexesOnCollection() requires that all + // indexes on that collection are done at once, so we use a map to group them together. + StringMap<IndexNameObjs> nsToIndexNameObjMap; + for (auto&& idxIdentifier : reconcileResult.indexesToRebuild) { + NamespaceString collNss = idxIdentifier.nss; + const std::string& indexName = idxIdentifier.indexName; + auto swIndexSpecs = + getIndexNameObjs(opCtx, idxIdentifier.catalogId, [&indexName](const std::string& name) { + return name == indexName; + }); + if (!swIndexSpecs.isOK() || swIndexSpecs.getValue().first.empty()) { + fassert(40590, + {ErrorCodes::InternalError, + str::stream() << "failed to get index spec for index " << indexName + << " in collection " << collNss.toString()}); + } + + auto& indexesToRebuild = swIndexSpecs.getValue(); + invariant(indexesToRebuild.first.size() == 1 && indexesToRebuild.second.size() == 1, + str::stream() << "Num Index Names: " << indexesToRebuild.first.size() + << " Num Index Objects: " << indexesToRebuild.second.size()); + auto& ino = nsToIndexNameObjMap[collNss.ns()]; + ino.first.emplace_back(std::move(indexesToRebuild.first.back())); + ino.second.emplace_back(std::move(indexesToRebuild.second.back())); + } + + for (const auto& entry : nsToIndexNameObjMap) { + NamespaceString collNss(entry.first); + + auto collection = CollectionCatalog::get(opCtx).lookupCollectionByNamespace(opCtx, collNss); + for (const auto& indexName : entry.second.first) { + LOGV2(21004, + "Rebuilding index. Collection: {collNss} Index: {indexName}", + "Rebuilding index", + "namespace"_attr = collNss, + "index"_attr = indexName); + } + + std::vector<BSONObj> indexSpecs = entry.second.second; + fassert(40592, rebuildIndexesOnCollection(opCtx, collection, indexSpecs, RepairData::kNo)); + } + + // Two-phase index builds depend on an eventually-replicated 'commitIndexBuild' oplog entry to + // complete. Therefore, when a replica set member is started in standalone mode, we cannot + // restart the index build because it will never complete. + if (getReplSetMemberInStandaloneMode(opCtx->getServiceContext())) { + LOGV2(21005, "Not restarting unfinished index builds because we are in standalone mode"); + return; + } + + // Once all unfinished indexes have been rebuilt, restart any unfinished index builds. This will + // not build any indexes to completion, but rather start the background thread to build the + // index, and wait for a replicated commit or abort oplog entry. + IndexBuildsCoordinator::get(opCtx)->restartIndexBuildsForRecovery( + opCtx, reconcileResult.indexBuildsToRestart); +} + +/** + * Sets the appropriate flag on the service context decorable 'replSetMemberInStandaloneMode' to + * 'true' if this is a replica set node running in standalone mode, otherwise 'false'. + */ +void setReplSetMemberInStandaloneMode(OperationContext* opCtx) { + const repl::ReplSettings& replSettings = + repl::ReplicationCoordinator::get(opCtx)->getSettings(); + + if (replSettings.usingReplSets()) { + // Not in standalone mode. + setReplSetMemberInStandaloneMode(opCtx->getServiceContext(), false); + return; + } + + invariant(opCtx->lockState()->isW()); + Collection* collection = CollectionCatalog::get(opCtx).lookupCollectionByNamespace( + opCtx, NamespaceString::kSystemReplSetNamespace); + if (collection && !collection->isEmpty(opCtx)) { + setReplSetMemberInStandaloneMode(opCtx->getServiceContext(), true); + return; + } + + setReplSetMemberInStandaloneMode(opCtx->getServiceContext(), false); +} + +// Perform startup procedures for --repair mode. +void startupRepair(OperationContext* opCtx, StorageEngine* storageEngine) { + invariant(!storageGlobalParams.readOnly); + + if (MONGO_unlikely(exitBeforeDataRepair.shouldFail())) { + LOGV2(21006, "Exiting because 'exitBeforeDataRepair' fail point was set."); + quickExit(EXIT_ABRUPT); + } + + // Repair, restore, and initialize the featureCompatibilityVersion document before allowing + // repair to potentially rebuild indexes on the remaining collections. This ensures any + // FCV-dependent features are rebuilt properly. Note that we don't try to prevent + // repairDatabase from repairing this collection again, because it only consists of one + // document. + if (auto fcvColl = CollectionCatalog::get(opCtx).lookupCollectionByNamespace( + opCtx, NamespaceString::kServerConfigurationNamespace)) { + auto databaseHolder = DatabaseHolder::get(opCtx); + databaseHolder->openDb(opCtx, fcvColl->ns().db()); + fassertNoTrace(4805000, + repair::repairCollection( + opCtx, storageEngine, NamespaceString::kServerConfigurationNamespace)); + } + uassertStatusOK(restoreMissingFeatureCompatibilityVersionDocument(opCtx)); + FeatureCompatibilityVersion::initializeForStartup(opCtx); + + // The local database should be repaired before any other replicated collections so we know + // whether not to rebuild unfinished two-phase index builds if this is a replica set node + // running in standalone mode. + auto dbNames = storageEngine->listDatabases(); + if (auto it = std::find(dbNames.begin(), dbNames.end(), NamespaceString::kLocalDb); + it != dbNames.end()) { + fassertNoTrace(4805001, repair::repairDatabase(opCtx, storageEngine, *it)); + + // This must be set before rebuilding index builds on replicated collections. + setReplSetMemberInStandaloneMode(opCtx); + dbNames.erase(it); + } + + // Repair the remaining databases. + for (const auto& dbName : dbNames) { + fassertNoTrace(18506, repair::repairDatabase(opCtx, storageEngine, dbName)); + } + + openDatabases(opCtx, storageEngine, [&](auto db) { + // Ensures all collections meet requirements such as having _id indexes, and corrects them + // if needed. + uassertStatusOK(ensureCollectionProperties(opCtx, db, EnsureIndexPolicy::kBuildMissing)); + }); + + if (MONGO_unlikely(exitBeforeRepairInvalidatesConfig.shouldFail())) { + LOGV2(21008, "Exiting because 'exitBeforeRepairInvalidatesConfig' fail point was set."); + quickExit(EXIT_ABRUPT); + } + + auto repairObserver = StorageRepairObserver::get(opCtx->getServiceContext()); + repairObserver->onRepairDone(opCtx); + if (repairObserver->getModifications().size() > 0) { + const auto& mods = repairObserver->getModifications(); + for (const auto& mod : mods) { + LOGV2_WARNING(21019, "repairModification", "description"_attr = mod.getDescription()); + } + } + if (repairObserver->isDataInvalidated()) { + if (hasReplSetConfigDoc(opCtx)) { + LOGV2_WARNING(21020, + "WARNING: Repair may have modified replicated data. This node will no " + "longer be able to join a replica set without a full re-sync"); + } + } + + // There were modifications, but only benign ones. + if (repairObserver->getModifications().size() > 0 && !repairObserver->isDataInvalidated()) { + LOGV2(21009, + "Repair has made modifications to unreplicated data. The data is healthy and " + "the node is eligible to be returned to the replica set."); + } +} + +// Perform startup procedures for read-only mode. +void startupRecoveryReadOnly(OperationContext* opCtx, StorageEngine* storageEngine) { + invariant(!storageGlobalParams.repair); + + setReplSetMemberInStandaloneMode(opCtx); + + FeatureCompatibilityVersion::initializeForStartup(opCtx); + + openDatabases(opCtx, storageEngine, [&](auto db) { + // Ensures all collections meet requirements such as having _id indexes. + uassertStatusOK(ensureCollectionProperties(opCtx, db, EnsureIndexPolicy::kError)); + }); +} + +// Perform routine startup recovery procedure. +void startupRecovery(OperationContext* opCtx, StorageEngine* storageEngine) { + invariant(!storageGlobalParams.readOnly && !storageGlobalParams.repair); + + // Determine whether this is a replica set node running in standalone mode. This must be set + // before determining whether to restart index builds. + setReplSetMemberInStandaloneMode(opCtx); + + // Initialize FCV before rebuilding indexes that may have features dependent on FCV. + FeatureCompatibilityVersion::initializeForStartup(opCtx); + + // Drops abandoned idents. Rebuilds unfinished indexes and restarts incomplete two-phase + // index builds. + reconcileCatalogAndRebuildUnfinishedIndexes(opCtx, storageEngine); + + const auto& replSettings = repl::ReplicationCoordinator::get(opCtx)->getSettings(); + + // On replica set members we only clear temp collections on DBs other than "local" during + // promotion to primary. On secondaries, they are only cleared when the oplog tells them to. The + // local DB is special because it is not replicated. See SERVER-10927 for more details. + const bool shouldClearNonLocalTmpCollections = + !(hasReplSetConfigDoc(opCtx) || replSettings.usingReplSets()); + + openDatabases(opCtx, storageEngine, [&](auto db) { + auto dbName = db->name(); + + // Ensures all collections meet requirements such as having _id indexes, and corrects them + // if needed. + uassertStatusOK(ensureCollectionProperties(opCtx, db, EnsureIndexPolicy::kBuildMissing)); + + if (replSettings.usingReplSets()) { + // We only care about _id indexes and drop-pending collections if we are in a replset. + db->checkForIdIndexesAndDropPendingCollections(opCtx); + // Ensure oplog is capped (mongodb does not guarantee order of inserts on noncapped + // collections) + if (db->name() == NamespaceString::kLocalDb) { + assertCappedOplog(opCtx, db); + } + } + + if (shouldClearNonLocalTmpCollections || dbName == NamespaceString::kLocalDb) { + db->clearTmpCollections(opCtx); + } + }); +} + +} // namespace + +namespace startup_recovery { +/** + * Recovers or repairs all databases from a previous shutdown. May throw a MustDowngrade error + * if data files are incompatible with the current binary version. + */ +void repairAndRecoverDatabases(OperationContext* opCtx) { + auto const storageEngine = opCtx->getServiceContext()->getStorageEngine(); + Lock::GlobalWrite lk(opCtx); + + // Create the FCV document for the first time, if necessary. Replica set nodes only initialize + // the FCV when the replica set is first initiated or by data replication. + const auto& replSettings = repl::ReplicationCoordinator::get(opCtx)->getSettings(); + if (isWriteableStorageEngine() && !replSettings.usingReplSets()) { + FeatureCompatibilityVersion::setIfCleanStartup(opCtx, repl::StorageInterface::get(opCtx)); + } + + if (storageGlobalParams.repair) { + startupRepair(opCtx, storageEngine); + } else if (storageGlobalParams.readOnly) { + startupRecoveryReadOnly(opCtx, storageEngine); + } else { + startupRecovery(opCtx, storageEngine); + } + + assertFilesCompatible(opCtx, storageEngine); +} +} // namespace startup_recovery +} // namespace mongo diff --git a/src/mongo/db/repair_database_and_check_version.h b/src/mongo/db/startup_recovery.h index 2749af11368..c2ccb7d45cb 100644 --- a/src/mongo/db/repair_database_and_check_version.h +++ b/src/mongo/db/startup_recovery.h @@ -1,5 +1,5 @@ /** - * Copyright (C) 2018-present MongoDB, Inc. + * Copyright (C) 2020-present MongoDB, Inc. * * This program is free software: you can redistribute it and/or modify * it under the terms of the Server Side Public License, version 1, @@ -29,16 +29,16 @@ #pragma once -#include "mongo/base/status_with.h" - namespace mongo { class OperationContext; +namespace startup_recovery { + /** - * Return whether there are non-local databases. If there was an error becauses the wrong mongod - * version was used for these datafiles, a DBException with status ErrorCodes::MustDowngrade is - * thrown. + * Recovers or repairs all databases from a previous shutdown. May throw a MustDowngrade error + * if data files are incompatible with the current binary version. */ -bool repairDatabasesAndCheckVersion(OperationContext* opCtx); +void repairAndRecoverDatabases(OperationContext* opCtx); +} // namespace startup_recovery } // namespace mongo diff --git a/src/mongo/db/storage/storage_engine_impl.cpp b/src/mongo/db/storage/storage_engine_impl.cpp index bcded20bb6d..903078e2264 100644 --- a/src/mongo/db/storage/storage_engine_impl.cpp +++ b/src/mongo/db/storage/storage_engine_impl.cpp @@ -858,7 +858,6 @@ bool StorageEngineImpl::supportsOplogStones() const { bool StorageEngineImpl::supportsResumableIndexBuilds() const { return enableResumableIndexBuilds && supportsReadConcernMajority() && !isEphemeral() && - serverGlobalParams.featureCompatibility.isVersionInitialized() && serverGlobalParams.featureCompatibility.getVersion() == ServerGlobalParams::FeatureCompatibility::Version::kVersion451 && !repl::ReplSettings::shouldRecoverFromOplogAsStandalone(); diff --git a/src/mongo/embedded/SConscript b/src/mongo/embedded/SConscript index eb6f8f86086..44ed64a0c2d 100644 --- a/src/mongo/embedded/SConscript +++ b/src/mongo/embedded/SConscript @@ -107,7 +107,6 @@ env.Library( '$BUILD_DIR/mongo/db/logical_session_cache_impl', '$BUILD_DIR/mongo/db/op_observer_impl', '$BUILD_DIR/mongo/db/pipeline/process_interface/mongod_process_interfaces', - '$BUILD_DIR/mongo/db/repair_database_and_check_version', '$BUILD_DIR/mongo/db/repl/repl_coordinator_interface', '$BUILD_DIR/mongo/db/repl/replica_set_messages', '$BUILD_DIR/mongo/db/repl/storage_interface_impl', @@ -120,6 +119,7 @@ env.Library( '$BUILD_DIR/mongo/db/service_entry_point_common', '$BUILD_DIR/mongo/db/service_liaison_mongod', '$BUILD_DIR/mongo/db/sessions_collection_standalone', + '$BUILD_DIR/mongo/db/startup_recovery', '$BUILD_DIR/mongo/db/storage/wiredtiger/storage_wiredtiger' if wiredtiger else [], '$BUILD_DIR/mongo/db/storage/storage_control', '$BUILD_DIR/mongo/db/storage/storage_engine_common', diff --git a/src/mongo/embedded/embedded.cpp b/src/mongo/embedded/embedded.cpp index 8866654cefb..8adea8014dd 100644 --- a/src/mongo/embedded/embedded.cpp +++ b/src/mongo/embedded/embedded.cpp @@ -52,11 +52,11 @@ #include "mongo/db/logical_session_cache_impl.h" #include "mongo/db/op_observer_impl.h" #include "mongo/db/op_observer_registry.h" -#include "mongo/db/repair_database_and_check_version.h" #include "mongo/db/repl/storage_interface_impl.h" #include "mongo/db/service_liaison_mongod.h" #include "mongo/db/session_killer.h" #include "mongo/db/sessions_collection_standalone.h" +#include "mongo/db/startup_recovery.h" #include "mongo/db/storage/control/storage_control.h" #include "mongo/db/storage/encryption_hooks.h" #include "mongo/db/storage/storage_engine_init.h" @@ -291,7 +291,7 @@ ServiceContext* initialize(const char* yaml_config) { } try { - repairDatabasesAndCheckVersion(startupOpCtx.get()); + startup_recovery::repairAndRecoverDatabases(startupOpCtx.get()); } catch (const ExceptionFor<ErrorCodes::MustDowngrade>& error) { LOGV2_FATAL_OPTIONS(22555, logv2::LogOptions(LogComponent::kControl, logv2::FatalMode::kContinue), @@ -300,14 +300,9 @@ ServiceContext* initialize(const char* yaml_config) { quickExit(EXIT_NEED_DOWNGRADE); } - // Assert that the in-memory featureCompatibilityVersion parameter has been explicitly set. - // If we are part of a replica set and are started up with no data files, we do not set the - // featureCompatibilityVersion until a primary is chosen. For this case, we expect the - // in-memory featureCompatibilityVersion parameter to still be uninitialized until after - // startup. - if (canCallFCVSetIfCleanStartup) { - invariant(serverGlobalParams.featureCompatibility.isVersionInitialized()); - } + // Ensure FCV document exists and is initialized in-memory. Fatally asserts if there is an + // error. + FeatureCompatibilityVersion::fassertInitializedAfterStartup(startupOpCtx.get()); if (storageGlobalParams.upgrade) { LOGV2(22553, "finished checking dbs"); |