diff options
23 files changed, 179 insertions, 155 deletions
diff --git a/src/mongo/db/SConscript b/src/mongo/db/SConscript index f80bd040b8f..9b71106974e 100644 --- a/src/mongo/db/SConscript +++ b/src/mongo/db/SConscript @@ -979,6 +979,7 @@ env.Library( 'db_raii', 'index_build_entry_helpers', '$BUILD_DIR/mongo/db/catalog/index_build_entry_idl', + '$BUILD_DIR/mongo/db/catalog/collection', '$BUILD_DIR/mongo/db/catalog/collection_catalog', '$BUILD_DIR/mongo/db/concurrency/lock_manager', '$BUILD_DIR/mongo/db/repl/repl_coordinator_interface', diff --git a/src/mongo/db/catalog/collection.cpp b/src/mongo/db/catalog/collection.cpp index dbd94df7d7e..685632a2f8d 100644 --- a/src/mongo/db/catalog/collection.cpp +++ b/src/mongo/db/catalog/collection.cpp @@ -73,6 +73,24 @@ bool CappedInsertNotifier::isDead() { // ---- +namespace { +const auto getFactory = ServiceContext::declareDecoration<std::unique_ptr<Collection::Factory>>(); +} + +Collection::Factory* Collection::Factory::get(ServiceContext* service) { + return getFactory(service).get(); +} + +Collection::Factory* Collection::Factory::get(OperationContext* opCtx) { + return getFactory(opCtx->getServiceContext()).get(); +}; + +void Collection::Factory::set(ServiceContext* service, + std::unique_ptr<Collection::Factory> newFactory) { + auto& factory = getFactory(service); + factory = std::move(newFactory); +} + // static Status Collection::parseValidationLevel(StringData newLevel) { if (newLevel == "") { diff --git a/src/mongo/db/catalog/collection.h b/src/mongo/db/catalog/collection.h index 6d30ab5a847..d9abc6816cd 100644 --- a/src/mongo/db/catalog/collection.h +++ b/src/mongo/db/catalog/collection.h @@ -162,6 +162,26 @@ public: }; /** + * A Collection::Factory is a factory class that constructs Collection objects. + */ + class Factory { + public: + Factory() = default; + virtual ~Factory() = default; + + static Factory* get(ServiceContext* service); + static Factory* get(OperationContext* opCtx); + static void set(ServiceContext* service, std::unique_ptr<Factory> factory); + + /** + * Constructs a Collection object. This does not persist any state to the storage engine, + * only constructs an in-memory representation of what already exists on disk. + */ + virtual std::unique_ptr<Collection> make( + OperationContext* opCtx, CollectionCatalogEntry* collectionCatalogEntry) const = 0; + }; + + /** * Callback function for callers of insertDocumentForBulkLoader(). */ using OnRecordInsertedFn = stdx::function<Status(const RecordId& loc)>; @@ -466,6 +486,12 @@ public: * onto the global lock in exclusive mode. */ virtual void establishOplogCollectionForLogging(OperationContext* opCtx) = 0; + + virtual void init(OperationContext* opCtx) {} + + virtual bool isInitialized() const { + return false; + } }; } // namespace mongo diff --git a/src/mongo/db/catalog/collection_impl.cpp b/src/mongo/db/catalog/collection_impl.cpp index d9cdc0a57dd..6052174f5ff 100644 --- a/src/mongo/db/catalog/collection_impl.cpp +++ b/src/mongo/db/catalog/collection_impl.cpp @@ -200,21 +200,14 @@ CollectionImpl::CollectionImpl(OperationContext* opCtx, _uuid(uuid), _details(details), _recordStore(recordStore), - _needCappedLock(supportsDocLocking() && _recordStore->isCapped() && _ns.db() != "local"), + _needCappedLock(supportsDocLocking() && _recordStore && _recordStore->isCapped() && + _ns.db() != "local"), _infoCache(std::make_unique<CollectionInfoCacheImpl>(this, _ns)), _indexCatalog( std::make_unique<IndexCatalogImpl>(this, getCatalogEntry()->getMaxAllowedIndexes())), - _collator(parseCollation(opCtx, _ns, _details->getCollectionOptions(opCtx).collation)), - _validatorDoc(_details->getCollectionOptions(opCtx).validator.getOwned()), - _validator(uassertStatusOK( - parseValidator(opCtx, _validatorDoc, MatchExpressionParser::kAllowAllSpecialFeatures))), - _validationAction(uassertStatusOK( - _parseValidationAction(_details->getCollectionOptions(opCtx).validationAction))), - _validationLevel(uassertStatusOK( - _parseValidationLevel(_details->getCollectionOptions(opCtx).validationLevel))), - _cappedNotifier(_recordStore->isCapped() ? stdx::make_unique<CappedInsertNotifier>() - : nullptr) { - + _cappedNotifier(_recordStore && _recordStore->isCapped() + ? stdx::make_unique<CappedInsertNotifier>() + : nullptr) { if (isCapped()) _recordStore->setCappedCallback(this); } @@ -233,6 +226,33 @@ CollectionImpl::~CollectionImpl() { _magic = 0; } +std::unique_ptr<Collection> CollectionImpl::FactoryImpl::make( + OperationContext* opCtx, CollectionCatalogEntry* collectionCatalogEntry) const { + auto rs = collectionCatalogEntry->getRecordStore(); + const auto uuid = collectionCatalogEntry->getCollectionOptions(opCtx).uuid; + const auto nss = collectionCatalogEntry->ns(); + return std::make_unique<CollectionImpl>(opCtx, nss.ns(), uuid, collectionCatalogEntry, rs); +} + +void CollectionImpl::init(OperationContext* opCtx) { + _collator = parseCollation(opCtx, _ns, _details->getCollectionOptions(opCtx).collation); + _validatorDoc = _details->getCollectionOptions(opCtx).validator.getOwned(); + _validator = uassertStatusOK( + parseValidator(opCtx, _validatorDoc, MatchExpressionParser::kAllowAllSpecialFeatures)); + _validationAction = uassertStatusOK( + _parseValidationAction(_details->getCollectionOptions(opCtx).validationAction)); + _validationLevel = uassertStatusOK( + _parseValidationLevel(_details->getCollectionOptions(opCtx).validationLevel)); + + getIndexCatalog()->init(opCtx).transitional_ignore(); + infoCache()->init(opCtx); + _initialized = true; +} + +bool CollectionImpl::isInitialized() const { + return _initialized; +} + bool CollectionImpl::requiresIdIndex() const { if (_ns.isVirtualized() || _ns.isOplog()) { // No indexes on virtual collections or the oplog. diff --git a/src/mongo/db/catalog/collection_impl.h b/src/mongo/db/catalog/collection_impl.h index f1283c8c1a3..8657a48589f 100644 --- a/src/mongo/db/catalog/collection_impl.h +++ b/src/mongo/db/catalog/collection_impl.h @@ -54,6 +54,12 @@ public: ~CollectionImpl(); + class FactoryImpl : public Factory { + public: + std::unique_ptr<Collection> make( + OperationContext* opCtx, CollectionCatalogEntry* collectionCatalogEntry) const final; + }; + bool ok() const final { return _magic == kMagicNumber; } @@ -366,6 +372,9 @@ public: void establishOplogCollectionForLogging(OperationContext* opCtx) final; + void init(OperationContext* opCtx) final; + bool isInitialized() const final; + private: /** * Returns a non-ok Status if document does not pass this collection's validator. @@ -391,6 +400,8 @@ private: NamespaceString _ns; OptionalCollectionUUID _uuid; CollectionCatalogEntry* const _details; + + // The RecordStore may be null during a repair operation. RecordStore* const _recordStore; const bool _needCappedLock; std::unique_ptr<CollectionInfoCache> _infoCache; @@ -420,5 +431,7 @@ private: // The earliest snapshot that is allowed to use this collection. boost::optional<Timestamp> _minVisibleSnapshot; + + bool _initialized = false; }; } // namespace mongo diff --git a/src/mongo/db/catalog/database_holder.h b/src/mongo/db/catalog/database_holder.h index a307e9c2ed7..53413a0e900 100644 --- a/src/mongo/db/catalog/database_holder.h +++ b/src/mongo/db/catalog/database_holder.h @@ -103,17 +103,6 @@ public: * Returns the set of existing database names that differ only in casing. */ virtual std::set<std::string> getNamesWithConflictingCasing(const StringData name) = 0; - - /** - * Returns a new Collection. - * This function supports rebuilding indexes during the repair process and should not be used - * for any other purpose. - */ - virtual std::unique_ptr<Collection> makeCollection(OperationContext* const opCtx, - const StringData fullNS, - OptionalCollectionUUID uuid, - CollectionCatalogEntry* const details, - RecordStore* const recordStore) = 0; }; } // namespace mongo diff --git a/src/mongo/db/catalog/database_holder_impl.cpp b/src/mongo/db/catalog/database_holder_impl.cpp index cd346130f0c..cab4b7bb007 100644 --- a/src/mongo/db/catalog/database_holder_impl.cpp +++ b/src/mongo/db/catalog/database_holder_impl.cpp @@ -267,13 +267,4 @@ void DatabaseHolderImpl::closeAll(OperationContext* opCtx) { } } -std::unique_ptr<Collection> DatabaseHolderImpl::makeCollection( - OperationContext* const opCtx, - const StringData fullNS, - OptionalCollectionUUID uuid, - CollectionCatalogEntry* const details, - RecordStore* const recordStore) { - return std::make_unique<CollectionImpl>(opCtx, fullNS, uuid, details, recordStore); -} - } // namespace mongo diff --git a/src/mongo/db/catalog/database_holder_impl.h b/src/mongo/db/catalog/database_holder_impl.h index c5dc6a8009e..5a53ad8ff5e 100644 --- a/src/mongo/db/catalog/database_holder_impl.h +++ b/src/mongo/db/catalog/database_holder_impl.h @@ -52,12 +52,6 @@ public: std::set<std::string> getNamesWithConflictingCasing(StringData name) override; - std::unique_ptr<Collection> makeCollection(OperationContext* const opCtx, - const StringData fullNS, - OptionalCollectionUUID uuid, - CollectionCatalogEntry* const details, - RecordStore* const recordStore) override; - private: std::set<std::string> _getNamesWithConflictingCasing_inlock(StringData name); diff --git a/src/mongo/db/catalog/database_holder_mock.h b/src/mongo/db/catalog/database_holder_mock.h index 69c4b97b1bd..bccfd4d1f2a 100644 --- a/src/mongo/db/catalog/database_holder_mock.h +++ b/src/mongo/db/catalog/database_holder_mock.h @@ -54,14 +54,6 @@ public: std::set<std::string> getNamesWithConflictingCasing(StringData name) override { return std::set<std::string>(); } - - std::unique_ptr<Collection> makeCollection(OperationContext* const opCtx, - const StringData fullNS, - OptionalCollectionUUID uuid, - CollectionCatalogEntry* const details, - RecordStore* const recordStore) override { - return {}; - } }; } // namespace mongo diff --git a/src/mongo/db/catalog/database_impl.cpp b/src/mongo/db/catalog/database_impl.cpp index 78067c9a48b..fc3f00c1511 100644 --- a/src/mongo/db/catalog/database_impl.cpp +++ b/src/mongo/db/catalog/database_impl.cpp @@ -84,22 +84,6 @@ namespace mongo { namespace { MONGO_FAIL_POINT_DEFINE(hangBeforeLoggingCreateCollection); -std::unique_ptr<Collection> _createCollectionInstance(OperationContext* opCtx, - const NamespaceString& nss) { - - auto cce = CollectionCatalog::get(opCtx).lookupCollectionCatalogEntryByNamespace(nss); - auto rs = cce->getRecordStore(); - auto uuid = cce->getCollectionOptions(opCtx).uuid; - invariant(rs, - str::stream() << "Record store did not exist. Collection: " << nss.ns() << " UUID: " - << uuid); - invariant(uuid); - - auto coll = std::make_unique<CollectionImpl>(opCtx, nss.ns(), uuid, cce, rs); - - return coll; -} - Status validateDBNameForWindows(StringData dbname) { const std::vector<std::string> windowsReservedNames = { "con", "prn", "aux", "nul", "com1", "com2", "com3", "com4", "com5", "com6", "com7", @@ -169,19 +153,12 @@ void DatabaseImpl::init(OperationContext* const opCtx) const { } auto& catalog = CollectionCatalog::get(opCtx); - for (const auto& nss : catalog.getAllCollectionNamesFromDb(opCtx, _name)) { - auto ownedCollection = _createCollectionInstance(opCtx, nss); - invariant(ownedCollection); - - // Call registerCollectionObject directly because we're not in a WUOW. - auto uuid = *(ownedCollection->uuid()); - catalog.registerCollectionObject(uuid, std::move(ownedCollection)); - } - for (const auto& uuid : catalog.getAllCollectionUUIDsFromDb(_name)) { auto collection = catalog.lookupCollectionByUUID(uuid); - collection->getIndexCatalog()->init(opCtx).transitional_ignore(); - collection->infoCache()->init(opCtx); + invariant(collection); + // If this is called from the repair path, the collection is already initialized. + if (!collection->isInitialized()) + collection->init(opCtx); } // At construction time of the viewCatalog, the CollectionCatalog map wasn't initialized yet, @@ -694,9 +671,9 @@ Collection* DatabaseImpl::createCollection(OperationContext* opCtx, // Create Collection object auto& catalog = CollectionCatalog::get(opCtx); - auto ownedCollection = _createCollectionInstance(opCtx, nss); - ownedCollection->getIndexCatalog()->init(opCtx).transitional_ignore(); - ownedCollection->infoCache()->init(opCtx); + auto catalogEntry = catalog.lookupCollectionCatalogEntryByUUID(optionsWithUUID.uuid.get()); + auto ownedCollection = Collection::Factory::get(opCtx)->make(opCtx, catalogEntry); + ownedCollection->init(opCtx); Collection* collection = ownedCollection.get(); catalog.onCreateCollection(opCtx, std::move(ownedCollection), *(collection->uuid())); opCtx->recoveryUnit()->onCommit([collection](auto commitTime) { diff --git a/src/mongo/db/db.cpp b/src/mongo/db/db.cpp index 32eb25ef96f..235a6e78546 100644 --- a/src/mongo/db/db.cpp +++ b/src/mongo/db/db.cpp @@ -51,6 +51,7 @@ #include "mongo/db/auth/sasl_options.h" #include "mongo/db/catalog/collection.h" #include "mongo/db/catalog/collection_catalog.h" +#include "mongo/db/catalog/collection_impl.h" #include "mongo/db/catalog/create_collection.h" #include "mongo/db/catalog/database.h" #include "mongo/db/catalog/database_holder_impl.h" @@ -784,6 +785,7 @@ void startupConfigActions(const std::vector<std::string>& args) { void setUpCatalog(ServiceContext* serviceContext) { DatabaseHolder::set(serviceContext, std::make_unique<DatabaseHolderImpl>()); IndexAccessMethodFactory::set(serviceContext, std::make_unique<IndexAccessMethodFactoryImpl>()); + Collection::Factory::set(serviceContext, std::make_unique<CollectionImpl::FactoryImpl>()); } auto makeReplicationExecutor(ServiceContext* serviceContext) { diff --git a/src/mongo/db/index_builds_coordinator.cpp b/src/mongo/db/index_builds_coordinator.cpp index dc556ad7e81..6ccdb4f8ec8 100644 --- a/src/mongo/db/index_builds_coordinator.cpp +++ b/src/mongo/db/index_builds_coordinator.cpp @@ -170,44 +170,46 @@ StatusWith<std::pair<long long, long long>> IndexBuildsCoordinator::startIndexRe indexNames.push_back(name); } - const auto& ns = cce->ns().ns(); - auto rs = cce->getRecordStore(); + const NamespaceString nss(cce->ns()); ReplIndexBuildState::IndexCatalogStats indexCatalogStats; - std::unique_ptr<Collection> collection; + auto& collectionCatalog = CollectionCatalog::get(getGlobalServiceContext()); + Collection* collection = collectionCatalog.lookupCollectionByNamespace(nss); + auto indexCatalog = collection->getIndexCatalog(); std::unique_ptr<MultiIndexBlock> indexer; { // These steps are combined into a single WUOW to ensure there are no commits without // the indexes. // 1) Drop all indexes. - // 2) Open the Collection + // 2) Re-create the Collection. // 3) Start the index build process. - WriteUnitOfWork wuow(opCtx); - { // 1 + // 1 + { for (size_t i = 0; i < indexNames.size(); i++) { - Status s = cce->removeIndex(opCtx, indexNames[i]); + auto descriptor = indexCatalog->findIndexByName(opCtx, indexNames[i], false); + if (!descriptor) { + // If it's unfinished index, drop it directly via removeIndex. + Status status = + collection->getCatalogEntry()->removeIndex(opCtx, indexNames[i]); + continue; + } + Status s = indexCatalog->dropIndex(opCtx, descriptor); if (!s.isOK()) { return s; } } } - // Indexes must be dropped before we open the Collection otherwise we could attempt to - // open a bad index and fail. - const auto uuid = cce->getCollectionOptions(opCtx).uuid; - auto databaseHolder = DatabaseHolder::get(opCtx); - collection = databaseHolder->makeCollection(opCtx, ns, uuid, cce, rs); - collection->getIndexCatalog()->init(opCtx).transitional_ignore(); - collection->infoCache()->init(opCtx); + // We need to initialize the collection to drop and rebuild the indexes. + collection->init(opCtx); // Register the index build. During recovery, collections may not have UUIDs present yet to // due upgrading. We don't require collection UUIDs during recovery except to create a // ReplIndexBuildState object. auto collectionUUID = UUID::gen(); - auto nss = collection->ns(); auto dbName = nss.db().toString(); // We run the index build using the single phase protocol as we already hold the global @@ -230,12 +232,12 @@ StatusWith<std::pair<long long, long long>> IndexBuildsCoordinator::startIndexRe // Setup the index build. indexCatalogStats.numIndexesBefore = - _getNumIndexesTotal(opCtx, collection.get()) + indexNames.size(); + _getNumIndexesTotal(opCtx, collection) + indexNames.size(); IndexBuildsManager::SetupOptions options; options.forRecovery = true; status = _indexBuildsManager.setUpIndexBuild( - opCtx, collection.get(), specs, buildUUID, MultiIndexBlock::kNoopOnInitFn, options); + opCtx, collection, specs, buildUUID, MultiIndexBlock::kNoopOnInitFn, options); if (!status.isOK()) { // An index build failure during recovery is fatal. logFailure(status, nss, replIndexBuildState); @@ -245,7 +247,7 @@ StatusWith<std::pair<long long, long long>> IndexBuildsCoordinator::startIndexRe wuow.commit(); } - return _runIndexRebuildForRecovery(opCtx, collection.get(), indexCatalogStats, buildUUID); + return _runIndexRebuildForRecovery(opCtx, collection, indexCatalogStats, buildUUID); } Future<void> IndexBuildsCoordinator::joinIndexBuilds(const NamespaceString& nss, diff --git a/src/mongo/db/index_builds_coordinator.h b/src/mongo/db/index_builds_coordinator.h index 28ddd788d46..3a473b43901 100644 --- a/src/mongo/db/index_builds_coordinator.h +++ b/src/mongo/db/index_builds_coordinator.h @@ -117,9 +117,8 @@ public: /** * Sets up the in-memory and persisted state of the index build. * - * This function should only be called when in recovery mode, because we use the DatabaseHolder - * to create a temporary collection using the collection catalog entry to allow us to rebuild - * the indexes on the collection without initializing it fully. + * This function should only be called when in recovery mode, because we create new Collection + * objects and replace old ones after dropping existing indexes. * * Returns the number of records and the size of the data iterated over, if successful. */ diff --git a/src/mongo/db/repair_database.cpp b/src/mongo/db/repair_database.cpp index 650dae1636d..193d5221eb5 100644 --- a/src/mongo/db/repair_database.cpp +++ b/src/mongo/db/repair_database.cpp @@ -138,8 +138,7 @@ Status rebuildIndexesOnCollection(OperationContext* opCtx, namespace { Status repairCollections(OperationContext* opCtx, StorageEngine* engine, - const std::string& dbName, - stdx::function<void(const std::string& dbName)> onRecordStoreRepair) { + const std::string& dbName) { auto colls = CollectionCatalog::get(opCtx).getAllCollectionNamesFromDb(opCtx, dbName); @@ -153,8 +152,6 @@ Status repairCollections(OperationContext* opCtx, return status; } - onRecordStoreRepair(dbName); - for (const auto& nss : colls) { opCtx->checkForInterrupt(); @@ -175,10 +172,7 @@ Status repairCollections(OperationContext* opCtx, } } // namespace -Status repairDatabase(OperationContext* opCtx, - StorageEngine* engine, - const std::string& dbName, - stdx::function<void(const std::string& dbName)> onRecordStoreRepair) { +Status repairDatabase(OperationContext* opCtx, StorageEngine* engine, const std::string& dbName) { DisableDocumentValidation validationDisabler(opCtx); // We must hold some form of lock here @@ -195,7 +189,7 @@ Status repairDatabase(OperationContext* opCtx, auto databaseHolder = DatabaseHolder::get(opCtx); databaseHolder->close(opCtx, dbName); - auto status = repairCollections(opCtx, engine, dbName, onRecordStoreRepair); + auto status = repairCollections(opCtx, engine, dbName); if (!status.isOK()) { severe() << "Failed to repair database " << dbName << ": " << status.reason(); } diff --git a/src/mongo/db/repair_database.h b/src/mongo/db/repair_database.h index af414d3d4be..25b518b6ac7 100644 --- a/src/mongo/db/repair_database.h +++ b/src/mongo/db/repair_database.h @@ -68,14 +68,8 @@ Status rebuildIndexesOnCollection(OperationContext* opCtx, * Some data may be lost or modified in the process but the output will * be structurally valid on successful return. * - * Calls 'onRecordStoreRepair' after repairing all the collection record stores for each database - * before rebuilding the appropriate indexes. - * * It is expected that the local database will be repaired first when running in repair mode. */ -Status repairDatabase(OperationContext* opCtx, - StorageEngine* engine, - const std::string& dbName, - stdx::function<void(const std::string& dbName)> onRecordStoreRepair); +Status repairDatabase(OperationContext* opCtx, StorageEngine* engine, const std::string& dbName); } // namespace mongo diff --git a/src/mongo/db/repair_database_and_check_version.cpp b/src/mongo/db/repair_database_and_check_version.cpp index 2c83af57982..4c0203f06fd 100644 --- a/src/mongo/db/repair_database_and_check_version.cpp +++ b/src/mongo/db/repair_database_and_check_version.cpp @@ -336,12 +336,9 @@ void setReplSetMemberInStandaloneMode(OperationContext* opCtx) { return; } - Lock::DBLock dbLock(opCtx, NamespaceString::kSystemReplSetNamespace.db(), MODE_X); - auto databaseHolder = DatabaseHolder::get(opCtx); - databaseHolder->openDb(opCtx, NamespaceString::kSystemReplSetNamespace.db()); - - AutoGetCollectionForRead autoCollection(opCtx, NamespaceString::kSystemReplSetNamespace); - Collection* collection = autoCollection.getCollection(); + invariant(opCtx->lockState()->isW()); + Collection* collection = CollectionCatalog::get(opCtx).lookupCollectionByNamespace( + NamespaceString::kSystemReplSetNamespace); if (collection && collection->numRecords(opCtx) > 0) { setReplSetMemberInStandaloneMode(opCtx->getServiceContext(), true); return; @@ -394,19 +391,13 @@ bool repairDatabasesAndCheckVersion(OperationContext* opCtx) { invariant(dbNames.front() == NamespaceString::kLocalDb); } - stdx::function<void(const std::string& dbName)> onRecordStoreRepair = - [opCtx](const std::string& dbName) { - if (dbName == NamespaceString::kLocalDb) { - setReplSetMemberInStandaloneMode(opCtx); - } - }; - for (const auto& dbName : dbNames) { LOG(1) << " Repairing database: " << dbName; - fassertNoTrace(18506, - repairDatabase(opCtx, storageEngine, dbName, onRecordStoreRepair)); + fassertNoTrace(18506, repairDatabase(opCtx, storageEngine, dbName)); } + setReplSetMemberInStandaloneMode(opCtx); + // All collections must have UUIDs before restoring the FCV document to a version that // requires UUIDs. uassertStatusOK(ensureCollectionProperties(opCtx, dbNames)); diff --git a/src/mongo/db/service_context_d_test_fixture.cpp b/src/mongo/db/service_context_d_test_fixture.cpp index 38a3e824620..873e558082e 100644 --- a/src/mongo/db/service_context_d_test_fixture.cpp +++ b/src/mongo/db/service_context_d_test_fixture.cpp @@ -36,6 +36,7 @@ #include "mongo/base/checked_cast.h" #include "mongo/db/catalog/catalog_control.h" #include "mongo/db/catalog/collection_catalog.h" +#include "mongo/db/catalog/collection_impl.h" #include "mongo/db/catalog/database_holder_impl.h" #include "mongo/db/concurrency/d_concurrency.h" #include "mongo/db/index/index_access_method_factory_impl.h" @@ -81,7 +82,7 @@ ServiceContextMongoDTest::ServiceContextMongoDTest(std::string engine, RepairAct DatabaseHolder::set(serviceContext, std::make_unique<DatabaseHolderImpl>()); IndexAccessMethodFactory::set(serviceContext, std::make_unique<IndexAccessMethodFactoryImpl>()); - + Collection::Factory::set(serviceContext, std::make_unique<CollectionImpl::FactoryImpl>()); IndexBuildsCoordinator::set(serviceContext, std::make_unique<IndexBuildsCoordinatorMongod>()); } diff --git a/src/mongo/db/storage/kv/kv_catalog.cpp b/src/mongo/db/storage/kv/kv_catalog.cpp index d268f64394e..a102a8707d4 100644 --- a/src/mongo/db/storage/kv/kv_catalog.cpp +++ b/src/mongo/db/storage/kv/kv_catalog.cpp @@ -707,15 +707,13 @@ StatusWith<std::string> KVCatalog::newOrphanedIdent(OperationContext* opCtx, std return StatusWith<std::string>(std::move(ns)); } -void KVCatalog::initCollection(OperationContext* opCtx, - const NamespaceString& nss, - bool forRepair) { +std::unique_ptr<CollectionCatalogEntry> KVCatalog::makeCollectionCatalogEntry( + OperationContext* opCtx, const NamespaceString& nss, bool forRepair) { BSONCollectionCatalogEntry::MetaData md = getMetaData(opCtx, nss); uassert(ErrorCodes::MustDowngrade, str::stream() << "Collection does not have UUID in KVCatalog. Collection: " << nss, md.options.uuid); - auto uuid = md.options.uuid.get(); auto ident = getCollectionIdent(nss); std::unique_ptr<RecordStore> rs; @@ -729,16 +727,8 @@ void KVCatalog::initCollection(OperationContext* opCtx, invariant(rs); } - CollectionCatalog::get(getGlobalServiceContext()) - .registerCatalogEntry(uuid, - std::make_unique<KVCollectionCatalogEntry>( - _engine, this, nss.ns(), ident, std::move(rs))); -} - -void KVCatalog::reinitCollectionAfterRepair(OperationContext* opCtx, const NamespaceString& nss) { - auto& catalog = CollectionCatalog::get(getGlobalServiceContext()); - catalog.deregisterCatalogEntry(catalog.lookupUUIDByNSS(nss).get()); - initCollection(opCtx, nss, false); + return std::make_unique<KVCollectionCatalogEntry>( + _engine, this, nss.ns(), ident, std::move(rs)); } Status KVCatalog::createCollection(OperationContext* opCtx, diff --git a/src/mongo/db/storage/kv/kv_catalog.h b/src/mongo/db/storage/kv/kv_catalog.h index 23b5c3cb561..b22b6cf68ff 100644 --- a/src/mongo/db/storage/kv/kv_catalog.h +++ b/src/mongo/db/storage/kv/kv_catalog.h @@ -110,9 +110,9 @@ public: */ std::string newInternalIdent(); - void initCollection(OperationContext* opCtx, const NamespaceString& nss, bool forRepair); - - void reinitCollectionAfterRepair(OperationContext* opCtx, const NamespaceString& nss); + std::unique_ptr<CollectionCatalogEntry> makeCollectionCatalogEntry(OperationContext* opCtx, + const NamespaceString& nss, + bool forRepair); Status createCollection(OperationContext* opCtx, const NamespaceString& nss, diff --git a/src/mongo/db/storage/kv/kv_storage_engine.cpp b/src/mongo/db/storage/kv/kv_storage_engine.cpp index ae7a10f561c..d295631076b 100644 --- a/src/mongo/db/storage/kv/kv_storage_engine.cpp +++ b/src/mongo/db/storage/kv/kv_storage_engine.cpp @@ -184,8 +184,7 @@ void KVStorageEngine::loadCatalog(OperationContext* opCtx) { } KVPrefix maxSeenPrefix = KVPrefix::kNotPrefixed; - for (const auto& coll : collectionsKnownToCatalog) { - NamespaceString nss(coll); + for (const auto& nss : collectionsKnownToCatalog) { std::string dbName = nss.db().toString(); if (loadingFromUncleanShutdownOrRepair) { @@ -193,7 +192,7 @@ void KVStorageEngine::loadCatalog(OperationContext* opCtx) { // possible that there are collections in the catalog that are unknown to the storage // engine. If we can't find a table in the list of storage engine idents, either // attempt to recover the ident or drop it. - const auto collectionIdent = _catalog->getCollectionIdent(coll); + const auto collectionIdent = _catalog->getCollectionIdent(nss); bool orphan = !std::binary_search(identsKnownToStorageEngine.begin(), identsKnownToStorageEngine.end(), collectionIdent); @@ -203,14 +202,14 @@ void KVStorageEngine::loadCatalog(OperationContext* opCtx) { if (orphan) { auto status = _recoverOrphanedCollection(opCtx, nss, collectionIdent); if (!status.isOK()) { - warning() << "Failed to recover orphaned data file for collection '" << coll + warning() << "Failed to recover orphaned data file for collection '" << nss << "': " << status; WriteUnitOfWork wuow(opCtx); - fassert(50716, _catalog->_removeEntry(opCtx, coll)); + fassert(50716, _catalog->_removeEntry(opCtx, nss)); if (_options.forRepair) { StorageRepairObserver::get(getGlobalServiceContext()) - ->onModification(str::stream() << "Collection " << coll << " dropped: " + ->onModification(str::stream() << "Collection " << nss << " dropped: " << status.reason()); } wuow.commit(); @@ -219,8 +218,8 @@ void KVStorageEngine::loadCatalog(OperationContext* opCtx) { } } - _catalog->initCollection(opCtx, coll, _options.forRepair); - auto maxPrefixForCollection = _catalog->getMetaData(opCtx, coll).getMaxPrefix(); + _initCollection(opCtx, nss, _options.forRepair); + auto maxPrefixForCollection = _catalog->getMetaData(opCtx, nss).getMaxPrefix(); maxSeenPrefix = std::max(maxSeenPrefix, maxPrefixForCollection); if (nss.isOrphanCollection()) { @@ -236,6 +235,20 @@ void KVStorageEngine::loadCatalog(OperationContext* opCtx) { startingAfterUncleanShutdown(getGlobalServiceContext()) = false; } +void KVStorageEngine::_initCollection(OperationContext* opCtx, + const NamespaceString& nss, + bool forRepair) { + auto catalogEntry = _catalog->makeCollectionCatalogEntry(opCtx, nss, forRepair); + auto uuid = catalogEntry->getCollectionOptions(opCtx).uuid.get(); + + auto collectionFactory = Collection::Factory::get(getGlobalServiceContext()); + auto collection = collectionFactory->make(opCtx, catalogEntry.get()); + + auto& collectionCatalog = CollectionCatalog::get(getGlobalServiceContext()); + collectionCatalog.registerCatalogEntry(uuid, std::move(catalogEntry)); + collectionCatalog.registerCollectionObject(uuid, std::move(collection)); +} + void KVStorageEngine::closeCatalog(OperationContext* opCtx) { dassert(opCtx->lockState()->isLocked()); if (shouldLog(::mongo::logger::LogComponent::kStorageRecovery, kCatalogLogLevel)) { @@ -651,8 +664,19 @@ Status KVStorageEngine::repairRecordStore(OperationContext* opCtx, const Namespa repairObserver->onModification(str::stream() << "Collection " << nss << ": " << status.reason()); } - _catalog->reinitCollectionAfterRepair(opCtx, nss); + auto& collectionCatalog = CollectionCatalog::get(getGlobalServiceContext()); + auto uuid = collectionCatalog.lookupUUIDByNSS(nss).get(); + + // It's possible the Collection may not already have been removed if the no DatabaseHolder was + // opened for a database. + if (collectionCatalog.lookupCollectionByUUID(uuid)) { + collectionCatalog.deregisterCollectionObject(uuid); + } + collectionCatalog.deregisterCatalogEntry(uuid); + + // After repairing, initialize the collection with a valid RecordStore. + _initCollection(opCtx, nss, false); return Status::OK(); } diff --git a/src/mongo/db/storage/kv/kv_storage_engine.h b/src/mongo/db/storage/kv/kv_storage_engine.h index 70c4923b80a..263c5cdde9e 100644 --- a/src/mongo/db/storage/kv/kv_storage_engine.h +++ b/src/mongo/db/storage/kv/kv_storage_engine.h @@ -348,6 +348,8 @@ public: private: using CollIter = std::list<std::string>::iterator; + void _initCollection(OperationContext* opCtx, const NamespaceString& nss, bool forRepair); + Status _dropCollectionsNoTimestamp(OperationContext* opCtx, std::vector<NamespaceString>& toDrop); diff --git a/src/mongo/dbtests/framework.cpp b/src/mongo/dbtests/framework.cpp index c39c01ddc3b..69760dd428e 100644 --- a/src/mongo/dbtests/framework.cpp +++ b/src/mongo/dbtests/framework.cpp @@ -38,6 +38,7 @@ #include "mongo/base/checked_cast.h" #include "mongo/base/status.h" #include "mongo/db/catalog/collection_catalog.h" +#include "mongo/db/catalog/collection_impl.h" #include "mongo/db/catalog/database_holder_impl.h" #include "mongo/db/client.h" #include "mongo/db/concurrency/lock_state.h" @@ -109,6 +110,7 @@ int runDbTests(int argc, char** argv) { DatabaseHolder::set(globalServiceContext, std::make_unique<DatabaseHolderImpl>()); IndexAccessMethodFactory::set(globalServiceContext, std::make_unique<IndexAccessMethodFactoryImpl>()); + Collection::Factory::set(globalServiceContext, std::make_unique<CollectionImpl::FactoryImpl>()); IndexBuildsCoordinator::set(globalServiceContext, std::make_unique<IndexBuildsCoordinatorMongod>()); auto registry = stdx::make_unique<OpObserverRegistry>(); diff --git a/src/mongo/embedded/embedded.cpp b/src/mongo/embedded/embedded.cpp index 158885e2d8a..db941660254 100644 --- a/src/mongo/embedded/embedded.cpp +++ b/src/mongo/embedded/embedded.cpp @@ -36,6 +36,7 @@ #include "mongo/base/initializer.h" #include "mongo/config.h" #include "mongo/db/catalog/collection_catalog.h" +#include "mongo/db/catalog/collection_impl.h" #include "mongo/db/catalog/database_holder_impl.h" #include "mongo/db/catalog/health_log.h" #include "mongo/db/catalog/index_key_validate.h" @@ -103,6 +104,7 @@ MONGO_INITIALIZER_GENERAL(ForkServer, ("EndStartupOptionHandling"), ("default")) void setUpCatalog(ServiceContext* serviceContext) { DatabaseHolder::set(serviceContext, std::make_unique<DatabaseHolderImpl>()); IndexAccessMethodFactory::set(serviceContext, std::make_unique<IndexAccessMethodFactoryImpl>()); + Collection::Factory::set(serviceContext, std::make_unique<CollectionImpl::FactoryImpl>()); } // Create a minimalistic replication coordinator to provide a limited interface for users. Not |