diff options
Diffstat (limited to 'src/mongo/db')
-rw-r--r-- | src/mongo/db/catalog/SConscript | 5 | ||||
-rw-r--r-- | src/mongo/db/catalog/catalog_control.cpp | 164 | ||||
-rw-r--r-- | src/mongo/db/catalog/catalog_control.h | 48 | ||||
-rw-r--r-- | src/mongo/db/catalog/database_holder_impl.cpp | 18 | ||||
-rw-r--r-- | src/mongo/db/commands/SConscript | 2 | ||||
-rw-r--r-- | src/mongo/db/commands/restart_catalog_command.cpp | 106 | ||||
-rw-r--r-- | src/mongo/db/repl/oplog.cpp | 6 | ||||
-rw-r--r-- | src/mongo/db/repl/oplog.h | 8 | ||||
-rw-r--r-- | src/mongo/db/storage/kv/kv_storage_engine.cpp | 39 | ||||
-rw-r--r-- | src/mongo/db/storage/kv/kv_storage_engine.h | 4 | ||||
-rw-r--r-- | src/mongo/db/storage/storage_engine.h | 9 |
11 files changed, 391 insertions, 18 deletions
diff --git a/src/mongo/db/catalog/SConscript b/src/mongo/db/catalog/SConscript index 0a66e9ba264..a6f6b3ce69f 100644 --- a/src/mongo/db/catalog/SConscript +++ b/src/mongo/db/catalog/SConscript @@ -221,13 +221,14 @@ env.CppUnitTest( env.Library( target='catalog_impl', source=[ + "catalog_control.cpp", "collection_compact.cpp", "collection_impl.cpp", "collection_info_cache_impl.cpp", - "database_impl.cpp", "database_holder_impl.cpp", - "index_catalog_impl.cpp", + "database_impl.cpp", "index_catalog_entry_impl.cpp", + "index_catalog_impl.cpp", "index_consistency.cpp", "index_create_impl.cpp", "private/record_store_validate_adaptor.cpp", diff --git a/src/mongo/db/catalog/catalog_control.cpp b/src/mongo/db/catalog/catalog_control.cpp new file mode 100644 index 00000000000..a9ca774c9a9 --- /dev/null +++ b/src/mongo/db/catalog/catalog_control.cpp @@ -0,0 +1,164 @@ +/** + * Copyright (C) 2018 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ +#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kStorage + +#include "mongo/platform/basic.h" + +#include "mongo/db/catalog/catalog_control.h" + +#include "mongo/db/catalog/collection.h" +#include "mongo/db/catalog/database.h" +#include "mongo/db/catalog/database_catalog_entry.h" +#include "mongo/db/catalog/database_holder.h" +#include "mongo/db/catalog/uuid_catalog.h" +#include "mongo/db/ftdc/ftdc_mongod.h" +#include "mongo/db/namespace_string.h" +#include "mongo/db/repair_database.h" +#include "mongo/util/log.h" + +namespace mongo { +namespace catalog { +void closeCatalog(OperationContext* opCtx) { + invariant(opCtx->lockState()->isW()); + + // Close all databases. + log() << "closeCatalog: closing all databases in dbholder"; + BSONObjBuilder closeDbsBuilder; + constexpr auto force = true; + constexpr auto reason = "closing databases for closeCatalog"; + uassert(40687, + str::stream() << "failed to close all databases; result of operation: " + << closeDbsBuilder.obj().jsonString(), + dbHolder().closeAll(opCtx, closeDbsBuilder, force, reason)); + + // Because we've force-closed the database, there should be no databases left open. + auto closeDbsResult = closeDbsBuilder.obj(); + invariant( + !closeDbsResult.hasField("nNotClosed"), + str::stream() << "expected no databases open after a force close; result of operation: " + << closeDbsResult.jsonString()); + + // Close the storage engine's catalog. + log() << "closeCatalog: closing storage engine catalog"; + opCtx->getServiceContext()->getGlobalStorageEngine()->closeCatalog(opCtx); +} + +void openCatalog(OperationContext* opCtx) { + invariant(opCtx->lockState()->isW()); + + // Load the catalog in the storage engine. + log() << "openCatalog: loading storage engine catalog"; + auto storageEngine = opCtx->getServiceContext()->getGlobalStorageEngine(); + storageEngine->loadCatalog(opCtx); + + log() << "openCatalog: reconciling catalog and idents"; + auto indexesToRebuild = storageEngine->reconcileCatalogAndIdents(opCtx); + fassertStatusOK(40688, indexesToRebuild.getStatus()); + + // Rebuild indexes if necessary. + for (auto indexNamespace : indexesToRebuild.getValue()) { + NamespaceString collNss(indexNamespace.first); + auto indexName = indexNamespace.second; + + auto dbCatalogEntry = storageEngine->getDatabaseCatalogEntry(opCtx, collNss.db()); + invariant(dbCatalogEntry, + str::stream() << "couldn't get database catalog entry for database " + << collNss.db()); + auto collCatalogEntry = dbCatalogEntry->getCollectionCatalogEntry(collNss.toString()); + invariant(collCatalogEntry, + str::stream() << "couldn't get collection catalog entry for collection " + << collNss.toString()); + + auto indexSpecs = getIndexNameObjs( + opCtx, dbCatalogEntry, collCatalogEntry, [&indexName](const std::string& name) { + return name == indexName; + }); + if (!indexSpecs.isOK() || indexSpecs.getValue().first.empty()) { + fassertStatusOK(40689, + {ErrorCodes::InternalError, + str::stream() << "failed to get index spec for index " << indexName + << " in collection " + << collNss.toString()}); + } + auto indexesToRebuild = indexSpecs.getValue(); + invariant( + indexesToRebuild.first.size() == 1, + str::stream() << "expected to find a list containing exactly 1 index name, but found " + << indexesToRebuild.first.size()); + invariant( + indexesToRebuild.second.size() == 1, + str::stream() << "expected to find a list containing exactly 1 index spec, but found " + << indexesToRebuild.second.size()); + + log() << "openCatalog: rebuilding index " << indexName << " in collection " + << collNss.toString(); + fassertStatusOK(40690, + rebuildIndexesOnCollection( + opCtx, dbCatalogEntry, collCatalogEntry, std::move(indexesToRebuild))); + } + + // Open all databases and repopulate the UUID catalog. + log() << "openCatalog: reopening all databases"; + auto& uuidCatalog = UUIDCatalog::get(opCtx); + std::vector<std::string> databasesToOpen; + storageEngine->listDatabases(&databasesToOpen); + for (auto&& dbName : databasesToOpen) { + LOG(1) << "openCatalog: dbholder reopening database " << dbName; + auto db = dbHolder().openDb(opCtx, dbName); + invariant(db, str::stream() << "failed to reopen database " << dbName); + + std::list<std::string> collections; + db->getDatabaseCatalogEntry()->getCollectionNamespaces(&collections); + for (auto&& collName : collections) { + // Note that the collection name already includes the database component. + NamespaceString collNss(collName); + auto collection = db->getCollection(opCtx, collName); + invariant(collection, + str::stream() << "failed to get valid collection pointer for namespace " + << collName); + + auto uuid = collection->uuid(); + // TODO (SERVER-32597): When the minimum featureCompatibilityVersion becomes 3.6, we + // can change this condition to be an invariant. + if (uuid) { + LOG(1) << "openCatalog: registering uuid " << uuid->toString() << " for collection " + << collName; + uuidCatalog.registerUUIDCatalogEntry(*uuid, collection); + } + + // If this is the oplog collection, re-establish the replication system's cached pointer + // to the oplog. + if (collNss.isOplog()) { + log() << "openCatalog: updating cached oplog pointer"; + repl::establishOplogCollectionForLogging(opCtx, collection); + } + } + } +} +} // namespace catalog +} // namespace mongo diff --git a/src/mongo/db/catalog/catalog_control.h b/src/mongo/db/catalog/catalog_control.h new file mode 100644 index 00000000000..85eff55ec25 --- /dev/null +++ b/src/mongo/db/catalog/catalog_control.h @@ -0,0 +1,48 @@ +/** + * Copyright (C) 2018 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/db/operation_context.h" + +namespace mongo { +namespace catalog { +/** + * Closes the catalog, destroying all associated in-memory data structures for all databases. After + * a call to this function, it is illegal to access the catalog before calling openCatalog(). + * + * Must be called with the global lock acquired in exclusive mode. + */ +void closeCatalog(OperationContext* opCtx); + +/** + * Restores the catalog and all in-memory state after a call to closeCatalog(). + * + * Must be called with the global lock acquired in exclusive mode. + */ +void openCatalog(OperationContext* opCtx); +} // namespace catalog +} // namespace mongo diff --git a/src/mongo/db/catalog/database_holder_impl.cpp b/src/mongo/db/catalog/database_holder_impl.cpp index af081c9d594..62f424e8b2a 100644 --- a/src/mongo/db/catalog/database_holder_impl.cpp +++ b/src/mongo/db/catalog/database_holder_impl.cpp @@ -42,6 +42,7 @@ #include "mongo/db/client.h" #include "mongo/db/clientcursor.h" #include "mongo/db/operation_context.h" +#include "mongo/db/repl/oplog.h" #include "mongo/db/service_context.h" #include "mongo/db/storage/storage_engine.h" #include "mongo/util/log.h" @@ -183,6 +184,15 @@ Database* DatabaseHolderImpl::openDb(OperationContext* opCtx, StringData ns, boo return it->second; } +namespace { +void evictDatabaseFromUUIDCatalog(OperationContext* opCtx, Database* db) { + UUIDCatalog::get(opCtx).onCloseDatabase(db); + for (auto&& coll : *db) { + NamespaceUUIDCache::get(opCtx).evictNamespace(coll->ns()); + } +} +} // namespace + void DatabaseHolderImpl::close(OperationContext* opCtx, StringData ns, const std::string& reason) { invariant(opCtx->lockState()->isW()); @@ -196,10 +206,8 @@ void DatabaseHolderImpl::close(OperationContext* opCtx, StringData ns, const std } auto db = it->second; - UUIDCatalog::get(opCtx).onCloseDatabase(db); - for (auto&& coll : *db) { - NamespaceUUIDCache::get(opCtx).evictNamespace(coll->ns()); - } + repl::oplogCheckCloseDatabase(opCtx, db); + evictDatabaseFromUUIDCatalog(opCtx, db); db->close(opCtx, reason); delete db; @@ -241,6 +249,8 @@ bool DatabaseHolderImpl::closeAll(OperationContext* opCtx, } Database* db = _dbs[name]; + repl::oplogCheckCloseDatabase(opCtx, db); + evictDatabaseFromUUIDCatalog(opCtx, db); db->close(opCtx, reason); delete db; diff --git a/src/mongo/db/commands/SConscript b/src/mongo/db/commands/SConscript index 2bf312a71a5..36107dce430 100644 --- a/src/mongo/db/commands/SConscript +++ b/src/mongo/db/commands/SConscript @@ -191,6 +191,7 @@ env.Library( "rename_collection_cmd.cpp", "repair_cursor.cpp", "resize_oplog.cpp", + "restart_catalog_command.cpp", "run_aggregate.cpp", "set_feature_compatibility_version_command.cpp", "snapshot_management.cpp", @@ -207,6 +208,7 @@ env.Library( '$BUILD_DIR/mongo/db/auth/authmongod', '$BUILD_DIR/mongo/db/background', '$BUILD_DIR/mongo/db/catalog/catalog_helpers', + '$BUILD_DIR/mongo/db/catalog/catalog_impl', '$BUILD_DIR/mongo/db/catalog/collection', '$BUILD_DIR/mongo/db/catalog/index_key_validate', '$BUILD_DIR/mongo/db/cloner', diff --git a/src/mongo/db/commands/restart_catalog_command.cpp b/src/mongo/db/commands/restart_catalog_command.cpp new file mode 100644 index 00000000000..6fb51d4fe56 --- /dev/null +++ b/src/mongo/db/commands/restart_catalog_command.cpp @@ -0,0 +1,106 @@ +/** + * Copyright (C) 2018 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kCommand + +#include "mongo/platform/basic.h" + +#include <string> +#include <vector> + +#include "mongo/db/catalog/catalog_control.h" +#include "mongo/db/commands.h" +#include "mongo/db/concurrency/d_concurrency.h" +#include "mongo/util/log.h" + +namespace mongo { +/** + * This testing-only command causes the server to close and reopen the catalog, rebuilding all + * in-memory data structures. + */ +class RestartCatalogCmd final : public BasicCommand { +public: + RestartCatalogCmd() : BasicCommand("restartCatalog") {} + + Status checkAuthForOperation(OperationContext* opCtx, + const std::string& dbname, + const BSONObj& cmdObj) final { + // No auth checks as this is a testing-only command. + return Status::OK(); + } + + bool adminOnly() const final { + return true; + } + + bool maintenanceMode() const final { + return true; + } + + bool maintenanceOk() const final { + return false; + } + + bool slaveOk() const final { + return true; + } + + bool supportsWriteConcern(const BSONObj& cmd) const final { + return false; + } + + std::string help() const final { + return "restart catalog\n" + "Internal command for testing only. Closes and restores the catalog, rebuilding\n" + "in-memory data structures as needed.\n"; + } + + bool run(OperationContext* opCtx, + const std::string& db, + const BSONObj& cmdObj, + BSONObjBuilder& result) final { + Lock::GlobalLock global(opCtx, MODE_X, UINT_MAX); + + log() << "Closing database catalog"; + catalog::closeCatalog(opCtx); + + log() << "Reopening database catalog"; + catalog::openCatalog(opCtx); + + return true; + } +}; + +MONGO_INITIALIZER(RegisterRestartCatalogCommand)(InitializerContext* ctx) { + if (Command::testCommandsEnabled) { + // Leaked intentionally: a Command registers itself when constructed. + new RestartCatalogCmd(); + } + return Status::OK(); +} +} // namespace mongo diff --git a/src/mongo/db/repl/oplog.cpp b/src/mongo/db/repl/oplog.cpp index 124e9c8ed78..16437a0552e 100644 --- a/src/mongo/db/repl/oplog.cpp +++ b/src/mongo/db/repl/oplog.cpp @@ -1687,6 +1687,12 @@ void acquireOplogCollectionForLogging(OperationContext* opCtx) { } } +void establishOplogCollectionForLogging(OperationContext* opCtx, Collection* oplog) { + invariant(opCtx->lockState()->isW()); + invariant(oplog); + _localOplogCollection = oplog; +} + void signalOplogWaiters() { if (_localOplogCollection) { _localOplogCollection->notifyCappedWaitersIfNeeded(); diff --git a/src/mongo/db/repl/oplog.h b/src/mongo/db/repl/oplog.h index 6a2de1c6bc4..f6ea00d0067 100644 --- a/src/mongo/db/repl/oplog.h +++ b/src/mongo/db/repl/oplog.h @@ -155,6 +155,14 @@ void oplogCheckCloseDatabase(OperationContext* opCtx, Database* db); */ void acquireOplogCollectionForLogging(OperationContext* opCtx); +/** + * Use 'oplog' as the new cached pointer to the local oplog. + * + * Called by catalog::openCatalog() to re-establish the oplog collection pointer while holding onto + * the global lock in exclusive mode. + */ +void establishOplogCollectionForLogging(OperationContext* opCtx, Collection* oplog); + using IncrementOpsAppliedStatsFn = stdx::function<void()>; /** * Take the object field of a BSONObj, the BSONObj, and the namespace of diff --git a/src/mongo/db/storage/kv/kv_storage_engine.cpp b/src/mongo/db/storage/kv/kv_storage_engine.cpp index e7f8ba7d377..436e4a47873 100644 --- a/src/mongo/db/storage/kv/kv_storage_engine.cpp +++ b/src/mongo/db/storage/kv/kv_storage_engine.cpp @@ -83,20 +83,23 @@ KVStorageEngine::KVStorageEngine( !(options.directoryPerDB && !engine->supportsDirectoryPerDB())); OperationContextNoop opCtx(_engine->newRecoveryUnit()); + loadCatalog(&opCtx); +} - bool catalogExists = engine->hasIdent(&opCtx, catalogInfo); - - if (options.forRepair && catalogExists) { +void KVStorageEngine::loadCatalog(OperationContext* opCtx) { + bool catalogExists = _engine->hasIdent(opCtx, catalogInfo); + if (_options.forRepair && catalogExists) { log() << "Repairing catalog metadata"; // TODO should also validate all BSON in the catalog. - engine->repairIdent(&opCtx, catalogInfo).transitional_ignore(); + _engine->repairIdent(opCtx, catalogInfo).transitional_ignore(); } if (!catalogExists) { - WriteUnitOfWork uow(&opCtx); + WriteUnitOfWork uow(opCtx); + + auto status = _engine->createGroupedRecordStore( + opCtx, catalogInfo, catalogInfo, CollectionOptions(), KVPrefix::kNotPrefixed); - Status status = _engine->createGroupedRecordStore( - &opCtx, catalogInfo, catalogInfo, CollectionOptions(), KVPrefix::kNotPrefixed); // BadValue is usually caused by invalid configuration string. // We still fassert() but without a stack trace. if (status.code() == ErrorCodes::BadValue) { @@ -107,10 +110,10 @@ KVStorageEngine::KVStorageEngine( } _catalogRecordStore = _engine->getGroupedRecordStore( - &opCtx, catalogInfo, catalogInfo, CollectionOptions(), KVPrefix::kNotPrefixed); + opCtx, catalogInfo, catalogInfo, CollectionOptions(), KVPrefix::kNotPrefixed); _catalog.reset(new KVCatalog( _catalogRecordStore.get(), _options.directoryPerDB, _options.directoryForIndexes)); - _catalog->init(&opCtx); + _catalog->init(opCtx); std::vector<std::string> collections; _catalog->getAllCollections(&collections); @@ -127,13 +130,25 @@ KVStorageEngine::KVStorageEngine( db = _databaseCatalogEntryFactory(dbName, this).release(); } - db->initCollection(&opCtx, coll, options.forRepair); - auto maxPrefixForCollection = _catalog->getMetaData(&opCtx, coll).getMaxPrefix(); + db->initCollection(opCtx, coll, _options.forRepair); + auto maxPrefixForCollection = _catalog->getMetaData(opCtx, coll).getMaxPrefix(); maxSeenPrefix = std::max(maxSeenPrefix, maxPrefixForCollection); } KVPrefix::setLargestPrefix(maxSeenPrefix); - opCtx.recoveryUnit()->abandonSnapshot(); + opCtx->recoveryUnit()->abandonSnapshot(); +} + +void KVStorageEngine::closeCatalog(OperationContext* opCtx) { + dassert(opCtx->lockState()->isLocked()); + stdx::lock_guard<stdx::mutex> lock(_dbsLock); + for (auto entry : _dbs) { + delete entry.second; + } + _dbs.clear(); + + _catalog.reset(nullptr); + _catalogRecordStore.reset(nullptr); } /** diff --git a/src/mongo/db/storage/kv/kv_storage_engine.h b/src/mongo/db/storage/kv/kv_storage_engine.h index c7d98cb7aad..4e08822cc73 100644 --- a/src/mongo/db/storage/kv/kv_storage_engine.h +++ b/src/mongo/db/storage/kv/kv_storage_engine.h @@ -153,6 +153,10 @@ public: StatusWith<std::vector<StorageEngine::CollectionIndexNamePair>> reconcileCatalogAndIdents( OperationContext* opCtx) override; + void loadCatalog(OperationContext* opCtx) final; + + void closeCatalog(OperationContext* opCtx) final; + private: using CollIter = std::list<std::string>::iterator; diff --git a/src/mongo/db/storage/storage_engine.h b/src/mongo/db/storage/storage_engine.h index 221f009edee..cd881198f03 100644 --- a/src/mongo/db/storage/storage_engine.h +++ b/src/mongo/db/storage/storage_engine.h @@ -203,6 +203,15 @@ public: } /** + * Populates and tears down in-memory data structures, respectively. Only required for storage + * engines that support recoverToStableTimestamp(). + * + * Must be called with the global lock acquired in exclusive mode. + */ + virtual void loadCatalog(OperationContext* opCtx) {} + virtual void closeCatalog(OperationContext* opCtx) {} + + /** * Closes all file handles associated with a database. */ virtual Status closeDatabase(OperationContext* opCtx, StringData db) = 0; |