diff options
21 files changed, 181 insertions, 4 deletions
diff --git a/src/mongo/db/storage/biggie/biggie_kv_engine.cpp b/src/mongo/db/storage/biggie/biggie_kv_engine.cpp index 6a412f24f03..3038ad06745 100644 --- a/src/mongo/db/storage/biggie/biggie_kv_engine.cpp +++ b/src/mongo/db/storage/biggie/biggie_kv_engine.cpp @@ -61,6 +61,15 @@ Status KVEngine::createRecordStore(OperationContext* opCtx, return Status::OK(); } +std::unique_ptr<mongo::RecordStore> KVEngine::makeTemporaryRecordStore(OperationContext* opCtx, + StringData ident) { + std::unique_ptr<mongo::RecordStore> recordStore = + std::make_unique<RecordStore>("", ident, false); + _idents[ident.toString()] = true; + return recordStore; +}; + + std::unique_ptr<mongo::RecordStore> KVEngine::getRecordStore(OperationContext* opCtx, StringData ns, StringData ident, diff --git a/src/mongo/db/storage/biggie/biggie_kv_engine.h b/src/mongo/db/storage/biggie/biggie_kv_engine.h index cb354a30798..b4b0adb09f2 100644 --- a/src/mongo/db/storage/biggie/biggie_kv_engine.h +++ b/src/mongo/db/storage/biggie/biggie_kv_engine.h @@ -64,6 +64,9 @@ public: StringData ident, const CollectionOptions& options); + virtual std::unique_ptr<mongo::RecordStore> makeTemporaryRecordStore(OperationContext* opCtx, + StringData ident) override; + virtual Status createSortedDataInterface(OperationContext* opCtx, StringData ident, const IndexDescriptor* desc); diff --git a/src/mongo/db/storage/devnull/devnull_kv_engine.cpp b/src/mongo/db/storage/devnull/devnull_kv_engine.cpp index 4a37c3b751b..759217faff9 100644 --- a/src/mongo/db/storage/devnull/devnull_kv_engine.cpp +++ b/src/mongo/db/storage/devnull/devnull_kv_engine.cpp @@ -258,6 +258,11 @@ std::unique_ptr<RecordStore> DevNullKVEngine::getRecordStore(OperationContext* o return stdx::make_unique<DevNullRecordStore>(ns, options); } +std::unique_ptr<RecordStore> DevNullKVEngine::makeTemporaryRecordStore(OperationContext* opCtx, + StringData ident) { + return stdx::make_unique<DevNullRecordStore>("", CollectionOptions()); +} + SortedDataInterface* DevNullKVEngine::getSortedDataInterface(OperationContext* opCtx, StringData ident, const IndexDescriptor* desc) { diff --git a/src/mongo/db/storage/devnull/devnull_kv_engine.h b/src/mongo/db/storage/devnull/devnull_kv_engine.h index a55e5d00ff7..5f9efa61fd2 100644 --- a/src/mongo/db/storage/devnull/devnull_kv_engine.h +++ b/src/mongo/db/storage/devnull/devnull_kv_engine.h @@ -64,6 +64,9 @@ public: StringData ident, const CollectionOptions& options); + virtual std::unique_ptr<RecordStore> makeTemporaryRecordStore(OperationContext* opCtx, + StringData ident) override; + virtual Status createSortedDataInterface(OperationContext* opCtx, StringData ident, const IndexDescriptor* desc) { diff --git a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_engine.cpp b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_engine.cpp index a2321724edf..aadeb5ac336 100644 --- a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_engine.cpp +++ b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_engine.cpp @@ -77,6 +77,13 @@ std::unique_ptr<RecordStore> EphemeralForTestEngine::getRecordStore( } } +std::unique_ptr<RecordStore> EphemeralForTestEngine::makeTemporaryRecordStore( + OperationContext* opCtx, StringData ident) { + stdx::lock_guard<stdx::mutex> lk(_mutex); + _dataMap[ident] = {}; + return stdx::make_unique<EphemeralForTestRecordStore>(ident, &_dataMap[ident]); +} + Status EphemeralForTestEngine::createSortedDataInterface(OperationContext* opCtx, StringData ident, const IndexDescriptor* desc) { diff --git a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_engine.h b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_engine.h index b3e1886ef82..7a6df032b0b 100644 --- a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_engine.h +++ b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_engine.h @@ -57,6 +57,9 @@ public: StringData ident, const CollectionOptions& options); + virtual std::unique_ptr<RecordStore> makeTemporaryRecordStore(OperationContext* opCtx, + StringData ident) override; + virtual Status createSortedDataInterface(OperationContext* opCtx, StringData ident, const IndexDescriptor* desc); diff --git a/src/mongo/db/storage/kv/kv_catalog.cpp b/src/mongo/db/storage/kv/kv_catalog.cpp index 7d827d48027..28736f8598e 100644 --- a/src/mongo/db/storage/kv/kv_catalog.cpp +++ b/src/mongo/db/storage/kv/kv_catalog.cpp @@ -340,6 +340,13 @@ bool KVCatalog::_hasEntryCollidingWithRand() const { return false; } +std::string KVCatalog::newTempIdent() { + StringBuilder buf; + buf << "temp-"; + buf << _next.fetchAndAdd(1) << '-' << _rand; + return buf.str(); +} + std::string KVCatalog::_newUniqueIdent(StringData ns, const char* kind) { // If this changes to not put _rand at the end, _hasEntryCollidingWithRand will need fixing. StringBuilder buf; @@ -621,12 +628,17 @@ std::vector<std::string> KVCatalog::getAllIdents(OperationContext* opCtx) const } bool KVCatalog::isUserDataIdent(StringData ident) const { + // Indexes, collections, and temporary RecordStore idents are all candidates for dropping when + // the storage engine's metadata does not align with the catalog metadata. return ident.find("index-") != std::string::npos || ident.find("index/") != std::string::npos || + ident.find("temp-") != std::string::npos || ident.find("collection-") != std::string::npos || ident.find("collection/") != std::string::npos; } bool KVCatalog::isCollectionIdent(StringData ident) const { + // Temporary RecordStores idents prefixed "temp-" should not be considered collections, because + // they are not eligible for orphan recovery. return ident.find("collection-") != std::string::npos || ident.find("collection/") != std::string::npos; } diff --git a/src/mongo/db/storage/kv/kv_catalog.h b/src/mongo/db/storage/kv/kv_catalog.h index 2bf2af72831..bc99e4e1e7c 100644 --- a/src/mongo/db/storage/kv/kv_catalog.h +++ b/src/mongo/db/storage/kv/kv_catalog.h @@ -113,6 +113,11 @@ public: */ StatusWith<std::string> newOrphanedIdent(OperationContext* opCtx, std::string ident); + /** + * Generate a temporary ident name. + */ + std::string newTempIdent(); + private: class AddIdentChange; class RemoveIdentChange; diff --git a/src/mongo/db/storage/kv/kv_engine.h b/src/mongo/db/storage/kv/kv_engine.h index 335046082fa..c3388aed0b5 100644 --- a/src/mongo/db/storage/kv/kv_engine.h +++ b/src/mongo/db/storage/kv/kv_engine.h @@ -122,6 +122,9 @@ public: StringData ident, const CollectionOptions& options) = 0; + virtual std::unique_ptr<RecordStore> makeTemporaryRecordStore(OperationContext* opCtx, + StringData ident) = 0; + /** * Create a RecordStore that MongoDB considers eligible to share space in an underlying table * with other RecordStores. 'prefix' is guaranteed to be 'KVPrefix::kNotPrefixed' when diff --git a/src/mongo/db/storage/kv/kv_engine_test_harness.cpp b/src/mongo/db/storage/kv/kv_engine_test_harness.cpp index 0ba7597fded..0bbc7781817 100644 --- a/src/mongo/db/storage/kv/kv_engine_test_harness.cpp +++ b/src/mongo/db/storage/kv/kv_engine_test_harness.cpp @@ -172,6 +172,43 @@ TEST(KVEngineTestHarness, SimpleSorted1) { } } +TEST(KVEngineTestHarness, TemporaryRecordStoreSimple) { + unique_ptr<KVHarnessHelper> helper(KVHarnessHelper::create()); + KVEngine* engine = helper->getEngine(); + ASSERT(engine); + + string ident = "temptemp"; + unique_ptr<RecordStore> rs; + { + MyOperationContext opCtx(engine); + rs = engine->makeTemporaryRecordStore(&opCtx, ident); + ASSERT(rs); + } + + RecordId loc; + { + MyOperationContext opCtx(engine); + WriteUnitOfWork uow(&opCtx); + StatusWith<RecordId> res = rs->insertRecord(&opCtx, "abc", 4, Timestamp()); + ASSERT_OK(res.getStatus()); + loc = res.getValue(); + uow.commit(); + } + + { + MyOperationContext opCtx(engine); + ASSERT_EQUALS(string("abc"), rs->dataFor(&opCtx, loc).data()); + + std::vector<std::string> all = engine->getAllIdents(&opCtx); + ASSERT_EQUALS(1U, all.size()); + ASSERT_EQUALS(ident, all[0]); + + WriteUnitOfWork wuow(&opCtx); + ASSERT_OK(engine->dropIdent(&opCtx, ident)); + wuow.commit(); + } +} + TEST(KVCatalogTest, Coll1) { unique_ptr<KVHarnessHelper> helper(KVHarnessHelper::create()); KVEngine* engine = helper->getEngine(); @@ -217,7 +254,6 @@ TEST(KVCatalogTest, Coll1) { ASSERT_NOT_EQUALS(ident, catalog->getCollectionIdent("a.b")); } - TEST(KVCatalogTest, Idx1) { unique_ptr<KVHarnessHelper> helper(KVHarnessHelper::create()); KVEngine* engine = helper->getEngine(); diff --git a/src/mongo/db/storage/kv/kv_storage_engine.cpp b/src/mongo/db/storage/kv/kv_storage_engine.cpp index c84fde79f43..7dc534c2942 100644 --- a/src/mongo/db/storage/kv/kv_storage_engine.cpp +++ b/src/mongo/db/storage/kv/kv_storage_engine.cpp @@ -627,6 +627,10 @@ Status KVStorageEngine::repairRecordStore(OperationContext* opCtx, const std::st return Status::OK(); } +std::unique_ptr<RecordStore> KVStorageEngine::makeTemporaryRecordStore(OperationContext* opCtx) { + return _engine->makeTemporaryRecordStore(opCtx, _catalog->newTempIdent()); +} + void KVStorageEngine::setJournalListener(JournalListener* jl) { _engine->setJournalListener(jl); } diff --git a/src/mongo/db/storage/kv/kv_storage_engine.h b/src/mongo/db/storage/kv/kv_storage_engine.h index 141546f3bae..b148f29b105 100644 --- a/src/mongo/db/storage/kv/kv_storage_engine.h +++ b/src/mongo/db/storage/kv/kv_storage_engine.h @@ -122,6 +122,8 @@ public: virtual Status repairRecordStore(OperationContext* opCtx, const std::string& ns); + virtual std::unique_ptr<RecordStore> makeTemporaryRecordStore(OperationContext* opCtx) override; + virtual void cleanShutdown(); virtual void setStableTimestamp(Timestamp stableTimestamp, diff --git a/src/mongo/db/storage/kv/kv_storage_engine_test.cpp b/src/mongo/db/storage/kv/kv_storage_engine_test.cpp index f68e355eba5..68c7098e689 100644 --- a/src/mongo/db/storage/kv/kv_storage_engine_test.cpp +++ b/src/mongo/db/storage/kv/kv_storage_engine_test.cpp @@ -76,6 +76,9 @@ public: return _storageEngine->getCatalog()->getCollectionIdent(ns.ns()); } + std::unique_ptr<RecordStore> makeTemporary(OperationContext* opCtx) { + return _storageEngine->makeTemporaryRecordStore(opCtx); + } /** * Create a collection table in the KVEngine not reflected in the KVCatalog. @@ -223,6 +226,21 @@ TEST_F(KVStorageEngineTest, LoadCatalogDropsOrphansAfterUncleanShutdown) { ASSERT(!collectionExists(opCtx.get(), collNs)); } +TEST_F(KVStorageEngineTest, ReconcileDropsTemporary) { + auto opCtx = cc().makeOperationContext(); + + auto rs = makeTemporary(opCtx.get()); + ASSERT(rs); + const std::string ident = rs->getIdent(); + + ASSERT(identExists(opCtx.get(), ident)); + + ASSERT_OK(reconcile(opCtx.get()).getStatus()); + + // The storage engine is responsible for dropping its temporary idents. + ASSERT(!identExists(opCtx.get(), ident)); +} + TEST_F(KVStorageEngineRepairTest, LoadCatalogRecoversOrphans) { auto opCtx = cc().makeOperationContext(); diff --git a/src/mongo/db/storage/mobile/mobile_kv_engine.cpp b/src/mongo/db/storage/mobile/mobile_kv_engine.cpp index 66d16c70d71..11670750a97 100644 --- a/src/mongo/db/storage/mobile/mobile_kv_engine.cpp +++ b/src/mongo/db/storage/mobile/mobile_kv_engine.cpp @@ -207,6 +207,14 @@ std::unique_ptr<RecordStore> MobileKVEngine::getRecordStore(OperationContext* op return stdx::make_unique<MobileRecordStore>(opCtx, ns, _path, ident.toString(), options); } +std::unique_ptr<RecordStore> MobileKVEngine::makeTemporaryRecordStore(OperationContext* opCtx, + StringData ident) { + MobileRecordStore::create(opCtx, ident.toString()); + return std::make_unique<MobileRecordStore>( + opCtx, "", _path, ident.toString(), CollectionOptions()); +} + + Status MobileKVEngine::createSortedDataInterface(OperationContext* opCtx, StringData ident, const IndexDescriptor* desc) { diff --git a/src/mongo/db/storage/mobile/mobile_kv_engine.h b/src/mongo/db/storage/mobile/mobile_kv_engine.h index 28b63301d10..a644d6db4b0 100644 --- a/src/mongo/db/storage/mobile/mobile_kv_engine.h +++ b/src/mongo/db/storage/mobile/mobile_kv_engine.h @@ -58,6 +58,9 @@ public: StringData ident, const CollectionOptions& options) override; + std::unique_ptr<RecordStore> makeTemporaryRecordStore(OperationContext* opCtx, + StringData ident) override; + Status createSortedDataInterface(OperationContext* opCtx, StringData ident, const IndexDescriptor* desc) override; diff --git a/src/mongo/db/storage/storage_engine.h b/src/mongo/db/storage/storage_engine.h index 795ee9f6397..456972b0997 100644 --- a/src/mongo/db/storage/storage_engine.h +++ b/src/mongo/db/storage/storage_engine.h @@ -45,6 +45,7 @@ namespace mongo { class DatabaseCatalogEntry; class JournalListener; class OperationContext; +class RecordStore; class RecoveryUnit; class SnapshotManager; struct StorageGlobalParams; @@ -287,6 +288,13 @@ public: virtual Status repairRecordStore(OperationContext* opCtx, const std::string& ns) = 0; /** + * Creates a temporary RecordStore on the storage engine. This record store should be dropped by + * the caller when done being used. The storage engine will drop any remaining temporary record + * stores on startup. + */ + virtual std::unique_ptr<RecordStore> makeTemporaryRecordStore(OperationContext* opCtx) = 0; + + /** * This method will be called before there is a clean shutdown. Storage engines should * override this method if they have clean-up to do that is different from unclean shutdown. * MongoDB will not call into the storage subsystem after calling this function. diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp index a8187e89699..0ee4e2c2781 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp @@ -1124,6 +1124,47 @@ SortedDataInterface* WiredTigerKVEngine::getGroupedSortedDataInterface(Operation return new WiredTigerIndexStandard(opCtx, _uri(ident), desc, prefix, _readOnly); } +std::unique_ptr<RecordStore> WiredTigerKVEngine::makeTemporaryRecordStore(OperationContext* opCtx, + StringData ident) { + invariant(!_readOnly); + + _ensureIdentPath(ident); + WiredTigerSession wtSession(_conn); + + CollectionOptions noOptions; + StatusWith<std::string> swConfig = WiredTigerRecordStore::generateCreateString( + _canonicalName, "" /* internal table */, noOptions, _rsOptions, false /* prefixed */); + uassertStatusOK(swConfig.getStatus()); + + std::string config = swConfig.getValue(); + + std::string uri = _uri(ident); + WT_SESSION* session = wtSession.getSession(); + LOG(2) << "WiredTigerKVEngine::createTemporaryRecordStore uri: " << uri + << " config: " << config; + uassertStatusOK(wtRCToStatus(session->create(session, uri.c_str(), config.c_str()))); + + WiredTigerRecordStore::Params params; + params.ns = ""; + params.uri = _uri(ident); + params.engineName = _canonicalName; + params.isCapped = false; + params.isEphemeral = _ephemeral; + params.cappedCallback = nullptr; + // Temporary collections do not need to persist size information to the size storer. + params.sizeStorer = nullptr; + params.isReadOnly = false; + + params.cappedMaxSize = -1; + params.cappedMaxDocs = -1; + + std::unique_ptr<WiredTigerRecordStore> rs; + rs = stdx::make_unique<StandardWiredTigerRecordStore>(this, opCtx, params); + rs->postConstructorInit(opCtx); + + return std::move(rs); +} + void WiredTigerKVEngine::alterIdentMetadata(OperationContext* opCtx, StringData ident, const IndexDescriptor* desc) { diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h index c55e7ca7a2f..95f9cda47aa 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h @@ -111,6 +111,9 @@ public: return getGroupedRecordStore(opCtx, ns, ident, options, KVPrefix::kNotPrefixed); } + virtual std::unique_ptr<RecordStore> makeTemporaryRecordStore(OperationContext* opCtx, + StringData ident) override; + virtual Status createSortedDataInterface(OperationContext* opCtx, StringData ident, const IndexDescriptor* desc) override { diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_prefixed_record_store_test.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_prefixed_record_store_test.cpp index 4f6c96f00c1..eb5077d48f8 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_prefixed_record_store_test.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_prefixed_record_store_test.cpp @@ -103,7 +103,7 @@ public: const bool prefixed = true; StatusWith<std::string> result = WiredTigerRecordStore::generateCreateString( - kWiredTigerEngineName, ns, CollectionOptions(), "", prefixed); + kWiredTigerEngineName, StringData(ns), CollectionOptions(), "", prefixed); ASSERT_TRUE(result.isOK()); std::string config = result.getValue(); @@ -149,7 +149,7 @@ public: KVPrefix prefix = KVPrefix::generateNextPrefix(); StatusWith<std::string> result = WiredTigerRecordStore::generateCreateString( - kWiredTigerEngineName, ns, options, "", prefix.isPrefixed()); + kWiredTigerEngineName, StringData(ns), options, "", prefix.isPrefixed()); ASSERT_TRUE(result.isOK()); std::string config = result.getValue(); diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp index 7efaefe2040..135d904f5ce 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp @@ -622,7 +622,10 @@ StatusWith<std::string> WiredTigerRecordStore::generateCreateString( bool replicatedWrites = getGlobalReplSettings().usingReplSets() || repl::ReplSettings::shouldRecoverFromOplogAsStandalone(); - if (WiredTigerUtil::useTableLogging(NamespaceString(ns), replicatedWrites)) { + + // Do not journal writes when 'ns' is an empty string, which is the case for internal-only + // temporary tables. + if (ns.size() && WiredTigerUtil::useTableLogging(NamespaceString(ns), replicatedWrites)) { ss << ",log=(enabled=true)"; } else { ss << ",log=(enabled=false)"; diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.h b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.h index f450193fe0c..019d7e2331c 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.h +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.h @@ -88,6 +88,7 @@ public: /** * Creates a configuration string suitable for 'config' parameter in WT_SESSION::create(). + * It is possible for 'ns' to be an empty string, in the case of internal-only temporary tables. * Configuration string is constructed from: * built-in defaults * storageEngine.wiredTiger.configString in 'options' |