SERVER-38187 Ability to create temporary RecordStore without a collection

21 files changed, 181 insertions, 4 deletions

diff --git a/src/mongo/db/storage/biggie/biggie_kv_engine.cpp b/src/mongo/db/storage/biggie/biggie_kv_engine.cpp
index 6a412f24f03..3038ad06745 100644
--- a/src/mongo/db/storage/biggie/biggie_kv_engine.cpp
+++ b/src/mongo/db/storage/biggie/biggie_kv_engine.cpp
@@ -61,6 +61,15 @@ Status KVEngine::createRecordStore(OperationContext* opCtx,
     return Status::OK();
 }
 
+std::unique_ptr<mongo::RecordStore> KVEngine::makeTemporaryRecordStore(OperationContext* opCtx,
+                                                                       StringData ident) {
+    std::unique_ptr<mongo::RecordStore> recordStore =
+        std::make_unique<RecordStore>("", ident, false);
+    _idents[ident.toString()] = true;
+    return recordStore;
+};
+
+
 std::unique_ptr<mongo::RecordStore> KVEngine::getRecordStore(OperationContext* opCtx,
                                                              StringData ns,
                                                              StringData ident,
diff --git a/src/mongo/db/storage/biggie/biggie_kv_engine.h b/src/mongo/db/storage/biggie/biggie_kv_engine.h
index cb354a30798..b4b0adb09f2 100644
--- a/src/mongo/db/storage/biggie/biggie_kv_engine.h
+++ b/src/mongo/db/storage/biggie/biggie_kv_engine.h
@@ -64,6 +64,9 @@ public:
                                                                StringData ident,
                                                                const CollectionOptions& options);
 
+    virtual std::unique_ptr<mongo::RecordStore> makeTemporaryRecordStore(OperationContext* opCtx,
+                                                                         StringData ident) override;
+
     virtual Status createSortedDataInterface(OperationContext* opCtx,
                                              StringData ident,
                                              const IndexDescriptor* desc);
diff --git a/src/mongo/db/storage/devnull/devnull_kv_engine.cpp b/src/mongo/db/storage/devnull/devnull_kv_engine.cpp
index 4a37c3b751b..759217faff9 100644
--- a/src/mongo/db/storage/devnull/devnull_kv_engine.cpp
+++ b/src/mongo/db/storage/devnull/devnull_kv_engine.cpp
@@ -258,6 +258,11 @@ std::unique_ptr<RecordStore> DevNullKVEngine::getRecordStore(OperationContext* o
     return stdx::make_unique<DevNullRecordStore>(ns, options);
 }
 
+std::unique_ptr<RecordStore> DevNullKVEngine::makeTemporaryRecordStore(OperationContext* opCtx,
+                                                                       StringData ident) {
+    return stdx::make_unique<DevNullRecordStore>("", CollectionOptions());
+}
+
 SortedDataInterface* DevNullKVEngine::getSortedDataInterface(OperationContext* opCtx,
                                                              StringData ident,
                                                              const IndexDescriptor* desc) {
diff --git a/src/mongo/db/storage/devnull/devnull_kv_engine.h b/src/mongo/db/storage/devnull/devnull_kv_engine.h
index a55e5d00ff7..5f9efa61fd2 100644
--- a/src/mongo/db/storage/devnull/devnull_kv_engine.h
+++ b/src/mongo/db/storage/devnull/devnull_kv_engine.h
@@ -64,6 +64,9 @@ public:
                                                         StringData ident,
                                                         const CollectionOptions& options);
 
+    virtual std::unique_ptr<RecordStore> makeTemporaryRecordStore(OperationContext* opCtx,
+                                                                  StringData ident) override;
+
     virtual Status createSortedDataInterface(OperationContext* opCtx,
                                              StringData ident,
                                              const IndexDescriptor* desc) {
diff --git a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_engine.cpp b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_engine.cpp
index a2321724edf..aadeb5ac336 100644
--- a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_engine.cpp
+++ b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_engine.cpp
@@ -77,6 +77,13 @@ std::unique_ptr<RecordStore> EphemeralForTestEngine::getRecordStore(
     }
 }
 
+std::unique_ptr<RecordStore> EphemeralForTestEngine::makeTemporaryRecordStore(
+    OperationContext* opCtx, StringData ident) {
+    stdx::lock_guard<stdx::mutex> lk(_mutex);
+    _dataMap[ident] = {};
+    return stdx::make_unique<EphemeralForTestRecordStore>(ident, &_dataMap[ident]);
+}
+
 Status EphemeralForTestEngine::createSortedDataInterface(OperationContext* opCtx,
                                                          StringData ident,
                                                          const IndexDescriptor* desc) {
diff --git a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_engine.h b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_engine.h
index b3e1886ef82..7a6df032b0b 100644
--- a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_engine.h
+++ b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_engine.h
@@ -57,6 +57,9 @@ public:
                                                         StringData ident,
                                                         const CollectionOptions& options);
 
+    virtual std::unique_ptr<RecordStore> makeTemporaryRecordStore(OperationContext* opCtx,
+                                                                  StringData ident) override;
+
     virtual Status createSortedDataInterface(OperationContext* opCtx,
                                              StringData ident,
                                              const IndexDescriptor* desc);
diff --git a/src/mongo/db/storage/kv/kv_catalog.cpp b/src/mongo/db/storage/kv/kv_catalog.cpp
index 7d827d48027..28736f8598e 100644
--- a/src/mongo/db/storage/kv/kv_catalog.cpp
+++ b/src/mongo/db/storage/kv/kv_catalog.cpp
@@ -340,6 +340,13 @@ bool KVCatalog::_hasEntryCollidingWithRand() const {
     return false;
 }
 
+std::string KVCatalog::newTempIdent() {
+    StringBuilder buf;
+    buf << "temp-";
+    buf << _next.fetchAndAdd(1) << '-' << _rand;
+    return buf.str();
+}
+
 std::string KVCatalog::_newUniqueIdent(StringData ns, const char* kind) {
     // If this changes to not put _rand at the end, _hasEntryCollidingWithRand will need fixing.
     StringBuilder buf;
@@ -621,12 +628,17 @@ std::vector<std::string> KVCatalog::getAllIdents(OperationContext* opCtx) const
 }
 
 bool KVCatalog::isUserDataIdent(StringData ident) const {
+    // Indexes, collections, and temporary RecordStore idents are all candidates for dropping when
+    // the storage engine's metadata does not align with the catalog metadata.
     return ident.find("index-") != std::string::npos || ident.find("index/") != std::string::npos ||
+        ident.find("temp-") != std::string::npos ||
         ident.find("collection-") != std::string::npos ||
         ident.find("collection/") != std::string::npos;
 }
 
 bool KVCatalog::isCollectionIdent(StringData ident) const {
+    // Temporary RecordStores idents prefixed "temp-" should not be considered collections, because
+    // they are not eligible for orphan recovery.
     return ident.find("collection-") != std::string::npos ||
         ident.find("collection/") != std::string::npos;
 }
diff --git a/src/mongo/db/storage/kv/kv_catalog.h b/src/mongo/db/storage/kv/kv_catalog.h
index 2bf2af72831..bc99e4e1e7c 100644
--- a/src/mongo/db/storage/kv/kv_catalog.h
+++ b/src/mongo/db/storage/kv/kv_catalog.h
@@ -113,6 +113,11 @@ public:
      */
     StatusWith<std::string> newOrphanedIdent(OperationContext* opCtx, std::string ident);
 
+    /**
+     * Generate a temporary ident name.
+     */
+    std::string newTempIdent();
+
 private:
     class AddIdentChange;
     class RemoveIdentChange;
diff --git a/src/mongo/db/storage/kv/kv_engine.h b/src/mongo/db/storage/kv/kv_engine.h
index 335046082fa..c3388aed0b5 100644
--- a/src/mongo/db/storage/kv/kv_engine.h
+++ b/src/mongo/db/storage/kv/kv_engine.h
@@ -122,6 +122,9 @@ public:
                                      StringData ident,
                                      const CollectionOptions& options) = 0;
 
+    virtual std::unique_ptr<RecordStore> makeTemporaryRecordStore(OperationContext* opCtx,
+                                                                  StringData ident) = 0;
+
     /**
      * Create a RecordStore that MongoDB considers eligible to share space in an underlying table
      * with other RecordStores. 'prefix' is guaranteed to be 'KVPrefix::kNotPrefixed' when
diff --git a/src/mongo/db/storage/kv/kv_engine_test_harness.cpp b/src/mongo/db/storage/kv/kv_engine_test_harness.cpp
index 0ba7597fded..0bbc7781817 100644
--- a/src/mongo/db/storage/kv/kv_engine_test_harness.cpp
+++ b/src/mongo/db/storage/kv/kv_engine_test_harness.cpp
@@ -172,6 +172,43 @@ TEST(KVEngineTestHarness, SimpleSorted1) {
     }
 }
 
+TEST(KVEngineTestHarness, TemporaryRecordStoreSimple) {
+    unique_ptr<KVHarnessHelper> helper(KVHarnessHelper::create());
+    KVEngine* engine = helper->getEngine();
+    ASSERT(engine);
+
+    string ident = "temptemp";
+    unique_ptr<RecordStore> rs;
+    {
+        MyOperationContext opCtx(engine);
+        rs = engine->makeTemporaryRecordStore(&opCtx, ident);
+        ASSERT(rs);
+    }
+
+    RecordId loc;
+    {
+        MyOperationContext opCtx(engine);
+        WriteUnitOfWork uow(&opCtx);
+        StatusWith<RecordId> res = rs->insertRecord(&opCtx, "abc", 4, Timestamp());
+        ASSERT_OK(res.getStatus());
+        loc = res.getValue();
+        uow.commit();
+    }
+
+    {
+        MyOperationContext opCtx(engine);
+        ASSERT_EQUALS(string("abc"), rs->dataFor(&opCtx, loc).data());
+
+        std::vector<std::string> all = engine->getAllIdents(&opCtx);
+        ASSERT_EQUALS(1U, all.size());
+        ASSERT_EQUALS(ident, all[0]);
+
+        WriteUnitOfWork wuow(&opCtx);
+        ASSERT_OK(engine->dropIdent(&opCtx, ident));
+        wuow.commit();
+    }
+}
+
 TEST(KVCatalogTest, Coll1) {
     unique_ptr<KVHarnessHelper> helper(KVHarnessHelper::create());
     KVEngine* engine = helper->getEngine();
@@ -217,7 +254,6 @@ TEST(KVCatalogTest, Coll1) {
     ASSERT_NOT_EQUALS(ident, catalog->getCollectionIdent("a.b"));
 }
 
-
 TEST(KVCatalogTest, Idx1) {
     unique_ptr<KVHarnessHelper> helper(KVHarnessHelper::create());
     KVEngine* engine = helper->getEngine();
diff --git a/src/mongo/db/storage/kv/kv_storage_engine.cpp b/src/mongo/db/storage/kv/kv_storage_engine.cpp
index c84fde79f43..7dc534c2942 100644
--- a/src/mongo/db/storage/kv/kv_storage_engine.cpp
+++ b/src/mongo/db/storage/kv/kv_storage_engine.cpp
@@ -627,6 +627,10 @@ Status KVStorageEngine::repairRecordStore(OperationContext* opCtx, const std::st
     return Status::OK();
 }
 
+std::unique_ptr<RecordStore> KVStorageEngine::makeTemporaryRecordStore(OperationContext* opCtx) {
+    return _engine->makeTemporaryRecordStore(opCtx, _catalog->newTempIdent());
+}
+
 void KVStorageEngine::setJournalListener(JournalListener* jl) {
     _engine->setJournalListener(jl);
 }
diff --git a/src/mongo/db/storage/kv/kv_storage_engine.h b/src/mongo/db/storage/kv/kv_storage_engine.h
index 141546f3bae..b148f29b105 100644
--- a/src/mongo/db/storage/kv/kv_storage_engine.h
+++ b/src/mongo/db/storage/kv/kv_storage_engine.h
@@ -122,6 +122,8 @@ public:
 
     virtual Status repairRecordStore(OperationContext* opCtx, const std::string& ns);
 
+    virtual std::unique_ptr<RecordStore> makeTemporaryRecordStore(OperationContext* opCtx) override;
+
     virtual void cleanShutdown();
 
     virtual void setStableTimestamp(Timestamp stableTimestamp,
diff --git a/src/mongo/db/storage/kv/kv_storage_engine_test.cpp b/src/mongo/db/storage/kv/kv_storage_engine_test.cpp
index f68e355eba5..68c7098e689 100644
--- a/src/mongo/db/storage/kv/kv_storage_engine_test.cpp
+++ b/src/mongo/db/storage/kv/kv_storage_engine_test.cpp
@@ -76,6 +76,9 @@ public:
         return _storageEngine->getCatalog()->getCollectionIdent(ns.ns());
     }
 
+    std::unique_ptr<RecordStore> makeTemporary(OperationContext* opCtx) {
+        return _storageEngine->makeTemporaryRecordStore(opCtx);
+    }
 
     /**
      * Create a collection table in the KVEngine not reflected in the KVCatalog.
@@ -223,6 +226,21 @@ TEST_F(KVStorageEngineTest, LoadCatalogDropsOrphansAfterUncleanShutdown) {
     ASSERT(!collectionExists(opCtx.get(), collNs));
 }
 
+TEST_F(KVStorageEngineTest, ReconcileDropsTemporary) {
+    auto opCtx = cc().makeOperationContext();
+
+    auto rs = makeTemporary(opCtx.get());
+    ASSERT(rs);
+    const std::string ident = rs->getIdent();
+
+    ASSERT(identExists(opCtx.get(), ident));
+
+    ASSERT_OK(reconcile(opCtx.get()).getStatus());
+
+    // The storage engine is responsible for dropping its temporary idents.
+    ASSERT(!identExists(opCtx.get(), ident));
+}
+
 TEST_F(KVStorageEngineRepairTest, LoadCatalogRecoversOrphans) {
     auto opCtx = cc().makeOperationContext();
 
diff --git a/src/mongo/db/storage/mobile/mobile_kv_engine.cpp b/src/mongo/db/storage/mobile/mobile_kv_engine.cpp
index 66d16c70d71..11670750a97 100644
--- a/src/mongo/db/storage/mobile/mobile_kv_engine.cpp
+++ b/src/mongo/db/storage/mobile/mobile_kv_engine.cpp
@@ -207,6 +207,14 @@ std::unique_ptr<RecordStore> MobileKVEngine::getRecordStore(OperationContext* op
     return stdx::make_unique<MobileRecordStore>(opCtx, ns, _path, ident.toString(), options);
 }
 
+std::unique_ptr<RecordStore> MobileKVEngine::makeTemporaryRecordStore(OperationContext* opCtx,
+                                                                      StringData ident) {
+    MobileRecordStore::create(opCtx, ident.toString());
+    return std::make_unique<MobileRecordStore>(
+        opCtx, "", _path, ident.toString(), CollectionOptions());
+}
+
+
 Status MobileKVEngine::createSortedDataInterface(OperationContext* opCtx,
                                                  StringData ident,
                                                  const IndexDescriptor* desc) {
diff --git a/src/mongo/db/storage/mobile/mobile_kv_engine.h b/src/mongo/db/storage/mobile/mobile_kv_engine.h
index 28b63301d10..a644d6db4b0 100644
--- a/src/mongo/db/storage/mobile/mobile_kv_engine.h
+++ b/src/mongo/db/storage/mobile/mobile_kv_engine.h
@@ -58,6 +58,9 @@ public:
                                                 StringData ident,
                                                 const CollectionOptions& options) override;
 
+    std::unique_ptr<RecordStore> makeTemporaryRecordStore(OperationContext* opCtx,
+                                                          StringData ident) override;
+
     Status createSortedDataInterface(OperationContext* opCtx,
                                      StringData ident,
                                      const IndexDescriptor* desc) override;
diff --git a/src/mongo/db/storage/storage_engine.h b/src/mongo/db/storage/storage_engine.h
index 795ee9f6397..456972b0997 100644
--- a/src/mongo/db/storage/storage_engine.h
+++ b/src/mongo/db/storage/storage_engine.h
@@ -45,6 +45,7 @@ namespace mongo {
 class DatabaseCatalogEntry;
 class JournalListener;
 class OperationContext;
+class RecordStore;
 class RecoveryUnit;
 class SnapshotManager;
 struct StorageGlobalParams;
@@ -287,6 +288,13 @@ public:
     virtual Status repairRecordStore(OperationContext* opCtx, const std::string& ns) = 0;
 
     /**
+     * Creates a temporary RecordStore on the storage engine. This record store should be dropped by
+     * the caller when done being used. The storage engine will drop any remaining temporary record
+     * stores on startup.
+     */
+    virtual std::unique_ptr<RecordStore> makeTemporaryRecordStore(OperationContext* opCtx) = 0;
+
+    /**
      * This method will be called before there is a clean shutdown.  Storage engines should
      * override this method if they have clean-up to do that is different from unclean shutdown.
      * MongoDB will not call into the storage subsystem after calling this function.
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
index a8187e89699..0ee4e2c2781 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
@@ -1124,6 +1124,47 @@ SortedDataInterface* WiredTigerKVEngine::getGroupedSortedDataInterface(Operation
     return new WiredTigerIndexStandard(opCtx, _uri(ident), desc, prefix, _readOnly);
 }
 
+std::unique_ptr<RecordStore> WiredTigerKVEngine::makeTemporaryRecordStore(OperationContext* opCtx,
+                                                                          StringData ident) {
+    invariant(!_readOnly);
+
+    _ensureIdentPath(ident);
+    WiredTigerSession wtSession(_conn);
+
+    CollectionOptions noOptions;
+    StatusWith<std::string> swConfig = WiredTigerRecordStore::generateCreateString(
+        _canonicalName, "" /* internal table */, noOptions, _rsOptions, false /* prefixed */);
+    uassertStatusOK(swConfig.getStatus());
+
+    std::string config = swConfig.getValue();
+
+    std::string uri = _uri(ident);
+    WT_SESSION* session = wtSession.getSession();
+    LOG(2) << "WiredTigerKVEngine::createTemporaryRecordStore uri: " << uri
+           << " config: " << config;
+    uassertStatusOK(wtRCToStatus(session->create(session, uri.c_str(), config.c_str())));
+
+    WiredTigerRecordStore::Params params;
+    params.ns = "";
+    params.uri = _uri(ident);
+    params.engineName = _canonicalName;
+    params.isCapped = false;
+    params.isEphemeral = _ephemeral;
+    params.cappedCallback = nullptr;
+    // Temporary collections do not need to persist size information to the size storer.
+    params.sizeStorer = nullptr;
+    params.isReadOnly = false;
+
+    params.cappedMaxSize = -1;
+    params.cappedMaxDocs = -1;
+
+    std::unique_ptr<WiredTigerRecordStore> rs;
+    rs = stdx::make_unique<StandardWiredTigerRecordStore>(this, opCtx, params);
+    rs->postConstructorInit(opCtx);
+
+    return std::move(rs);
+}
+
 void WiredTigerKVEngine::alterIdentMetadata(OperationContext* opCtx,
                                             StringData ident,
                                             const IndexDescriptor* desc) {
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
index c55e7ca7a2f..95f9cda47aa 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
@@ -111,6 +111,9 @@ public:
         return getGroupedRecordStore(opCtx, ns, ident, options, KVPrefix::kNotPrefixed);
     }
 
+    virtual std::unique_ptr<RecordStore> makeTemporaryRecordStore(OperationContext* opCtx,
+                                                                  StringData ident) override;
+
     virtual Status createSortedDataInterface(OperationContext* opCtx,
                                              StringData ident,
                                              const IndexDescriptor* desc) override {
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_prefixed_record_store_test.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_prefixed_record_store_test.cpp
index 4f6c96f00c1..eb5077d48f8 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_prefixed_record_store_test.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_prefixed_record_store_test.cpp
@@ -103,7 +103,7 @@ public:
 
         const bool prefixed = true;
         StatusWith<std::string> result = WiredTigerRecordStore::generateCreateString(
-            kWiredTigerEngineName, ns, CollectionOptions(), "", prefixed);
+            kWiredTigerEngineName, StringData(ns), CollectionOptions(), "", prefixed);
         ASSERT_TRUE(result.isOK());
         std::string config = result.getValue();
 
@@ -149,7 +149,7 @@ public:
 
         KVPrefix prefix = KVPrefix::generateNextPrefix();
         StatusWith<std::string> result = WiredTigerRecordStore::generateCreateString(
-            kWiredTigerEngineName, ns, options, "", prefix.isPrefixed());
+            kWiredTigerEngineName, StringData(ns), options, "", prefix.isPrefixed());
         ASSERT_TRUE(result.isOK());
         std::string config = result.getValue();
 
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp
index 7efaefe2040..135d904f5ce 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp
@@ -622,7 +622,10 @@ StatusWith<std::string> WiredTigerRecordStore::generateCreateString(
 
     bool replicatedWrites = getGlobalReplSettings().usingReplSets() ||
         repl::ReplSettings::shouldRecoverFromOplogAsStandalone();
-    if (WiredTigerUtil::useTableLogging(NamespaceString(ns), replicatedWrites)) {
+
+    // Do not journal writes when 'ns' is an empty string, which is the case for internal-only
+    // temporary tables.
+    if (ns.size() && WiredTigerUtil::useTableLogging(NamespaceString(ns), replicatedWrites)) {
         ss << ",log=(enabled=true)";
     } else {
         ss << ",log=(enabled=false)";
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.h b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.h
index f450193fe0c..019d7e2331c 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.h
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.h
@@ -88,6 +88,7 @@ public:
 
     /**
      * Creates a configuration string suitable for 'config' parameter in WT_SESSION::create().
+     * It is possible for 'ns' to be an empty string, in the case of internal-only temporary tables.
      * Configuration string is constructed from:
      *     built-in defaults
      *     storageEngine.wiredTiger.configString in 'options'