From bef2975bdd054b9c9826b7a4bac1fda2cb174be3 Mon Sep 17 00:00:00 2001 From: Yoonsoo Kim Date: Wed, 28 Sep 2022 17:59:27 +0000 Subject: SERVER-69685 Add `createVirtualCollection` API --- jstests/noPassthrough/compute_mode.js | 15 + src/mongo/db/catalog/SConscript | 2 + src/mongo/db/catalog/collection_impl.cpp | 57 ++- src/mongo/db/catalog/collection_impl.h | 7 + src/mongo/db/catalog/create_collection.cpp | 31 +- src/mongo/db/catalog/create_collection.h | 8 + src/mongo/db/catalog/create_collection_test.cpp | 144 +++++- src/mongo/db/catalog/database.h | 22 + src/mongo/db/catalog/database_impl.cpp | 131 ++++-- src/mongo/db/catalog/database_impl.h | 21 + src/mongo/db/catalog/virtual_collection_impl.cpp | 72 +++ src/mongo/db/catalog/virtual_collection_impl.h | 544 ++++++++++++++++++++++ src/mongo/db/catalog/virtual_collection_options.h | 100 ++++ src/mongo/db/storage/SConscript | 4 +- src/mongo/db/storage/external_record_store.cpp | 43 ++ src/mongo/db/storage/external_record_store.h | 148 ++++++ 16 files changed, 1279 insertions(+), 70 deletions(-) create mode 100644 src/mongo/db/catalog/virtual_collection_impl.cpp create mode 100644 src/mongo/db/catalog/virtual_collection_impl.h create mode 100644 src/mongo/db/catalog/virtual_collection_options.h create mode 100644 src/mongo/db/storage/external_record_store.cpp create mode 100644 src/mongo/db/storage/external_record_store.h diff --git a/jstests/noPassthrough/compute_mode.js b/jstests/noPassthrough/compute_mode.js index 3f732495db9..16da986f6b9 100644 --- a/jstests/noPassthrough/compute_mode.js +++ b/jstests/noPassthrough/compute_mode.js @@ -20,6 +20,17 @@ const configsvrOption = { setParameter: "enableComputeMode=true" }; +(function testStandaloneMongod() { + const conn = MongoRunner.runMongod(); + const db = conn.getDB(jsTestName()); + + assert.commandFailedWithCode(db.createCollection("a", {virtual: {}}), + 40415, + "ERROR from virtual collection creation in regular mode"); + + MongoRunner.stopMongod(conn); +})(); + (function testComputeModeInStandaloneMongod() { const conn = MongoRunner.runMongod(mongodOption); @@ -36,6 +47,10 @@ const configsvrOption = { }); assert(warningMsgFound, "ERROR from warning message match"); + assert.commandFailedWithCode(db.createCollection("a", {virtual: {}}), + 40415, + "ERROR from virtual collection creation in compute mode"); + MongoRunner.stopMongod(conn); })(); diff --git a/src/mongo/db/catalog/SConscript b/src/mongo/db/catalog/SConscript index 37f9252b47a..9b23aff0efc 100644 --- a/src/mongo/db/catalog/SConscript +++ b/src/mongo/db/catalog/SConscript @@ -370,6 +370,7 @@ env.Library( 'index_catalog_entry_impl.cpp', 'index_catalog_impl.cpp', 'index_consistency.cpp', + 'virtual_collection_impl.cpp', ], LIBDEPS_PRIVATE=[ '$BUILD_DIR/mongo/base', @@ -676,6 +677,7 @@ if wiredtiger: '$BUILD_DIR/mongo/util/pcre_wrapper', 'catalog_control', 'catalog_helpers', + 'catalog_impl', 'catalog_test_fixture', 'collection', 'collection_catalog', diff --git a/src/mongo/db/catalog/collection_impl.cpp b/src/mongo/db/catalog/collection_impl.cpp index ee764810669..a15c2276aaf 100644 --- a/src/mongo/db/catalog/collection_impl.cpp +++ b/src/mongo/db/catalog/collection_impl.cpp @@ -79,36 +79,6 @@ MONGO_FAIL_POINT_DEFINE(allowSettingMalformedCollectionValidators); MONGO_FAIL_POINT_DEFINE(skipCappedDeletes); -// Uses the collator factory to convert the BSON representation of a collator to a -// CollatorInterface. Returns null if the BSONObj is empty. We expect the stored collation to be -// valid, since it gets validated on collection create. -std::unique_ptr parseCollation(OperationContext* opCtx, - const NamespaceString& nss, - BSONObj collationSpec) { - if (collationSpec.isEmpty()) { - return {nullptr}; - } - - auto collator = - CollatorFactoryInterface::get(opCtx->getServiceContext())->makeFromBSON(collationSpec); - - // If the collection's default collator has a version not currently supported by our ICU - // integration, shut down the server. Errors other than IncompatibleCollationVersion should not - // be possible, so these are an invariant rather than fassert. - if (collator == ErrorCodes::IncompatibleCollationVersion) { - LOGV2(20288, - "Collection {namespace} has a default collation which is incompatible with this " - "version: {collationSpec}" - "Collection has a default collation incompatible with this version", - logAttrs(nss), - "collationSpec"_attr = collationSpec); - fassertFailedNoTrace(40144); - } - invariant(collator.getStatus()); - - return std::move(collator.getValue()); -} - Status checkValidatorCanBeUsedOnNs(const BSONObj& validator, const NamespaceString& nss, const UUID& uuid) { @@ -273,6 +243,33 @@ bool isIndexCompatible(std::shared_ptr index, Timestamp readT } // namespace +std::unique_ptr CollectionImpl::parseCollation(OperationContext* opCtx, + const NamespaceString& nss, + BSONObj collationSpec) { + if (collationSpec.isEmpty()) { + return nullptr; + } + + auto collator = + CollatorFactoryInterface::get(opCtx->getServiceContext())->makeFromBSON(collationSpec); + + // If the collection's default collator has a version not currently supported by our ICU + // integration, shut down the server. Errors other than IncompatibleCollationVersion should not + // be possible, so these are an invariant rather than fassert. + if (collator == ErrorCodes::IncompatibleCollationVersion) { + LOGV2(20288, + "Collection {namespace} has a default collation which is incompatible with this " + "version: {collationSpec}" + "Collection has a default collation incompatible with this version", + logAttrs(nss), + "collationSpec"_attr = collationSpec); + fassertFailedNoTrace(40144); + } + invariant(collator.getStatus()); + + return std::move(collator.getValue()); +} + CollectionImpl::SharedState::SharedState(CollectionImpl* collection, std::unique_ptr recordStore, const CollectionOptions& options) diff --git a/src/mongo/db/catalog/collection_impl.h b/src/mongo/db/catalog/collection_impl.h index 0594d3c289e..43799d81489 100644 --- a/src/mongo/db/catalog/collection_impl.h +++ b/src/mongo/db/catalog/collection_impl.h @@ -36,6 +36,13 @@ namespace mongo { class CollectionImpl final : public Collection { public: + // Uses the collator factory to convert the BSON representation of a collator to a + // CollatorInterface. Returns null if the BSONObj is empty. We expect the stored collation to be + // valid, since it gets validated on collection create. + static std::unique_ptr parseCollation(OperationContext* opCtx, + const NamespaceString& nss, + BSONObj collationSpec); + // TODO SERVER-56999: We should just need one API to create Collections explicit CollectionImpl(OperationContext* opCtx, const NamespaceString& nss, diff --git a/src/mongo/db/catalog/create_collection.cpp b/src/mongo/db/catalog/create_collection.cpp index 75cda7ad9d9..b190ef52dc2 100644 --- a/src/mongo/db/catalog/create_collection.cpp +++ b/src/mongo/db/catalog/create_collection.cpp @@ -36,9 +36,11 @@ #include "mongo/db/catalog/clustered_collection_util.h" #include "mongo/db/catalog/collection_catalog.h" #include "mongo/db/catalog/collection_catalog_helper.h" +#include "mongo/db/catalog/collection_options.h" #include "mongo/db/catalog/database_holder.h" #include "mongo/db/catalog/index_key_validate.h" #include "mongo/db/catalog/unique_collection_name.h" +#include "mongo/db/catalog/virtual_collection_options.h" #include "mongo/db/commands.h" #include "mongo/db/commands/create_gen.h" #include "mongo/db/concurrency/exception_util.h" @@ -52,6 +54,7 @@ #include "mongo/db/operation_context.h" #include "mongo/db/ops/insert.h" #include "mongo/db/query/collation/collator_factory_interface.h" +#include "mongo/db/query/query_knobs_gen.h" #include "mongo/db/repl/replication_coordinator.h" #include "mongo/db/s/collection_sharding_state.h" #include "mongo/db/storage/storage_parameters_gen.h" @@ -493,10 +496,12 @@ Status _createTimeseries(OperationContext* opCtx, return ret; } -Status _createCollection(OperationContext* opCtx, - const NamespaceString& nss, - const CollectionOptions& collectionOptions, - const boost::optional& idIndex) { +Status _createCollection( + OperationContext* opCtx, + const NamespaceString& nss, + const CollectionOptions& collectionOptions, + const boost::optional& idIndex, + const boost::optional& virtualCollectionOptions = boost::none) { return writeConflictRetry(opCtx, "create", nss.ns(), [&] { AutoGetDb autoDb(opCtx, nss.dbName(), MODE_IX); Lock::CollectionLock collLock(opCtx, nss, MODE_IX); @@ -577,14 +582,15 @@ Status _createCollection(OperationContext* opCtx, // Even though 'collectionOptions' is passed by rvalue reference, it is not safe to move // because 'userCreateNS' may throw a WriteConflictException. if (idIndex == boost::none || collectionOptions.clusteredIndex) { - status = db->userCreateNS(opCtx, nss, collectionOptions, /*createIdIndex=*/false); + status = virtualCollectionOptions + ? db->userCreateVirtualNS(opCtx, nss, collectionOptions, *virtualCollectionOptions) + : db->userCreateNS(opCtx, nss, collectionOptions, /*createIdIndex=*/false); } else { bool createIdIndex = true; if (MONGO_unlikely(skipIdIndex.shouldFail())) { createIdIndex = false; } - status = db->userCreateNS( - opCtx, nss, collectionOptions, /*createIdIndex=*/createIdIndex, *idIndex); + status = db->userCreateNS(opCtx, nss, collectionOptions, createIdIndex, *idIndex); } if (!status.isOK()) { return status; @@ -885,4 +891,15 @@ Status createCollection(OperationContext* opCtx, } } +Status createVirtualCollection(OperationContext* opCtx, + const NamespaceString& ns, + const VirtualCollectionOptions& vopts) { + tassert(6968504, + "Virtual collection is available when the compute mode is enabled", + computeModeEnabled); + CollectionOptions options; + options.setNoIdIndex(); + return _createCollection(opCtx, ns, options, boost::none, vopts); +} + } // namespace mongo diff --git a/src/mongo/db/catalog/create_collection.h b/src/mongo/db/catalog/create_collection.h index 7812510484b..6b8b4b51801 100644 --- a/src/mongo/db/catalog/create_collection.h +++ b/src/mongo/db/catalog/create_collection.h @@ -33,6 +33,7 @@ #include "mongo/base/status.h" #include "mongo/bson/bsonobj.h" #include "mongo/db/catalog/collection_options.h" +#include "mongo/db/catalog/virtual_collection_options.h" #include "mongo/db/commands/create_gen.h" namespace mongo { @@ -62,6 +63,13 @@ Status createCollection(OperationContext* opCtx, const CollectionOptions& options, const boost::optional& idIndex); +/** + * Creates a virtual collection as described by 'vopts'. + */ +Status createVirtualCollection(OperationContext* opCtx, + const NamespaceString& ns, + const VirtualCollectionOptions& vopts); + /** * As above, but only used by replication to apply operations. This allows recreating collections * with specific UUIDs (if ui is given). If ui is given and and a collection exists with the same diff --git a/src/mongo/db/catalog/create_collection_test.cpp b/src/mongo/db/catalog/create_collection_test.cpp index 63c472f1ecd..c1b47d20166 100644 --- a/src/mongo/db/catalog/create_collection_test.cpp +++ b/src/mongo/db/catalog/create_collection_test.cpp @@ -27,28 +27,32 @@ * it in the license file. */ +#include "mongo/base/error_codes.h" #include "mongo/db/catalog/collection_catalog.h" #include "mongo/db/catalog/collection_write_path.h" #include "mongo/db/catalog/create_collection.h" #include "mongo/db/catalog/database_holder.h" +#include "mongo/db/catalog/virtual_collection_impl.h" +#include "mongo/db/catalog/virtual_collection_options.h" #include "mongo/db/concurrency/exception_util.h" #include "mongo/db/db_raii.h" #include "mongo/db/jsobj.h" #include "mongo/db/repl/replication_coordinator_mock.h" #include "mongo/db/repl/storage_interface_impl.h" #include "mongo/db/service_context_d_test_fixture.h" +#include "mongo/stdx/utility.h" #include "mongo/unittest/unittest.h" +#include "mongo/util/assert_util.h" #include "mongo/util/uuid.h" namespace mongo { namespace { class CreateCollectionTest : public ServiceContextMongoDTest { -private: +protected: void setUp() override; void tearDown() override; -protected: void validateValidator(const std::string& validatorStr, int expectedError); // Use StorageInterface to access storage features below catalog interface. @@ -76,6 +80,18 @@ void CreateCollectionTest::tearDown() { ServiceContextMongoDTest::tearDown(); } +class CreateVirtualCollectionTest : public CreateCollectionTest { +private: + void setUp() override { + CreateCollectionTest::setUp(); + computeModeEnabled = true; + } + void tearDown() override { + computeModeEnabled = false; + CreateCollectionTest::tearDown(); + } +}; + /** * Creates an OperationContext. */ @@ -125,6 +141,26 @@ CollectionOptions getCollectionOptions(OperationContext* opCtx, const NamespaceS return collection->getCollectionOptions(); } +/** + * Returns a VirtualCollectionImpl if the underlying implementation object is a virtual collection. + */ +const VirtualCollectionImpl* getVirtualCollection(OperationContext* opCtx, + const NamespaceString& nss) { + AutoGetCollectionForRead collection(opCtx, nss); + return dynamic_cast(collection.getCollection().get()); +} + +/** + * Returns virtual collection options. + */ +VirtualCollectionOptions getVirtualCollectionOptions(OperationContext* opCtx, + const NamespaceString& nss) { + auto vcollPtr = getVirtualCollection(opCtx, nss); + ASSERT_TRUE(vcollPtr) << "Collection must be a virtual collection"; + + return vcollPtr->getVirtualCollectionOptions(); +} + /** * Returns UUID of collection. */ @@ -288,5 +324,109 @@ TEST_F(CreateCollectionTest, ValidationDisabledForTemporaryReshardingCollection) ASSERT_OK(status); } +const auto kValidUrl1 = + ExternalDataSourceMetadata::kDefaultFileUrlPrefix.toString() + "named_pipe1"; +const auto kValidUrl2 = + ExternalDataSourceMetadata::kDefaultFileUrlPrefix.toString() + "named_pipe2"; + +TEST_F(CreateVirtualCollectionTest, VirtualCollectionOptionsWithOneSource) { + NamespaceString vcollNss("myDb", "vcoll.name"); + auto opCtx = makeOpCtx(); + + Lock::GlobalLock lk(opCtx.get(), MODE_X); // Satisfy low-level locking invariants. + VirtualCollectionOptions reqVcollOpts; + reqVcollOpts.dataSources.emplace_back(kValidUrl1, StorageType::pipe, FileType::bson); + ASSERT_OK(createVirtualCollection(opCtx.get(), vcollNss, reqVcollOpts)); + ASSERT_TRUE(getVirtualCollection(opCtx.get(), vcollNss)); + + ASSERT_EQ(stdx::to_underlying(getCollectionOptions(opCtx.get(), vcollNss).autoIndexId), + stdx::to_underlying(CollectionOptions::NO)); + + auto vcollOpts = getVirtualCollectionOptions(opCtx.get(), vcollNss); + ASSERT_EQ(vcollOpts.dataSources.size(), 1); + ASSERT_EQ(vcollOpts.dataSources[0].url, kValidUrl1); + ASSERT_EQ(stdx::to_underlying(vcollOpts.dataSources[0].storageType), + stdx::to_underlying(StorageType::pipe)); + ASSERT_EQ(stdx::to_underlying(vcollOpts.dataSources[0].fileType), + stdx::to_underlying(FileType::bson)); +} + +TEST_F(CreateVirtualCollectionTest, VirtualCollectionOptionsWithMultiSource) { + NamespaceString vcollNss("myDb", "vcoll.name"); + auto opCtx = makeOpCtx(); + + Lock::GlobalLock lk(opCtx.get(), MODE_X); // Satisfy low-level locking invariants. + VirtualCollectionOptions reqVcollOpts; + reqVcollOpts.dataSources.emplace_back(kValidUrl1, StorageType::pipe, FileType::bson); + reqVcollOpts.dataSources.emplace_back(kValidUrl2, StorageType::pipe, FileType::bson); + + ASSERT_OK(createVirtualCollection(opCtx.get(), vcollNss, reqVcollOpts)); + ASSERT_TRUE(getVirtualCollection(opCtx.get(), vcollNss)); + + ASSERT_EQ(stdx::to_underlying(getCollectionOptions(opCtx.get(), vcollNss).autoIndexId), + stdx::to_underlying(CollectionOptions::NO)); + + auto vcollOpts = getVirtualCollectionOptions(opCtx.get(), vcollNss); + ASSERT_EQ(vcollOpts.dataSources.size(), 2); + for (int i = 0; i < 2; ++i) { + ASSERT_EQ(vcollOpts.dataSources[i].url, reqVcollOpts.dataSources[i].url); + ASSERT_EQ(stdx::to_underlying(vcollOpts.dataSources[i].storageType), + stdx::to_underlying(StorageType::pipe)); + ASSERT_EQ(stdx::to_underlying(vcollOpts.dataSources[i].fileType), + stdx::to_underlying(FileType::bson)); + } +} + +TEST_F(CreateVirtualCollectionTest, InvalidVirtualCollectionOptions) { + using namespace fmt::literals; + + NamespaceString vcollNss("myDb", "vcoll.name"); + auto opCtx = makeOpCtx(); + + Lock::GlobalLock lk(opCtx.get(), MODE_X); // Satisfy low-level locking invariants. + + { + bool exceptionOccurred = false; + VirtualCollectionOptions reqVcollOpts; + constexpr auto kInvalidUrl = "file:///abc/named_pipe"_sd; + try { + reqVcollOpts.dataSources.emplace_back(kInvalidUrl, StorageType::pipe, FileType::bson); + } catch (const DBException&) { + exceptionOccurred = true; + } + + ASSERT_TRUE(exceptionOccurred) + << "Invalid 'url': {} must fail but succeeded"_format(kInvalidUrl); + } + + { + bool exceptionOccurred = false; + VirtualCollectionOptions reqVcollOpts; + constexpr auto kInvalidStorageType = StorageType(2); + try { + reqVcollOpts.dataSources.emplace_back(kValidUrl1, kInvalidStorageType, FileType::bson); + } catch (const DBException&) { + exceptionOccurred = true; + } + + ASSERT_TRUE(exceptionOccurred) + << "Unknown 'storageType': {} must fail but succeeded"_format( + stdx::to_underlying(kInvalidStorageType)); + } + + { + bool exceptionOccurred = false; + VirtualCollectionOptions reqVcollOpts; + constexpr auto kInvalidFileType = FileType(2); + try { + reqVcollOpts.dataSources.emplace_back(kValidUrl1, StorageType::pipe, kInvalidFileType); + } catch (const DBException&) { + exceptionOccurred = true; + } + + ASSERT_TRUE(exceptionOccurred) << "Unknown 'fileType': {} must fail but succeeded"_format( + stdx::to_underlying(kInvalidFileType)); + } +} } // namespace } // namespace mongo diff --git a/src/mongo/db/catalog/database.h b/src/mongo/db/catalog/database.h index 06b7456c81c..bd8ea3dbed1 100644 --- a/src/mongo/db/catalog/database.h +++ b/src/mongo/db/catalog/database.h @@ -35,6 +35,7 @@ #include "mongo/db/catalog/collection.h" #include "mongo/db/catalog/collection_catalog.h" #include "mongo/db/catalog/collection_options.h" +#include "mongo/db/catalog/virtual_collection_options.h" #include "mongo/db/database_name.h" #include "mongo/db/dbcommands_gen.h" #include "mongo/db/namespace_string.h" @@ -64,6 +65,14 @@ public: const BSONObj& idIndex = BSONObj(), bool fromMigrate = false) const = 0; + /** + * Creates the virtual namespace 'fullns' according to 'opts' and 'vopts'. + */ + virtual Status userCreateVirtualNS(OperationContext* opCtx, + const NamespaceString& fullns, + CollectionOptions opts, + const VirtualCollectionOptions& vopts) const = 0; + Database() = default; // must call close first @@ -133,6 +142,19 @@ public: const BSONObj& idIndex = BSONObj(), bool fromMigrate = false) const = 0; + /** + * A MODE_IX collection lock must be held for this call. Throws a WriteConflictException error + * if the collection already exists (say if another thread raced to create it). + * + * Surrounding writeConflictRetry loops must encompass checking that the collection exists as + * well as creating it. Otherwise the loop will endlessly throw WCEs: the caller must check that + * the collection exists to break free. + */ + virtual Collection* createVirtualCollection(OperationContext* opCtx, + const NamespaceString& nss, + const CollectionOptions& opts, + const VirtualCollectionOptions& vopts) const = 0; + virtual Status createView(OperationContext* opCtx, const NamespaceString& viewName, const CollectionOptions& options) const = 0; diff --git a/src/mongo/db/catalog/database_impl.cpp b/src/mongo/db/catalog/database_impl.cpp index bda9e976636..84d11b97f6f 100644 --- a/src/mongo/db/catalog/database_impl.cpp +++ b/src/mongo/db/catalog/database_impl.cpp @@ -43,6 +43,8 @@ #include "mongo/db/catalog/drop_indexes.h" #include "mongo/db/catalog/index_catalog.h" #include "mongo/db/catalog/uncommitted_catalog_updates.h" +#include "mongo/db/catalog/virtual_collection_impl.h" +#include "mongo/db/catalog/virtual_collection_options.h" #include "mongo/db/clientcursor.h" #include "mongo/db/concurrency/d_concurrency.h" #include "mongo/db/concurrency/exception_util.h" @@ -771,6 +773,31 @@ Collection* DatabaseImpl::createCollection(OperationContext* opCtx, bool createIdIndex, const BSONObj& idIndex, bool fromMigrate) const { + return _createCollection( + opCtx, nss, options, createIdIndex, idIndex, fromMigrate, /*vopts=*/boost::none); +} + +Collection* DatabaseImpl::createVirtualCollection(OperationContext* opCtx, + const NamespaceString& nss, + const CollectionOptions& opts, + const VirtualCollectionOptions& vopts) const { + return _createCollection(opCtx, + nss, + opts, + /*createIdIndex=*/false, + /*idIndex=*/BSONObj(), + /*fromMigrate=*/false, + vopts); +} + +Collection* DatabaseImpl::_createCollection( + OperationContext* opCtx, + const NamespaceString& nss, + const CollectionOptions& options, + bool createIdIndex, + const BSONObj& idIndex, + bool fromMigrate, + const boost::optional& vopts) const { invariant(!options.isView()); invariant(opCtx->lockState()->isCollectionLockedForMode(nss, MODE_IX)); @@ -833,13 +860,21 @@ Collection* DatabaseImpl::createCollection(OperationContext* opCtx, "options"_attr = options); // Create Collection object - auto storageEngine = opCtx->getServiceContext()->getStorageEngine(); - std::pair> catalogIdRecordStorePair = - uassertStatusOK(storageEngine->getCatalog()->createCollection( - opCtx, nss, optionsWithUUID, true /*allocateDefaultSpace*/)); - auto& catalogId = catalogIdRecordStorePair.first; - std::shared_ptr ownedCollection = Collection::Factory::get(opCtx)->make( - opCtx, nss, catalogId, optionsWithUUID, std::move(catalogIdRecordStorePair.second)); + auto ownedCollection = [&]() -> std::shared_ptr { + if (!vopts) { + auto storageEngine = opCtx->getServiceContext()->getStorageEngine(); + std::pair> catalogIdRecordStorePair = + uassertStatusOK(storageEngine->getCatalog()->createCollection( + opCtx, nss, optionsWithUUID, true /*allocateDefaultSpace*/)); + auto& catalogId = catalogIdRecordStorePair.first; + return Collection::Factory::get(opCtx)->make( + opCtx, nss, catalogId, optionsWithUUID, std::move(catalogIdRecordStorePair.second)); + } else { + // Virtual collection stays only in memory and its metadata need not persist on disk and + // therefore we bypass DurableCatalog. + return VirtualCollectionImpl::make(opCtx, nss, optionsWithUUID, *vopts); + } + }(); auto collection = ownedCollection.get(); ownedCollection->init(opCtx); ownedCollection->setCommitted(false); @@ -899,25 +934,12 @@ Collection* DatabaseImpl::createCollection(OperationContext* opCtx, return collection; } -Status DatabaseImpl::userCreateNS(OperationContext* opCtx, - const NamespaceString& nss, - CollectionOptions collectionOptions, - bool createDefaultIndexes, - const BSONObj& idIndex, - bool fromMigrate) const { - LOGV2_DEBUG(20324, - 1, - "create collection {namespace} {collectionOptions}", - "namespace"_attr = nss, - "collectionOptions"_attr = collectionOptions.toBSON()); - if (!NamespaceString::validCollectionComponent(nss.ns())) - return Status(ErrorCodes::InvalidNamespace, str::stream() << "invalid ns: " << nss); - - // Validate the collation, if there is one. +StatusWith> DatabaseImpl::_validateCollator( + OperationContext* opCtx, CollectionOptions& opts) const { std::unique_ptr collator; - if (!collectionOptions.collation.isEmpty()) { - auto collatorWithStatus = CollatorFactoryInterface::get(opCtx->getServiceContext()) - ->makeFromBSON(collectionOptions.collation); + if (!opts.collation.isEmpty()) { + auto collatorWithStatus = + CollatorFactoryInterface::get(opCtx->getServiceContext())->makeFromBSON(opts.collation); if (!collatorWithStatus.isOK()) { return collatorWithStatus.getStatus(); @@ -933,12 +955,35 @@ Status DatabaseImpl::userCreateNS(OperationContext* opCtx, // we simply unset the "collation" from the collection options. This ensures that // collections created on versions which do not support the collation feature have the same // format for representing the simple collation as collections created on this version. - collectionOptions.collation = collator ? collator->getSpec().toBSON() : BSONObj(); + opts.collation = collator ? collator->getSpec().toBSON() : BSONObj(); + } + + return {std::move(collator)}; +} + +Status DatabaseImpl::userCreateNS(OperationContext* opCtx, + const NamespaceString& nss, + CollectionOptions collectionOptions, + bool createDefaultIndexes, + const BSONObj& idIndex, + bool fromMigrate) const { + LOGV2_DEBUG(20324, + 1, + "create collection {namespace} {collectionOptions}", + "namespace"_attr = nss, + "collectionOptions"_attr = collectionOptions.toBSON()); + if (!NamespaceString::validCollectionComponent(nss.ns())) + return Status(ErrorCodes::InvalidNamespace, str::stream() << "invalid ns: " << nss); + + // Validate the collation, if there is one. + auto swCollator = _validateCollator(opCtx, collectionOptions); + if (!swCollator.isOK()) { + return swCollator.getStatus(); } if (!collectionOptions.validator.isEmpty()) { boost::intrusive_ptr expCtx( - new ExpressionContext(opCtx, std::move(collator), nss)); + new ExpressionContext(opCtx, std::move(swCollator.getValue()), nss)); // If the feature compatibility version is not kLatest, and we are validating features as // primary, ban the use of new agg features introduced in kLatest to prevent them from being @@ -1004,7 +1049,7 @@ Status DatabaseImpl::userCreateNS(OperationContext* opCtx, uassertStatusOK(createView(opCtx, nss, collectionOptions)); } else { - invariant(createCollection( + invariant(_createCollection( opCtx, nss, collectionOptions, createDefaultIndexes, idIndex, fromMigrate), str::stream() << "Collection creation failed after validating options: " << nss << ". Options: " << collectionOptions.toBSON()); @@ -1013,4 +1058,34 @@ Status DatabaseImpl::userCreateNS(OperationContext* opCtx, return Status::OK(); } +Status DatabaseImpl::userCreateVirtualNS(OperationContext* opCtx, + const NamespaceString& nss, + CollectionOptions opts, + const VirtualCollectionOptions& vopts) const { + LOGV2_DEBUG(6968505, + 1, + "create collection {namespace} {collectionOptions}", + "namespace"_attr = nss, + "collectionOptions"_attr = opts.toBSON()); + if (!NamespaceString::validCollectionComponent(nss.ns())) + return Status(ErrorCodes::InvalidNamespace, str::stream() << "invalid ns: " << nss); + + // Validate the collation, if there is one. + if (auto swCollator = _validateCollator(opCtx, opts); !swCollator.isOK()) { + return swCollator.getStatus(); + } + + invariant(_createCollection(opCtx, + nss, + opts, + /*createDefaultIndexes=*/false, + /*idIndex=*/BSONObj(), + /*fromMigrate=*/false, + vopts), + str::stream() << "Collection creation failed after validating options: " << nss + << ". Options: " << opts.toBSON()); + + return Status::OK(); +} + } // namespace mongo diff --git a/src/mongo/db/catalog/database_impl.h b/src/mongo/db/catalog/database_impl.h index 0da06082628..489df90ab64 100644 --- a/src/mongo/db/catalog/database_impl.h +++ b/src/mongo/db/catalog/database_impl.h @@ -81,6 +81,11 @@ public: const BSONObj& idIndex, bool fromMigrate) const final; + Status userCreateVirtualNS(OperationContext* opCtx, + const NamespaceString& fullns, + CollectionOptions opts, + const VirtualCollectionOptions& vopts) const final; + Collection* createCollection(OperationContext* opCtx, const NamespaceString& nss, const CollectionOptions& options = CollectionOptions(), @@ -88,6 +93,11 @@ public: const BSONObj& idIndex = BSONObj(), bool fromMigrate = false) const final; + Collection* createVirtualCollection(OperationContext* opCtx, + const NamespaceString& nss, + const CollectionOptions& opts, + const VirtualCollectionOptions& vopts) const final; + Status createView(OperationContext* opCtx, const NamespaceString& viewName, const CollectionOptions& options) const final; @@ -106,6 +116,17 @@ public: private: friend class DatabaseHolderImpl; + StatusWith> _validateCollator(OperationContext* opCtx, + CollectionOptions& opts) const; + Collection* _createCollection( + OperationContext* opCtx, + const NamespaceString& nss, + const CollectionOptions& opts = CollectionOptions(), + bool createDefaultIndexes = true, + const BSONObj& idIndex = BSONObj(), + bool fromMigrate = false, + const boost::optional& vopts = boost::none) const; + /** * Throws if there is a reason 'ns' cannot be created as a user collection. Namespace pattern * matching checks should be added to userAllowedCreateNS(). diff --git a/src/mongo/db/catalog/virtual_collection_impl.cpp b/src/mongo/db/catalog/virtual_collection_impl.cpp new file mode 100644 index 00000000000..1c7e2f4e341 --- /dev/null +++ b/src/mongo/db/catalog/virtual_collection_impl.cpp @@ -0,0 +1,72 @@ +/** + * Copyright (C) 2022-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/catalog/virtual_collection_impl.h" + +#include "mongo/db/catalog/collection_impl.h" +#include "mongo/db/catalog/collection_options.h" +#include "mongo/db/operation_context.h" +#include "mongo/db/storage/external_record_store.h" + +#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kStorage + +namespace mongo { +VirtualCollectionImpl::VirtualCollectionImpl(OperationContext* opCtx, + const NamespaceString& nss, + const CollectionOptions& options, + std::unique_ptr recordStore) + : _nss(nss), + _options(options), + _recordStore(std::move(recordStore)), + _collator(CollectionImpl::parseCollation(opCtx, nss, options.collation)) { + tassert(6968503, + "Cannot create _id index for a virtual collection", + options.autoIndexId == CollectionOptions::NO && options.idIndex.isEmpty()); +} + +std::shared_ptr VirtualCollectionImpl::make(OperationContext* opCtx, + const NamespaceString& nss, + const CollectionOptions& options, + const VirtualCollectionOptions& vopts) { + return std::make_shared( + opCtx, nss, options, std::make_unique(nss.ns(), vopts)); +} + +std::unique_ptr VirtualCollectionImpl::makePlanExecutor( + OperationContext* opCtx, + const CollectionPtr& yieldableCollection, + PlanYieldPolicy::YieldPolicy yieldPolicy, + ScanDirection scanDirection, + const boost::optional& resumeAfterRecordId) const { + // TODO SERVER-69683 Implement this function when implementing MultiBsonStreamCursor since + // we can scan when it's done. We don't support scan yet. + unimplementedTasserted(); + return nullptr; +} +} // namespace mongo diff --git a/src/mongo/db/catalog/virtual_collection_impl.h b/src/mongo/db/catalog/virtual_collection_impl.h new file mode 100644 index 00000000000..4283a7e53ed --- /dev/null +++ b/src/mongo/db/catalog/virtual_collection_impl.h @@ -0,0 +1,544 @@ +/** + * Copyright (C) 2022-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/catalog/collection.h" +#include "mongo/db/catalog/index_catalog.h" +#include "mongo/db/catalog/virtual_collection_options.h" +#include "mongo/db/storage/external_record_store.h" +#include "mongo/db/storage/snapshot.h" +#include "mongo/util/assert_util.h" + +namespace mongo { +class VirtualCollectionImpl final : public Collection { +public: + static std::shared_ptr make(OperationContext* opCtx, + const NamespaceString& nss, + const CollectionOptions& options, + const VirtualCollectionOptions& vopts); + + // Constructor for a virtual collection. + explicit VirtualCollectionImpl(OperationContext* opCtx, + const NamespaceString& nss, + const CollectionOptions& options, + std::unique_ptr recordStore); + + ~VirtualCollectionImpl() = default; + + const VirtualCollectionOptions& getVirtualCollectionOptions() const { + return _recordStore->getOptions(); + } + + std::shared_ptr clone() const final { + unimplementedTasserted(); + return nullptr; + } + + SharedCollectionDecorations* getSharedDecorations() const final { + return nullptr; + } + + Status initFromExisting(OperationContext* opCtx, + std::shared_ptr collection, + Timestamp readTimestamp) final { + unimplementedTasserted(); + return Status(ErrorCodes::UnknownError, "unknown"); + }; + + void setCommitted(bool) {} + + bool isInitialized() const final { + return true; + }; + + const NamespaceString& ns() const final { + return _nss; + } + + Status rename(OperationContext* opCtx, const NamespaceString& nss, bool stayTemp) final { + unimplementedTasserted(); + return Status(ErrorCodes::UnknownError, "unknown"); + } + + RecordId getCatalogId() const final { + return RecordId(); + } + + UUID uuid() const final { + return *_options.uuid; + } + + const IndexCatalog* getIndexCatalog() const final { + return nullptr; + } + + IndexCatalog* getIndexCatalog() final { + return nullptr; + } + + RecordStore* getRecordStore() const final { + return _recordStore.get(); + } + + std::shared_ptr getSharedIdent() const final { + return _recordStore->getSharedIdent(); + } + + void setIdent(std::shared_ptr newIdent) final { + unimplementedTasserted(); + } + + BSONObj getValidatorDoc() const final { + return BSONObj(); + } + + std::pair checkValidation(OperationContext* opCtx, + const BSONObj& document) const final { + unimplementedTasserted(); + return {SchemaValidationResult::kError, Status(ErrorCodes::UnknownError, "unknown")}; + } + + Status checkValidationAndParseResult(OperationContext* opCtx, + const BSONObj& document) const final { + unimplementedTasserted(); + return Status(ErrorCodes::UnknownError, "unknown"); + } + + bool requiresIdIndex() const final { + return false; + }; + + Snapshotted docFor(OperationContext* opCtx, const RecordId& loc) const final { + unimplementedTasserted(); + return Snapshotted(); + } + + bool findDoc(OperationContext* opCtx, + const RecordId& loc, + Snapshotted* out) const final { + unimplementedTasserted(); + return false; + } + + std::unique_ptr getCursor(OperationContext* opCtx, + bool forward = true) const final { + return _recordStore->getCursor(opCtx, forward); + } + + void deleteDocument( + OperationContext* opCtx, + StmtId stmtId, + const RecordId& loc, + OpDebug* opDebug, + bool fromMigrate = false, + bool noWarn = false, + Collection::StoreDeletedDoc storeDeletedDoc = Collection::StoreDeletedDoc::Off, + CheckRecordId checkRecordId = CheckRecordId::Off) const final { + unimplementedTasserted(); + } + + void deleteDocument( + OperationContext* opCtx, + Snapshotted doc, + StmtId stmtId, + const RecordId& loc, + OpDebug* opDebug, + bool fromMigrate = false, + bool noWarn = false, + Collection::StoreDeletedDoc storeDeletedDoc = Collection::StoreDeletedDoc::Off, + CheckRecordId checkRecordId = CheckRecordId::Off) const final { + unimplementedTasserted(); + } + + RecordId updateDocument(OperationContext* opCtx, + const RecordId& oldLocation, + const Snapshotted& oldDoc, + const BSONObj& newDoc, + bool indexesAffected, + OpDebug* opDebug, + CollectionUpdateArgs* args) const final { + unimplementedTasserted(); + return RecordId(); + } + + bool updateWithDamagesSupported() const final { + unimplementedTasserted(); + return false; + } + + StatusWith updateDocumentWithDamages(OperationContext* opCtx, + const RecordId& loc, + const Snapshotted& oldDoc, + const char* damageSource, + const mutablebson::DamageVector& damages, + bool indexesAffected, + OpDebug* opDebug, + CollectionUpdateArgs* args) const final { + unimplementedTasserted(); + return Status(ErrorCodes::UnknownError, "unknown"); + } + + Status truncate(OperationContext* opCtx) final { + unimplementedTasserted(); + return Status(ErrorCodes::UnknownError, "unknown"); + } + + Validator parseValidator(OperationContext* opCtx, + const BSONObj& validator, + MatchExpressionParser::AllowedFeatureSet allowedFeatures, + boost::optional + maxFeatureCompatibilityVersion = boost::none) const final { + unimplementedTasserted(); + return Validator(); + } + + void setValidator(OperationContext* opCtx, Validator validator) final { + unimplementedTasserted(); + } + + Status setValidationLevel(OperationContext* opCtx, ValidationLevelEnum newLevel) final { + unimplementedTasserted(); + return Status(ErrorCodes::UnknownError, "unknown"); + } + + Status setValidationAction(OperationContext* opCtx, ValidationActionEnum newAction) final { + unimplementedTasserted(); + return Status(ErrorCodes::UnknownError, "unknown"); + } + + boost::optional getValidationLevel() const final { + unimplementedTasserted(); + return boost::none; + } + + boost::optional getValidationAction() const final { + unimplementedTasserted(); + return boost::none; + } + + Status updateValidator(OperationContext* opCtx, + BSONObj newValidator, + boost::optional newLevel, + boost::optional newAction) final { + unimplementedTasserted(); + return Status(ErrorCodes::UnknownError, "unknown"); + } + + Status checkValidatorAPIVersionCompatability(OperationContext* opCtx) const final { + unimplementedTasserted(); + return Status(ErrorCodes::UnknownError, "unknown"); + } + + bool isChangeStreamPreAndPostImagesEnabled() const final { + unimplementedTasserted(); + return false; + } + + void setChangeStreamPreAndPostImages(OperationContext* opCtx, + ChangeStreamPreAndPostImagesOptions val) final { + unimplementedTasserted(); + } + + bool isTemporary() const final { + return true; + } + + boost::optional getTimeseriesBucketsMayHaveMixedSchemaData() const final { + unimplementedTasserted(); + return boost::none; + } + + void setTimeseriesBucketsMayHaveMixedSchemaData(OperationContext* opCtx, + boost::optional setting) final { + unimplementedTasserted(); + } + + bool doesTimeseriesBucketsDocContainMixedSchemaData(const BSONObj& bucketsDoc) const final { + unimplementedTasserted(); + return false; + } + + bool getRequiresTimeseriesExtendedRangeSupport() const final { + unimplementedTasserted(); + return false; + } + + void setRequiresTimeseriesExtendedRangeSupport(OperationContext* opCtx) const final { + unimplementedTasserted(); + } + + bool isClustered() const final { + return false; + } + + boost::optional getClusteredInfo() const final { + unimplementedTasserted(); + return boost::none; + } + + void updateClusteredIndexTTLSetting(OperationContext* opCtx, + boost::optional expireAfterSeconds) final { + unimplementedTasserted(); + } + + Status updateCappedSize(OperationContext* opCtx, + boost::optional newCappedSize, + boost::optional newCappedMax) final { + unimplementedTasserted(); + return Status(ErrorCodes::UnknownError, "unknown"); + } + + StatusWith checkMetaDataForIndex(const std::string& indexName, + const BSONObj& spec) const final { + unimplementedTasserted(); + return Status(ErrorCodes::UnknownError, "unknown"); + } + + void updateTTLSetting(OperationContext* opCtx, + StringData idxName, + long long newExpireSeconds) final { + unimplementedTasserted(); + } + + void updateHiddenSetting(OperationContext* opCtx, StringData idxName, bool hidden) final { + unimplementedTasserted(); + } + + void updateUniqueSetting(OperationContext* opCtx, StringData idxName, bool unique) final { + unimplementedTasserted(); + } + + void updatePrepareUniqueSetting(OperationContext* opCtx, + StringData idxName, + bool prepareUnique) final { + unimplementedTasserted(); + } + + std::vector repairInvalidIndexOptions(OperationContext* opCtx) final { + unimplementedTasserted(); + return {}; + } + + void setIsTemp(OperationContext* opCtx, bool isTemp) final { + unimplementedTasserted(); + } + + void removeIndex(OperationContext* opCtx, StringData indexName) final { + unimplementedTasserted(); + } + + Status prepareForIndexBuild(OperationContext* opCtx, + const IndexDescriptor* spec, + boost::optional buildUUID, + bool isBackgroundSecondaryBuild) final { + unimplementedTasserted(); + return Status(ErrorCodes::UnknownError, "unknown"); + } + + boost::optional getIndexBuildUUID(StringData indexName) const final { + unimplementedTasserted(); + return boost::none; + } + + bool isIndexMultikey(OperationContext* opCtx, + StringData indexName, + MultikeyPaths* multikeyPaths, + int indexOffset) const final { + unimplementedTasserted(); + return false; + } + + bool setIndexIsMultikey(OperationContext* opCtx, + StringData indexName, + const MultikeyPaths& multikeyPaths, + int indexOffset) const final { + unimplementedTasserted(); + return false; + } + + void forceSetIndexIsMultikey(OperationContext* opCtx, + const IndexDescriptor* desc, + bool isMultikey, + const MultikeyPaths& multikeyPaths) const final { + unimplementedTasserted(); + } + + int getTotalIndexCount() const final { + unimplementedTasserted(); + return 0; + } + + int getCompletedIndexCount() const final { + unimplementedTasserted(); + return 0; + } + + BSONObj getIndexSpec(StringData indexName) const final { + unimplementedTasserted(); + return BSONObj(); + } + + void getAllIndexes(std::vector* names) const final { + unimplementedTasserted(); + } + + void getReadyIndexes(std::vector* names) const final { + unimplementedTasserted(); + } + + bool isIndexPresent(StringData indexName) const final { + unimplementedTasserted(); + return false; + } + + bool isIndexReady(StringData indexName) const final { + unimplementedTasserted(); + return false; + } + + void replaceMetadata(OperationContext* opCtx, + std::shared_ptr md) final { + unimplementedTasserted(); + } + + bool needsCappedLock() const final { + unimplementedTasserted(); + return false; + } + + bool isCappedAndNeedsDelete(OperationContext* opCtx) const final { + unimplementedTasserted(); + return false; + } + + bool isCapped() const final { + return false; + } + + long long getCappedMaxDocs() const final { + return 0; + } + + long long getCappedMaxSize() const final { + return 0; + } + + long long numRecords(OperationContext* opCtx) const final { + return _recordStore->numRecords(opCtx); + } + + long long dataSize(OperationContext* opCtx) const final { + return _recordStore->dataSize(opCtx); + } + + bool isEmpty(OperationContext* opCtx) const final { + return _recordStore->dataSize(opCtx) == 0LL; + } + + inline int averageObjectSize(OperationContext* opCtx) const { + return 0; + } + + uint64_t getIndexSize(OperationContext* opCtx, + BSONObjBuilder* details = nullptr, + int scale = 1) const final { + return 0; + } + + uint64_t getIndexFreeStorageBytes(OperationContext* opCtx) const final { + return 0; + } + + boost::optional getMinimumVisibleSnapshot() const final { + return boost::none; + } + + boost::optional getMinimumValidSnapshot() const final { + return boost::none; + } + + void setMinimumVisibleSnapshot(Timestamp newMinimumVisibleSnapshot) final {} + + void setMinimumValidSnapshot(Timestamp newMinimumValidSnapshot) final {} + + boost::optional getTimeseriesOptions() const final { + return boost::none; + } + + void setTimeseriesOptions(OperationContext* opCtx, const TimeseriesOptions& tsOptions) final { + unimplementedTasserted(); + } + + /** + * Get a pointer to the collection's default collator. The pointer must not be used after this + * Collection is destroyed. + */ + const CollatorInterface* getDefaultCollator() const final { + return _collator.get(); + } + + const CollectionOptions& getCollectionOptions() const final { + return _options; + } + + StatusWith> addCollationDefaultsToIndexSpecsForCreate( + OperationContext* opCtx, const std::vector& indexSpecs) const final { + unimplementedTasserted(); + return Status(ErrorCodes::UnknownError, "unknown"); + } + + std::unique_ptr makePlanExecutor( + OperationContext* opCtx, + const CollectionPtr& yieldableCollection, + PlanYieldPolicy::YieldPolicy yieldPolicy, + ScanDirection scanDirection, + const boost::optional& resumeAfterRecordId) const final; + + void indexBuildSuccess(OperationContext* opCtx, IndexCatalogEntry* index) final { + unimplementedTasserted(); + } + + void onDeregisterFromCatalog(OperationContext* opCtx) final {} + +private: + void unimplementedTasserted() const { + MONGO_UNIMPLEMENTED_TASSERT(6968504); + } + + NamespaceString _nss; + CollectionOptions _options; + std::unique_ptr _recordStore; + + // The default collation which is applied to operations and indices which have no collation + // of their own. The collection's validator will respect this collation. If null, the + // default collation is simple binary compare. + std::unique_ptr _collator; +}; +} // namespace mongo diff --git a/src/mongo/db/catalog/virtual_collection_options.h b/src/mongo/db/catalog/virtual_collection_options.h new file mode 100644 index 00000000000..301036bf71e --- /dev/null +++ b/src/mongo/db/catalog/virtual_collection_options.h @@ -0,0 +1,100 @@ +/** + * Copyright (C) 2022-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include +#include +#include +#include + +#include "mongo/base/string_data.h" +#include "mongo/util/assert_util.h" + +namespace mongo { +/** + * Storage type for external data source. + */ +enum class StorageType : std::int32_t { + pipe, +}; + +/** + * File type for external data source. + */ +enum class FileType : std::int32_t { + bson, +}; + +/** + * Metadata for external data source. + */ +struct ExternalDataSourceMetadata { + static constexpr auto kDefaultFileUrlPrefix = "file:///tmp/"_sd; + + ExternalDataSourceMetadata(StringData url, StorageType storageType, FileType fileType) + : url(url), storageType(storageType), fileType(fileType) { + using namespace fmt::literals; + uassert(6968500, + "File url must start with {}"_format(kDefaultFileUrlPrefix), + url.startsWith(kDefaultFileUrlPrefix)); + uassert(6968501, "Storage type must be 'pipe'", storageType == StorageType::pipe); + uassert(6968502, "File type must be 'bson'", fileType == FileType::bson); + } + + /** + * Url for an external data source + */ + StringData url; + + /** + * Storage type for an external data source + */ + StorageType storageType; + + /** + * File type for an external data source + */ + FileType fileType; +}; + +/** + * Options for virtual collection. + */ +struct VirtualCollectionOptions { + VirtualCollectionOptions() : dataSources() {} + VirtualCollectionOptions(std::vector dataSources) + : dataSources(dataSources) {} + + /** + * Specification for external data sources. + */ + std::vector dataSources; +}; +} // namespace mongo diff --git a/src/mongo/db/storage/SConscript b/src/mongo/db/storage/SConscript index 4d07e90204c..83c697dab61 100644 --- a/src/mongo/db/storage/SConscript +++ b/src/mongo/db/storage/SConscript @@ -66,9 +66,7 @@ env.Library( env.Library( target='record_store_base', - source=[ - 'record_store.cpp', - ], + source=['record_store.cpp', 'external_record_store.cpp'], LIBDEPS=[ '$BUILD_DIR/mongo/base', ], diff --git a/src/mongo/db/storage/external_record_store.cpp b/src/mongo/db/storage/external_record_store.cpp new file mode 100644 index 00000000000..ee65c8a861c --- /dev/null +++ b/src/mongo/db/storage/external_record_store.cpp @@ -0,0 +1,43 @@ +/** + * Copyright (C) 2022-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/storage/external_record_store.h" + +#include "mongo/db/storage/record_store.h" + +namespace mongo { +ExternalRecordStore::ExternalRecordStore(StringData ns, const VirtualCollectionOptions& vopts) + : RecordStore(ns, /*identName=*/ns, /*isCapped=*/false), _vopts(vopts) {} + +std::unique_ptr ExternalRecordStore::getCursor(OperationContext* opCtx, + bool forward) const { + // TODO SERVER-69683 Implement this method together when implementing MultiBsonStreamCursor + return nullptr; +} +} // namespace mongo diff --git a/src/mongo/db/storage/external_record_store.h b/src/mongo/db/storage/external_record_store.h new file mode 100644 index 00000000000..ac0c61392c6 --- /dev/null +++ b/src/mongo/db/storage/external_record_store.h @@ -0,0 +1,148 @@ +/** + * Copyright (C) 2022-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/base/error_codes.h" +#include "mongo/db/catalog/virtual_collection_options.h" +#include "mongo/db/storage/record_store.h" +#include "mongo/util/assert_util.h" + +namespace mongo { +class ExternalRecordStore : public RecordStore { +public: + ExternalRecordStore(StringData ns, const VirtualCollectionOptions& vopts); + + const VirtualCollectionOptions& getOptions() const { + return _vopts; + } + + const char* name() const { + return "external"; + } + + bool isTemp() const { + return true; + } + + KeyFormat keyFormat() const final { + return KeyFormat::Long; + } + + long long dataSize(OperationContext*) const final { + return 0LL; + } + + long long numRecords(OperationContext*) const final { + return 0LL; + } + + int64_t storageSize(OperationContext*, BSONObjBuilder*, int) const final { + return 0LL; + } + + bool findRecord(OperationContext*, const RecordId&, RecordData*) const final { + unimplementedTasserted(); + return false; + } + + bool updateWithDamagesSupported() const final { + return false; + } + + void printRecordMetadata(OperationContext*, const RecordId&) const final { + unimplementedTasserted(); + } + + std::unique_ptr getCursor(OperationContext* opCtx, + bool forward = true) const final; + + std::unique_ptr getRandomCursor(OperationContext* opCtx) const final { + unimplementedTasserted(); + return nullptr; + } + + void appendNumericCustomStats(OperationContext*, BSONObjBuilder*, double) const final {} + + void updateStatsAfterRepair(OperationContext* opCtx, + long long numRecords, + long long dataSize) final { + unimplementedTasserted(); + } + +protected: + void doDeleteRecord(OperationContext*, const RecordId&) final { + unimplementedTasserted(); + } + + Status doInsertRecords(OperationContext*, + std::vector*, + const std::vector&) final { + unimplementedTasserted(); + return {ErrorCodes::Error::UnknownError, "Unknown error"}; + } + + Status doUpdateRecord(OperationContext*, const RecordId&, const char*, int) final { + unimplementedTasserted(); + return {ErrorCodes::Error::UnknownError, "Unknown error"}; + } + + StatusWith doUpdateWithDamages(OperationContext*, + const RecordId&, + const RecordData&, + const char*, + const mutablebson::DamageVector&) final { + unimplementedTasserted(); + return {ErrorCodes::Error::UnknownError, "Unknown error"}; + } + + Status doTruncate(OperationContext* opCtx) final { + unimplementedTasserted(); + return {ErrorCodes::Error::UnknownError, "Unknown error"}; + } + + void doCappedTruncateAfter(OperationContext*, + const RecordId&, + bool, + const AboutToDeleteRecordCallback&) final { + unimplementedTasserted(); + } + + void waitForAllEarlierOplogWritesToBeVisibleImpl(OperationContext*) const final { + unimplementedTasserted(); + } + +private: + void unimplementedTasserted() const { + MONGO_UNIMPLEMENTED_TASSERT(6968600); + } + + VirtualCollectionOptions _vopts; +}; +} // namespace mongo -- cgit v1.2.1