diff options
author | Daniel Gottlieb <daniel.gottlieb@10gen.com> | 2017-05-05 15:23:14 -0400 |
---|---|---|
committer | Daniel Gottlieb <daniel.gottlieb@10gen.com> | 2017-05-15 14:06:46 -0400 |
commit | b69aed9d10ef66de42880fd379b0a593419b6e47 (patch) | |
tree | 26afd158f751c711682ab801ff028e562b4503af /src/mongo | |
parent | c31686212e0011909bbe13f8740fe4f45b8117ef (diff) | |
download | mongo-b69aed9d10ef66de42880fd379b0a593419b6e47.tar.gz |
SERVER-28742: Assign prefixes to collections/indexes when --groupCollections is engaged
Diffstat (limited to 'src/mongo')
34 files changed, 545 insertions, 77 deletions
diff --git a/src/mongo/db/catalog/collection_catalog_entry.h b/src/mongo/db/catalog/collection_catalog_entry.h index 16e7002db05..41d932f9e83 100644 --- a/src/mongo/db/catalog/collection_catalog_entry.h +++ b/src/mongo/db/catalog/collection_catalog_entry.h @@ -35,6 +35,7 @@ #include "mongo/db/index/multikey_paths.h" #include "mongo/db/namespace_string.h" #include "mongo/db/record_id.h" +#include "mongo/db/storage/kv/kv_prefix.h" namespace mongo { @@ -100,6 +101,8 @@ public: virtual bool isIndexReady(OperationContext* opCtx, StringData indexName) const = 0; + virtual KVPrefix getIndexPrefix(OperationContext* opCtx, StringData indexName) const = 0; + virtual Status removeIndex(OperationContext* opCtx, StringData indexName) = 0; virtual Status prepareForIndexBuild(OperationContext* opCtx, const IndexDescriptor* spec) = 0; diff --git a/src/mongo/db/catalog/index_catalog_entry.h b/src/mongo/db/catalog/index_catalog_entry.h index 3c1f1ebd6ac..9a503d317c0 100644 --- a/src/mongo/db/catalog/index_catalog_entry.h +++ b/src/mongo/db/catalog/index_catalog_entry.h @@ -35,6 +35,7 @@ #include "mongo/bson/ordering.h" #include "mongo/db/index/multikey_paths.h" #include "mongo/db/record_id.h" +#include "mongo/db/storage/kv/kv_prefix.h" #include "mongo/db/storage/snapshot_name.h" #include "mongo/platform/atomic_word.h" #include "mongo/stdx/functional.h" @@ -94,6 +95,8 @@ public: virtual bool isReady(OperationContext* opCtx) const = 0; + virtual KVPrefix getPrefix() const = 0; + virtual boost::optional<SnapshotName> getMinimumVisibleSnapshot() = 0; virtual void setMinimumVisibleSnapshot(SnapshotName name) = 0; @@ -221,6 +224,10 @@ public: return this->_impl().isReady(opCtx); } + KVPrefix getPrefix() const { + return this->_impl().getPrefix(); + } + /** * If return value is not boost::none, reads with majority read concern using an older snapshot * must treat this index as unfinished. diff --git a/src/mongo/db/catalog/index_catalog_entry_impl.cpp b/src/mongo/db/catalog/index_catalog_entry_impl.cpp index 7b0295d234b..bee9a4e0c87 100644 --- a/src/mongo/db/catalog/index_catalog_entry_impl.cpp +++ b/src/mongo/db/catalog/index_catalog_entry_impl.cpp @@ -99,7 +99,8 @@ IndexCatalogEntryImpl::IndexCatalogEntryImpl(IndexCatalogEntry* const this_, _infoCache(infoCache), _headManager(stdx::make_unique<HeadManagerImpl>(this_)), _ordering(Ordering::make(_descriptor->keyPattern())), - _isReady(false) { + _isReady(false), + _prefix(collection->getIndexPrefix(opCtx, _descriptor->indexName())) { _descriptor->_cachedEntry = this_; _isReady = _catalogIsReady(opCtx); @@ -323,4 +324,9 @@ bool IndexCatalogEntryImpl::_catalogIsMultikey(OperationContext* opCtx, MultikeyPaths* multikeyPaths) const { return _collection->isIndexMultikey(opCtx, _descriptor->indexName(), multikeyPaths); } + +KVPrefix IndexCatalogEntryImpl::_catalogGetPrefix(OperationContext* opCtx) const { + return _collection->getIndexPrefix(opCtx, _descriptor->indexName()); +} + } // namespace mongo diff --git a/src/mongo/db/catalog/index_catalog_entry_impl.h b/src/mongo/db/catalog/index_catalog_entry_impl.h index 593c74dc3f1..66349668a5b 100644 --- a/src/mongo/db/catalog/index_catalog_entry_impl.h +++ b/src/mongo/db/catalog/index_catalog_entry_impl.h @@ -36,6 +36,7 @@ #include "mongo/db/catalog/index_catalog_entry.h" #include "mongo/db/index/multikey_paths.h" #include "mongo/db/record_id.h" +#include "mongo/db/storage/kv/kv_prefix.h" #include "mongo/db/storage/snapshot_name.h" #include "mongo/platform/atomic_word.h" #include "mongo/stdx/mutex.h" @@ -142,6 +143,10 @@ public: // if this ready is ready for queries bool isReady(OperationContext* opCtx) const final; + KVPrefix getPrefix() const final { + return _prefix; + } + /** * If return value is not boost::none, reads with majority read concern using an older snapshot * must treat this index as unfinished. @@ -168,6 +173,8 @@ private: */ bool _catalogIsMultikey(OperationContext* opCtx, MultikeyPaths* multikeyPaths) const; + KVPrefix _catalogGetPrefix(OperationContext* opCtx) const; + // ----- std::string _ns; @@ -213,6 +220,10 @@ private: // causes the index to be multikey. MultikeyPaths _indexMultikeyPaths; + // KVPrefix used to differentiate between index entries in different logical indexes sharing the + // same underlying sorted data interface. + const KVPrefix _prefix; + // The earliest snapshot that is allowed to read this index. boost::optional<SnapshotName> _minVisibleSnapshot; }; diff --git a/src/mongo/db/storage/SConscript b/src/mongo/db/storage/SConscript index be6290defcb..05da73b079b 100644 --- a/src/mongo/db/storage/SConscript +++ b/src/mongo/db/storage/SConscript @@ -54,6 +54,7 @@ env.Library( '$BUILD_DIR/mongo/db/catalog/collection_options', '$BUILD_DIR/mongo/db/common', '$BUILD_DIR/mongo/db/service_context', + '$BUILD_DIR/mongo/db/storage/kv/kv_prefix', ], ) diff --git a/src/mongo/db/storage/bson_collection_catalog_entry.cpp b/src/mongo/db/storage/bson_collection_catalog_entry.cpp index 9bd57ee6941..48e8c41bc50 100644 --- a/src/mongo/db/storage/bson_collection_catalog_entry.cpp +++ b/src/mongo/db/storage/bson_collection_catalog_entry.cpp @@ -31,6 +31,7 @@ #include "mongo/db/storage/bson_collection_catalog_entry.h" #include <algorithm> +#include <numeric> #include "mongo/db/field_ref.h" @@ -175,6 +176,14 @@ bool BSONCollectionCatalogEntry::isIndexReady(OperationContext* opCtx, StringDat return md.indexes[offset].ready; } +KVPrefix BSONCollectionCatalogEntry::getIndexPrefix(OperationContext* opCtx, + StringData indexName) const { + MetaData md = _getMetaData(opCtx); + int offset = md.findIndexOffset(indexName); + invariant(offset >= 0); + return md.indexes[offset].prefix; +} + // -------------------------- void BSONCollectionCatalogEntry::IndexMetaData::updateTTLSetting(long long newExpireSeconds) { @@ -228,6 +237,15 @@ void BSONCollectionCatalogEntry::MetaData::rename(StringData toNS) { } } +KVPrefix BSONCollectionCatalogEntry::MetaData::getMaxPrefix() const { + // Use the collection prefix as the initial max value seen. Then compare it with each index + // prefix. Note the oplog has no indexes so the vector of 'IndexMetaData' may be empty. + return std::accumulate( + indexes.begin(), indexes.end(), prefix, [](KVPrefix max, IndexMetaData index) { + return max < index.prefix ? index.prefix : max; + }); +} + BSONObj BSONCollectionCatalogEntry::MetaData::toBSON() const { BSONObjBuilder b; b.append("ns", ns); @@ -249,10 +267,12 @@ BSONObj BSONCollectionCatalogEntry::MetaData::toBSON() const { } sub.append("head", static_cast<long long>(indexes[i].head.repr())); + sub.append("prefix", indexes[i].prefix.toBSONValue()); sub.doneFast(); } arr.doneFast(); } + b.append("prefix", prefix.toBSONValue()); return b.obj(); } @@ -282,8 +302,11 @@ void BSONCollectionCatalogEntry::MetaData::parse(const BSONObj& obj) { parseMultikeyPathsFromBytes(multikeyPathsElem.Obj(), &imd.multikeyPaths); } + imd.prefix = KVPrefix::fromBSONElement(idx["prefix"]); indexes.push_back(imd); } } + + prefix = KVPrefix::fromBSONElement(obj["prefix"]); } } diff --git a/src/mongo/db/storage/bson_collection_catalog_entry.h b/src/mongo/db/storage/bson_collection_catalog_entry.h index f2dbd891dd7..8806829cd0b 100644 --- a/src/mongo/db/storage/bson_collection_catalog_entry.h +++ b/src/mongo/db/storage/bson_collection_catalog_entry.h @@ -35,6 +35,7 @@ #include "mongo/db/catalog/collection_catalog_entry.h" #include "mongo/db/index/multikey_paths.h" +#include "mongo/db/storage/kv/kv_prefix.h" namespace mongo { @@ -66,12 +67,14 @@ public: virtual bool isIndexReady(OperationContext* opCtx, StringData indexName) const; + virtual KVPrefix getIndexPrefix(OperationContext* opCtx, StringData indexName) const; + // ------ for implementors struct IndexMetaData { IndexMetaData() {} - IndexMetaData(BSONObj s, bool r, RecordId h, bool m) - : spec(s), ready(r), head(h), multikey(m) {} + IndexMetaData(BSONObj s, bool r, RecordId h, bool m, KVPrefix prefix) + : spec(s), ready(r), head(h), multikey(m), prefix(prefix) {} void updateTTLSetting(long long newExpireSeconds); @@ -83,6 +86,7 @@ public: bool ready; RecordId head; bool multikey; + KVPrefix prefix = KVPrefix::kNotPrefixed; // If non-empty, 'multikeyPaths' is a vector with size equal to the number of elements in // the index key pattern. Each element in the vector is an ordered set of positions @@ -105,9 +109,12 @@ public: void rename(StringData toNS); + KVPrefix getMaxPrefix() const; + std::string ns; CollectionOptions options; std::vector<IndexMetaData> indexes; + KVPrefix prefix = KVPrefix::kNotPrefixed; }; protected: diff --git a/src/mongo/db/storage/devnull/SConscript b/src/mongo/db/storage/devnull/SConscript index a11aad58065..564ec403783 100644 --- a/src/mongo/db/storage/devnull/SConscript +++ b/src/mongo/db/storage/devnull/SConscript @@ -11,6 +11,7 @@ env.Library( ], LIBDEPS=[ '$BUILD_DIR/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_record_store', + '$BUILD_DIR/mongo/db/storage/kv/kv_prefix', ], ) diff --git a/src/mongo/db/storage/ephemeral_for_test/SConscript b/src/mongo/db/storage/ephemeral_for_test/SConscript index c95e2e6d223..f96900c9375 100644 --- a/src/mongo/db/storage/ephemeral_for_test/SConscript +++ b/src/mongo/db/storage/ephemeral_for_test/SConscript @@ -31,6 +31,7 @@ env.Library( '$BUILD_DIR/mongo/db/repl/repl_coordinator_global', '$BUILD_DIR/mongo/db/storage/index_entry_comparison', '$BUILD_DIR/mongo/db/storage/journal_listener', + '$BUILD_DIR/mongo/db/storage/kv/kv_prefix', ] ) diff --git a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_init.cpp b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_init.cpp index 4b11094babc..df936cfacb8 100644 --- a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_init.cpp +++ b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_init.cpp @@ -46,6 +46,10 @@ public: virtual ~EphemeralForTestFactory() {} virtual StorageEngine* create(const StorageGlobalParams& params, const StorageEngineLockFile* lockFile) const { + uassert(ErrorCodes::InvalidOptions, + "ephemeralForTest does not support --groupCollections", + !params.groupCollections); + KVStorageEngineOptions options; options.directoryPerDB = params.directoryperdb; options.forRepair = params.repair; diff --git a/src/mongo/db/storage/kv/SConscript b/src/mongo/db/storage/kv/SConscript index 47217f149fe..88af38faf43 100644 --- a/src/mongo/db/storage/kv/SConscript +++ b/src/mongo/db/storage/kv/SConscript @@ -4,6 +4,14 @@ Import("env") env = env.Clone() +env.Library( + target='kv_prefix', + source=['kv_prefix.cpp'], + LIBDEPS=[ + '$BUILD_DIR/mongo/db/storage/storage_options', + ], +) + # Should not be referenced outside this SConscript file. env.Library( target='kv_engine_core', @@ -18,6 +26,7 @@ env.Library( '$BUILD_DIR/mongo/db/index_names', '$BUILD_DIR/mongo/db/namespace_string', '$BUILD_DIR/mongo/db/storage/bson_collection_catalog_entry', + 'kv_prefix', ], ) @@ -85,6 +94,7 @@ env.Library( '$BUILD_DIR/mongo/unittest/unittest', '$BUILD_DIR/mongo/util/clock_source_mock', 'kv_engine_mock', + 'kv_prefix', ], ) diff --git a/src/mongo/db/storage/kv/kv_catalog.cpp b/src/mongo/db/storage/kv/kv_catalog.cpp index 588e52e24ad..5e23a27ff00 100644 --- a/src/mongo/db/storage/kv/kv_catalog.cpp +++ b/src/mongo/db/storage/kv/kv_catalog.cpp @@ -337,7 +337,8 @@ void KVCatalog::getAllCollections(std::vector<std::string>* out) const { Status KVCatalog::newCollection(OperationContext* opCtx, StringData ns, - const CollectionOptions& options) { + const CollectionOptions& options, + KVPrefix prefix) { invariant(opCtx->lockState() == NULL || opCtx->lockState()->isDbLockedForMode(nsToDatabaseSubstring(ns), MODE_X)); @@ -359,6 +360,7 @@ Status KVCatalog::newCollection(OperationContext* opCtx, BSONCollectionCatalogEntry::MetaData md; md.ns = ns.toString(); md.options = options; + md.prefix = prefix; b.append("md", md.toBSON()); obj = b.obj(); } diff --git a/src/mongo/db/storage/kv/kv_catalog.h b/src/mongo/db/storage/kv/kv_catalog.h index 8f05b52564d..aded9fd18a3 100644 --- a/src/mongo/db/storage/kv/kv_catalog.h +++ b/src/mongo/db/storage/kv/kv_catalog.h @@ -38,6 +38,7 @@ #include "mongo/db/catalog/collection_options.h" #include "mongo/db/record_id.h" #include "mongo/db/storage/bson_collection_catalog_entry.h" +#include "mongo/db/storage/kv/kv_prefix.h" #include "mongo/stdx/mutex.h" namespace mongo { @@ -65,7 +66,10 @@ public: /** * @return error or ident for instance */ - Status newCollection(OperationContext* opCtx, StringData ns, const CollectionOptions& options); + Status newCollection(OperationContext* opCtx, + StringData ns, + const CollectionOptions& options, + KVPrefix prefix); std::string getCollectionIdent(StringData ns) const; diff --git a/src/mongo/db/storage/kv/kv_collection_catalog_entry.cpp b/src/mongo/db/storage/kv/kv_collection_catalog_entry.cpp index 7ab0d15beb0..70ae05fe666 100644 --- a/src/mongo/db/storage/kv/kv_collection_catalog_entry.cpp +++ b/src/mongo/db/storage/kv/kv_collection_catalog_entry.cpp @@ -179,7 +179,9 @@ Status KVCollectionCatalogEntry::removeIndex(OperationContext* opCtx, StringData Status KVCollectionCatalogEntry::prepareForIndexBuild(OperationContext* opCtx, const IndexDescriptor* spec) { MetaData md = _getMetaData(opCtx); - IndexMetaData imd(spec->infoObj(), false, RecordId(), false); + + KVPrefix prefix = KVPrefix::getNextPrefix(ns()); + IndexMetaData imd(spec->infoObj(), false, RecordId(), false, prefix); if (indexTypeSupportsPathLevelMultikeyTracking(spec->getAccessMethodName())) { const auto feature = KVCatalog::FeatureTracker::RepairableFeature::kPathLevelMultikeyTracking; @@ -202,7 +204,7 @@ Status KVCollectionCatalogEntry::prepareForIndexBuild(OperationContext* opCtx, string ident = _catalog->getIndexIdent(opCtx, ns().ns(), spec->indexName()); - const Status status = _engine->createSortedDataInterface(opCtx, ident, spec); + const Status status = _engine->createGroupedSortedDataInterface(opCtx, ident, spec, prefix); if (status.isOK()) { opCtx->recoveryUnit()->registerChange(new AddIndexChange(opCtx, this, ident)); } diff --git a/src/mongo/db/storage/kv/kv_database_catalog_entry.cpp b/src/mongo/db/storage/kv/kv_database_catalog_entry.cpp index a0dfc0e7559..9dea5a31d18 100644 --- a/src/mongo/db/storage/kv/kv_database_catalog_entry.cpp +++ b/src/mongo/db/storage/kv/kv_database_catalog_entry.cpp @@ -62,7 +62,8 @@ IndexAccessMethod* KVDatabaseCatalogEntry::getIndex(OperationContext* opCtx, std::string ident = _engine->getCatalog()->getIndexIdent(opCtx, collection->ns().ns(), desc->indexName()); - SortedDataInterface* sdi = _engine->getEngine()->getSortedDataInterface(opCtx, ident, desc); + SortedDataInterface* sdi = + _engine->getEngine()->getGroupedSortedDataInterface(opCtx, ident, desc, index->getPrefix()); if ("" == type) return new BtreeAccessMethod(index, sdi); diff --git a/src/mongo/db/storage/kv/kv_database_catalog_entry_base.cpp b/src/mongo/db/storage/kv/kv_database_catalog_entry_base.cpp index 76031197e80..cc6b9a7829a 100644 --- a/src/mongo/db/storage/kv/kv_database_catalog_entry_base.cpp +++ b/src/mongo/db/storage/kv/kv_database_catalog_entry_base.cpp @@ -32,6 +32,7 @@ #include "mongo/db/storage/kv/kv_database_catalog_entry.h" +#include "mongo/db/namespace_string.h" #include "mongo/db/operation_context.h" #include "mongo/db/storage/kv/kv_catalog_feature_tracker.h" #include "mongo/db/storage/kv/kv_collection_catalog_entry.h" @@ -206,14 +207,16 @@ Status KVDatabaseCatalogEntryBase::createCollection(OperationContext* opCtx, return Status(ErrorCodes::NamespaceExists, "collection already exists"); } + KVPrefix prefix = KVPrefix::getNextPrefix(NamespaceString(ns)); + // need to create it - Status status = _engine->getCatalog()->newCollection(opCtx, ns, options); + Status status = _engine->getCatalog()->newCollection(opCtx, ns, options, prefix); if (!status.isOK()) return status; string ident = _engine->getCatalog()->getCollectionIdent(ns); - status = _engine->getEngine()->createRecordStore(opCtx, ns, ident, options); + status = _engine->getEngine()->createGroupedRecordStore(opCtx, ns, ident, options, prefix); if (!status.isOK()) return status; @@ -229,7 +232,7 @@ Status KVDatabaseCatalogEntryBase::createCollection(OperationContext* opCtx, opCtx->recoveryUnit()->registerChange(new AddCollectionChange(opCtx, this, ns, ident, true)); - auto rs = _engine->getEngine()->getRecordStore(opCtx, ns, ident, options); + auto rs = _engine->getEngine()->getGroupedRecordStore(opCtx, ns, ident, options, prefix); invariant(rs); _collections[ns.toString()] = new KVCollectionCatalogEntry( @@ -252,7 +255,7 @@ void KVDatabaseCatalogEntryBase::initCollection(OperationContext* opCtx, rs = nullptr; } else { BSONCollectionCatalogEntry::MetaData md = _engine->getCatalog()->getMetaData(opCtx, ns); - rs = _engine->getEngine()->getRecordStore(opCtx, ns, ident, md.options); + rs = _engine->getEngine()->getGroupedRecordStore(opCtx, ns, ident, md.options, md.prefix); invariant(rs); } @@ -318,7 +321,8 @@ Status KVDatabaseCatalogEntryBase::renameCollection(OperationContext* opCtx, opCtx->recoveryUnit()->registerChange( new AddCollectionChange(opCtx, this, toNS, identTo, false)); - auto rs = _engine->getEngine()->getRecordStore(opCtx, toNS, identTo, md.options); + auto rs = + _engine->getEngine()->getGroupedRecordStore(opCtx, toNS, identTo, md.options, md.prefix); _collections[toNS.toString()] = new KVCollectionCatalogEntry( _engine->getEngine(), _engine->getCatalog(), toNS, identTo, std::move(rs)); diff --git a/src/mongo/db/storage/kv/kv_database_catalog_entry_test.cpp b/src/mongo/db/storage/kv/kv_database_catalog_entry_test.cpp index 3645a2ded1f..0e122cf5603 100644 --- a/src/mongo/db/storage/kv/kv_database_catalog_entry_test.cpp +++ b/src/mongo/db/storage/kv/kv_database_catalog_entry_test.cpp @@ -34,6 +34,7 @@ #include "mongo/db/catalog/collection_options.h" #include "mongo/db/operation_context_noop.h" #include "mongo/db/storage/devnull/devnull_kv_engine.h" +#include "mongo/db/storage/kv/kv_prefix.h" #include "mongo/db/storage/kv/kv_storage_engine.h" #include "mongo/stdx/memory.h" #include "mongo/unittest/unittest.h" diff --git a/src/mongo/db/storage/kv/kv_engine.h b/src/mongo/db/storage/kv/kv_engine.h index f76ef68bb0e..6c15a9519e5 100644 --- a/src/mongo/db/storage/kv/kv_engine.h +++ b/src/mongo/db/storage/kv/kv_engine.h @@ -37,13 +37,14 @@ #include "mongo/base/status.h" #include "mongo/base/string_data.h" #include "mongo/db/catalog/collection_options.h" +#include "mongo/db/storage/kv/kv_prefix.h" +#include "mongo/db/storage/record_store.h" namespace mongo { class IndexDescriptor; class JournalListener; class OperationContext; -class RecordStore; class RecoveryUnit; class SortedDataInterface; class SnapshotManager; @@ -55,35 +56,106 @@ public: // --------- /** - * Having multiple out for the same ns is a rules violation; - * Calling on a non-created ident is invalid and may crash. + * Having multiple out for the same ns is a rules violation; Calling on a non-created ident is + * invalid and may crash. */ virtual std::unique_ptr<RecordStore> getRecordStore(OperationContext* opCtx, StringData ns, StringData ident, const CollectionOptions& options) = 0; + /** + * Get a RecordStore that may share an underlying table with other RecordStores. 'prefix' is + * guaranteed to be 'KVPrefix::kNotPrefixed' when 'groupCollections' is not enabled. + * + * @param prefix dictates the value keys for the RecordStore should be prefixed with to + * distinguish between RecordStores sharing an underlying table. A value of + * `KVPrefix::kNotPrefixed` guarantees the index is the sole resident of the table. + */ + virtual std::unique_ptr<RecordStore> getGroupedRecordStore(OperationContext* opCtx, + StringData ns, + StringData ident, + const CollectionOptions& options, + KVPrefix prefix) { + invariant(prefix == KVPrefix::kNotPrefixed); + return getRecordStore(opCtx, ns, ident, options); + } + virtual SortedDataInterface* getSortedDataInterface(OperationContext* opCtx, StringData ident, const IndexDescriptor* desc) = 0; - // - // The create and drop methods on KVEngine are not transactional. Transactional semantics - // are provided by the KVStorageEngine code that calls these. For example, drop will be - // called if a create is rolled back. A higher-level drop operation will only propagate to a - // drop call on the KVEngine once the WUOW commits. Therefore drops will never be rolled - // back and it is safe to immediately reclaim storage. - // + /** + * Get a SortedDataInterface that may share an underlying table with other + * SortedDataInterface. 'prefix' is guaranteed to be 'KVPrefix::kNotPrefixed' when + * 'groupCollections' is not enabled. + * + * @param prefix dictates the value keys for the index should be prefixed with to distinguish + * between indexes sharing an underlying table. A value of `KVPrefix::kNotPrefixed` + * guarantees the index is the sole resident of the table. + */ + virtual SortedDataInterface* getGroupedSortedDataInterface(OperationContext* opCtx, + StringData ident, + const IndexDescriptor* desc, + KVPrefix prefix) { + invariant(prefix == KVPrefix::kNotPrefixed); + return getSortedDataInterface(opCtx, ident, desc); + } + /** + * The create and drop methods on KVEngine are not transactional. Transactional semantics + * are provided by the KVStorageEngine code that calls these. For example, drop will be + * called if a create is rolled back. A higher-level drop operation will only propagate to a + * drop call on the KVEngine once the WUOW commits. Therefore drops will never be rolled + * back and it is safe to immediately reclaim storage. + */ virtual Status createRecordStore(OperationContext* opCtx, StringData ns, StringData ident, const CollectionOptions& options) = 0; + /** + * Create a RecordStore that MongoDB considers eligible to share space in an underlying table + * with other RecordStores. 'prefix' is guaranteed to be 'KVPrefix::kNotPrefixed' when + * 'groupCollections' is not enabled. + * + * @param prefix signals whether the RecordStore may be shared by an underlying table. A + * prefix of `KVPrefix::kNotPrefixed` must remain isolated in its own table. Otherwise + * the storage engine implementation ultimately chooses which RecordStores share a + * table. Sharing RecordStores belonging to different databases within the same table + * is forbidden. + */ + virtual Status createGroupedRecordStore(OperationContext* opCtx, + StringData ns, + StringData ident, + const CollectionOptions& options, + KVPrefix prefix) { + invariant(prefix == KVPrefix::kNotPrefixed); + return createRecordStore(opCtx, ns, ident, options); + } + virtual Status createSortedDataInterface(OperationContext* opCtx, StringData ident, const IndexDescriptor* desc) = 0; + /** + * Create a SortedDataInterface that MongoDB considers eligible to share space in an + * underlying table with other SortedDataInterfaces. 'prefix' is guaranteed to be + * 'KVPrefix::kNotPrefixed' when 'groupCollections' is not enabled. + * + * @param prefix signals whether the SortedDataInterface (index) may be shared by an + * underlying table. A prefix of `KVPrefix::kNotPrefixed` must remain isolated in its own + * table. Otherwise the storage engine implementation ultimately chooses which indexes + * share a table. Sharing indexes belonging to different databases is forbidden. + */ + virtual Status createGroupedSortedDataInterface(OperationContext* opCtx, + StringData ident, + const IndexDescriptor* desc, + KVPrefix prefix) { + invariant(prefix == KVPrefix::kNotPrefixed); + return createSortedDataInterface(opCtx, ident, desc); + } + virtual int64_t getIdentSize(OperationContext* opCtx, StringData ident) = 0; virtual Status repairIdent(OperationContext* opCtx, StringData ident) = 0; diff --git a/src/mongo/db/storage/kv/kv_engine_test_harness.cpp b/src/mongo/db/storage/kv/kv_engine_test_harness.cpp index 886cd1d556f..11f3fa50c60 100644 --- a/src/mongo/db/storage/kv/kv_engine_test_harness.cpp +++ b/src/mongo/db/storage/kv/kv_engine_test_harness.cpp @@ -34,11 +34,13 @@ #include "mongo/db/operation_context_noop.h" #include "mongo/db/storage/kv/kv_catalog.h" #include "mongo/db/storage/kv/kv_engine.h" +#include "mongo/db/storage/kv/kv_prefix.h" #include "mongo/db/storage/record_store.h" #include "mongo/db/storage/sorted_data_interface.h" #include "mongo/unittest/unittest.h" #include "mongo/util/assert_util.h" #include "mongo/util/clock_source_mock.h" +#include "mongo/util/scopeguard.h" namespace mongo { namespace { @@ -184,7 +186,8 @@ TEST(KVCatalogTest, Coll1) { { MyOperationContext opCtx(engine); WriteUnitOfWork uow(&opCtx); - ASSERT_OK(catalog->newCollection(&opCtx, "a.b", CollectionOptions())); + ASSERT_OK( + catalog->newCollection(&opCtx, "a.b", CollectionOptions(), KVPrefix::kNotPrefixed)); ASSERT_NOT_EQUALS("a.b", catalog->getCollectionIdent("a.b")); uow.commit(); } @@ -203,7 +206,7 @@ TEST(KVCatalogTest, Coll1) { MyOperationContext opCtx(engine); WriteUnitOfWork uow(&opCtx); catalog->dropCollection(&opCtx, "a.b"); - catalog->newCollection(&opCtx, "a.b", CollectionOptions()); + catalog->newCollection(&opCtx, "a.b", CollectionOptions(), KVPrefix::kNotPrefixed); uow.commit(); } ASSERT_NOT_EQUALS(ident, catalog->getCollectionIdent("a.b")); @@ -228,7 +231,8 @@ TEST(KVCatalogTest, Idx1) { { MyOperationContext opCtx(engine); WriteUnitOfWork uow(&opCtx); - ASSERT_OK(catalog->newCollection(&opCtx, "a.b", CollectionOptions())); + ASSERT_OK( + catalog->newCollection(&opCtx, "a.b", CollectionOptions(), KVPrefix::kNotPrefixed)); ASSERT_NOT_EQUALS("a.b", catalog->getCollectionIdent("a.b")); ASSERT_TRUE(catalog->isUserDataIdent(catalog->getCollectionIdent("a.b"))); uow.commit(); @@ -244,7 +248,8 @@ TEST(KVCatalogTest, Idx1) { << "foo"), false, RecordId(), - false)); + false, + KVPrefix::kNotPrefixed)); catalog->putMetaData(&opCtx, "a.b", md); uow.commit(); } @@ -272,7 +277,8 @@ TEST(KVCatalogTest, Idx1) { << "foo"), false, RecordId(), - false)); + false, + KVPrefix::kNotPrefixed)); catalog->putMetaData(&opCtx, "a.b", md); uow.commit(); } @@ -301,7 +307,8 @@ TEST(KVCatalogTest, DirectoryPerDb1) { { // collection MyOperationContext opCtx(engine); WriteUnitOfWork uow(&opCtx); - ASSERT_OK(catalog->newCollection(&opCtx, "a.b", CollectionOptions())); + ASSERT_OK( + catalog->newCollection(&opCtx, "a.b", CollectionOptions(), KVPrefix::kNotPrefixed)); ASSERT_STRING_CONTAINS(catalog->getCollectionIdent("a.b"), "a/"); ASSERT_TRUE(catalog->isUserDataIdent(catalog->getCollectionIdent("a.b"))); uow.commit(); @@ -317,7 +324,8 @@ TEST(KVCatalogTest, DirectoryPerDb1) { << "foo"), false, RecordId(), - false)); + false, + KVPrefix::kNotPrefixed)); catalog->putMetaData(&opCtx, "a.b", md); ASSERT_STRING_CONTAINS(catalog->getIndexIdent(&opCtx, "a.b", "foo"), "a/"); ASSERT_TRUE(catalog->isUserDataIdent(catalog->getIndexIdent(&opCtx, "a.b", "foo"))); @@ -343,7 +351,8 @@ TEST(KVCatalogTest, Split1) { { MyOperationContext opCtx(engine); WriteUnitOfWork uow(&opCtx); - ASSERT_OK(catalog->newCollection(&opCtx, "a.b", CollectionOptions())); + ASSERT_OK( + catalog->newCollection(&opCtx, "a.b", CollectionOptions(), KVPrefix::kNotPrefixed)); ASSERT_STRING_CONTAINS(catalog->getCollectionIdent("a.b"), "collection/"); ASSERT_TRUE(catalog->isUserDataIdent(catalog->getCollectionIdent("a.b"))); uow.commit(); @@ -359,7 +368,8 @@ TEST(KVCatalogTest, Split1) { << "foo"), false, RecordId(), - false)); + false, + KVPrefix::kNotPrefixed)); catalog->putMetaData(&opCtx, "a.b", md); ASSERT_STRING_CONTAINS(catalog->getIndexIdent(&opCtx, "a.b", "foo"), "index/"); ASSERT_TRUE(catalog->isUserDataIdent(catalog->getIndexIdent(&opCtx, "a.b", "foo"))); @@ -385,7 +395,8 @@ TEST(KVCatalogTest, DirectoryPerAndSplit1) { { MyOperationContext opCtx(engine); WriteUnitOfWork uow(&opCtx); - ASSERT_OK(catalog->newCollection(&opCtx, "a.b", CollectionOptions())); + ASSERT_OK( + catalog->newCollection(&opCtx, "a.b", CollectionOptions(), KVPrefix::kNotPrefixed)); ASSERT_STRING_CONTAINS(catalog->getCollectionIdent("a.b"), "a/collection/"); ASSERT_TRUE(catalog->isUserDataIdent(catalog->getCollectionIdent("a.b"))); uow.commit(); @@ -401,7 +412,8 @@ TEST(KVCatalogTest, DirectoryPerAndSplit1) { << "foo"), false, RecordId(), - false)); + false, + KVPrefix::kNotPrefixed)); catalog->putMetaData(&opCtx, "a.b", md); ASSERT_STRING_CONTAINS(catalog->getIndexIdent(&opCtx, "a.b", "foo"), "a/index/"); ASSERT_TRUE(catalog->isUserDataIdent(catalog->getIndexIdent(&opCtx, "a.b", "foo"))); @@ -409,6 +421,70 @@ TEST(KVCatalogTest, DirectoryPerAndSplit1) { } } +TEST(KVCatalogTest, RestartForPrefixes) { + storageGlobalParams.groupCollections = true; + ON_BLOCK_EXIT([&] { storageGlobalParams.groupCollections = false; }); + + KVPrefix abCollPrefix = KVPrefix::getNextPrefix(NamespaceString("a.b")); + KVPrefix fooIndexPrefix = KVPrefix::getNextPrefix(NamespaceString("a.b")); + + unique_ptr<KVHarnessHelper> helper(KVHarnessHelper::create()); + KVEngine* engine = helper->getEngine(); + { + unique_ptr<RecordStore> rs; + unique_ptr<KVCatalog> catalog; + { + MyOperationContext opCtx(engine); + WriteUnitOfWork uow(&opCtx); + ASSERT_OK(engine->createRecordStore(&opCtx, "catalog", "catalog", CollectionOptions())); + rs = engine->getRecordStore(&opCtx, "catalog", "catalog", CollectionOptions()); + catalog.reset(new KVCatalog(rs.get(), false, false)); + uow.commit(); + } + + { + MyOperationContext opCtx(engine); + WriteUnitOfWork uow(&opCtx); + ASSERT_OK(catalog->newCollection(&opCtx, "a.b", CollectionOptions(), abCollPrefix)); + ASSERT_NOT_EQUALS("a.b", catalog->getCollectionIdent("a.b")); + ASSERT_TRUE(catalog->isUserDataIdent(catalog->getCollectionIdent("a.b"))); + uow.commit(); + } + + { + MyOperationContext opCtx(engine); + WriteUnitOfWork uow(&opCtx); + + BSONCollectionCatalogEntry::MetaData md; + md.ns = "a.b"; + md.indexes.push_back(BSONCollectionCatalogEntry::IndexMetaData(BSON("name" + << "foo"), + false, + RecordId(), + false, + fooIndexPrefix)); + md.prefix = abCollPrefix; + catalog->putMetaData(&opCtx, "a.b", md); + uow.commit(); + } + } + + engine = helper->restartEngine(); + { + MyOperationContext opCtx(engine); + WriteUnitOfWork uow(&opCtx); + unique_ptr<RecordStore> rs = + engine->getRecordStore(&opCtx, "catalog", "catalog", CollectionOptions()); + unique_ptr<KVCatalog> catalog = stdx::make_unique<KVCatalog>(rs.get(), false, false); + catalog->init(&opCtx); + + const BSONCollectionCatalogEntry::MetaData md = catalog->getMetaData(&opCtx, "a.b"); + ASSERT_EQ("a.b", md.ns); + ASSERT_EQ(abCollPrefix, md.prefix); + ASSERT_EQ(fooIndexPrefix, md.indexes[md.findIndexOffset("foo")].prefix); + } +} + } // namespace std::unique_ptr<KVHarnessHelper> KVHarnessHelper::create() { diff --git a/src/mongo/db/storage/kv/kv_prefix.cpp b/src/mongo/db/storage/kv/kv_prefix.cpp new file mode 100644 index 00000000000..fabc2d99108 --- /dev/null +++ b/src/mongo/db/storage/kv/kv_prefix.cpp @@ -0,0 +1,68 @@ +/** + * Copyright (C) 2017 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/storage/kv/kv_prefix.h" + +namespace mongo { +int64_t KVPrefix::_nextValue = 0; +stdx::mutex KVPrefix::_nextValueMutex; +const KVPrefix KVPrefix::kNotPrefixed = KVPrefix(-1); + +std::string KVPrefix::toString() const { + StackStringBuilder ss; + ss << "KVPrefix(" << _value << ")"; + return ss.str(); +} + + +/* static */ KVPrefix KVPrefix::fromBSONElement(const BSONElement value) { + if (value.eoo()) { + return kNotPrefixed; + } + + return KVPrefix(static_cast<int64_t>(value.Long())); +} + +/* static */ void KVPrefix::setLargestPrefix(KVPrefix largestPrefix) { + if (!storageGlobalParams.groupCollections) { + return; + } + + stdx::lock_guard<stdx::mutex> lk(_nextValueMutex); + _nextValue = largestPrefix._value + 1; +} + +/* static */ KVPrefix KVPrefix::getNextPrefix(const NamespaceString& ns) { + if (!storageGlobalParams.groupCollections || ns.isOplog()) { + return kNotPrefixed; + } + + stdx::lock_guard<stdx::mutex> lk(_nextValueMutex); + return KVPrefix(_nextValue++); +} +} diff --git a/src/mongo/db/storage/kv/kv_prefix.h b/src/mongo/db/storage/kv/kv_prefix.h new file mode 100644 index 00000000000..240e00cc7c5 --- /dev/null +++ b/src/mongo/db/storage/kv/kv_prefix.h @@ -0,0 +1,88 @@ +/** + * Copyright (C) 2017 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/bson/bsonelement.h" +#include "mongo/bson/util/builder.h" +#include "mongo/db/namespace_string.h" +#include "mongo/db/storage/storage_options.h" +#include "mongo/stdx/mutex.h" + +namespace mongo { + +/** + * A KVPrefix may be prepended to the keys of entries in an underlying KV store. Prefixing keys as + * such allows multiple MongoDB collections share an underlying table. This can be a beneficial + * tradeoff for workloads that create many collections. + */ +class KVPrefix { +public: + // Represents a table that is not grouped and should not have its keys prefixed. + static const KVPrefix kNotPrefixed; + + bool isPrefixed() const { + return _value >= 0; + } + + int64_t toBSONValue() const { + return _value; + } + + std::string toString() const; + + inline bool operator<(const KVPrefix& rhs) const { + return _value < rhs._value; + } + + inline bool operator==(const KVPrefix& rhs) const { + return _value == rhs._value; + } + + inline bool operator!=(const KVPrefix& rhs) const { + return _value != rhs._value; + } + + static KVPrefix fromBSONElement(const BSONElement value); + + static void setLargestPrefix(KVPrefix largestPrefix); + + static KVPrefix getNextPrefix(const NamespaceString& ns); + +private: + explicit KVPrefix(int64_t value) : _value(value) {} + int64_t _value; + + static stdx::mutex _nextValueMutex; + static int64_t _nextValue; +}; + +inline std::ostream& operator<<(std::ostream& s, const KVPrefix& prefix) { + return (s << prefix.toString()); +} +} diff --git a/src/mongo/db/storage/kv/kv_storage_engine.cpp b/src/mongo/db/storage/kv/kv_storage_engine.cpp index 23e06da3ab4..462b6214243 100644 --- a/src/mongo/db/storage/kv/kv_storage_engine.cpp +++ b/src/mongo/db/storage/kv/kv_storage_engine.cpp @@ -30,6 +30,8 @@ #include "mongo/db/storage/kv/kv_storage_engine.h" +#include <algorithm> + #include "mongo/db/operation_context_noop.h" #include "mongo/db/storage/kv/kv_database_catalog_entry.h" #include "mongo/db/storage/kv/kv_engine.h" @@ -90,8 +92,8 @@ KVStorageEngine::KVStorageEngine( if (!catalogExists) { WriteUnitOfWork uow(&opCtx); - Status status = - _engine->createRecordStore(&opCtx, catalogInfo, catalogInfo, CollectionOptions()); + Status status = _engine->createGroupedRecordStore( + &opCtx, catalogInfo, catalogInfo, CollectionOptions(), KVPrefix::kNotPrefixed); // BadValue is usually caused by invalid configuration string. // We still fassert() but without a stack trace. if (status.code() == ErrorCodes::BadValue) { @@ -101,8 +103,8 @@ KVStorageEngine::KVStorageEngine( uow.commit(); } - _catalogRecordStore = - _engine->getRecordStore(&opCtx, catalogInfo, catalogInfo, CollectionOptions()); + _catalogRecordStore = _engine->getGroupedRecordStore( + &opCtx, catalogInfo, catalogInfo, CollectionOptions(), KVPrefix::kNotPrefixed); _catalog.reset(new KVCatalog( _catalogRecordStore.get(), _options.directoryPerDB, _options.directoryForIndexes)); _catalog->init(&opCtx); @@ -110,6 +112,7 @@ KVStorageEngine::KVStorageEngine( std::vector<std::string> collections; _catalog->getAllCollections(&collections); + KVPrefix maxSeenPrefix = KVPrefix::kNotPrefixed; for (size_t i = 0; i < collections.size(); i++) { std::string coll = collections[i]; NamespaceString nss(coll); @@ -122,8 +125,11 @@ KVStorageEngine::KVStorageEngine( } db->initCollection(&opCtx, coll, options.forRepair); + auto maxPrefixForCollection = _catalog->getMetaData(&opCtx, coll).getMaxPrefix(); + maxSeenPrefix = std::max(maxSeenPrefix, maxPrefixForCollection); } + KVPrefix::setLargestPrefix(maxSeenPrefix); opCtx.recoveryUnit()->abandonSnapshot(); // now clean up orphaned idents diff --git a/src/mongo/db/storage/mmap_v1/catalog/namespace_details_collection_entry.cpp b/src/mongo/db/storage/mmap_v1/catalog/namespace_details_collection_entry.cpp index ee2031d4a7e..617a6739cfe 100644 --- a/src/mongo/db/storage/mmap_v1/catalog/namespace_details_collection_entry.cpp +++ b/src/mongo/db/storage/mmap_v1/catalog/namespace_details_collection_entry.cpp @@ -186,6 +186,11 @@ bool NamespaceDetailsCollectionCatalogEntry::isIndexReady(OperationContext* opCt return idxNo < getCompletedIndexCount(opCtx); } +KVPrefix NamespaceDetailsCollectionCatalogEntry::getIndexPrefix(OperationContext* opCtx, + StringData indexName) const { + return KVPrefix::kNotPrefixed; +} + int NamespaceDetailsCollectionCatalogEntry::_findIndexNumber(OperationContext* opCtx, StringData idxName) const { NamespaceDetails::IndexIterator i = _details->ii(true); diff --git a/src/mongo/db/storage/mmap_v1/catalog/namespace_details_collection_entry.h b/src/mongo/db/storage/mmap_v1/catalog/namespace_details_collection_entry.h index 06b370bfa1f..f4e4410e317 100644 --- a/src/mongo/db/storage/mmap_v1/catalog/namespace_details_collection_entry.h +++ b/src/mongo/db/storage/mmap_v1/catalog/namespace_details_collection_entry.h @@ -83,6 +83,8 @@ public: bool isIndexReady(OperationContext* opCtx, StringData indexName) const final; + KVPrefix getIndexPrefix(OperationContext* opCtx, StringData indexName) const final; + Status removeIndex(OperationContext* opCtx, StringData indexName) final; Status prepareForIndexBuild(OperationContext* opCtx, const IndexDescriptor* spec) final; diff --git a/src/mongo/db/storage/wiredtiger/SConscript b/src/mongo/db/storage/wiredtiger/SConscript index d5f1957d1d6..29d64cdc3ba 100644 --- a/src/mongo/db/storage/wiredtiger/SConscript +++ b/src/mongo/db/storage/wiredtiger/SConscript @@ -45,7 +45,6 @@ if wiredtiger: 'wiredtiger_util.cpp', ], LIBDEPS= [ - 'storage_wiredtiger_customization_hooks', '$BUILD_DIR/mongo/base', '$BUILD_DIR/mongo/db/bson/dotted_path_support', '$BUILD_DIR/mongo/db/catalog/collection', @@ -58,14 +57,16 @@ if wiredtiger: '$BUILD_DIR/mongo/db/storage/index_entry_comparison', '$BUILD_DIR/mongo/db/storage/journal_listener', '$BUILD_DIR/mongo/db/storage/key_string', + '$BUILD_DIR/mongo/db/storage/kv/kv_prefix', '$BUILD_DIR/mongo/db/storage/oplog_hack', '$BUILD_DIR/mongo/db/storage/storage_options', '$BUILD_DIR/mongo/util/concurrency/ticketholder', '$BUILD_DIR/mongo/util/elapsed_tracker', '$BUILD_DIR/mongo/util/processinfo', - '$BUILD_DIR/third_party/shim_wiredtiger', '$BUILD_DIR/third_party/shim_snappy', + '$BUILD_DIR/third_party/shim_wiredtiger', '$BUILD_DIR/third_party/shim_zlib', + 'storage_wiredtiger_customization_hooks', ], ) @@ -115,8 +116,9 @@ if wiredtiger: source=['wiredtiger_record_store_test.cpp', ], LIBDEPS=[ - 'storage_wiredtiger_mock', + '$BUILD_DIR/mongo/db/storage/kv/kv_engine_core', '$BUILD_DIR/mongo/db/storage/record_store_test_harness', + 'storage_wiredtiger_mock', ], ) @@ -125,8 +127,9 @@ if wiredtiger: source=['wiredtiger_index_test.cpp', ], LIBDEPS=[ - 'storage_wiredtiger_mock', + '$BUILD_DIR/mongo/db/storage/kv/kv_engine_core', '$BUILD_DIR/mongo/db/storage/sorted_data_interface_test_harness', + 'storage_wiredtiger_mock', ], ) @@ -146,6 +149,7 @@ if wiredtiger: ], LIBDEPS=[ '$BUILD_DIR/mongo/db/service_context', + '$BUILD_DIR/mongo/db/storage/kv/kv_engine_core', 'storage_wiredtiger_mock', ], ) diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp index 532447eeda0..bb045b4bf54 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp @@ -221,12 +221,14 @@ int WiredTigerIndex::Create(OperationContext* opCtx, WiredTigerIndex::WiredTigerIndex(OperationContext* ctx, const std::string& uri, - const IndexDescriptor* desc) + const IndexDescriptor* desc, + KVPrefix prefix) : _ordering(Ordering::make(desc->keyPattern())), _uri(uri), _tableId(WiredTigerSession::genTableId()), _collectionNamespace(desc->parentNS()), - _indexName(desc->indexName()) { + _indexName(desc->indexName()), + _prefix(prefix) { auto version = WiredTigerUtil::checkApplicationMetadataFormatVersion( ctx, uri, kMinimumIndexVersion, kMaximumIndexVersion); if (!version.isOK()) { @@ -1003,8 +1005,9 @@ public: WiredTigerIndexUnique::WiredTigerIndexUnique(OperationContext* ctx, const std::string& uri, - const IndexDescriptor* desc) - : WiredTigerIndex(ctx, uri, desc), _partial(desc->isPartial()) {} + const IndexDescriptor* desc, + KVPrefix prefix) + : WiredTigerIndex(ctx, uri, desc, prefix), _partial(desc->isPartial()) {} std::unique_ptr<SortedDataInterface::Cursor> WiredTigerIndexUnique::newCursor( OperationContext* opCtx, bool forward) const { @@ -1188,8 +1191,9 @@ void WiredTigerIndexUnique::_unindex(WT_CURSOR* c, WiredTigerIndexStandard::WiredTigerIndexStandard(OperationContext* ctx, const std::string& uri, - const IndexDescriptor* desc) - : WiredTigerIndex(ctx, uri, desc) {} + const IndexDescriptor* desc, + KVPrefix prefix) + : WiredTigerIndex(ctx, uri, desc, prefix) {} std::unique_ptr<SortedDataInterface::Cursor> WiredTigerIndexStandard::newCursor( OperationContext* opCtx, bool forward) const { diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_index.h b/src/mongo/db/storage/wiredtiger/wiredtiger_index.h index 78d8e788b07..e9fe8c07941 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_index.h +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_index.h @@ -33,6 +33,7 @@ #include "mongo/base/status_with.h" #include "mongo/db/storage/index_entry_comparison.h" #include "mongo/db/storage/key_string.h" +#include "mongo/db/storage/kv/kv_prefix.h" #include "mongo/db/storage/sorted_data_interface.h" #include "mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.h" @@ -74,7 +75,10 @@ public: */ static int Create(OperationContext* opCtx, const std::string& uri, const std::string& config); - WiredTigerIndex(OperationContext* ctx, const std::string& uri, const IndexDescriptor* desc); + WiredTigerIndex(OperationContext* ctx, + const std::string& uri, + const IndexDescriptor* desc, + KVPrefix prefix); virtual Status insert(OperationContext* opCtx, const BSONObj& key, @@ -153,6 +157,7 @@ protected: uint64_t _tableId; std::string _collectionNamespace; std::string _indexName; + KVPrefix _prefix; }; @@ -160,7 +165,8 @@ class WiredTigerIndexUnique : public WiredTigerIndex { public: WiredTigerIndexUnique(OperationContext* ctx, const std::string& uri, - const IndexDescriptor* desc); + const IndexDescriptor* desc, + KVPrefix prefix); std::unique_ptr<SortedDataInterface::Cursor> newCursor(OperationContext* opCtx, bool forward) const override; @@ -183,7 +189,8 @@ class WiredTigerIndexStandard : public WiredTigerIndex { public: WiredTigerIndexStandard(OperationContext* ctx, const std::string& uri, - const IndexDescriptor* desc); + const IndexDescriptor* desc, + KVPrefix prefix); std::unique_ptr<SortedDataInterface::Cursor> newCursor(OperationContext* opCtx, bool forward) const override; diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_index_test.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_index_test.cpp index eceeb4e28d7..0c715cf694e 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_index_test.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_index_test.cpp @@ -38,6 +38,7 @@ #include "mongo/db/index/index_descriptor.h" #include "mongo/db/json.h" #include "mongo/db/operation_context_noop.h" +#include "mongo/db/storage/kv/kv_prefix.h" #include "mongo/db/storage/sorted_data_interface_test_harness.h" #include "mongo/db/storage/wiredtiger/wiredtiger_index.h" #include "mongo/db/storage/wiredtiger/wiredtiger_record_store.h" @@ -87,8 +88,10 @@ public: invariantWTOK(WiredTigerIndex::Create(&opCtx, uri, result.getValue())); if (unique) - return stdx::make_unique<WiredTigerIndexUnique>(&opCtx, uri, &desc); - return stdx::make_unique<WiredTigerIndexStandard>(&opCtx, uri, &desc); + return stdx::make_unique<WiredTigerIndexUnique>( + &opCtx, uri, &desc, KVPrefix::kNotPrefixed); + return stdx::make_unique<WiredTigerIndexStandard>( + &opCtx, uri, &desc, KVPrefix::kNotPrefixed); } std::unique_ptr<RecoveryUnit> newRecoveryUnit() final { diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp index 7dd943f8d33..63ac8dfc7ad 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp @@ -459,10 +459,11 @@ void WiredTigerKVEngine::setSortedDataInterfaceExtraOptions(const std::string& o _indexOptions = options; } -Status WiredTigerKVEngine::createRecordStore(OperationContext* opCtx, - StringData ns, - StringData ident, - const CollectionOptions& options) { +Status WiredTigerKVEngine::createGroupedRecordStore(OperationContext* opCtx, + StringData ns, + StringData ident, + const CollectionOptions& options, + KVPrefix prefix) { _checkIdentPath(ident); WiredTigerSession session(_conn); @@ -479,10 +480,12 @@ Status WiredTigerKVEngine::createRecordStore(OperationContext* opCtx, return wtRCToStatus(s->create(s, uri.c_str(), config.c_str())); } -std::unique_ptr<RecordStore> WiredTigerKVEngine::getRecordStore(OperationContext* opCtx, - StringData ns, - StringData ident, - const CollectionOptions& options) { +std::unique_ptr<RecordStore> WiredTigerKVEngine::getGroupedRecordStore( + OperationContext* opCtx, + StringData ns, + StringData ident, + const CollectionOptions& options, + KVPrefix prefix) { if (options.capped) { return stdx::make_unique<WiredTigerRecordStore>( opCtx, @@ -494,7 +497,8 @@ std::unique_ptr<RecordStore> WiredTigerKVEngine::getRecordStore(OperationContext options.cappedSize ? options.cappedSize : 4096, options.cappedMaxDocs ? options.cappedMaxDocs : -1, nullptr, - _sizeStorer.get()); + _sizeStorer.get(), + prefix); } else { return stdx::make_unique<WiredTigerRecordStore>(opCtx, ns, @@ -505,7 +509,8 @@ std::unique_ptr<RecordStore> WiredTigerKVEngine::getRecordStore(OperationContext -1, -1, nullptr, - _sizeStorer.get()); + _sizeStorer.get(), + prefix); } } @@ -513,9 +518,10 @@ string WiredTigerKVEngine::_uri(StringData ident) const { return string("table:") + ident.toString(); } -Status WiredTigerKVEngine::createSortedDataInterface(OperationContext* opCtx, - StringData ident, - const IndexDescriptor* desc) { +Status WiredTigerKVEngine::createGroupedSortedDataInterface(OperationContext* opCtx, + StringData ident, + const IndexDescriptor* desc, + KVPrefix prefix) { _checkIdentPath(ident); std::string collIndexOptions; @@ -548,12 +554,13 @@ Status WiredTigerKVEngine::createSortedDataInterface(OperationContext* opCtx, return wtRCToStatus(WiredTigerIndex::Create(opCtx, _uri(ident), config)); } -SortedDataInterface* WiredTigerKVEngine::getSortedDataInterface(OperationContext* opCtx, - StringData ident, - const IndexDescriptor* desc) { +SortedDataInterface* WiredTigerKVEngine::getGroupedSortedDataInterface(OperationContext* opCtx, + StringData ident, + const IndexDescriptor* desc, + KVPrefix prefix) { if (desc->unique()) - return new WiredTigerIndexUnique(opCtx, _uri(ident), desc); - return new WiredTigerIndexStandard(opCtx, _uri(ident), desc); + return new WiredTigerIndexUnique(opCtx, _uri(ident), desc, prefix); + return new WiredTigerIndexStandard(opCtx, _uri(ident), desc, prefix); } Status WiredTigerKVEngine::dropIdent(OperationContext* opCtx, StringData ident) { diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h index 1386eb7808d..19663b2b8fa 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h @@ -85,20 +85,50 @@ public: virtual Status createRecordStore(OperationContext* opCtx, StringData ns, StringData ident, - const CollectionOptions& options); + const CollectionOptions& options) { + return createGroupedRecordStore(opCtx, ns, ident, options, KVPrefix::kNotPrefixed); + } virtual std::unique_ptr<RecordStore> getRecordStore(OperationContext* opCtx, StringData ns, StringData ident, - const CollectionOptions& options); + const CollectionOptions& options) { + return getGroupedRecordStore(opCtx, ns, ident, options, KVPrefix::kNotPrefixed); + } virtual Status createSortedDataInterface(OperationContext* opCtx, StringData ident, - const IndexDescriptor* desc); + const IndexDescriptor* desc) { + return createGroupedSortedDataInterface(opCtx, ident, desc, KVPrefix::kNotPrefixed); + } virtual SortedDataInterface* getSortedDataInterface(OperationContext* opCtx, StringData ident, - const IndexDescriptor* desc); + const IndexDescriptor* desc) { + return getGroupedSortedDataInterface(opCtx, ident, desc, KVPrefix::kNotPrefixed); + } + + virtual Status createGroupedRecordStore(OperationContext* opCtx, + StringData ns, + StringData ident, + const CollectionOptions& options, + KVPrefix prefix); + + virtual std::unique_ptr<RecordStore> getGroupedRecordStore(OperationContext* opCtx, + StringData ns, + StringData ident, + const CollectionOptions& options, + KVPrefix prefix); + + virtual Status createGroupedSortedDataInterface(OperationContext* opCtx, + StringData ident, + const IndexDescriptor* desc, + KVPrefix prefix); + + virtual SortedDataInterface* getGroupedSortedDataInterface(OperationContext* opCtx, + StringData ident, + const IndexDescriptor* desc, + KVPrefix prefix); virtual Status dropIdent(OperationContext* opCtx, StringData ident); diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp index 72f22f176ca..e9f778715e2 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp @@ -770,7 +770,8 @@ WiredTigerRecordStore::WiredTigerRecordStore(OperationContext* ctx, int64_t cappedMaxSize, int64_t cappedMaxDocs, CappedCallback* cappedCallback, - WiredTigerSizeStorer* sizeStorer) + WiredTigerSizeStorer* sizeStorer, + KVPrefix prefix) : RecordStore(ns), _uri(uri.toString()), _tableId(WiredTigerSession::genTableId()), @@ -788,6 +789,7 @@ WiredTigerRecordStore::WiredTigerRecordStore(OperationContext* ctx, _useOplogHack(shouldUseOplogHack(ctx, _uri)), _sizeStorer(sizeStorer), _sizeStorerCounter(0), + _prefix(prefix), _shuttingDown(false) { Status versionStatus = WiredTigerUtil::checkApplicationMetadataFormatVersion( ctx, uri, kMinimumRecordStoreVersion, kMaximumRecordStoreVersion) diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.h b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.h index 5e11b67e90a..cb7c8374633 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.h +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.h @@ -37,6 +37,7 @@ #include "mongo/db/catalog/collection_options.h" #include "mongo/db/storage/capped_callback.h" +#include "mongo/db/storage/kv/kv_prefix.h" #include "mongo/db/storage/record_store.h" #include "mongo/platform/atomic_word.h" #include "mongo/stdx/condition_variable.h" @@ -102,7 +103,8 @@ public: int64_t cappedMaxSize = -1, int64_t cappedMaxDocs = -1, CappedCallback* cappedCallback = nullptr, - WiredTigerSizeStorer* sizeStorer = nullptr); + WiredTigerSizeStorer* sizeStorer = nullptr, + KVPrefix prefix = KVPrefix::kNotPrefixed); virtual ~WiredTigerRecordStore(); @@ -320,6 +322,8 @@ private: WiredTigerSizeStorer* _sizeStorer; // not owned, can be NULL int _sizeStorerCounter; + KVPrefix _prefix; + bool _shuttingDown; // Non-null if this record store is underlying the active oplog. diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store_test.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store_test.cpp index e47072cbe58..24fd17b30e0 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store_test.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store_test.cpp @@ -39,6 +39,7 @@ #include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/json.h" #include "mongo/db/operation_context_noop.h" +#include "mongo/db/storage/kv/kv_prefix.h" #include "mongo/db/storage/record_store_test_harness.h" #include "mongo/db/storage/wiredtiger/wiredtiger_record_store.h" #include "mongo/db/storage/wiredtiger/wiredtiger_record_store_oplog_stones.h" diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_util_test.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_util_test.cpp index 8c856bba938..d03b2826208 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_util_test.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_util_test.cpp @@ -35,6 +35,7 @@ #include "mongo/base/string_data.h" #include "mongo/db/operation_context_noop.h" +#include "mongo/db/storage/kv/kv_prefix.h" #include "mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.h" #include "mongo/db/storage/wiredtiger/wiredtiger_session_cache.h" #include "mongo/db/storage/wiredtiger/wiredtiger_util.h" |