diff options
author | Louis Williams <louis.williams@mongodb.com> | 2019-01-25 17:09:09 -0500 |
---|---|---|
committer | Louis Williams <louis.williams@mongodb.com> | 2019-01-31 18:25:19 -0500 |
commit | f4656acfee11569a796e06d14e4825ab54d39ecc (patch) | |
tree | 6e0c71fe0e1f4fd32f95e6d7dc8f3b1e3b17b569 /src/mongo | |
parent | 79142496a53ae70d875a8797defbb4cdd699ce4f (diff) | |
download | mongo-f4656acfee11569a796e06d14e4825ab54d39ecc.tar.gz |
SERVER-37645 Add parsing for new index build fields in catalog
Diffstat (limited to 'src/mongo')
-rw-r--r-- | src/mongo/db/catalog/collection_catalog_entry.h | 70 | ||||
-rw-r--r-- | src/mongo/db/catalog/index_build_block.cpp | 20 | ||||
-rw-r--r-- | src/mongo/db/index/duplicate_key_tracker.h | 4 | ||||
-rw-r--r-- | src/mongo/db/index/index_build_interceptor.cpp | 9 | ||||
-rw-r--r-- | src/mongo/db/index/index_build_interceptor.h | 6 | ||||
-rw-r--r-- | src/mongo/db/storage/bson_collection_catalog_entry.cpp | 29 | ||||
-rw-r--r-- | src/mongo/db/storage/bson_collection_catalog_entry.h | 24 | ||||
-rw-r--r-- | src/mongo/db/storage/kv/kv_collection_catalog_entry.cpp | 95 | ||||
-rw-r--r-- | src/mongo/db/storage/kv/kv_collection_catalog_entry.h | 21 | ||||
-rw-r--r-- | src/mongo/db/storage/kv/kv_collection_catalog_entry_test.cpp | 108 | ||||
-rw-r--r-- | src/mongo/db/storage/kv/kv_engine_test_harness.cpp | 102 | ||||
-rw-r--r-- | src/mongo/db/storage/kv/kv_storage_engine_test.cpp | 4 |
12 files changed, 430 insertions, 62 deletions
diff --git a/src/mongo/db/catalog/collection_catalog_entry.h b/src/mongo/db/catalog/collection_catalog_entry.h index 729adb86bb9..35a9227b9f8 100644 --- a/src/mongo/db/catalog/collection_catalog_entry.h +++ b/src/mongo/db/catalog/collection_catalog_entry.h @@ -46,8 +46,30 @@ class Collection; class IndexDescriptor; class OperationContext; +// Indicates which protocol an index build is using. +enum class IndexBuildProtocol { + /** + * Refers to the pre-FCV 4.2 index build protocol for building indexes in replica sets. + * Index builds must complete on the primary before replicating, and are not resumable in + * any scenario. + */ + kSinglePhase, + /** + * Refers to the FCV 4.2 two-phase index build protocol for building indexes in replica + * sets. Indexes are built simultaneously on all nodes and are resumable during the draining + * phase. + */ + kTwoPhase +}; + class CollectionCatalogEntry { public: + /** + * Incremented when breaking changes are made to the index build procedure so that other servers + * know whether or not to resume or discard unfinished index builds. + */ + static const int kIndexBuildVersion = 1; + CollectionCatalogEntry(StringData ns) : _ns(ns) {} virtual ~CollectionCatalogEntry() {} @@ -123,10 +145,58 @@ public: virtual Status prepareForIndexBuild(OperationContext* opCtx, const IndexDescriptor* spec, + IndexBuildProtocol indexBuildProtocol, bool isBackgroundSecondaryBuild) = 0; + /** + * Returns whether or not the index is being built with the two-phase index build procedure. + */ + virtual bool isTwoPhaseIndexBuild(OperationContext* opCtx, StringData indexName) const = 0; + + /** + * Returns the server-compatibility version of the index build procedure. + */ + virtual long getIndexBuildVersion(OperationContext* opCtx, StringData indexName) const = 0; + + /** + * Indicate that a build index is now in the "scanning" phase of a hybrid index build. The + * 'constraintViolationsIdent' is only used for unique indexes. + * + * It is only valid to call this when the index is using the kTwoPhase IndexBuildProtocol. + */ + virtual void setIndexBuildScanning(OperationContext* opCtx, + StringData indexName, + std::string sideWritesIdent, + boost::optional<std::string> constraintViolationsIdent) = 0; + + /** + * Returns whether or not this index is building in the "scanning" phase. + */ + virtual bool isIndexBuildScanning(OperationContext* opCtx, StringData indexName) const = 0; + + /** + * Indicate that a build index is now in the "draining" phase of a hybrid index build. + * + * It is only valid to call this when the index is using the kTwoPhase IndexBuildProtocol. + */ + virtual void setIndexBuildDraining(OperationContext* opCtx, StringData indexName) = 0; + + /** + * Returns whether or not this index is building in the "draining" phase. + */ + virtual bool isIndexBuildDraining(OperationContext* opCtx, StringData indexName) const = 0; + + /** + * Indicate that an index build is completed and the index is ready to use. + */ virtual void indexBuildSuccess(OperationContext* opCtx, StringData indexName) = 0; + virtual boost::optional<std::string> getSideWritesIdent(OperationContext* opCtx, + StringData indexName) const = 0; + + virtual boost::optional<std::string> getConstraintViolationsIdent( + OperationContext* opCtx, StringData indexName) const = 0; + /* Updates the expireAfterSeconds field of the given index to the value in newExpireSecs. * The specified index must already contain an expireAfterSeconds field, and the value in * that field and newExpireSecs must both be numeric. diff --git a/src/mongo/db/catalog/index_build_block.cpp b/src/mongo/db/catalog/index_build_block.cpp index 51294456a36..22f60db88d3 100644 --- a/src/mongo/db/catalog/index_build_block.cpp +++ b/src/mongo/db/catalog/index_build_block.cpp @@ -83,12 +83,12 @@ Status IndexCatalogImpl::IndexBuildBlock::init() { } // Setup on-disk structures. + const auto protocol = IndexBuildProtocol::kTwoPhase; Status status = _collection->getCatalogEntry()->prepareForIndexBuild( - _opCtx, descriptor.get(), isBackgroundSecondaryBuild); + _opCtx, descriptor.get(), protocol, isBackgroundSecondaryBuild); if (!status.isOK()) return status; - auto* const descriptorPtr = descriptor.get(); const bool initFromDisk = false; const bool isReadyIndex = false; _entry = _catalog->_setupInMemoryStructures( @@ -98,6 +98,20 @@ Status IndexCatalogImpl::IndexBuildBlock::init() { _indexBuildInterceptor = stdx::make_unique<IndexBuildInterceptor>(_opCtx, _entry); _entry->setIndexBuildInterceptor(_indexBuildInterceptor.get()); + const auto sideWritesIdent = _indexBuildInterceptor->getSideWritesTableIdent(); + // Only unique indexes have a constraint violations table. + const auto constraintsIdent = (_entry->descriptor()->unique()) + ? boost::optional<std::string>( + _indexBuildInterceptor->getConstraintViolationsTableIdent()) + : boost::none; + + if (IndexBuildProtocol::kTwoPhase == protocol) { + _collection->getCatalogEntry()->setIndexBuildScanning( + _opCtx, _entry->descriptor()->indexName(), sideWritesIdent, constraintsIdent); + } + } + + if (isBackgroundIndex) { _opCtx->recoveryUnit()->onCommit( [ opCtx = _opCtx, entry = _entry, collection = _collection ]( boost::optional<Timestamp> commitTime) { @@ -112,7 +126,7 @@ Status IndexCatalogImpl::IndexBuildBlock::init() { // Register this index with the CollectionInfoCache to regenerate the cache. This way, updates // occurring while an index is being build in the background will be aware of whether or not // they need to modify any indexes. - _collection->infoCache()->addedIndex(_opCtx, descriptorPtr); + _collection->infoCache()->addedIndex(_opCtx, _entry->descriptor()); return Status::OK(); } diff --git a/src/mongo/db/index/duplicate_key_tracker.h b/src/mongo/db/index/duplicate_key_tracker.h index d8047a7001d..8246397a508 100644 --- a/src/mongo/db/index/duplicate_key_tracker.h +++ b/src/mongo/db/index/duplicate_key_tracker.h @@ -72,6 +72,10 @@ public: */ bool areAllConstraintsChecked(OperationContext* opCtx) const; + const std::string& getConstraintsTableIdent() const { + return _keyConstraintsTable->rs()->getIdent(); + } + private: const IndexCatalogEntry* _indexCatalogEntry; diff --git a/src/mongo/db/index/index_build_interceptor.cpp b/src/mongo/db/index/index_build_interceptor.cpp index 8cea8b1ef2e..41fea74abba 100644 --- a/src/mongo/db/index/index_build_interceptor.cpp +++ b/src/mongo/db/index/index_build_interceptor.cpp @@ -80,6 +80,15 @@ bool IndexBuildInterceptor::areAllConstraintsChecked(OperationContext* opCtx) co return _duplicateKeyTracker->areAllConstraintsChecked(opCtx); } +const std::string& IndexBuildInterceptor::getSideWritesTableIdent() const { + return _sideWritesTable->rs()->getIdent(); +} + +const std::string& IndexBuildInterceptor::getConstraintViolationsTableIdent() const { + return _duplicateKeyTracker->getConstraintsTableIdent(); +} + + Status IndexBuildInterceptor::drainWritesIntoIndex(OperationContext* opCtx, const InsertDeleteOptions& options) { invariant(!opCtx->lockState()->inAWriteUnitOfWork()); diff --git a/src/mongo/db/index/index_build_interceptor.h b/src/mongo/db/index/index_build_interceptor.h index c3213827e62..39db95418f1 100644 --- a/src/mongo/db/index/index_build_interceptor.h +++ b/src/mongo/db/index/index_build_interceptor.h @@ -70,7 +70,7 @@ public: /** * Given a set of duplicate keys, record the keys for later verification by a call to - * checkConstraints(); + * checkDuplicateKeyConstraints(); */ Status recordDuplicateKeys(OperationContext* opCtx, const std::vector<BSONObj>& keys); @@ -109,6 +109,10 @@ public: */ boost::optional<MultikeyPaths> getMultikeyPaths() const; + const std::string& getSideWritesTableIdent() const; + + const std::string& getConstraintViolationsTableIdent() const; + private: using SideWriteRecord = std::pair<RecordId, BSONObj>; diff --git a/src/mongo/db/storage/bson_collection_catalog_entry.cpp b/src/mongo/db/storage/bson_collection_catalog_entry.cpp index ab81524ad19..631eea8237d 100644 --- a/src/mongo/db/storage/bson_collection_catalog_entry.cpp +++ b/src/mongo/db/storage/bson_collection_catalog_entry.cpp @@ -102,6 +102,9 @@ void parseMultikeyPathsFromBytes(BSONObj multikeyPathsObj, MultikeyPaths* multik } // namespace +const StringData BSONCollectionCatalogEntry::kIndexBuildScanning = "scanning"_sd; +const StringData BSONCollectionCatalogEntry::kIndexBuildDraining = "draining"_sd; + BSONCollectionCatalogEntry::BSONCollectionCatalogEntry(StringData ns) : CollectionCatalogEntry(ns) {} @@ -300,6 +303,18 @@ BSONObj BSONCollectionCatalogEntry::MetaData::toBSON() const { sub.append("head", static_cast<long long>(indexes[i].head.repr())); sub.append("prefix", indexes[i].prefix.toBSONValue()); sub.append("backgroundSecondary", indexes[i].isBackgroundSecondaryBuild); + + sub.append("runTwoPhaseBuild", indexes[i].runTwoPhaseBuild); + sub.append("versionOfBuild", indexes[i].versionOfBuild); + if (indexes[i].buildPhase) { + sub.append("buildPhase", *indexes[i].buildPhase); + } + if (indexes[i].constraintViolationsIdent) { + sub.append("constraintViolationsIdent", *indexes[i].constraintViolationsIdent); + } + if (indexes[i].sideWritesIdent) { + sub.append("sideWritesIdent", *indexes[i].sideWritesIdent); + } sub.doneFast(); } arr.doneFast(); @@ -339,6 +354,20 @@ void BSONCollectionCatalogEntry::MetaData::parse(const BSONObj& obj) { auto bgSecondary = BSONElement(idx["backgroundSecondary"]); // Opt-in to rebuilding behavior for old-format index catalog objects. imd.isBackgroundSecondaryBuild = bgSecondary.eoo() || bgSecondary.trueValue(); + + imd.runTwoPhaseBuild = idx["runTwoPhaseBuild"].trueValue(); + if (idx.hasField("versionOfBuild")) { + imd.versionOfBuild = idx["versionOfBuild"].numberLong(); + } + if (idx["buildPhase"]) { + imd.buildPhase = idx["buildPhase"].str(); + } + if (idx["constraintViolationsIdent"]) { + imd.constraintViolationsIdent = idx["constraintViolationsIdent"].str(); + } + if (idx["sideWritesIdent"]) { + imd.sideWritesIdent = idx["sideWritesIdent"].str(); + } indexes.push_back(imd); } } diff --git a/src/mongo/db/storage/bson_collection_catalog_entry.h b/src/mongo/db/storage/bson_collection_catalog_entry.h index dc30044f1db..499edfa044f 100644 --- a/src/mongo/db/storage/bson_collection_catalog_entry.h +++ b/src/mongo/db/storage/bson_collection_catalog_entry.h @@ -47,6 +47,9 @@ namespace mongo { */ class BSONCollectionCatalogEntry : public CollectionCatalogEntry { public: + static const StringData kIndexBuildScanning; + static const StringData kIndexBuildDraining; + BSONCollectionCatalogEntry(StringData ns); virtual ~BSONCollectionCatalogEntry() {} @@ -82,14 +85,6 @@ public: struct IndexMetaData { IndexMetaData() {} - IndexMetaData( - BSONObj s, bool r, RecordId h, bool m, KVPrefix prefix, bool isBackgroundSecondaryBuild) - : spec(s), - ready(r), - head(h), - multikey(m), - prefix(prefix), - isBackgroundSecondaryBuild(isBackgroundSecondaryBuild) {} void updateTTLSetting(long long newExpireSeconds); @@ -98,11 +93,18 @@ public: } BSONObj spec; - bool ready; + bool ready = false; RecordId head; - bool multikey; + bool multikey = false; KVPrefix prefix = KVPrefix::kNotPrefixed; - bool isBackgroundSecondaryBuild; + bool isBackgroundSecondaryBuild = false; + + long versionOfBuild = kIndexBuildVersion; + // If true, a two-phase index build is in progress, false otherwise. + bool runTwoPhaseBuild = false; + boost::optional<std::string> buildPhase; + boost::optional<std::string> constraintViolationsIdent; + boost::optional<std::string> sideWritesIdent; // If non-empty, 'multikeyPaths' is a vector with size equal to the number of elements in // the index key pattern. Each element in the vector is an ordered set of positions diff --git a/src/mongo/db/storage/kv/kv_collection_catalog_entry.cpp b/src/mongo/db/storage/kv/kv_collection_catalog_entry.cpp index 48411771d55..1ac5b6fbe3f 100644 --- a/src/mongo/db/storage/kv/kv_collection_catalog_entry.cpp +++ b/src/mongo/db/storage/kv/kv_collection_catalog_entry.cpp @@ -217,12 +217,20 @@ Status KVCollectionCatalogEntry::removeIndex(OperationContext* opCtx, StringData Status KVCollectionCatalogEntry::prepareForIndexBuild(OperationContext* opCtx, const IndexDescriptor* spec, + IndexBuildProtocol indexBuildProtocol, bool isBackgroundSecondaryBuild) { MetaData md = _getMetaData(opCtx); KVPrefix prefix = KVPrefix::getNextPrefix(ns()); - IndexMetaData imd( - spec->infoObj(), false, RecordId(), false, prefix, isBackgroundSecondaryBuild); + IndexMetaData imd; + imd.spec = spec->infoObj(); + imd.ready = false; + imd.head = RecordId(); + imd.multikey = false; + imd.prefix = prefix; + imd.isBackgroundSecondaryBuild = isBackgroundSecondaryBuild; + imd.runTwoPhaseBuild = indexBuildProtocol == IndexBuildProtocol::kTwoPhase; + if (indexTypeSupportsPathLevelMultikeyTracking(spec->getAccessMethodName())) { const auto feature = KVCatalog::FeatureTracker::RepairableFeature::kPathLevelMultikeyTracking; @@ -254,14 +262,97 @@ Status KVCollectionCatalogEntry::prepareForIndexBuild(OperationContext* opCtx, return status; } +bool KVCollectionCatalogEntry::isTwoPhaseIndexBuild(OperationContext* opCtx, + StringData indexName) const { + MetaData md = _getMetaData(opCtx); + int offset = md.findIndexOffset(indexName); + invariant(offset >= 0); + return md.indexes[offset].runTwoPhaseBuild; +} + +long KVCollectionCatalogEntry::getIndexBuildVersion(OperationContext* opCtx, + StringData indexName) const { + MetaData md = _getMetaData(opCtx); + int offset = md.findIndexOffset(indexName); + invariant(offset >= 0); + return md.indexes[offset].versionOfBuild; +} + +void KVCollectionCatalogEntry::setIndexBuildScanning( + OperationContext* opCtx, + StringData indexName, + std::string sideWritesIdent, + boost::optional<std::string> constraintViolationsIdent) { + MetaData md = _getMetaData(opCtx); + int offset = md.findIndexOffset(indexName); + invariant(offset >= 0); + invariant(!md.indexes[offset].ready); + invariant(!md.indexes[offset].buildPhase); + invariant(md.indexes[offset].runTwoPhaseBuild); + + md.indexes[offset].buildPhase = kIndexBuildScanning.toString(); + md.indexes[offset].sideWritesIdent = sideWritesIdent; + md.indexes[offset].constraintViolationsIdent = constraintViolationsIdent; + _catalog->putMetaData(opCtx, ns().toString(), md); +} + +bool KVCollectionCatalogEntry::isIndexBuildScanning(OperationContext* opCtx, + StringData indexName) const { + MetaData md = _getMetaData(opCtx); + int offset = md.findIndexOffset(indexName); + invariant(offset >= 0); + return md.indexes[offset].buildPhase == kIndexBuildScanning.toString(); +} + +void KVCollectionCatalogEntry::setIndexBuildDraining(OperationContext* opCtx, + StringData indexName) { + MetaData md = _getMetaData(opCtx); + int offset = md.findIndexOffset(indexName); + invariant(offset >= 0); + invariant(!md.indexes[offset].ready); + invariant(md.indexes[offset].runTwoPhaseBuild); + invariant(md.indexes[offset].buildPhase == kIndexBuildScanning.toString()); + + md.indexes[offset].buildPhase = kIndexBuildDraining.toString(); + _catalog->putMetaData(opCtx, ns().toString(), md); +} + +bool KVCollectionCatalogEntry::isIndexBuildDraining(OperationContext* opCtx, + StringData indexName) const { + MetaData md = _getMetaData(opCtx); + int offset = md.findIndexOffset(indexName); + invariant(offset >= 0); + return md.indexes[offset].buildPhase == kIndexBuildDraining.toString(); +} + void KVCollectionCatalogEntry::indexBuildSuccess(OperationContext* opCtx, StringData indexName) { MetaData md = _getMetaData(opCtx); int offset = md.findIndexOffset(indexName); invariant(offset >= 0); md.indexes[offset].ready = true; + md.indexes[offset].runTwoPhaseBuild = false; + md.indexes[offset].buildPhase = boost::none; + md.indexes[offset].sideWritesIdent = boost::none; + md.indexes[offset].constraintViolationsIdent = boost::none; _catalog->putMetaData(opCtx, ns().toString(), md); } +boost::optional<std::string> KVCollectionCatalogEntry::getSideWritesIdent( + OperationContext* opCtx, StringData indexName) const { + MetaData md = _getMetaData(opCtx); + int offset = md.findIndexOffset(indexName); + invariant(offset >= 0); + return md.indexes[offset].sideWritesIdent; +} + +boost::optional<std::string> KVCollectionCatalogEntry::getConstraintViolationsIdent( + OperationContext* opCtx, StringData indexName) const { + MetaData md = _getMetaData(opCtx); + int offset = md.findIndexOffset(indexName); + invariant(offset >= 0); + return md.indexes[offset].constraintViolationsIdent; +} + void KVCollectionCatalogEntry::updateTTLSetting(OperationContext* opCtx, StringData idxName, long long newExpireSeconds) { diff --git a/src/mongo/db/storage/kv/kv_collection_catalog_entry.h b/src/mongo/db/storage/kv/kv_collection_catalog_entry.h index f59c512e3c1..3b53b513031 100644 --- a/src/mongo/db/storage/kv/kv_collection_catalog_entry.h +++ b/src/mongo/db/storage/kv/kv_collection_catalog_entry.h @@ -72,10 +72,31 @@ public: Status prepareForIndexBuild(OperationContext* opCtx, const IndexDescriptor* spec, + IndexBuildProtocol indexBuildProtocol, bool isBackgroundSecondaryBuild) final; + bool isTwoPhaseIndexBuild(OperationContext* opCtx, StringData indexName) const final; + + long getIndexBuildVersion(OperationContext* opCtx, StringData indexName) const final; + + void setIndexBuildScanning(OperationContext* opCtx, + StringData indexName, + std::string sideWritesIdent, + boost::optional<std::string> constraintViolationsIdent) final; + + bool isIndexBuildScanning(OperationContext* opCtx, StringData indexName) const final; + + void setIndexBuildDraining(OperationContext* opCtx, StringData indexName) final; + + bool isIndexBuildDraining(OperationContext* opCtx, StringData indexName) const final; + void indexBuildSuccess(OperationContext* opCtx, StringData indexName) final; + boost::optional<std::string> getSideWritesIdent(OperationContext* opCtx, + StringData indexName) const final; + + boost::optional<std::string> getConstraintViolationsIdent(OperationContext* opCtx, + StringData indexName) const final; void updateTTLSetting(OperationContext* opCtx, StringData idxName, long long newExpireSeconds) final; diff --git a/src/mongo/db/storage/kv/kv_collection_catalog_entry_test.cpp b/src/mongo/db/storage/kv/kv_collection_catalog_entry_test.cpp index c82888375b2..666f2f88c70 100644 --- a/src/mongo/db/storage/kv/kv_collection_catalog_entry_test.cpp +++ b/src/mongo/db/storage/kv/kv_collection_catalog_entry_test.cpp @@ -30,6 +30,7 @@ #include "mongo/platform/basic.h" +#include <boost/optional/optional_io.hpp> #include <iostream> #include <string> @@ -51,6 +52,12 @@ namespace mongo { namespace { +static std::string kSideWritesTableIdent("sideWrites"); +static std::string kConstraintViolationsTableIdent("constraintViolations"); + +// Update version as breaking changes are introduced into the index build procedure. +static const long kExpectedVersion = 1; + class KVCollectionCatalogEntryTest : public ServiceContextTest { public: KVCollectionCatalogEntryTest() @@ -89,7 +96,9 @@ public: return dbEntry->getCollectionCatalogEntry(_nss.ns()); } - std::string createIndex(BSONObj keyPattern, std::string indexType = IndexNames::BTREE) { + std::string createIndex(BSONObj keyPattern, + std::string indexType = IndexNames::BTREE, + IndexBuildProtocol protocol = IndexBuildProtocol::kSinglePhase) { auto opCtx = newOperationContext(); std::string indexName = "idx" + std::to_string(numIndexesCreated); @@ -102,7 +111,7 @@ public: WriteUnitOfWork wuow(opCtx.get()); const bool isSecondaryBackgroundIndexBuild = false; ASSERT_OK(getCollectionCatalogEntry()->prepareForIndexBuild( - opCtx.get(), &desc, isSecondaryBackgroundIndexBuild)); + opCtx.get(), &desc, protocol, isSecondaryBackgroundIndexBuild)); wuow.commit(); } @@ -336,6 +345,101 @@ TEST_F(KVCollectionCatalogEntryTest, NoOpWhenEntireIndexAlreadySetAsMultikey) { } } +TEST_F(KVCollectionCatalogEntryTest, SinglePhaseIndexBuild) { + std::string indexName = createIndex(BSON("a" << 1)); + CollectionCatalogEntry* collEntry = getCollectionCatalogEntry(); + + auto opCtx = newOperationContext(); + + ASSERT_EQ(kExpectedVersion, collEntry->getIndexBuildVersion(opCtx.get(), indexName)); + ASSERT_FALSE(collEntry->isIndexReady(opCtx.get(), indexName)); + ASSERT_FALSE(collEntry->isTwoPhaseIndexBuild(opCtx.get(), indexName)); + ASSERT_FALSE(collEntry->isIndexBuildScanning(opCtx.get(), indexName)); + ASSERT_FALSE(collEntry->isIndexBuildDraining(opCtx.get(), indexName)); + ASSERT_FALSE(collEntry->getSideWritesIdent(opCtx.get(), indexName)); + ASSERT_FALSE(collEntry->getConstraintViolationsIdent(opCtx.get(), indexName)); + + collEntry->indexBuildSuccess(opCtx.get(), indexName); + + ASSERT_EQ(kExpectedVersion, collEntry->getIndexBuildVersion(opCtx.get(), indexName)); + ASSERT_TRUE(collEntry->isIndexReady(opCtx.get(), indexName)); + ASSERT_FALSE(collEntry->isTwoPhaseIndexBuild(opCtx.get(), indexName)); + ASSERT_FALSE(collEntry->isIndexBuildScanning(opCtx.get(), indexName)); + ASSERT_FALSE(collEntry->isIndexBuildDraining(opCtx.get(), indexName)); + ASSERT_FALSE(collEntry->getSideWritesIdent(opCtx.get(), indexName)); + ASSERT_FALSE(collEntry->getConstraintViolationsIdent(opCtx.get(), indexName)); +} + +TEST_F(KVCollectionCatalogEntryTest, TwoPhaseIndexBuild) { + std::string indexName = + createIndex(BSON("a" << 1), IndexNames::BTREE, IndexBuildProtocol::kTwoPhase); + CollectionCatalogEntry* collEntry = getCollectionCatalogEntry(); + + auto opCtx = newOperationContext(); + + ASSERT_EQ(kExpectedVersion, collEntry->getIndexBuildVersion(opCtx.get(), indexName)); + ASSERT_FALSE(collEntry->isIndexReady(opCtx.get(), indexName)); + ASSERT_TRUE(collEntry->isTwoPhaseIndexBuild(opCtx.get(), indexName)); + ASSERT_FALSE(collEntry->isIndexBuildScanning(opCtx.get(), indexName)); + ASSERT_FALSE(collEntry->isIndexBuildDraining(opCtx.get(), indexName)); + ASSERT_FALSE(collEntry->getSideWritesIdent(opCtx.get(), indexName)); + ASSERT_FALSE(collEntry->getConstraintViolationsIdent(opCtx.get(), indexName)); + + collEntry->setIndexBuildScanning( + opCtx.get(), indexName, kSideWritesTableIdent, kConstraintViolationsTableIdent); + + ASSERT_EQ(kExpectedVersion, collEntry->getIndexBuildVersion(opCtx.get(), indexName)); + ASSERT_FALSE(collEntry->isIndexReady(opCtx.get(), indexName)); + ASSERT_TRUE(collEntry->isTwoPhaseIndexBuild(opCtx.get(), indexName)); + ASSERT_TRUE(collEntry->isIndexBuildScanning(opCtx.get(), indexName)); + ASSERT_FALSE(collEntry->isIndexBuildDraining(opCtx.get(), indexName)); + ASSERT_EQ(kSideWritesTableIdent, collEntry->getSideWritesIdent(opCtx.get(), indexName)); + ASSERT_EQ(kConstraintViolationsTableIdent, + collEntry->getConstraintViolationsIdent(opCtx.get(), indexName)); + + collEntry->setIndexBuildDraining(opCtx.get(), indexName); + + ASSERT_EQ(kExpectedVersion, collEntry->getIndexBuildVersion(opCtx.get(), indexName)); + ASSERT_FALSE(collEntry->isIndexReady(opCtx.get(), indexName)); + ASSERT_TRUE(collEntry->isTwoPhaseIndexBuild(opCtx.get(), indexName)); + ASSERT_FALSE(collEntry->isIndexBuildScanning(opCtx.get(), indexName)); + ASSERT_TRUE(collEntry->isIndexBuildDraining(opCtx.get(), indexName)); + ASSERT_EQ(kSideWritesTableIdent, collEntry->getSideWritesIdent(opCtx.get(), indexName)); + ASSERT_EQ(kConstraintViolationsTableIdent, + collEntry->getConstraintViolationsIdent(opCtx.get(), indexName)); + + collEntry->indexBuildSuccess(opCtx.get(), indexName); + + ASSERT_EQ(kExpectedVersion, collEntry->getIndexBuildVersion(opCtx.get(), indexName)); + ASSERT(collEntry->isIndexReady(opCtx.get(), indexName)); + ASSERT_FALSE(collEntry->isIndexBuildScanning(opCtx.get(), indexName)); + ASSERT_FALSE(collEntry->isIndexBuildDraining(opCtx.get(), indexName)); + ASSERT_FALSE(collEntry->isTwoPhaseIndexBuild(opCtx.get(), indexName)); + ASSERT_FALSE(collEntry->getSideWritesIdent(opCtx.get(), indexName)); + ASSERT_FALSE(collEntry->getConstraintViolationsIdent(opCtx.get(), indexName)); +} + +DEATH_TEST_F(KVCollectionCatalogEntryTest, + SinglePhaseIllegalScanPhase, + "Invariant failure md.indexes[offset].runTwoPhaseBuild") { + std::string indexName = createIndex(BSON("a" << 1)); + CollectionCatalogEntry* collEntry = getCollectionCatalogEntry(); + + auto opCtx = newOperationContext(); + collEntry->setIndexBuildScanning( + opCtx.get(), indexName, kSideWritesTableIdent, kConstraintViolationsTableIdent); +} + +DEATH_TEST_F(KVCollectionCatalogEntryTest, + SinglePhaseIllegalDrainPhase, + "Invariant failure md.indexes[offset].runTwoPhaseBuild") { + std::string indexName = createIndex(BSON("a" << 1)); + CollectionCatalogEntry* collEntry = getCollectionCatalogEntry(); + + auto opCtx = newOperationContext(); + collEntry->setIndexBuildDraining(opCtx.get(), indexName); +} + DEATH_TEST_F(KVCollectionCatalogEntryTest, CannotSetIndividualPathComponentsOfTextIndexAsMultikey, "Invariant failure multikeyPaths.empty()") { diff --git a/src/mongo/db/storage/kv/kv_engine_test_harness.cpp b/src/mongo/db/storage/kv/kv_engine_test_harness.cpp index 5a1f8ad15a6..ae702567e81 100644 --- a/src/mongo/db/storage/kv/kv_engine_test_harness.cpp +++ b/src/mongo/db/storage/kv/kv_engine_test_harness.cpp @@ -358,13 +358,16 @@ TEST(KVCatalogTest, Idx1) { BSONCollectionCatalogEntry::MetaData md; md.ns = "a.b"; - md.indexes.push_back(BSONCollectionCatalogEntry::IndexMetaData(BSON("name" - << "foo"), - false, - RecordId(), - false, - KVPrefix::kNotPrefixed, - false)); + + BSONCollectionCatalogEntry::IndexMetaData imd; + imd.spec = BSON("name" + << "foo"); + imd.ready = false; + imd.head = RecordId(); + imd.multikey = false; + imd.prefix = KVPrefix::kNotPrefixed; + imd.isBackgroundSecondaryBuild = false; + md.indexes.push_back(imd); catalog->putMetaData(&opCtx, "a.b", md); uow.commit(); } @@ -388,13 +391,16 @@ TEST(KVCatalogTest, Idx1) { BSONCollectionCatalogEntry::MetaData md; md.ns = "a.b"; catalog->putMetaData(&opCtx, "a.b", md); // remove index - md.indexes.push_back(BSONCollectionCatalogEntry::IndexMetaData(BSON("name" - << "foo"), - false, - RecordId(), - false, - KVPrefix::kNotPrefixed, - false)); + + BSONCollectionCatalogEntry::IndexMetaData imd; + imd.spec = BSON("name" + << "foo"); + imd.ready = false; + imd.head = RecordId(); + imd.multikey = false; + imd.prefix = KVPrefix::kNotPrefixed; + imd.isBackgroundSecondaryBuild = false; + md.indexes.push_back(imd); catalog->putMetaData(&opCtx, "a.b", md); uow.commit(); } @@ -436,13 +442,16 @@ TEST(KVCatalogTest, DirectoryPerDb1) { BSONCollectionCatalogEntry::MetaData md; md.ns = "a.b"; - md.indexes.push_back(BSONCollectionCatalogEntry::IndexMetaData(BSON("name" - << "foo"), - false, - RecordId(), - false, - KVPrefix::kNotPrefixed, - false)); + + BSONCollectionCatalogEntry::IndexMetaData imd; + imd.spec = BSON("name" + << "foo"); + imd.ready = false; + imd.head = RecordId(); + imd.multikey = false; + imd.prefix = KVPrefix::kNotPrefixed; + imd.isBackgroundSecondaryBuild = false; + md.indexes.push_back(imd); catalog->putMetaData(&opCtx, "a.b", md); ASSERT_STRING_CONTAINS(catalog->getIndexIdent(&opCtx, "a.b", "foo"), "a/"); ASSERT_TRUE(catalog->isUserDataIdent(catalog->getIndexIdent(&opCtx, "a.b", "foo"))); @@ -481,13 +490,16 @@ TEST(KVCatalogTest, Split1) { BSONCollectionCatalogEntry::MetaData md; md.ns = "a.b"; - md.indexes.push_back(BSONCollectionCatalogEntry::IndexMetaData(BSON("name" - << "foo"), - false, - RecordId(), - false, - KVPrefix::kNotPrefixed, - false)); + + BSONCollectionCatalogEntry::IndexMetaData imd; + imd.spec = BSON("name" + << "foo"); + imd.ready = false; + imd.head = RecordId(); + imd.multikey = false; + imd.prefix = KVPrefix::kNotPrefixed; + imd.isBackgroundSecondaryBuild = false; + md.indexes.push_back(imd); catalog->putMetaData(&opCtx, "a.b", md); ASSERT_STRING_CONTAINS(catalog->getIndexIdent(&opCtx, "a.b", "foo"), "index/"); ASSERT_TRUE(catalog->isUserDataIdent(catalog->getIndexIdent(&opCtx, "a.b", "foo"))); @@ -526,13 +538,16 @@ TEST(KVCatalogTest, DirectoryPerAndSplit1) { BSONCollectionCatalogEntry::MetaData md; md.ns = "a.b"; - md.indexes.push_back(BSONCollectionCatalogEntry::IndexMetaData(BSON("name" - << "foo"), - false, - RecordId(), - false, - KVPrefix::kNotPrefixed, - false)); + + BSONCollectionCatalogEntry::IndexMetaData imd; + imd.spec = BSON("name" + << "foo"); + imd.ready = false; + imd.head = RecordId(); + imd.multikey = false; + imd.prefix = KVPrefix::kNotPrefixed; + imd.isBackgroundSecondaryBuild = false; + md.indexes.push_back(imd); catalog->putMetaData(&opCtx, "a.b", md); ASSERT_STRING_CONTAINS(catalog->getIndexIdent(&opCtx, "a.b", "foo"), "a/index/"); ASSERT_TRUE(catalog->isUserDataIdent(catalog->getIndexIdent(&opCtx, "a.b", "foo"))); @@ -576,13 +591,16 @@ TEST(KVCatalogTest, RestartForPrefixes) { BSONCollectionCatalogEntry::MetaData md; md.ns = "a.b"; - md.indexes.push_back(BSONCollectionCatalogEntry::IndexMetaData(BSON("name" - << "foo"), - false, - RecordId(), - false, - fooIndexPrefix, - false)); + + BSONCollectionCatalogEntry::IndexMetaData imd; + imd.spec = BSON("name" + << "foo"); + imd.ready = false; + imd.head = RecordId(); + imd.multikey = false; + imd.prefix = fooIndexPrefix; + imd.isBackgroundSecondaryBuild = false; + md.indexes.push_back(imd); md.prefix = abCollPrefix; catalog->putMetaData(&opCtx, "a.b", md); uow.commit(); diff --git a/src/mongo/db/storage/kv/kv_storage_engine_test.cpp b/src/mongo/db/storage/kv/kv_storage_engine_test.cpp index 0001988d0b4..5531b6b4f5b 100644 --- a/src/mongo/db/storage/kv/kv_storage_engine_test.cpp +++ b/src/mongo/db/storage/kv/kv_storage_engine_test.cpp @@ -140,7 +140,9 @@ public: DatabaseCatalogEntry* dbce = _storageEngine->getDatabaseCatalogEntry(opCtx, collNs.db()); CollectionCatalogEntry* cce = dbce->getCollectionCatalogEntry(collNs.ns()); const bool isBackgroundSecondaryBuild = false; - auto ret = cce->prepareForIndexBuild(opCtx, descriptor.get(), isBackgroundSecondaryBuild); + const auto protocol = IndexBuildProtocol::kSinglePhase; + auto ret = cce->prepareForIndexBuild( + opCtx, descriptor.get(), protocol, isBackgroundSecondaryBuild); if (!ret.isOK()) { return ret; } |