diff options
Diffstat (limited to 'src/mongo/db')
-rw-r--r-- | src/mongo/db/catalog/database_test.cpp | 2 | ||||
-rw-r--r-- | src/mongo/db/catalog/index_build_block.cpp | 28 | ||||
-rw-r--r-- | src/mongo/db/catalog/index_build_block.h | 5 | ||||
-rw-r--r-- | src/mongo/db/catalog/index_builds_manager.cpp | 3 | ||||
-rw-r--r-- | src/mongo/db/catalog/index_builds_manager.h | 2 | ||||
-rw-r--r-- | src/mongo/db/catalog/index_catalog_impl.cpp | 4 | ||||
-rw-r--r-- | src/mongo/db/catalog/multi_index_block.cpp | 2 | ||||
-rw-r--r-- | src/mongo/db/catalog/multi_index_block.h | 10 | ||||
-rw-r--r-- | src/mongo/db/index_builds_coordinator.cpp | 1 | ||||
-rw-r--r-- | src/mongo/db/repl_index_build_state.h | 2 | ||||
-rw-r--r-- | src/mongo/db/storage/bson_collection_catalog_entry.cpp | 6 | ||||
-rw-r--r-- | src/mongo/db/storage/bson_collection_catalog_entry.h | 1 | ||||
-rw-r--r-- | src/mongo/db/storage/durable_catalog.h | 9 | ||||
-rw-r--r-- | src/mongo/db/storage/durable_catalog_impl.cpp | 19 | ||||
-rw-r--r-- | src/mongo/db/storage/durable_catalog_impl.h | 6 | ||||
-rw-r--r-- | src/mongo/db/storage/kv/durable_catalog_test.cpp | 11 | ||||
-rw-r--r-- | src/mongo/db/storage/storage_engine_test_fixture.h | 3 |
17 files changed, 78 insertions, 36 deletions
diff --git a/src/mongo/db/catalog/database_test.cpp b/src/mongo/db/catalog/database_test.cpp index 47f2acd3d97..efcb5fec842 100644 --- a/src/mongo/db/catalog/database_test.cpp +++ b/src/mongo/db/catalog/database_test.cpp @@ -304,7 +304,7 @@ void _testDropCollectionThrowsExceptionIfThereAreIndexesInProgress(OperationCont << "a_1"); auto indexBuildBlock = std::make_unique<IndexBuildBlock>( - indexCatalog, collection->ns(), indexInfoObj, IndexBuildMethod::kHybrid); + indexCatalog, collection->ns(), indexInfoObj, IndexBuildMethod::kHybrid, UUID::gen()); { WriteUnitOfWork wuow(opCtx); ASSERT_OK(indexBuildBlock->init(opCtx, collection)); diff --git a/src/mongo/db/catalog/index_build_block.cpp b/src/mongo/db/catalog/index_build_block.cpp index 77c7f04fb60..a8cb11c8823 100644 --- a/src/mongo/db/catalog/index_build_block.cpp +++ b/src/mongo/db/catalog/index_build_block.cpp @@ -54,11 +54,13 @@ class IndexCatalog; IndexBuildBlock::IndexBuildBlock(IndexCatalog* indexCatalog, const NamespaceString& nss, const BSONObj& spec, - IndexBuildMethod method) + IndexBuildMethod method, + boost::optional<UUID> indexBuildUUID) : _indexCatalog(indexCatalog), _nss(nss), _spec(spec.getOwned()), _method(method), + _buildUUID(indexBuildUUID), _indexCatalogEntry(nullptr) {} void IndexBuildBlock::deleteTemporaryTables(OperationContext* opCtx) { @@ -93,9 +95,11 @@ Status IndexBuildBlock::init(OperationContext* opCtx, Collection* collection) { } // Setup on-disk structures. - const auto protocol = IndexBuildProtocol::kSinglePhase; - Status status = DurableCatalog::get(opCtx)->prepareForIndexBuild( - opCtx, collection->getCatalogId(), descriptor.get(), protocol, isBackgroundSecondaryBuild); + Status status = DurableCatalog::get(opCtx)->prepareForIndexBuild(opCtx, + collection->getCatalogId(), + descriptor.get(), + _buildUUID, + isBackgroundSecondaryBuild); if (!status.isOK()) return status; @@ -107,22 +111,6 @@ Status IndexBuildBlock::init(OperationContext* opCtx, Collection* collection) { if (_method == IndexBuildMethod::kHybrid) { _indexBuildInterceptor = std::make_unique<IndexBuildInterceptor>(opCtx, _indexCatalogEntry); _indexCatalogEntry->setIndexBuildInterceptor(_indexBuildInterceptor.get()); - - if (IndexBuildProtocol::kTwoPhase == protocol) { - const auto sideWritesIdent = _indexBuildInterceptor->getSideWritesTableIdent(); - // Only unique indexes have a constraint violations table. - const auto constraintsIdent = (_indexCatalogEntry->descriptor()->unique()) - ? boost::optional<std::string>( - _indexBuildInterceptor->getConstraintViolationsTableIdent()) - : boost::none; - - DurableCatalog::get(opCtx)->setIndexBuildScanning( - opCtx, - collection->getCatalogId(), - _indexCatalogEntry->descriptor()->indexName(), - sideWritesIdent, - constraintsIdent); - } } if (isBackgroundIndex) { diff --git a/src/mongo/db/catalog/index_build_block.h b/src/mongo/db/catalog/index_build_block.h index 99cb2d9fda6..9ed2ec5759d 100644 --- a/src/mongo/db/catalog/index_build_block.h +++ b/src/mongo/db/catalog/index_build_block.h @@ -44,7 +44,9 @@ public: IndexBuildBlock(IndexCatalog* indexCatalog, const NamespaceString& nss, const BSONObj& spec, - IndexBuildMethod method); + IndexBuildMethod method, + // The index build UUID is only required for persisting to the catalog. + boost::optional<UUID> indexBuildUUID); ~IndexBuildBlock(); @@ -109,6 +111,7 @@ private: BSONObj _spec; IndexBuildMethod _method; + boost::optional<UUID> _buildUUID; std::string _indexName; std::string _indexNamespace; diff --git a/src/mongo/db/catalog/index_builds_manager.cpp b/src/mongo/db/catalog/index_builds_manager.cpp index 9dff10efdef..7cfb138219a 100644 --- a/src/mongo/db/catalog/index_builds_manager.cpp +++ b/src/mongo/db/catalog/index_builds_manager.cpp @@ -89,6 +89,9 @@ Status IndexBuildsManager::setUpIndexBuild(OperationContext* opCtx, << nss.ns() << " is not locked in exclusive mode."); auto builder = _getBuilder(buildUUID); + if (options.protocol == IndexBuildProtocol::kTwoPhase) { + builder->setTwoPhaseBuildUUID(buildUUID); + } // Ignore uniqueness constraint violations when relaxed (on secondaries). Secondaries can // complete index builds in the middle of batches, which creates the potential for finding diff --git a/src/mongo/db/catalog/index_builds_manager.h b/src/mongo/db/catalog/index_builds_manager.h index 0e0a393ec69..1843177d5ae 100644 --- a/src/mongo/db/catalog/index_builds_manager.h +++ b/src/mongo/db/catalog/index_builds_manager.h @@ -36,6 +36,7 @@ #include "mongo/db/catalog/multi_index_block.h" #include "mongo/db/namespace_string.h" +#include "mongo/db/repl_index_build_state.h" #include "mongo/platform/mutex.h" namespace mongo { @@ -67,6 +68,7 @@ public: struct SetupOptions { SetupOptions(); IndexConstraints indexConstraints = IndexConstraints::kEnforce; + IndexBuildProtocol protocol = IndexBuildProtocol::kSinglePhase; bool forRecovery = false; }; diff --git a/src/mongo/db/catalog/index_catalog_impl.cpp b/src/mongo/db/catalog/index_catalog_impl.cpp index 88ca5640675..958f286703e 100644 --- a/src/mongo/db/catalog/index_catalog_impl.cpp +++ b/src/mongo/db/catalog/index_catalog_impl.cpp @@ -406,7 +406,9 @@ StatusWith<BSONObj> IndexCatalogImpl::createIndexOnEmptyCollection(OperationCont spec = statusWithSpec.getValue(); // now going to touch disk - IndexBuildBlock indexBuildBlock(this, _collection->ns(), spec, IndexBuildMethod::kForeground); + boost::optional<UUID> buildUUID = boost::none; + IndexBuildBlock indexBuildBlock( + this, _collection->ns(), spec, IndexBuildMethod::kForeground, buildUUID); status = indexBuildBlock.init(opCtx, _collection); if (!status.isOK()) return status; diff --git a/src/mongo/db/catalog/multi_index_block.cpp b/src/mongo/db/catalog/multi_index_block.cpp index 15ac455da05..4b8b50a2ff9 100644 --- a/src/mongo/db/catalog/multi_index_block.cpp +++ b/src/mongo/db/catalog/multi_index_block.cpp @@ -310,7 +310,7 @@ StatusWith<std::vector<BSONObj>> MultiIndexBlock::init(OperationContext* opCtx, IndexToBuild index; index.block = std::make_unique<IndexBuildBlock>( - collection->getIndexCatalog(), collection->ns(), info, _method); + collection->getIndexCatalog(), collection->ns(), info, _method, _buildUUID); status = index.block->init(opCtx, collection); if (!status.isOK()) return status; diff --git a/src/mongo/db/catalog/multi_index_block.h b/src/mongo/db/catalog/multi_index_block.h index 37f5d45bab9..7ffb1470d77 100644 --- a/src/mongo/db/catalog/multi_index_block.h +++ b/src/mongo/db/catalog/multi_index_block.h @@ -111,6 +111,14 @@ public: void ignoreUniqueConstraint(); /** + * Sets an index build UUID associated with the indexes for this builder. This call is required + * for two-phase index builds. + */ + void setTwoPhaseBuildUUID(UUID indexBuildUUID) { + _buildUUID = indexBuildUUID; + } + + /** * Prepares the index(es) for building and returns the canonicalized form of the requested index * specifications. * @@ -355,6 +363,8 @@ private: // Duplicate key constraints should be checked at least once in the MultiIndexBlock. bool _constraintsChecked = false; + boost::optional<UUID> _buildUUID; + // Protects member variables of this class declared below. mutable Mutex _mutex = MONGO_MAKE_LATCH("MultiIndexBlock::_mutex"); diff --git a/src/mongo/db/index_builds_coordinator.cpp b/src/mongo/db/index_builds_coordinator.cpp index 0eaa5962d85..aea56abc00b 100644 --- a/src/mongo/db/index_builds_coordinator.cpp +++ b/src/mongo/db/index_builds_coordinator.cpp @@ -979,6 +979,7 @@ IndexBuildsCoordinator::_registerAndSetUpIndexBuild( repl::ReplicationCoordinator::get(opCtx)->shouldRelaxIndexConstraints(opCtx, nss) ? IndexBuildsManager::IndexConstraints::kRelax : IndexBuildsManager::IndexConstraints::kEnforce; + options.protocol = protocol; status = _indexBuildsManager.setUpIndexBuild( opCtx, collection, filteredSpecs, replIndexBuildState->buildUUID, onInitFn, options); diff --git a/src/mongo/db/repl_index_build_state.h b/src/mongo/db/repl_index_build_state.h index 66ab914807c..e1dafd9c6d5 100644 --- a/src/mongo/db/repl_index_build_state.h +++ b/src/mongo/db/repl_index_build_state.h @@ -55,7 +55,7 @@ enum class IndexBuildProtocol { kSinglePhase, /** * Refers to the two-phase index build protocol for building indexes in replica sets. Indexes - * are built simultaneously on all nodes and are resumable during the draining phase. + * are built simultaneously on all nodes. */ kTwoPhase }; diff --git a/src/mongo/db/storage/bson_collection_catalog_entry.cpp b/src/mongo/db/storage/bson_collection_catalog_entry.cpp index 7210678dea6..a3228528ea5 100644 --- a/src/mongo/db/storage/bson_collection_catalog_entry.cpp +++ b/src/mongo/db/storage/bson_collection_catalog_entry.cpp @@ -181,6 +181,9 @@ BSONObj BSONCollectionCatalogEntry::MetaData::toBSON() const { if (indexes[i].buildPhase) { sub.append("buildPhase", *indexes[i].buildPhase); } + if (indexes[i].buildUUID) { + indexes[i].buildUUID->appendToBuilder(&sub, "buildUUID"); + } if (indexes[i].constraintViolationsIdent) { sub.append("constraintViolationsIdent", *indexes[i].constraintViolationsIdent); } @@ -229,6 +232,9 @@ void BSONCollectionCatalogEntry::MetaData::parse(const BSONObj& obj) { if (idx["buildPhase"]) { imd.buildPhase = idx["buildPhase"].str(); } + if (idx["buildUUID"]) { + imd.buildUUID = fassert(31353, UUID::parse(idx["buildUUID"])); + } if (idx["constraintViolationsIdent"]) { imd.constraintViolationsIdent = idx["constraintViolationsIdent"].str(); } diff --git a/src/mongo/db/storage/bson_collection_catalog_entry.h b/src/mongo/db/storage/bson_collection_catalog_entry.h index 4f71435937c..a33e472d8ef 100644 --- a/src/mongo/db/storage/bson_collection_catalog_entry.h +++ b/src/mongo/db/storage/bson_collection_catalog_entry.h @@ -73,6 +73,7 @@ public: bool multikey = false; KVPrefix prefix = KVPrefix::kNotPrefixed; bool isBackgroundSecondaryBuild = false; + boost::optional<UUID> buildUUID; long versionOfBuild = kIndexBuildVersion; // If true, a two-phase index build is in progress, false otherwise. diff --git a/src/mongo/db/storage/durable_catalog.h b/src/mongo/db/storage/durable_catalog.h index a13527268d7..48d792f13f6 100644 --- a/src/mongo/db/storage/durable_catalog.h +++ b/src/mongo/db/storage/durable_catalog.h @@ -34,7 +34,6 @@ #include "mongo/db/index/index_descriptor.h" #include "mongo/db/namespace_string.h" #include "mongo/db/operation_context.h" -#include "mongo/db/repl_index_build_state.h" #include "mongo/db/storage/bson_collection_catalog_entry.h" #include "mongo/db/storage/kv/kv_prefix.h" #include "mongo/db/storage/storage_engine.h" @@ -186,11 +185,13 @@ public: /** * Updates the persisted catalog entry for 'ns' with the new index and creates the index on * disk. + * + * A passed 'buildUUID' implies that the index is part of a two-phase index build. */ virtual Status prepareForIndexBuild(OperationContext* opCtx, RecordId catalogId, const IndexDescriptor* spec, - IndexBuildProtocol indexBuildProtocol, + boost::optional<UUID> buildUUID, bool isBackgroundSecondaryBuild) = 0; /** @@ -213,6 +214,10 @@ public: boost::optional<std::string> constraintViolationsIdent) = 0; + virtual boost::optional<UUID> getIndexBuildUUID(OperationContext* opCtx, + RecordId catalogId, + StringData indexName) const = 0; + /** * Returns whether or not this index is building in the "scanning" phase. */ diff --git a/src/mongo/db/storage/durable_catalog_impl.cpp b/src/mongo/db/storage/durable_catalog_impl.cpp index f6511da8264..42a1207672b 100644 --- a/src/mongo/db/storage/durable_catalog_impl.cpp +++ b/src/mongo/db/storage/durable_catalog_impl.cpp @@ -595,6 +595,9 @@ void DurableCatalogImpl::putMetaData(OperationContext* opCtx, const auto index = md.indexes[i]; string name = index.name(); + // All indexes with buildUUIDs must be ready:false. + invariant(!(index.buildUUID && index.ready), str::stream() << md.toBSON()); + // fix ident map BSONElement e = oldIdentMap[name]; if (e.type() == String) { @@ -942,7 +945,7 @@ Status DurableCatalogImpl::removeIndex(OperationContext* opCtx, Status DurableCatalogImpl::prepareForIndexBuild(OperationContext* opCtx, RecordId catalogId, const IndexDescriptor* spec, - IndexBuildProtocol indexBuildProtocol, + boost::optional<UUID> buildUUID, bool isBackgroundSecondaryBuild) { BSONCollectionCatalogEntry::MetaData md = getMetaData(opCtx, catalogId); @@ -953,7 +956,8 @@ Status DurableCatalogImpl::prepareForIndexBuild(OperationContext* opCtx, imd.multikey = false; imd.prefix = prefix; imd.isBackgroundSecondaryBuild = isBackgroundSecondaryBuild; - imd.runTwoPhaseBuild = indexBuildProtocol == IndexBuildProtocol::kTwoPhase; + imd.runTwoPhaseBuild = buildUUID.is_initialized(); + imd.buildUUID = buildUUID; if (indexTypeSupportsPathLevelMultikeyTracking(spec->getAccessMethodName())) { const auto feature = FeatureTracker::RepairableFeature::kPathLevelMultikeyTracking; @@ -996,6 +1000,15 @@ bool DurableCatalogImpl::isTwoPhaseIndexBuild(OperationContext* opCtx, return md.indexes[offset].runTwoPhaseBuild; } +boost::optional<UUID> DurableCatalogImpl::getIndexBuildUUID(OperationContext* opCtx, + RecordId catalogId, + StringData indexName) const { + BSONCollectionCatalogEntry::MetaData md = getMetaData(opCtx, catalogId); + int offset = md.findIndexOffset(indexName); + invariant(offset >= 0); + return md.indexes[offset].buildUUID; +} + void DurableCatalogImpl::setIndexBuildScanning( OperationContext* opCtx, RecordId catalogId, @@ -1059,6 +1072,7 @@ void DurableCatalogImpl::indexBuildSuccess(OperationContext* opCtx, md.indexes[offset].ready = true; md.indexes[offset].runTwoPhaseBuild = false; md.indexes[offset].buildPhase = boost::none; + md.indexes[offset].buildUUID = boost::none; md.indexes[offset].sideWritesIdent = boost::none; md.indexes[offset].constraintViolationsIdent = boost::none; putMetaData(opCtx, catalogId, md); @@ -1143,6 +1157,7 @@ boost::optional<std::string> DurableCatalogImpl::getConstraintViolationsIdent( return md.indexes[offset].constraintViolationsIdent; } + long DurableCatalogImpl::getIndexBuildVersion(OperationContext* opCtx, RecordId catalogId, StringData indexName) const { diff --git a/src/mongo/db/storage/durable_catalog_impl.h b/src/mongo/db/storage/durable_catalog_impl.h index 9767f1bb637..07a02b16d27 100644 --- a/src/mongo/db/storage/durable_catalog_impl.h +++ b/src/mongo/db/storage/durable_catalog_impl.h @@ -145,13 +145,17 @@ public: Status prepareForIndexBuild(OperationContext* opCtx, RecordId catalogId, const IndexDescriptor* spec, - IndexBuildProtocol indexBuildProtocol, + boost::optional<UUID> buildUUID, bool isBackgroundSecondaryBuild); bool isTwoPhaseIndexBuild(OperationContext* opCtx, RecordId catalogId, StringData indexName) const; + boost::optional<UUID> getIndexBuildUUID(OperationContext* opCtx, + RecordId catalogId, + StringData indexName) const; + void setIndexBuildScanning(OperationContext* opCtx, RecordId catalogId, StringData indexName, diff --git a/src/mongo/db/storage/kv/durable_catalog_test.cpp b/src/mongo/db/storage/kv/durable_catalog_test.cpp index d8c2be64ff2..5a2da09c0ea 100644 --- a/src/mongo/db/storage/kv/durable_catalog_test.cpp +++ b/src/mongo/db/storage/kv/durable_catalog_test.cpp @@ -109,7 +109,7 @@ public: std::string createIndex(BSONObj keyPattern, std::string indexType = IndexNames::BTREE, - IndexBuildProtocol protocol = IndexBuildProtocol::kSinglePhase) { + bool twoPhase = false) { auto opCtx = newOperationContext(); std::string indexName = "idx" + std::to_string(numIndexesCreated); @@ -121,8 +121,9 @@ public: { WriteUnitOfWork wuow(opCtx.get()); const bool isSecondaryBackgroundIndexBuild = false; + boost::optional<UUID> buildUUID(twoPhase, UUID::gen()); ASSERT_OK(_storageEngine.getCatalog()->prepareForIndexBuild( - opCtx.get(), _catalogId, &desc, protocol, isSecondaryBackgroundIndexBuild)); + opCtx.get(), _catalogId, &desc, buildUUID, isSecondaryBackgroundIndexBuild)); wuow.commit(); } @@ -377,8 +378,8 @@ TEST_F(DurableCatalogTest, SinglePhaseIndexBuild) { } TEST_F(DurableCatalogTest, TwoPhaseIndexBuild) { - std::string indexName = - createIndex(BSON("a" << 1), IndexNames::BTREE, IndexBuildProtocol::kTwoPhase); + bool twoPhase = true; + std::string indexName = createIndex(BSON("a" << 1), IndexNames::BTREE, twoPhase); auto opCtx = newOperationContext(); DurableCatalog* catalog = getCatalog(); @@ -386,6 +387,7 @@ TEST_F(DurableCatalogTest, TwoPhaseIndexBuild) { catalog->getIndexBuildVersion(opCtx.get(), getCatalogId(), indexName)); ASSERT_FALSE(catalog->isIndexReady(opCtx.get(), getCatalogId(), indexName)); ASSERT_TRUE(catalog->isTwoPhaseIndexBuild(opCtx.get(), getCatalogId(), indexName)); + ASSERT_TRUE(catalog->getIndexBuildUUID(opCtx.get(), getCatalogId(), indexName)); ASSERT_FALSE(catalog->isIndexBuildScanning(opCtx.get(), getCatalogId(), indexName)); ASSERT_FALSE(catalog->isIndexBuildDraining(opCtx.get(), getCatalogId(), indexName)); ASSERT_FALSE(catalog->getSideWritesIdent(opCtx.get(), getCatalogId(), indexName)); @@ -429,6 +431,7 @@ TEST_F(DurableCatalogTest, TwoPhaseIndexBuild) { ASSERT_FALSE(catalog->isIndexBuildScanning(opCtx.get(), getCatalogId(), indexName)); ASSERT_FALSE(catalog->isIndexBuildDraining(opCtx.get(), getCatalogId(), indexName)); ASSERT_FALSE(catalog->isTwoPhaseIndexBuild(opCtx.get(), getCatalogId(), indexName)); + ASSERT_FALSE(catalog->getIndexBuildUUID(opCtx.get(), getCatalogId(), indexName)); ASSERT_FALSE(catalog->getSideWritesIdent(opCtx.get(), getCatalogId(), indexName)); ASSERT_FALSE(catalog->getConstraintViolationsIdent(opCtx.get(), getCatalogId(), indexName)); } diff --git a/src/mongo/db/storage/storage_engine_test_fixture.h b/src/mongo/db/storage/storage_engine_test_fixture.h index de657637577..22945fc184e 100644 --- a/src/mongo/db/storage/storage_engine_test_fixture.h +++ b/src/mongo/db/storage/storage_engine_test_fixture.h @@ -140,11 +140,10 @@ public: auto descriptor = std::make_unique<IndexDescriptor>(collection, IndexNames::findPluginName(spec), spec); - const auto protocol = IndexBuildProtocol::kTwoPhase; auto ret = DurableCatalog::get(opCtx)->prepareForIndexBuild(opCtx, collection->getCatalogId(), descriptor.get(), - protocol, + UUID::gen(), isBackgroundSecondaryBuild); return ret; } |