diff options
Diffstat (limited to 'src/mongo/db')
26 files changed, 256 insertions, 87 deletions
diff --git a/src/mongo/db/auth/auth_op_observer.cpp b/src/mongo/db/auth/auth_op_observer.cpp index e17c8e4f614..7ce32cca827 100644 --- a/src/mongo/db/auth/auth_op_observer.cpp +++ b/src/mongo/db/auth/auth_op_observer.cpp @@ -116,11 +116,11 @@ void AuthOpObserver::onCollMod(OperationContext* opCtx, OptionalCollectionUUID uuid, const BSONObj& collModCmd, const CollectionOptions& oldCollOptions, - boost::optional<TTLCollModInfo> ttlInfo) { + boost::optional<IndexCollModInfo> indexInfo) { const auto cmdNss = nss.getCommandNS(); // Create the 'o' field object. - const auto cmdObj = makeCollModCmdObj(collModCmd, oldCollOptions, ttlInfo); + const auto cmdObj = makeCollModCmdObj(collModCmd, oldCollOptions, indexInfo); AuthorizationManager::get(opCtx->getServiceContext()) ->logOp(opCtx, "c", cmdNss, cmdObj, nullptr); diff --git a/src/mongo/db/auth/auth_op_observer.h b/src/mongo/db/auth/auth_op_observer.h index 47293e12e09..45f9d56b099 100644 --- a/src/mongo/db/auth/auth_op_observer.h +++ b/src/mongo/db/auth/auth_op_observer.h @@ -113,7 +113,7 @@ public: OptionalCollectionUUID uuid, const BSONObj& collModCmd, const CollectionOptions& oldCollOptions, - boost::optional<TTLCollModInfo> ttlInfo) final; + boost::optional<IndexCollModInfo> indexInfo) final; void onDropDatabase(OperationContext* opCtx, const std::string& dbName) final; diff --git a/src/mongo/db/catalog/coll_mod.cpp b/src/mongo/db/catalog/coll_mod.cpp index 49382234f0a..8fce56e7416 100644 --- a/src/mongo/db/catalog/coll_mod.cpp +++ b/src/mongo/db/catalog/coll_mod.cpp @@ -69,6 +69,7 @@ MONGO_FAIL_POINT_DEFINE(assertAfterIndexUpdate); struct CollModRequest { const IndexDescriptor* idx = nullptr; BSONElement indexExpireAfterSeconds = {}; + BSONElement indexHidden = {}; BSONElement viewPipeLine = {}; std::string viewOn = {}; BSONElement collValidator = {}; @@ -125,14 +126,18 @@ StatusWith<CollModRequest> parseCollModRequest(OperationContext* opCtx, } cmr.indexExpireAfterSeconds = indexObj["expireAfterSeconds"]; - if (cmr.indexExpireAfterSeconds.eoo()) { - return Status(ErrorCodes::InvalidOptions, "no expireAfterSeconds field"); + cmr.indexHidden = indexObj["hidden"]; + + if (cmr.indexExpireAfterSeconds.eoo() && cmr.indexHidden.eoo()) { + return Status(ErrorCodes::InvalidOptions, "no expireAfterSeconds or hidden field"); } - if (!cmr.indexExpireAfterSeconds.isNumber()) { + if (!cmr.indexExpireAfterSeconds.eoo() && !cmr.indexExpireAfterSeconds.isNumber()) { return Status(ErrorCodes::InvalidOptions, "expireAfterSeconds field must be a number"); } - + if (!cmr.indexHidden.eoo() && !cmr.indexHidden.isBoolean()) { + return Status(ErrorCodes::InvalidOptions, "hidden field must be a boolean"); + } if (!indexName.empty()) { cmr.idx = coll->getIndexCatalog()->findIndexByName(opCtx, indexName); if (!cmr.idx) { @@ -161,15 +166,17 @@ StatusWith<CollModRequest> parseCollModRequest(OperationContext* opCtx, cmr.idx = indexes[0]; } - BSONElement oldExpireSecs = cmr.idx->infoObj().getField("expireAfterSeconds"); - if (oldExpireSecs.eoo()) { - return Status(ErrorCodes::InvalidOptions, "no expireAfterSeconds field to update"); - } - if (!oldExpireSecs.isNumber()) { - return Status(ErrorCodes::InvalidOptions, - "existing expireAfterSeconds field is not a number"); + if (!cmr.indexExpireAfterSeconds.eoo()) { + BSONElement oldExpireSecs = cmr.idx->infoObj().getField("expireAfterSeconds"); + if (oldExpireSecs.eoo()) { + return Status(ErrorCodes::InvalidOptions, + "no expireAfterSeconds field to update"); + } + if (!oldExpireSecs.isNumber()) { + return Status(ErrorCodes::InvalidOptions, + "existing expireAfterSeconds field is not a number"); + } } - } else if (fieldName == "validator" && !isView) { // Save this to a variable to avoid reading the atomic variable multiple times. const auto currentFCV = serverGlobalParams.featureCompatibility.getVersion(); @@ -250,13 +257,26 @@ class CollModResultChange : public RecoveryUnit::Change { public: CollModResultChange(const BSONElement& oldExpireSecs, const BSONElement& newExpireSecs, + const BSONElement& oldHidden, + const BSONElement& newHidden, BSONObjBuilder* result) - : _oldExpireSecs(oldExpireSecs), _newExpireSecs(newExpireSecs), _result(result) {} + : _oldExpireSecs(oldExpireSecs), + _newExpireSecs(newExpireSecs), + _oldHidden(oldHidden), + _newHidden(newHidden), + _result(result) {} void commit(boost::optional<Timestamp>) override { // add the fields to BSONObjBuilder result - _result->appendAs(_oldExpireSecs, "expireAfterSeconds_old"); - _result->appendAs(_newExpireSecs, "expireAfterSeconds_new"); + if (!_oldExpireSecs.eoo()) { + _result->appendAs(_oldExpireSecs, "expireAfterSeconds_old"); + _result->appendAs(_newExpireSecs, "expireAfterSeconds_new"); + } + if (!_newHidden.eoo()) { + bool oldValue = _oldHidden.eoo() ? false : _oldHidden.booleanSafe(); + _result->append("hidden_old", oldValue); + _result->appendAs(_newHidden, "hidden_new"); + } } void rollback() override {} @@ -264,6 +284,8 @@ public: private: const BSONElement _oldExpireSecs; const BSONElement _newExpireSecs; + const BSONElement _oldHidden; + const BSONElement _newHidden; BSONObjBuilder* _result; }; @@ -326,6 +348,19 @@ Status _collModInternal(OperationContext* opCtx, const CollModRequest cmrOld = statusW.getValue(); CollModRequest cmrNew = statusW.getValue(); + if (!cmrOld.indexHidden.eoo()) { + + if (serverGlobalParams.featureCompatibility.getVersion() < + ServerGlobalParams::FeatureCompatibility::Version::kFullyUpgradedTo46 && + cmrOld.indexHidden.booleanSafe()) { + return Status(ErrorCodes::BadValue, "Hidden indexes can only be created with FCV 4.6"); + } + if (coll->ns().isSystem()) + return Status(ErrorCodes::BadValue, "Can't hide index on system collection"); + if (cmrOld.idx->isIdIndex()) + return Status(ErrorCodes::BadValue, "can't hide _id index"); + } + return writeConflictRetry(opCtx, "collMod", nss.ns(), [&] { WriteUnitOfWork wunit(opCtx); @@ -362,40 +397,64 @@ Status _collModInternal(OperationContext* opCtx, CollectionOptions oldCollOptions = DurableCatalog::get(opCtx)->getCollectionOptions(opCtx, coll->getCatalogId()); - boost::optional<TTLCollModInfo> ttlInfo; + boost::optional<IndexCollModInfo> indexCollModInfo; // Handle collMod operation type appropriately. - // TTLIndex - if (!cmrOld.indexExpireAfterSeconds.eoo()) { - BSONElement newExpireSecs = cmrOld.indexExpireAfterSeconds; - BSONElement oldExpireSecs = cmrOld.idx->infoObj().getField("expireAfterSeconds"); - - if (SimpleBSONElementComparator::kInstance.evaluate(oldExpireSecs != newExpireSecs)) { - // Change the value of "expireAfterSeconds" on disk. - DurableCatalog::get(opCtx)->updateTTLSetting(opCtx, - coll->getCatalogId(), - cmrOld.idx->indexName(), - newExpireSecs.safeNumberLong()); - - // Notify the index catalog that the definition of this index changed. This will - // invalidate the idx pointer in cmrOld. On rollback of this WUOW, the idx pointer - // in cmrNew will be invalidated and the idx pointer in cmrOld will be valid again. - cmrNew.idx = coll->getIndexCatalog()->refreshEntry(opCtx, cmrOld.idx); - opCtx->recoveryUnit()->registerChange( - std::make_unique<CollModResultChange>(oldExpireSecs, newExpireSecs, result)); - - if (MONGO_unlikely(assertAfterIndexUpdate.shouldFail())) { - LOGV2(20307, "collMod - assertAfterIndexUpdate fail point enabled."); - uasserted(50970, "trigger rollback after the index update"); + if (!cmrOld.indexExpireAfterSeconds.eoo() || !cmrOld.indexHidden.eoo()) { + BSONElement newExpireSecs = {}; + BSONElement oldExpireSecs = {}; + BSONElement newHidden = {}; + BSONElement oldHidden = {}; + // TTL Index + if (!cmrOld.indexExpireAfterSeconds.eoo()) { + newExpireSecs = cmrOld.indexExpireAfterSeconds; + oldExpireSecs = cmrOld.idx->infoObj().getField("expireAfterSeconds"); + if (SimpleBSONElementComparator::kInstance.evaluate(oldExpireSecs != + newExpireSecs)) { + // Change the value of "expireAfterSeconds" on disk. + DurableCatalog::get(opCtx)->updateTTLSetting(opCtx, + coll->getCatalogId(), + cmrOld.idx->indexName(), + newExpireSecs.safeNumberLong()); + } + } + + // User wants to hide or unhide index. + if (!cmrOld.indexHidden.eoo()) { + newHidden = cmrOld.indexHidden; + oldHidden = cmrOld.idx->infoObj().getField("hidden"); + // Make sure when we set 'hidden' to false, we can remove the hidden field from + // catalog. + if (SimpleBSONElementComparator::kInstance.evaluate(oldHidden != newHidden)) { + DurableCatalog::get(opCtx)->updateHiddenSetting(opCtx, + coll->getCatalogId(), + cmrOld.idx->indexName(), + newHidden.booleanSafe()); } } - // Save previous TTL index expiration. - ttlInfo = TTLCollModInfo{Seconds(newExpireSecs.safeNumberLong()), - Seconds(oldExpireSecs.safeNumberLong()), - cmrNew.idx->indexName()}; + indexCollModInfo = IndexCollModInfo{ + cmrOld.indexExpireAfterSeconds.eoo() ? boost::optional<Seconds>() + : Seconds(newExpireSecs.safeNumberLong()), + cmrOld.indexExpireAfterSeconds.eoo() ? boost::optional<Seconds>() + : Seconds(oldExpireSecs.safeNumberLong()), + cmrOld.indexHidden.eoo() ? boost::optional<bool>() : newHidden.booleanSafe(), + cmrOld.indexHidden.eoo() ? boost::optional<bool>() : oldHidden.booleanSafe(), + cmrNew.idx->indexName()}; + + // Notify the index catalog that the definition of this index changed. This will + // invalidate the idx pointer in cmrOld. On rollback of this WUOW, the idx pointer + // in cmrNew will be invalidated and the idx pointer in cmrOld will be valid again. + cmrNew.idx = coll->getIndexCatalog()->refreshEntry(opCtx, cmrOld.idx); + opCtx->recoveryUnit()->registerChange(std::make_unique<CollModResultChange>( + oldExpireSecs, newExpireSecs, oldHidden, newHidden, result)); + + if (MONGO_unlikely(assertAfterIndexUpdate.shouldFail())) { + LOGV2(20307, "collMod - assertAfterIndexUpdate fail point enabled."); + uasserted(50970, "trigger rollback after the index update"); + } } // The Validator, ValidationAction and ValidationLevel are already parsed and must be OK. @@ -412,8 +471,10 @@ Status _collModInternal(OperationContext* opCtx, // Only observe non-view collMods, as view operations are observed as operations on the // system.views collection. + auto* const opObserver = opCtx->getServiceContext()->getOpObserver(); - opObserver->onCollMod(opCtx, nss, coll->uuid(), oplogEntryObj, oldCollOptions, ttlInfo); + opObserver->onCollMod( + opCtx, nss, coll->uuid(), oplogEntryObj, oldCollOptions, indexCollModInfo); wunit.commit(); return Status::OK(); diff --git a/src/mongo/db/catalog/index_key_validate.cpp b/src/mongo/db/catalog/index_key_validate.cpp index e6ec964bf25..2d991b12da3 100644 --- a/src/mongo/db/catalog/index_key_validate.cpp +++ b/src/mongo/db/catalog/index_key_validate.cpp @@ -83,6 +83,7 @@ static std::set<StringData> allowedFieldNames = { IndexDescriptor::kDropDuplicatesFieldName, IndexDescriptor::kExpireAfterSecondsFieldName, IndexDescriptor::kGeoHaystackBucketSize, + IndexDescriptor::kHiddenFieldName, IndexDescriptor::kIndexNameFieldName, IndexDescriptor::kIndexVersionFieldName, IndexDescriptor::kKeyPatternFieldName, @@ -270,6 +271,12 @@ StatusWith<BSONObj> validateIndexSpec( boost::optional<IndexVersion> resolvedIndexVersion; + // Allow hidden index only if FCV is 4.6. + const auto isFeatureDisabled = + (!featureCompatibility.isVersionInitialized() || + featureCompatibility.getVersion() < + ServerGlobalParams::FeatureCompatibility::Version::kFullyUpgradedTo46); + for (auto&& indexSpecElem : indexSpec) { auto indexSpecElemFieldName = indexSpecElem.fieldNameStringData(); if (IndexDescriptor::kKeyPatternFieldName == indexSpecElemFieldName) { @@ -307,6 +314,7 @@ StatusWith<BSONObj> validateIndexSpec( << "Values in the index key pattern cannot be empty strings"}; } } + hasKeyPatternField = true; } else if (IndexDescriptor::kIndexNameFieldName == indexSpecElemFieldName) { if (indexSpecElem.type() != BSONType::String) { @@ -317,6 +325,18 @@ StatusWith<BSONObj> validateIndexSpec( } hasIndexNameField = true; + } else if (IndexDescriptor::kHiddenFieldName == indexSpecElemFieldName) { + if (isFeatureDisabled) { + return {ErrorCodes::Error(31449), + "Hidden indexes can only be created with FCV 4.6"}; + } + if (indexSpecElem.type() != BSONType::Bool) { + return {ErrorCodes::TypeMismatch, + str::stream() + << "The field '" << IndexDescriptor::kIndexNameFieldName + << "' must be a bool, but got " << typeName(indexSpecElem.type())}; + } + } else if (IndexDescriptor::kNamespaceFieldName == indexSpecElemFieldName) { hasNamespaceField = true; } else if (IndexDescriptor::kIndexVersionFieldName == indexSpecElemFieldName) { @@ -498,6 +518,10 @@ Status validateIdIndexSpec(const BSONObj& indexSpec) { << keyPatternElem.Obj()}; } + if (!indexSpec[IndexDescriptor::kHiddenFieldName].eoo()) { + return Status(ErrorCodes::BadValue, "can't hide _id index"); + } + return Status::OK(); } diff --git a/src/mongo/db/commands/create_indexes.cpp b/src/mongo/db/commands/create_indexes.cpp index fe1370a0827..f4c2297eadc 100644 --- a/src/mongo/db/commands/create_indexes.cpp +++ b/src/mongo/db/commands/create_indexes.cpp @@ -166,6 +166,8 @@ StatusWith<std::vector<BSONObj>> parseAndValidateIndexSpecs( // entry with a '*' as an index name means "drop all indexes in this // collection". We disallow creation of such indexes to avoid this conflict. return {ErrorCodes::BadValue, "The index name '*' is not valid."}; + } else if (ns.isSystem() && !indexSpec[IndexDescriptor::kHiddenFieldName].eoo()) { + return {ErrorCodes::BadValue, "Can't hide index on system collection"}; } indexSpecs.push_back(std::move(indexSpec)); diff --git a/src/mongo/db/create_indexes.idl b/src/mongo/db/create_indexes.idl index 59c37157b13..20b589660ca 100644 --- a/src/mongo/db/create_indexes.idl +++ b/src/mongo/db/create_indexes.idl @@ -50,6 +50,9 @@ structs: unique: type: bool optional: true + hidden: + type: bool + optional: true partialFilterExpression: type: object optional: true diff --git a/src/mongo/db/free_mon/free_mon_op_observer.h b/src/mongo/db/free_mon/free_mon_op_observer.h index 21478a822e5..4408d1d76ac 100644 --- a/src/mongo/db/free_mon/free_mon_op_observer.h +++ b/src/mongo/db/free_mon/free_mon_op_observer.h @@ -113,7 +113,7 @@ public: OptionalCollectionUUID uuid, const BSONObj& collModCmd, const CollectionOptions& oldCollOptions, - boost::optional<TTLCollModInfo> ttlInfo) final {} + boost::optional<IndexCollModInfo> indexInfo) final {} void onDropDatabase(OperationContext* opCtx, const std::string& dbName) final {} diff --git a/src/mongo/db/index/index_descriptor.cpp b/src/mongo/db/index/index_descriptor.cpp index 248e87e8b91..17f765d87cf 100644 --- a/src/mongo/db/index/index_descriptor.cpp +++ b/src/mongo/db/index/index_descriptor.cpp @@ -61,7 +61,8 @@ void populateOptionsMap(std::map<StringData, BSONElement>& theMap, const BSONObj IndexDescriptor::kBackgroundFieldName || // this is a creation time option only fieldName == IndexDescriptor::kDropDuplicatesFieldName || // this is now ignored fieldName == IndexDescriptor::kSparseFieldName || // checked specially - fieldName == IndexDescriptor::kUniqueFieldName // check specially + fieldName == IndexDescriptor::kHiddenFieldName || // not considered for equivalence + fieldName == IndexDescriptor::kUniqueFieldName // check specially ) { continue; } @@ -93,6 +94,7 @@ constexpr StringData IndexDescriptor::kSparseFieldName; constexpr StringData IndexDescriptor::kStorageEngineFieldName; constexpr StringData IndexDescriptor::kTextVersionFieldName; constexpr StringData IndexDescriptor::kUniqueFieldName; +constexpr StringData IndexDescriptor::kHiddenFieldName; constexpr StringData IndexDescriptor::kWeightsFieldName; IndexDescriptor::IndexDescriptor(Collection* collection, @@ -109,6 +111,7 @@ IndexDescriptor::IndexDescriptor(Collection* collection, _isIdIndex(isIdIndexPattern(_keyPattern)), _sparse(infoObj[IndexDescriptor::kSparseFieldName].trueValue()), _unique(_isIdIndex || infoObj[kUniqueFieldName].trueValue()), + _hidden(infoObj[kHiddenFieldName].trueValue()), _partial(!infoObj[kPartialFilterExprFieldName].eoo()), _cachedEntry(nullptr) { BSONElement e = _infoObj[IndexDescriptor::kIndexVersionFieldName]; diff --git a/src/mongo/db/index/index_descriptor.h b/src/mongo/db/index/index_descriptor.h index b8d61b20564..4eab168a65a 100644 --- a/src/mongo/db/index/index_descriptor.h +++ b/src/mongo/db/index/index_descriptor.h @@ -82,6 +82,7 @@ public: static constexpr StringData kStorageEngineFieldName = "storageEngine"_sd; static constexpr StringData kTextVersionFieldName = "textIndexVersion"_sd; static constexpr StringData kUniqueFieldName = "unique"_sd; + static constexpr StringData kHiddenFieldName = "hidden"_sd; static constexpr StringData kWeightsFieldName = "weights"_sd; /** @@ -174,6 +175,10 @@ public: return _unique; } + bool hidden() const { + return _hidden; + } + // Is this index sparse? bool isSparse() const { return _sparse; @@ -255,6 +260,7 @@ private: bool _isIdIndex; bool _sparse; bool _unique; + bool _hidden; bool _partial; IndexVersion _version; BSONObj _collation; diff --git a/src/mongo/db/op_observer.h b/src/mongo/db/op_observer.h index 00fba51f1c5..b970643444a 100644 --- a/src/mongo/db/op_observer.h +++ b/src/mongo/db/op_observer.h @@ -60,9 +60,11 @@ struct OplogUpdateEntryArgs { : updateArgs(std::move(updateArgs)), nss(std::move(nss)), uuid(std::move(uuid)) {} }; -struct TTLCollModInfo { - Seconds expireAfterSeconds; - Seconds oldExpireAfterSeconds; +struct IndexCollModInfo { + boost::optional<Seconds> expireAfterSeconds; + boost::optional<Seconds> oldExpireAfterSeconds; + boost::optional<bool> hidden; + boost::optional<bool> oldHidden; std::string indexName; }; @@ -208,7 +210,7 @@ public: OptionalCollectionUUID uuid, const BSONObj& collModCmd, const CollectionOptions& oldCollOptions, - boost::optional<TTLCollModInfo> ttlInfo) = 0; + boost::optional<IndexCollModInfo> indexInfo) = 0; virtual void onDropDatabase(OperationContext* opCtx, const std::string& dbName) = 0; /** diff --git a/src/mongo/db/op_observer_impl.cpp b/src/mongo/db/op_observer_impl.cpp index 4551f8f1a32..f630951baa1 100644 --- a/src/mongo/db/op_observer_impl.cpp +++ b/src/mongo/db/op_observer_impl.cpp @@ -655,7 +655,7 @@ void OpObserverImpl::onCollMod(OperationContext* opCtx, OptionalCollectionUUID uuid, const BSONObj& collModCmd, const CollectionOptions& oldCollOptions, - boost::optional<TTLCollModInfo> ttlInfo) { + boost::optional<IndexCollModInfo> indexInfo) { if (!nss.isSystemDotProfile()) { // do not replicate system.profile modifications @@ -663,16 +663,23 @@ void OpObserverImpl::onCollMod(OperationContext* opCtx, // Create the 'o2' field object. We save the old collection metadata and TTL expiration. BSONObjBuilder o2Builder; o2Builder.append("collectionOptions_old", oldCollOptions.toBSON()); - if (ttlInfo) { - auto oldExpireAfterSeconds = durationCount<Seconds>(ttlInfo->oldExpireAfterSeconds); - o2Builder.append("expireAfterSeconds_old", oldExpireAfterSeconds); + if (indexInfo) { + if (indexInfo->oldExpireAfterSeconds) { + auto oldExpireAfterSeconds = + durationCount<Seconds>(indexInfo->oldExpireAfterSeconds.get()); + o2Builder.append("expireAfterSeconds_old", oldExpireAfterSeconds); + } + if (indexInfo->oldHidden) { + auto oldHidden = indexInfo->oldHidden.get(); + o2Builder.append("hidden_old", oldHidden); + } } MutableOplogEntry oplogEntry; oplogEntry.setOpType(repl::OpTypeEnum::kCommand); oplogEntry.setNss(nss.getCommandNS()); oplogEntry.setUuid(uuid); - oplogEntry.setObject(makeCollModCmdObj(collModCmd, oldCollOptions, ttlInfo)); + oplogEntry.setObject(makeCollModCmdObj(collModCmd, oldCollOptions, indexInfo)); oplogEntry.setObject2(o2Builder.done()); logOperation(opCtx, &oplogEntry); } diff --git a/src/mongo/db/op_observer_impl.h b/src/mongo/db/op_observer_impl.h index 51d6ebd6c99..e61f14e6e2c 100644 --- a/src/mongo/db/op_observer_impl.h +++ b/src/mongo/db/op_observer_impl.h @@ -102,7 +102,7 @@ public: OptionalCollectionUUID uuid, const BSONObj& collModCmd, const CollectionOptions& oldCollOptions, - boost::optional<TTLCollModInfo> ttlInfo) final; + boost::optional<IndexCollModInfo> indexInfo) final; void onDropDatabase(OperationContext* opCtx, const std::string& dbName) final; repl::OpTime onDropCollection(OperationContext* opCtx, const NamespaceString& collectionName, diff --git a/src/mongo/db/op_observer_impl_test.cpp b/src/mongo/db/op_observer_impl_test.cpp index 56949f04d88..8e4b093fafa 100644 --- a/src/mongo/db/op_observer_impl_test.cpp +++ b/src/mongo/db/op_observer_impl_test.cpp @@ -273,16 +273,16 @@ TEST_F(OpObserverTest, CollModWithCollectionOptionsAndTTLInfo) { oldCollOpts.validationLevel = "strict"; oldCollOpts.validationAction = "error"; - TTLCollModInfo ttlInfo; - ttlInfo.expireAfterSeconds = Seconds(10); - ttlInfo.oldExpireAfterSeconds = Seconds(5); - ttlInfo.indexName = "name_of_index"; + IndexCollModInfo indexInfo; + indexInfo.expireAfterSeconds = Seconds(10); + indexInfo.oldExpireAfterSeconds = Seconds(5); + indexInfo.indexName = "name_of_index"; // Write to the oplog. { AutoGetCollection autoColl(opCtx.get(), nss, MODE_X); WriteUnitOfWork wunit(opCtx.get()); - opObserver.onCollMod(opCtx.get(), nss, uuid, collModCmd, oldCollOpts, ttlInfo); + opObserver.onCollMod(opCtx.get(), nss, uuid, collModCmd, oldCollOpts, indexInfo); wunit.commit(); } @@ -290,14 +290,14 @@ TEST_F(OpObserverTest, CollModWithCollectionOptionsAndTTLInfo) { // Ensure that collMod fields were properly added to the oplog entry. auto o = oplogEntry.getObjectField("o"); - auto oExpected = - BSON("collMod" << nss.coll() << "validationLevel" - << "off" - << "validationAction" - << "warn" - << "index" - << BSON("name" << ttlInfo.indexName << "expireAfterSeconds" - << durationCount<Seconds>(ttlInfo.expireAfterSeconds))); + auto oExpected = BSON( + "collMod" << nss.coll() << "validationLevel" + << "off" + << "validationAction" + << "warn" + << "index" + << BSON("name" << indexInfo.indexName << "expireAfterSeconds" + << durationCount<Seconds>(indexInfo.expireAfterSeconds.get()))); ASSERT_BSONOBJ_EQ(oExpected, o); // Ensure that the old collection metadata was saved. @@ -306,7 +306,8 @@ TEST_F(OpObserverTest, CollModWithCollectionOptionsAndTTLInfo) { BSON("collectionOptions_old" << BSON("validationLevel" << oldCollOpts.validationLevel << "validationAction" << oldCollOpts.validationAction) - << "expireAfterSeconds_old" << durationCount<Seconds>(ttlInfo.oldExpireAfterSeconds)); + << "expireAfterSeconds_old" + << durationCount<Seconds>(indexInfo.oldExpireAfterSeconds.get())); ASSERT_BSONOBJ_EQ(o2Expected, o2); } diff --git a/src/mongo/db/op_observer_noop.h b/src/mongo/db/op_observer_noop.h index 5c5816e7c0b..3684625cc27 100644 --- a/src/mongo/db/op_observer_noop.h +++ b/src/mongo/db/op_observer_noop.h @@ -98,7 +98,7 @@ public: OptionalCollectionUUID uuid, const BSONObj& collModCmd, const CollectionOptions& oldCollOptions, - boost::optional<TTLCollModInfo> ttlInfo) override {} + boost::optional<IndexCollModInfo> indexInfo) override {} void onDropDatabase(OperationContext* opCtx, const std::string& dbName) override {} repl::OpTime onDropCollection(OperationContext* opCtx, const NamespaceString& collectionName, diff --git a/src/mongo/db/op_observer_registry.h b/src/mongo/db/op_observer_registry.h index 2470baa74cc..9bc29820269 100644 --- a/src/mongo/db/op_observer_registry.h +++ b/src/mongo/db/op_observer_registry.h @@ -175,10 +175,10 @@ public: OptionalCollectionUUID uuid, const BSONObj& collModCmd, const CollectionOptions& oldCollOptions, - boost::optional<TTLCollModInfo> ttlInfo) override { + boost::optional<IndexCollModInfo> indexInfo) override { ReservedTimes times{opCtx}; for (auto& o : _observers) - o->onCollMod(opCtx, nss, uuid, collModCmd, oldCollOptions, ttlInfo); + o->onCollMod(opCtx, nss, uuid, collModCmd, oldCollOptions, indexInfo); } void onDropDatabase(OperationContext* const opCtx, const std::string& dbName) override { diff --git a/src/mongo/db/op_observer_util.cpp b/src/mongo/db/op_observer_util.cpp index 9c03a37a3db..7d69555618f 100644 --- a/src/mongo/db/op_observer_util.cpp +++ b/src/mongo/db/op_observer_util.cpp @@ -42,21 +42,24 @@ namespace mongo { */ BSONObj makeCollModCmdObj(const BSONObj& collModCmd, const CollectionOptions& oldCollOptions, - boost::optional<TTLCollModInfo> ttlInfo) { + boost::optional<IndexCollModInfo> indexInfo) { BSONObjBuilder cmdObjBuilder; - std::string ttlIndexFieldName = "index"; + std::string indexFieldName = "index"; // Add all fields from the original collMod command. for (auto elem : collModCmd) { // We normalize all TTL collMod oplog entry objects to use the index name, even if the // command used an index key pattern. - if (elem.fieldNameStringData() == ttlIndexFieldName && ttlInfo) { - BSONObjBuilder ttlIndexObjBuilder; - ttlIndexObjBuilder.append("name", ttlInfo->indexName); - ttlIndexObjBuilder.append("expireAfterSeconds", - durationCount<Seconds>(ttlInfo->expireAfterSeconds)); + if (elem.fieldNameStringData() == indexFieldName && indexInfo) { + BSONObjBuilder indexObjBuilder; + indexObjBuilder.append("name", indexInfo->indexName); + if (indexInfo->expireAfterSeconds) + indexObjBuilder.append("expireAfterSeconds", + durationCount<Seconds>(indexInfo->expireAfterSeconds.get())); + if (indexInfo->hidden) + indexObjBuilder.append("hidden", indexInfo->hidden.get()); - cmdObjBuilder.append(ttlIndexFieldName, ttlIndexObjBuilder.obj()); + cmdObjBuilder.append(indexFieldName, indexObjBuilder.obj()); } else { cmdObjBuilder.append(elem); } diff --git a/src/mongo/db/op_observer_util.h b/src/mongo/db/op_observer_util.h index 800a1486d4d..d603d3fb177 100644 --- a/src/mongo/db/op_observer_util.h +++ b/src/mongo/db/op_observer_util.h @@ -36,5 +36,5 @@ namespace mongo { BSONObj makeCollModCmdObj(const BSONObj& collModCmd, const CollectionOptions& oldCollOptions, - boost::optional<TTLCollModInfo> ttlInfo); + boost::optional<IndexCollModInfo> indexInfo); } // namespace mongo diff --git a/src/mongo/db/query/get_executor.cpp b/src/mongo/db/query/get_executor.cpp index db1ead9711d..8fa38f09a9a 100644 --- a/src/mongo/db/query/get_executor.cpp +++ b/src/mongo/db/query/get_executor.cpp @@ -260,6 +260,10 @@ void fillOutPlannerParams(OperationContext* opCtx, collection->getIndexCatalog()->getIndexIterator(opCtx, false); while (ii->more()) { const IndexCatalogEntry* ice = ii->next(); + + // Skip the addition of hidden indexes to prevent use in query planning. + if (ice->descriptor()->hidden()) + continue; plannerParams->indices.push_back( indexEntryFromIndexCatalogEntry(opCtx, *ice, canonicalQuery)); } @@ -1469,6 +1473,10 @@ QueryPlannerParams fillOutPlannerParamsForDistinct(OperationContext* opCtx, while (ii->more()) { const IndexCatalogEntry* ice = ii->next(); const IndexDescriptor* desc = ice->descriptor(); + + // Skip the addition of hidden indexes to prevent use in query planning. + if (desc->hidden()) + continue; if (desc->keyPattern().hasField(parsedDistinct.getKey())) { if (!mayUnwindArrays && isAnyComponentOfPathMultikey(desc->keyPattern(), diff --git a/src/mongo/db/s/config_server_op_observer.h b/src/mongo/db/s/config_server_op_observer.h index d274dfef21a..ca17153d814 100644 --- a/src/mongo/db/s/config_server_op_observer.h +++ b/src/mongo/db/s/config_server_op_observer.h @@ -114,7 +114,7 @@ public: OptionalCollectionUUID uuid, const BSONObj& collModCmd, const CollectionOptions& oldCollOptions, - boost::optional<TTLCollModInfo> ttlInfo) override {} + boost::optional<IndexCollModInfo> indexInfo) override {} void onDropDatabase(OperationContext* opCtx, const std::string& dbName) override {} diff --git a/src/mongo/db/s/shard_server_op_observer.cpp b/src/mongo/db/s/shard_server_op_observer.cpp index 07f2bcdc2be..2b303782a9c 100644 --- a/src/mongo/db/s/shard_server_op_observer.cpp +++ b/src/mongo/db/s/shard_server_op_observer.cpp @@ -508,7 +508,7 @@ void ShardServerOpObserver::onCollMod(OperationContext* opCtx, OptionalCollectionUUID uuid, const BSONObj& collModCmd, const CollectionOptions& oldCollOptions, - boost::optional<TTLCollModInfo> ttlInfo) { + boost::optional<IndexCollModInfo> indexInfo) { abortOngoingMigrationIfNeeded(opCtx, nss); }; diff --git a/src/mongo/db/s/shard_server_op_observer.h b/src/mongo/db/s/shard_server_op_observer.h index 68a0f401ec6..481b4619836 100644 --- a/src/mongo/db/s/shard_server_op_observer.h +++ b/src/mongo/db/s/shard_server_op_observer.h @@ -113,7 +113,7 @@ public: OptionalCollectionUUID uuid, const BSONObj& collModCmd, const CollectionOptions& oldCollOptions, - boost::optional<TTLCollModInfo> ttlInfo) override; + boost::optional<IndexCollModInfo> indexInfo) override; void onDropDatabase(OperationContext* opCtx, const std::string& dbName) override {} diff --git a/src/mongo/db/storage/bson_collection_catalog_entry.cpp b/src/mongo/db/storage/bson_collection_catalog_entry.cpp index 95290131e9c..1cf0ed5b6a1 100644 --- a/src/mongo/db/storage/bson_collection_catalog_entry.cpp +++ b/src/mongo/db/storage/bson_collection_catalog_entry.cpp @@ -120,6 +120,26 @@ void BSONCollectionCatalogEntry::IndexMetaData::updateTTLSetting(long long newEx spec = b.obj(); } + +void BSONCollectionCatalogEntry::IndexMetaData::updateHiddenSetting(bool hidden) { + // If hidden == false, we remove this field from catalog rather than add a field with false. + // or else, the old binary can't startup due to the unknown field. + BSONObjBuilder b; + for (BSONObjIterator bi(spec); bi.more();) { + BSONElement e = bi.next(); + if (e.fieldNameStringData() == "hidden") { + continue; + } + b.append(e); + } + + if (hidden) { + b.append("hidden", hidden); + } + spec = b.obj(); +} + + // -------------------------- int BSONCollectionCatalogEntry::MetaData::findIndexOffset(StringData name) const { diff --git a/src/mongo/db/storage/bson_collection_catalog_entry.h b/src/mongo/db/storage/bson_collection_catalog_entry.h index b22a3290b07..74bdcc2732d 100644 --- a/src/mongo/db/storage/bson_collection_catalog_entry.h +++ b/src/mongo/db/storage/bson_collection_catalog_entry.h @@ -64,6 +64,8 @@ public: void updateTTLSetting(long long newExpireSeconds); + void updateHiddenSetting(bool hidden); + std::string name() const { return spec["name"].String(); } diff --git a/src/mongo/db/storage/durable_catalog.h b/src/mongo/db/storage/durable_catalog.h index 29e1fa394cb..17860449ba6 100644 --- a/src/mongo/db/storage/durable_catalog.h +++ b/src/mongo/db/storage/durable_catalog.h @@ -151,6 +151,15 @@ public: StringData idxName, long long newExpireSeconds) = 0; + /* + * Hide or unhide the given index. A hidden index will not be considered for use by the + * query planner. + */ + virtual void updateHiddenSetting(OperationContext* opCtx, + RecordId catalogId, + StringData idxName, + bool hidden) = 0; + /** Compares the UUID argument to the UUID obtained from the metadata. Returns true if they are * equal, false otherwise. */ diff --git a/src/mongo/db/storage/durable_catalog_impl.cpp b/src/mongo/db/storage/durable_catalog_impl.cpp index 46e4f3ce5ad..1ad66f2f2bd 100644 --- a/src/mongo/db/storage/durable_catalog_impl.cpp +++ b/src/mongo/db/storage/durable_catalog_impl.cpp @@ -920,6 +920,19 @@ void DurableCatalogImpl::updateTTLSetting(OperationContext* opCtx, putMetaData(opCtx, catalogId, md); } +void DurableCatalogImpl::updateHiddenSetting(OperationContext* opCtx, + RecordId catalogId, + StringData idxName, + bool hidden) { + + BSONCollectionCatalogEntry::MetaData md = getMetaData(opCtx, catalogId); + int offset = md.findIndexOffset(idxName); + invariant(offset >= 0); + md.indexes[offset].updateHiddenSetting(hidden); + putMetaData(opCtx, catalogId, md); +} + + bool DurableCatalogImpl::isEqualToMetadataUUID(OperationContext* opCtx, RecordId catalogId, OptionalCollectionUUID uuid) { diff --git a/src/mongo/db/storage/durable_catalog_impl.h b/src/mongo/db/storage/durable_catalog_impl.h index f11e5b445f9..5c674af4824 100644 --- a/src/mongo/db/storage/durable_catalog_impl.h +++ b/src/mongo/db/storage/durable_catalog_impl.h @@ -124,6 +124,11 @@ public: StringData idxName, long long newExpireSeconds); + void updateHiddenSetting(OperationContext* opCtx, + RecordId catalogId, + StringData idxName, + bool hidden); + bool isEqualToMetadataUUID(OperationContext* opCtx, RecordId catalogId, OptionalCollectionUUID uuid); |