diff options
author | Zhihui Fan <yizhi.fzh@alibaba-inc.com> | 2020-02-04 08:09:01 +0800 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2020-04-24 16:55:37 +0000 |
commit | 340222f5e2005fcf734a65e269c52572f9b0b2cb (patch) | |
tree | a5a76cf454f92e5edfdec45384c71276799962e7 | |
parent | 67124b21abd481d4c96cf6090d5438b7e034b280 (diff) | |
download | mongo-340222f5e2005fcf734a65e269c52572f9b0b2cb.tar.gz |
SERVER-9306 Ability to temporarily forbid query optimizer from using index ("Hidden Index") SERVER-47275 Take over and complete Hidden Indexes PR
Co-authored-by: Ruoxin Xu <ruoxin.xu@mongodb.com>
(cherry picked from commit bad7c538e7efbc996a6089e1569681edf24e6b33)
29 files changed, 444 insertions, 87 deletions
diff --git a/jstests/core/hidden_index.js b/jstests/core/hidden_index.js new file mode 100644 index 00000000000..a5ea2d09f17 --- /dev/null +++ b/jstests/core/hidden_index.js @@ -0,0 +1,126 @@ +/** + * Test expected behavior for hidden indexes. A hidden index is invisible to the query planner so + * it will not be used in planning. It is handled in the same way as other indexes by the index + * catalog and for TTL purposes. + * @tags: [ + * multiversion_incompatible, + * requires_fcv_44, + * requires_non_retryable_commands, // CollMod is not retryable. + * ] + */ + +(function() { +'use strict'; +load("jstests/libs/analyze_plan.js"); // For getPlanStages. +load("jstests/libs/collection_drop_recreate.js"); // For assert[Drop|Create]Collection. +load("jstests/libs/fixture_helpers.js"); // For FixtureHelpers. +load("jstests/libs/get_index_helpers.js"); // For GetIndexHelpers.findByName. + +const collName = "hidden_index"; +let coll = assertDropAndRecreateCollection(db, collName); + +function numOfUsedIXSCAN(query) { + const explain = assert.commandWorked(coll.find(query).explain()); + const ixScans = getPlanStages(explain.queryPlanner.winningPlan, "IXSCAN"); + return ixScans.length; +} + +function validateHiddenIndexBehaviour(query, index_type, wildcard) { + let index_name; + if (wildcard) + index_name = 'a.$**_' + index_type; + else + index_name = 'a_' + index_type; + + if (wildcard) + assert.commandWorked(coll.createIndex({"a.$**": index_type})); + else + assert.commandWorked(coll.createIndex({"a": index_type})); + + let idxSpec = GetIndexHelpers.findByName(coll.getIndexes(), index_name); + assert.eq(idxSpec.hidden, undefined); + assert.gt(numOfUsedIXSCAN(query), 0); + + assert.commandWorked(coll.hideIndex(index_name)); + idxSpec = GetIndexHelpers.findByName(coll.getIndexes(), index_name); + assert(idxSpec.hidden); + if (index_type === "text") { + assert.commandFailedWithCode(coll.runCommand("find", {filter: query}, {hint: {a: 1}}), 291); + assert.commandWorked(coll.dropIndexes()); + return; + } + assert.eq(numOfUsedIXSCAN(query), 0); + + assert.commandWorked(coll.unhideIndex(index_name)); + idxSpec = GetIndexHelpers.findByName(coll.getIndexes(), index_name); + assert.eq(idxSpec.hidden, undefined); + assert.gt(numOfUsedIXSCAN(query), 0); + + assert.commandWorked(coll.dropIndex(index_name)); + + if (wildcard) + assert.commandWorked(coll.createIndex({"a.$**": index_type}, {hidden: true})); + else + assert.commandWorked(coll.createIndex({"a": index_type}, {hidden: true})); + + idxSpec = GetIndexHelpers.findByName(coll.getIndexes(), index_name); + assert(idxSpec.hidden); + assert.eq(numOfUsedIXSCAN(query), 0); + assert.commandWorked(coll.dropIndexes()); +} + +// Normal index testing. +validateHiddenIndexBehaviour({a: 1}, 1); + +// GEO index testing. +validateHiddenIndexBehaviour({ + a: {$geoWithin: {$geometry: {type: "Polygon", coordinates: [[[0, 0], [3, 6], [6, 1], [0, 0]]]}}} +}, + "2dsphere"); + +// Fts index. +validateHiddenIndexBehaviour({$text: {$search: "java"}}, "text"); + +// Wildcard index. +validateHiddenIndexBehaviour({"a.f": 1}, 1, true); + +// Hidden index on capped collection. +if (!FixtureHelpers.isMongos(db)) { + coll = assertDropAndRecreateCollection(db, collName, {capped: true, size: 100}); + validateHiddenIndexBehaviour({a: 1}, 1); + coll = assertDropAndRecreateCollection(db, collName); +} +// Test that index 'hidden' status can be found in listIndexes command. +assert.commandWorked(coll.createIndex({lsIdx: 1}, {hidden: true})); +let res = assert.commandWorked(db.runCommand({"listIndexes": collName})); +let idxSpec = GetIndexHelpers.findByName(res.cursor.firstBatch, "lsIdx_1"); +assert.eq(idxSpec.hidden, true); + +// Can't hide any index in a system collection. +const systemColl = db.getSiblingDB('admin').system.version; +assert.commandWorked(systemColl.createIndex({a: 1})); +assert.commandFailedWithCode(systemColl.hideIndex("a_1"), 2); +assert.commandFailedWithCode(systemColl.createIndex({a: 1}, {hidden: true}), 2); + +// Can't hide the '_id' index. +assert.commandFailed(coll.hideIndex("_id_")); + +// Can't 'hint' a hidden index. +assert.commandWorked(coll.createIndex({"a": 1}, {"hidden": true})); +assert.commandFailedWithCode(coll.runCommand("find", {hint: {a: 1}}), 2); + +// We can change ttl index and hide info at the same time. +assert.commandWorked(coll.dropIndexes()); +assert.commandWorked(coll.createIndex({"tm": 1}, {expireAfterSeconds: 10})); +idxSpec = GetIndexHelpers.findByName(coll.getIndexes(), "tm_1"); +assert.eq(idxSpec.hidden, undefined); +assert.eq(idxSpec.expireAfterSeconds, 10); + +db.runCommand({ + "collMod": coll.getName(), + "index": {"name": "tm_1", "expireAfterSeconds": 1, "hidden": true} +}); +idxSpec = GetIndexHelpers.findByName(coll.getIndexes(), "tm_1"); +assert(idxSpec.hidden); +assert.eq(idxSpec.expireAfterSeconds, 1); +})(); diff --git a/jstests/noPassthrough/ttl_hidden_index.js b/jstests/noPassthrough/ttl_hidden_index.js new file mode 100644 index 00000000000..80c783dad33 --- /dev/null +++ b/jstests/noPassthrough/ttl_hidden_index.js @@ -0,0 +1,29 @@ +// Make sure the TTL index still work after we hide it +(function() { +"use strict"; +let runner = MongoRunner.runMongod({setParameter: "ttlMonitorSleepSecs=1"}); +let coll = runner.getDB("test").ttl_hiddenl_index; +coll.drop(); + +// Create TTL index. +assert.commandWorked(coll.ensureIndex({x: 1}, {expireAfterSeconds: 0})); +let now = new Date(); + +assert.commandWorked(coll.hideIndex("x_1")); + +// Insert docs after having set hidden index in order to prevent inserted docs being expired out +// before the hidden index is set. +assert.commandWorked(coll.insert({x: now})); +assert.commandWorked(coll.insert({x: now})); + +// Wait for the TTL monitor to run at least twice (in case we weren't finished setting up our +// collection when it ran the first time). +var ttlPass = coll.getDB().serverStatus().metrics.ttl.passes; +assert.soon(function() { + return coll.getDB().serverStatus().metrics.ttl.passes >= ttlPass + 2; +}, "TTL monitor didn't run before timing out."); + +assert.eq(coll.count(), 0, "We should get 0 documents after TTL monitor run"); + +MongoRunner.stopMongod(runner); +})(); diff --git a/src/mongo/db/auth/auth_op_observer.cpp b/src/mongo/db/auth/auth_op_observer.cpp index 8043a10c7f7..a1532e1533f 100644 --- a/src/mongo/db/auth/auth_op_observer.cpp +++ b/src/mongo/db/auth/auth_op_observer.cpp @@ -114,11 +114,11 @@ void AuthOpObserver::onCollMod(OperationContext* opCtx, OptionalCollectionUUID uuid, const BSONObj& collModCmd, const CollectionOptions& oldCollOptions, - boost::optional<TTLCollModInfo> ttlInfo) { + boost::optional<IndexCollModInfo> indexInfo) { const auto cmdNss = nss.getCommandNS(); // Create the 'o' field object. - const auto cmdObj = makeCollModCmdObj(collModCmd, oldCollOptions, ttlInfo); + const auto cmdObj = makeCollModCmdObj(collModCmd, oldCollOptions, indexInfo); AuthorizationManager::get(opCtx->getServiceContext()) ->logOp(opCtx, "c", cmdNss, cmdObj, nullptr); diff --git a/src/mongo/db/auth/auth_op_observer.h b/src/mongo/db/auth/auth_op_observer.h index 47293e12e09..45f9d56b099 100644 --- a/src/mongo/db/auth/auth_op_observer.h +++ b/src/mongo/db/auth/auth_op_observer.h @@ -113,7 +113,7 @@ public: OptionalCollectionUUID uuid, const BSONObj& collModCmd, const CollectionOptions& oldCollOptions, - boost::optional<TTLCollModInfo> ttlInfo) final; + boost::optional<IndexCollModInfo> indexInfo) final; void onDropDatabase(OperationContext* opCtx, const std::string& dbName) final; diff --git a/src/mongo/db/catalog/coll_mod.cpp b/src/mongo/db/catalog/coll_mod.cpp index afb64d12476..6f2f5dcfafd 100644 --- a/src/mongo/db/catalog/coll_mod.cpp +++ b/src/mongo/db/catalog/coll_mod.cpp @@ -69,6 +69,7 @@ MONGO_FAIL_POINT_DEFINE(assertAfterIndexUpdate); struct CollModRequest { const IndexDescriptor* idx = nullptr; BSONElement indexExpireAfterSeconds = {}; + BSONElement indexHidden = {}; BSONElement viewPipeLine = {}; std::string viewOn = {}; BSONElement collValidator = {}; @@ -125,14 +126,18 @@ StatusWith<CollModRequest> parseCollModRequest(OperationContext* opCtx, } cmr.indexExpireAfterSeconds = indexObj["expireAfterSeconds"]; - if (cmr.indexExpireAfterSeconds.eoo()) { - return Status(ErrorCodes::InvalidOptions, "no expireAfterSeconds field"); + cmr.indexHidden = indexObj["hidden"]; + + if (cmr.indexExpireAfterSeconds.eoo() && cmr.indexHidden.eoo()) { + return Status(ErrorCodes::InvalidOptions, "no expireAfterSeconds or hidden field"); } - if (!cmr.indexExpireAfterSeconds.isNumber()) { + if (!cmr.indexExpireAfterSeconds.eoo() && !cmr.indexExpireAfterSeconds.isNumber()) { return Status(ErrorCodes::InvalidOptions, "expireAfterSeconds field must be a number"); } - + if (!cmr.indexHidden.eoo() && !cmr.indexHidden.isBoolean()) { + return Status(ErrorCodes::InvalidOptions, "hidden field must be a boolean"); + } if (!indexName.empty()) { cmr.idx = coll->getIndexCatalog()->findIndexByName(opCtx, indexName); if (!cmr.idx) { @@ -161,15 +166,17 @@ StatusWith<CollModRequest> parseCollModRequest(OperationContext* opCtx, cmr.idx = indexes[0]; } - BSONElement oldExpireSecs = cmr.idx->infoObj().getField("expireAfterSeconds"); - if (oldExpireSecs.eoo()) { - return Status(ErrorCodes::InvalidOptions, "no expireAfterSeconds field to update"); - } - if (!oldExpireSecs.isNumber()) { - return Status(ErrorCodes::InvalidOptions, - "existing expireAfterSeconds field is not a number"); + if (!cmr.indexExpireAfterSeconds.eoo()) { + BSONElement oldExpireSecs = cmr.idx->infoObj().getField("expireAfterSeconds"); + if (oldExpireSecs.eoo()) { + return Status(ErrorCodes::InvalidOptions, + "no expireAfterSeconds field to update"); + } + if (!oldExpireSecs.isNumber()) { + return Status(ErrorCodes::InvalidOptions, + "existing expireAfterSeconds field is not a number"); + } } - } else if (fieldName == "validator" && !isView) { // Save this to a variable to avoid reading the atomic variable multiple times. const auto currentFCV = serverGlobalParams.featureCompatibility.getVersion(); @@ -250,13 +257,26 @@ class CollModResultChange : public RecoveryUnit::Change { public: CollModResultChange(const BSONElement& oldExpireSecs, const BSONElement& newExpireSecs, + const BSONElement& oldHidden, + const BSONElement& newHidden, BSONObjBuilder* result) - : _oldExpireSecs(oldExpireSecs), _newExpireSecs(newExpireSecs), _result(result) {} + : _oldExpireSecs(oldExpireSecs), + _newExpireSecs(newExpireSecs), + _oldHidden(oldHidden), + _newHidden(newHidden), + _result(result) {} void commit(boost::optional<Timestamp>) override { // add the fields to BSONObjBuilder result - _result->appendAs(_oldExpireSecs, "expireAfterSeconds_old"); - _result->appendAs(_newExpireSecs, "expireAfterSeconds_new"); + if (!_oldExpireSecs.eoo()) { + _result->appendAs(_oldExpireSecs, "expireAfterSeconds_old"); + _result->appendAs(_newExpireSecs, "expireAfterSeconds_new"); + } + if (!_newHidden.eoo()) { + bool oldValue = _oldHidden.eoo() ? false : _oldHidden.booleanSafe(); + _result->append("hidden_old", oldValue); + _result->appendAs(_newHidden, "hidden_new"); + } } void rollback() override {} @@ -264,6 +284,8 @@ public: private: const BSONElement _oldExpireSecs; const BSONElement _newExpireSecs; + const BSONElement _oldHidden; + const BSONElement _newHidden; BSONObjBuilder* _result; }; @@ -326,6 +348,19 @@ Status _collModInternal(OperationContext* opCtx, const CollModRequest cmrOld = statusW.getValue(); CollModRequest cmrNew = statusW.getValue(); + if (!cmrOld.indexHidden.eoo()) { + + if (serverGlobalParams.featureCompatibility.getVersion() < + ServerGlobalParams::FeatureCompatibility::Version::kFullyUpgradedTo44 && + cmrOld.indexHidden.booleanSafe()) { + return Status(ErrorCodes::BadValue, "Hidden indexes can only be created with FCV 4.4"); + } + if (coll->ns().isSystem()) + return Status(ErrorCodes::BadValue, "Can't hide index on system collection"); + if (cmrOld.idx->isIdIndex()) + return Status(ErrorCodes::BadValue, "can't hide _id index"); + } + return writeConflictRetry(opCtx, "collMod", nss.ns(), [&] { WriteUnitOfWork wunit(opCtx); @@ -362,40 +397,64 @@ Status _collModInternal(OperationContext* opCtx, CollectionOptions oldCollOptions = DurableCatalog::get(opCtx)->getCollectionOptions(opCtx, coll->getCatalogId()); - boost::optional<TTLCollModInfo> ttlInfo; + boost::optional<IndexCollModInfo> indexCollModInfo; // Handle collMod operation type appropriately. - // TTLIndex - if (!cmrOld.indexExpireAfterSeconds.eoo()) { - BSONElement newExpireSecs = cmrOld.indexExpireAfterSeconds; - BSONElement oldExpireSecs = cmrOld.idx->infoObj().getField("expireAfterSeconds"); - - if (SimpleBSONElementComparator::kInstance.evaluate(oldExpireSecs != newExpireSecs)) { - // Change the value of "expireAfterSeconds" on disk. - DurableCatalog::get(opCtx)->updateTTLSetting(opCtx, - coll->getCatalogId(), - cmrOld.idx->indexName(), - newExpireSecs.safeNumberLong()); - - // Notify the index catalog that the definition of this index changed. This will - // invalidate the idx pointer in cmrOld. On rollback of this WUOW, the idx pointer - // in cmrNew will be invalidated and the idx pointer in cmrOld will be valid again. - cmrNew.idx = coll->getIndexCatalog()->refreshEntry(opCtx, cmrOld.idx); - opCtx->recoveryUnit()->registerChange( - std::make_unique<CollModResultChange>(oldExpireSecs, newExpireSecs, result)); - - if (MONGO_unlikely(assertAfterIndexUpdate.shouldFail())) { - LOGV2(20307, "collMod - assertAfterIndexUpdate fail point enabled."); - uasserted(50970, "trigger rollback after the index update"); + if (!cmrOld.indexExpireAfterSeconds.eoo() || !cmrOld.indexHidden.eoo()) { + BSONElement newExpireSecs = {}; + BSONElement oldExpireSecs = {}; + BSONElement newHidden = {}; + BSONElement oldHidden = {}; + // TTL Index + if (!cmrOld.indexExpireAfterSeconds.eoo()) { + newExpireSecs = cmrOld.indexExpireAfterSeconds; + oldExpireSecs = cmrOld.idx->infoObj().getField("expireAfterSeconds"); + if (SimpleBSONElementComparator::kInstance.evaluate(oldExpireSecs != + newExpireSecs)) { + // Change the value of "expireAfterSeconds" on disk. + DurableCatalog::get(opCtx)->updateTTLSetting(opCtx, + coll->getCatalogId(), + cmrOld.idx->indexName(), + newExpireSecs.safeNumberLong()); + } + } + + // User wants to hide or unhide index. + if (!cmrOld.indexHidden.eoo()) { + newHidden = cmrOld.indexHidden; + oldHidden = cmrOld.idx->infoObj().getField("hidden"); + // Make sure when we set 'hidden' to false, we can remove the hidden field from + // catalog. + if (SimpleBSONElementComparator::kInstance.evaluate(oldHidden != newHidden)) { + DurableCatalog::get(opCtx)->updateHiddenSetting(opCtx, + coll->getCatalogId(), + cmrOld.idx->indexName(), + newHidden.booleanSafe()); } } - // Save previous TTL index expiration. - ttlInfo = TTLCollModInfo{Seconds(newExpireSecs.safeNumberLong()), - Seconds(oldExpireSecs.safeNumberLong()), - cmrNew.idx->indexName()}; + indexCollModInfo = IndexCollModInfo{ + cmrOld.indexExpireAfterSeconds.eoo() ? boost::optional<Seconds>() + : Seconds(newExpireSecs.safeNumberLong()), + cmrOld.indexExpireAfterSeconds.eoo() ? boost::optional<Seconds>() + : Seconds(oldExpireSecs.safeNumberLong()), + cmrOld.indexHidden.eoo() ? boost::optional<bool>() : newHidden.booleanSafe(), + cmrOld.indexHidden.eoo() ? boost::optional<bool>() : oldHidden.booleanSafe(), + cmrNew.idx->indexName()}; + + // Notify the index catalog that the definition of this index changed. This will + // invalidate the idx pointer in cmrOld. On rollback of this WUOW, the idx pointer + // in cmrNew will be invalidated and the idx pointer in cmrOld will be valid again. + cmrNew.idx = coll->getIndexCatalog()->refreshEntry(opCtx, cmrOld.idx); + opCtx->recoveryUnit()->registerChange(std::make_unique<CollModResultChange>( + oldExpireSecs, newExpireSecs, oldHidden, newHidden, result)); + + if (MONGO_unlikely(assertAfterIndexUpdate.shouldFail())) { + LOGV2(20307, "collMod - assertAfterIndexUpdate fail point enabled."); + uasserted(50970, "trigger rollback after the index update"); + } } // The Validator, ValidationAction and ValidationLevel are already parsed and must be OK. @@ -412,8 +471,10 @@ Status _collModInternal(OperationContext* opCtx, // Only observe non-view collMods, as view operations are observed as operations on the // system.views collection. + auto* const opObserver = opCtx->getServiceContext()->getOpObserver(); - opObserver->onCollMod(opCtx, nss, coll->uuid(), oplogEntryObj, oldCollOptions, ttlInfo); + opObserver->onCollMod( + opCtx, nss, coll->uuid(), oplogEntryObj, oldCollOptions, indexCollModInfo); wunit.commit(); return Status::OK(); diff --git a/src/mongo/db/catalog/index_key_validate.cpp b/src/mongo/db/catalog/index_key_validate.cpp index ffe98157d91..3fc9b93a388 100644 --- a/src/mongo/db/catalog/index_key_validate.cpp +++ b/src/mongo/db/catalog/index_key_validate.cpp @@ -83,6 +83,7 @@ static std::set<StringData> allowedFieldNames = { IndexDescriptor::kDropDuplicatesFieldName, IndexDescriptor::kExpireAfterSecondsFieldName, IndexDescriptor::kGeoHaystackBucketSize, + IndexDescriptor::kHiddenFieldName, IndexDescriptor::kIndexNameFieldName, IndexDescriptor::kIndexVersionFieldName, IndexDescriptor::kKeyPatternFieldName, @@ -269,6 +270,12 @@ StatusWith<BSONObj> validateIndexSpec( boost::optional<IndexVersion> resolvedIndexVersion; + // Allow hidden index only if FCV is 4.4. + const auto isFeatureDisabled = + (!featureCompatibility.isVersionInitialized() || + featureCompatibility.getVersion() < + ServerGlobalParams::FeatureCompatibility::Version::kFullyUpgradedTo44); + for (auto&& indexSpecElem : indexSpec) { auto indexSpecElemFieldName = indexSpecElem.fieldNameStringData(); if (IndexDescriptor::kKeyPatternFieldName == indexSpecElemFieldName) { @@ -329,6 +336,18 @@ StatusWith<BSONObj> validateIndexSpec( } hasIndexNameField = true; + } else if (IndexDescriptor::kHiddenFieldName == indexSpecElemFieldName) { + if (isFeatureDisabled) { + return {ErrorCodes::Error(31449), + "Hidden indexes can only be created with FCV 4.4"}; + } + if (indexSpecElem.type() != BSONType::Bool) { + return {ErrorCodes::TypeMismatch, + str::stream() + << "The field '" << IndexDescriptor::kIndexNameFieldName + << "' must be a bool, but got " << typeName(indexSpecElem.type())}; + } + } else if (IndexDescriptor::kNamespaceFieldName == indexSpecElemFieldName) { hasNamespaceField = true; } else if (IndexDescriptor::kIndexVersionFieldName == indexSpecElemFieldName) { @@ -510,6 +529,10 @@ Status validateIdIndexSpec(const BSONObj& indexSpec) { << keyPatternElem.Obj()}; } + if (!indexSpec[IndexDescriptor::kHiddenFieldName].eoo()) { + return Status(ErrorCodes::BadValue, "can't hide _id index"); + } + return Status::OK(); } diff --git a/src/mongo/db/commands/create_indexes.cpp b/src/mongo/db/commands/create_indexes.cpp index da1dd52f056..9c2f4826026 100644 --- a/src/mongo/db/commands/create_indexes.cpp +++ b/src/mongo/db/commands/create_indexes.cpp @@ -167,6 +167,8 @@ StatusWith<std::vector<BSONObj>> parseAndValidateIndexSpecs( // entry with a '*' as an index name means "drop all indexes in this // collection". We disallow creation of such indexes to avoid this conflict. return {ErrorCodes::BadValue, "The index name '*' is not valid."}; + } else if (ns.isSystem() && !indexSpec[IndexDescriptor::kHiddenFieldName].eoo()) { + return {ErrorCodes::BadValue, "Can't hide index on system collection"}; } indexSpecs.push_back(std::move(indexSpec)); diff --git a/src/mongo/db/create_indexes.idl b/src/mongo/db/create_indexes.idl index 59c37157b13..20b589660ca 100644 --- a/src/mongo/db/create_indexes.idl +++ b/src/mongo/db/create_indexes.idl @@ -50,6 +50,9 @@ structs: unique: type: bool optional: true + hidden: + type: bool + optional: true partialFilterExpression: type: object optional: true diff --git a/src/mongo/db/free_mon/free_mon_op_observer.h b/src/mongo/db/free_mon/free_mon_op_observer.h index 21478a822e5..4408d1d76ac 100644 --- a/src/mongo/db/free_mon/free_mon_op_observer.h +++ b/src/mongo/db/free_mon/free_mon_op_observer.h @@ -113,7 +113,7 @@ public: OptionalCollectionUUID uuid, const BSONObj& collModCmd, const CollectionOptions& oldCollOptions, - boost::optional<TTLCollModInfo> ttlInfo) final {} + boost::optional<IndexCollModInfo> indexInfo) final {} void onDropDatabase(OperationContext* opCtx, const std::string& dbName) final {} diff --git a/src/mongo/db/index/index_descriptor.cpp b/src/mongo/db/index/index_descriptor.cpp index 248e87e8b91..17f765d87cf 100644 --- a/src/mongo/db/index/index_descriptor.cpp +++ b/src/mongo/db/index/index_descriptor.cpp @@ -61,7 +61,8 @@ void populateOptionsMap(std::map<StringData, BSONElement>& theMap, const BSONObj IndexDescriptor::kBackgroundFieldName || // this is a creation time option only fieldName == IndexDescriptor::kDropDuplicatesFieldName || // this is now ignored fieldName == IndexDescriptor::kSparseFieldName || // checked specially - fieldName == IndexDescriptor::kUniqueFieldName // check specially + fieldName == IndexDescriptor::kHiddenFieldName || // not considered for equivalence + fieldName == IndexDescriptor::kUniqueFieldName // check specially ) { continue; } @@ -93,6 +94,7 @@ constexpr StringData IndexDescriptor::kSparseFieldName; constexpr StringData IndexDescriptor::kStorageEngineFieldName; constexpr StringData IndexDescriptor::kTextVersionFieldName; constexpr StringData IndexDescriptor::kUniqueFieldName; +constexpr StringData IndexDescriptor::kHiddenFieldName; constexpr StringData IndexDescriptor::kWeightsFieldName; IndexDescriptor::IndexDescriptor(Collection* collection, @@ -109,6 +111,7 @@ IndexDescriptor::IndexDescriptor(Collection* collection, _isIdIndex(isIdIndexPattern(_keyPattern)), _sparse(infoObj[IndexDescriptor::kSparseFieldName].trueValue()), _unique(_isIdIndex || infoObj[kUniqueFieldName].trueValue()), + _hidden(infoObj[kHiddenFieldName].trueValue()), _partial(!infoObj[kPartialFilterExprFieldName].eoo()), _cachedEntry(nullptr) { BSONElement e = _infoObj[IndexDescriptor::kIndexVersionFieldName]; diff --git a/src/mongo/db/index/index_descriptor.h b/src/mongo/db/index/index_descriptor.h index b8d61b20564..4eab168a65a 100644 --- a/src/mongo/db/index/index_descriptor.h +++ b/src/mongo/db/index/index_descriptor.h @@ -82,6 +82,7 @@ public: static constexpr StringData kStorageEngineFieldName = "storageEngine"_sd; static constexpr StringData kTextVersionFieldName = "textIndexVersion"_sd; static constexpr StringData kUniqueFieldName = "unique"_sd; + static constexpr StringData kHiddenFieldName = "hidden"_sd; static constexpr StringData kWeightsFieldName = "weights"_sd; /** @@ -174,6 +175,10 @@ public: return _unique; } + bool hidden() const { + return _hidden; + } + // Is this index sparse? bool isSparse() const { return _sparse; @@ -255,6 +260,7 @@ private: bool _isIdIndex; bool _sparse; bool _unique; + bool _hidden; bool _partial; IndexVersion _version; BSONObj _collation; diff --git a/src/mongo/db/op_observer.h b/src/mongo/db/op_observer.h index d10620418cf..fe9c6ce9f48 100644 --- a/src/mongo/db/op_observer.h +++ b/src/mongo/db/op_observer.h @@ -61,9 +61,11 @@ struct OplogUpdateEntryArgs { : updateArgs(std::move(updateArgs)), nss(std::move(nss)), uuid(std::move(uuid)) {} }; -struct TTLCollModInfo { - Seconds expireAfterSeconds; - Seconds oldExpireAfterSeconds; +struct IndexCollModInfo { + boost::optional<Seconds> expireAfterSeconds; + boost::optional<Seconds> oldExpireAfterSeconds; + boost::optional<bool> hidden; + boost::optional<bool> oldHidden; std::string indexName; }; @@ -212,7 +214,7 @@ public: OptionalCollectionUUID uuid, const BSONObj& collModCmd, const CollectionOptions& oldCollOptions, - boost::optional<TTLCollModInfo> ttlInfo) = 0; + boost::optional<IndexCollModInfo> indexInfo) = 0; virtual void onDropDatabase(OperationContext* opCtx, const std::string& dbName) = 0; /** diff --git a/src/mongo/db/op_observer_impl.cpp b/src/mongo/db/op_observer_impl.cpp index 40802b19abf..dbaf1984947 100644 --- a/src/mongo/db/op_observer_impl.cpp +++ b/src/mongo/db/op_observer_impl.cpp @@ -659,7 +659,7 @@ void OpObserverImpl::onCollMod(OperationContext* opCtx, OptionalCollectionUUID uuid, const BSONObj& collModCmd, const CollectionOptions& oldCollOptions, - boost::optional<TTLCollModInfo> ttlInfo) { + boost::optional<IndexCollModInfo> indexInfo) { if (!nss.isSystemDotProfile()) { // do not replicate system.profile modifications @@ -667,16 +667,23 @@ void OpObserverImpl::onCollMod(OperationContext* opCtx, // Create the 'o2' field object. We save the old collection metadata and TTL expiration. BSONObjBuilder o2Builder; o2Builder.append("collectionOptions_old", oldCollOptions.toBSON()); - if (ttlInfo) { - auto oldExpireAfterSeconds = durationCount<Seconds>(ttlInfo->oldExpireAfterSeconds); - o2Builder.append("expireAfterSeconds_old", oldExpireAfterSeconds); + if (indexInfo) { + if (indexInfo->oldExpireAfterSeconds) { + auto oldExpireAfterSeconds = + durationCount<Seconds>(indexInfo->oldExpireAfterSeconds.get()); + o2Builder.append("expireAfterSeconds_old", oldExpireAfterSeconds); + } + if (indexInfo->oldHidden) { + auto oldHidden = indexInfo->oldHidden.get(); + o2Builder.append("hidden_old", oldHidden); + } } MutableOplogEntry oplogEntry; oplogEntry.setOpType(repl::OpTypeEnum::kCommand); oplogEntry.setNss(nss.getCommandNS()); oplogEntry.setUuid(uuid); - oplogEntry.setObject(makeCollModCmdObj(collModCmd, oldCollOptions, ttlInfo)); + oplogEntry.setObject(makeCollModCmdObj(collModCmd, oldCollOptions, indexInfo)); oplogEntry.setObject2(o2Builder.done()); logOperation(opCtx, &oplogEntry); } diff --git a/src/mongo/db/op_observer_impl.h b/src/mongo/db/op_observer_impl.h index 51d6ebd6c99..e61f14e6e2c 100644 --- a/src/mongo/db/op_observer_impl.h +++ b/src/mongo/db/op_observer_impl.h @@ -102,7 +102,7 @@ public: OptionalCollectionUUID uuid, const BSONObj& collModCmd, const CollectionOptions& oldCollOptions, - boost::optional<TTLCollModInfo> ttlInfo) final; + boost::optional<IndexCollModInfo> indexInfo) final; void onDropDatabase(OperationContext* opCtx, const std::string& dbName) final; repl::OpTime onDropCollection(OperationContext* opCtx, const NamespaceString& collectionName, diff --git a/src/mongo/db/op_observer_impl_test.cpp b/src/mongo/db/op_observer_impl_test.cpp index 56949f04d88..8e4b093fafa 100644 --- a/src/mongo/db/op_observer_impl_test.cpp +++ b/src/mongo/db/op_observer_impl_test.cpp @@ -273,16 +273,16 @@ TEST_F(OpObserverTest, CollModWithCollectionOptionsAndTTLInfo) { oldCollOpts.validationLevel = "strict"; oldCollOpts.validationAction = "error"; - TTLCollModInfo ttlInfo; - ttlInfo.expireAfterSeconds = Seconds(10); - ttlInfo.oldExpireAfterSeconds = Seconds(5); - ttlInfo.indexName = "name_of_index"; + IndexCollModInfo indexInfo; + indexInfo.expireAfterSeconds = Seconds(10); + indexInfo.oldExpireAfterSeconds = Seconds(5); + indexInfo.indexName = "name_of_index"; // Write to the oplog. { AutoGetCollection autoColl(opCtx.get(), nss, MODE_X); WriteUnitOfWork wunit(opCtx.get()); - opObserver.onCollMod(opCtx.get(), nss, uuid, collModCmd, oldCollOpts, ttlInfo); + opObserver.onCollMod(opCtx.get(), nss, uuid, collModCmd, oldCollOpts, indexInfo); wunit.commit(); } @@ -290,14 +290,14 @@ TEST_F(OpObserverTest, CollModWithCollectionOptionsAndTTLInfo) { // Ensure that collMod fields were properly added to the oplog entry. auto o = oplogEntry.getObjectField("o"); - auto oExpected = - BSON("collMod" << nss.coll() << "validationLevel" - << "off" - << "validationAction" - << "warn" - << "index" - << BSON("name" << ttlInfo.indexName << "expireAfterSeconds" - << durationCount<Seconds>(ttlInfo.expireAfterSeconds))); + auto oExpected = BSON( + "collMod" << nss.coll() << "validationLevel" + << "off" + << "validationAction" + << "warn" + << "index" + << BSON("name" << indexInfo.indexName << "expireAfterSeconds" + << durationCount<Seconds>(indexInfo.expireAfterSeconds.get()))); ASSERT_BSONOBJ_EQ(oExpected, o); // Ensure that the old collection metadata was saved. @@ -306,7 +306,8 @@ TEST_F(OpObserverTest, CollModWithCollectionOptionsAndTTLInfo) { BSON("collectionOptions_old" << BSON("validationLevel" << oldCollOpts.validationLevel << "validationAction" << oldCollOpts.validationAction) - << "expireAfterSeconds_old" << durationCount<Seconds>(ttlInfo.oldExpireAfterSeconds)); + << "expireAfterSeconds_old" + << durationCount<Seconds>(indexInfo.oldExpireAfterSeconds.get())); ASSERT_BSONOBJ_EQ(o2Expected, o2); } diff --git a/src/mongo/db/op_observer_noop.h b/src/mongo/db/op_observer_noop.h index 5c5816e7c0b..3684625cc27 100644 --- a/src/mongo/db/op_observer_noop.h +++ b/src/mongo/db/op_observer_noop.h @@ -98,7 +98,7 @@ public: OptionalCollectionUUID uuid, const BSONObj& collModCmd, const CollectionOptions& oldCollOptions, - boost::optional<TTLCollModInfo> ttlInfo) override {} + boost::optional<IndexCollModInfo> indexInfo) override {} void onDropDatabase(OperationContext* opCtx, const std::string& dbName) override {} repl::OpTime onDropCollection(OperationContext* opCtx, const NamespaceString& collectionName, diff --git a/src/mongo/db/op_observer_registry.h b/src/mongo/db/op_observer_registry.h index 2470baa74cc..9bc29820269 100644 --- a/src/mongo/db/op_observer_registry.h +++ b/src/mongo/db/op_observer_registry.h @@ -175,10 +175,10 @@ public: OptionalCollectionUUID uuid, const BSONObj& collModCmd, const CollectionOptions& oldCollOptions, - boost::optional<TTLCollModInfo> ttlInfo) override { + boost::optional<IndexCollModInfo> indexInfo) override { ReservedTimes times{opCtx}; for (auto& o : _observers) - o->onCollMod(opCtx, nss, uuid, collModCmd, oldCollOptions, ttlInfo); + o->onCollMod(opCtx, nss, uuid, collModCmd, oldCollOptions, indexInfo); } void onDropDatabase(OperationContext* const opCtx, const std::string& dbName) override { diff --git a/src/mongo/db/op_observer_util.cpp b/src/mongo/db/op_observer_util.cpp index 9c03a37a3db..7d69555618f 100644 --- a/src/mongo/db/op_observer_util.cpp +++ b/src/mongo/db/op_observer_util.cpp @@ -42,21 +42,24 @@ namespace mongo { */ BSONObj makeCollModCmdObj(const BSONObj& collModCmd, const CollectionOptions& oldCollOptions, - boost::optional<TTLCollModInfo> ttlInfo) { + boost::optional<IndexCollModInfo> indexInfo) { BSONObjBuilder cmdObjBuilder; - std::string ttlIndexFieldName = "index"; + std::string indexFieldName = "index"; // Add all fields from the original collMod command. for (auto elem : collModCmd) { // We normalize all TTL collMod oplog entry objects to use the index name, even if the // command used an index key pattern. - if (elem.fieldNameStringData() == ttlIndexFieldName && ttlInfo) { - BSONObjBuilder ttlIndexObjBuilder; - ttlIndexObjBuilder.append("name", ttlInfo->indexName); - ttlIndexObjBuilder.append("expireAfterSeconds", - durationCount<Seconds>(ttlInfo->expireAfterSeconds)); + if (elem.fieldNameStringData() == indexFieldName && indexInfo) { + BSONObjBuilder indexObjBuilder; + indexObjBuilder.append("name", indexInfo->indexName); + if (indexInfo->expireAfterSeconds) + indexObjBuilder.append("expireAfterSeconds", + durationCount<Seconds>(indexInfo->expireAfterSeconds.get())); + if (indexInfo->hidden) + indexObjBuilder.append("hidden", indexInfo->hidden.get()); - cmdObjBuilder.append(ttlIndexFieldName, ttlIndexObjBuilder.obj()); + cmdObjBuilder.append(indexFieldName, indexObjBuilder.obj()); } else { cmdObjBuilder.append(elem); } diff --git a/src/mongo/db/op_observer_util.h b/src/mongo/db/op_observer_util.h index 800a1486d4d..d603d3fb177 100644 --- a/src/mongo/db/op_observer_util.h +++ b/src/mongo/db/op_observer_util.h @@ -36,5 +36,5 @@ namespace mongo { BSONObj makeCollModCmdObj(const BSONObj& collModCmd, const CollectionOptions& oldCollOptions, - boost::optional<TTLCollModInfo> ttlInfo); + boost::optional<IndexCollModInfo> indexInfo); } // namespace mongo diff --git a/src/mongo/db/query/get_executor.cpp b/src/mongo/db/query/get_executor.cpp index ce5dafd80f6..0d7eb0f6978 100644 --- a/src/mongo/db/query/get_executor.cpp +++ b/src/mongo/db/query/get_executor.cpp @@ -260,6 +260,10 @@ void fillOutPlannerParams(OperationContext* opCtx, collection->getIndexCatalog()->getIndexIterator(opCtx, false); while (ii->more()) { const IndexCatalogEntry* ice = ii->next(); + + // Skip the addition of hidden indexes to prevent use in query planning. + if (ice->descriptor()->hidden()) + continue; plannerParams->indices.push_back( indexEntryFromIndexCatalogEntry(opCtx, *ice, canonicalQuery)); } @@ -1466,6 +1470,10 @@ QueryPlannerParams fillOutPlannerParamsForDistinct(OperationContext* opCtx, while (ii->more()) { const IndexCatalogEntry* ice = ii->next(); const IndexDescriptor* desc = ice->descriptor(); + + // Skip the addition of hidden indexes to prevent use in query planning. + if (desc->hidden()) + continue; if (desc->keyPattern().hasField(parsedDistinct.getKey())) { if (!mayUnwindArrays && isAnyComponentOfPathMultikey(desc->keyPattern(), diff --git a/src/mongo/db/s/config_server_op_observer.h b/src/mongo/db/s/config_server_op_observer.h index d274dfef21a..ca17153d814 100644 --- a/src/mongo/db/s/config_server_op_observer.h +++ b/src/mongo/db/s/config_server_op_observer.h @@ -114,7 +114,7 @@ public: OptionalCollectionUUID uuid, const BSONObj& collModCmd, const CollectionOptions& oldCollOptions, - boost::optional<TTLCollModInfo> ttlInfo) override {} + boost::optional<IndexCollModInfo> indexInfo) override {} void onDropDatabase(OperationContext* opCtx, const std::string& dbName) override {} diff --git a/src/mongo/db/s/shard_server_op_observer.cpp b/src/mongo/db/s/shard_server_op_observer.cpp index 6832dd1bc0e..7ce56dee550 100644 --- a/src/mongo/db/s/shard_server_op_observer.cpp +++ b/src/mongo/db/s/shard_server_op_observer.cpp @@ -517,7 +517,7 @@ void ShardServerOpObserver::onCollMod(OperationContext* opCtx, OptionalCollectionUUID uuid, const BSONObj& collModCmd, const CollectionOptions& oldCollOptions, - boost::optional<TTLCollModInfo> ttlInfo) { + boost::optional<IndexCollModInfo> indexInfo) { abortOngoingMigrationIfNeeded(opCtx, nss); }; diff --git a/src/mongo/db/s/shard_server_op_observer.h b/src/mongo/db/s/shard_server_op_observer.h index 2b16ac88150..b84dcd989a6 100644 --- a/src/mongo/db/s/shard_server_op_observer.h +++ b/src/mongo/db/s/shard_server_op_observer.h @@ -114,7 +114,7 @@ public: OptionalCollectionUUID uuid, const BSONObj& collModCmd, const CollectionOptions& oldCollOptions, - boost::optional<TTLCollModInfo> ttlInfo) override; + boost::optional<IndexCollModInfo> indexInfo) override; void onDropDatabase(OperationContext* opCtx, const std::string& dbName) override {} diff --git a/src/mongo/db/storage/bson_collection_catalog_entry.cpp b/src/mongo/db/storage/bson_collection_catalog_entry.cpp index 1e016ff8f40..52b7f251bd7 100644 --- a/src/mongo/db/storage/bson_collection_catalog_entry.cpp +++ b/src/mongo/db/storage/bson_collection_catalog_entry.cpp @@ -120,6 +120,26 @@ void BSONCollectionCatalogEntry::IndexMetaData::updateTTLSetting(long long newEx spec = b.obj(); } + +void BSONCollectionCatalogEntry::IndexMetaData::updateHiddenSetting(bool hidden) { + // If hidden == false, we remove this field from catalog rather than add a field with false. + // or else, the old binary can't startup due to the unknown field. + BSONObjBuilder b; + for (BSONObjIterator bi(spec); bi.more();) { + BSONElement e = bi.next(); + if (e.fieldNameStringData() == "hidden") { + continue; + } + b.append(e); + } + + if (hidden) { + b.append("hidden", hidden); + } + spec = b.obj(); +} + + // -------------------------- int BSONCollectionCatalogEntry::MetaData::findIndexOffset(StringData name) const { diff --git a/src/mongo/db/storage/bson_collection_catalog_entry.h b/src/mongo/db/storage/bson_collection_catalog_entry.h index 9bb34165636..bf4598b22a4 100644 --- a/src/mongo/db/storage/bson_collection_catalog_entry.h +++ b/src/mongo/db/storage/bson_collection_catalog_entry.h @@ -64,6 +64,8 @@ public: void updateTTLSetting(long long newExpireSeconds); + void updateHiddenSetting(bool hidden); + std::string name() const { return spec["name"].String(); } diff --git a/src/mongo/db/storage/durable_catalog.h b/src/mongo/db/storage/durable_catalog.h index 29e1fa394cb..17860449ba6 100644 --- a/src/mongo/db/storage/durable_catalog.h +++ b/src/mongo/db/storage/durable_catalog.h @@ -151,6 +151,15 @@ public: StringData idxName, long long newExpireSeconds) = 0; + /* + * Hide or unhide the given index. A hidden index will not be considered for use by the + * query planner. + */ + virtual void updateHiddenSetting(OperationContext* opCtx, + RecordId catalogId, + StringData idxName, + bool hidden) = 0; + /** Compares the UUID argument to the UUID obtained from the metadata. Returns true if they are * equal, false otherwise. */ diff --git a/src/mongo/db/storage/durable_catalog_impl.cpp b/src/mongo/db/storage/durable_catalog_impl.cpp index 349cd7ced5a..bb2032e3315 100644 --- a/src/mongo/db/storage/durable_catalog_impl.cpp +++ b/src/mongo/db/storage/durable_catalog_impl.cpp @@ -931,6 +931,19 @@ void DurableCatalogImpl::updateTTLSetting(OperationContext* opCtx, putMetaData(opCtx, catalogId, md); } +void DurableCatalogImpl::updateHiddenSetting(OperationContext* opCtx, + RecordId catalogId, + StringData idxName, + bool hidden) { + + BSONCollectionCatalogEntry::MetaData md = getMetaData(opCtx, catalogId); + int offset = md.findIndexOffset(idxName); + invariant(offset >= 0); + md.indexes[offset].updateHiddenSetting(hidden); + putMetaData(opCtx, catalogId, md); +} + + bool DurableCatalogImpl::isEqualToMetadataUUID(OperationContext* opCtx, RecordId catalogId, OptionalCollectionUUID uuid) { diff --git a/src/mongo/db/storage/durable_catalog_impl.h b/src/mongo/db/storage/durable_catalog_impl.h index f11e5b445f9..5c674af4824 100644 --- a/src/mongo/db/storage/durable_catalog_impl.h +++ b/src/mongo/db/storage/durable_catalog_impl.h @@ -124,6 +124,11 @@ public: StringData idxName, long long newExpireSeconds); + void updateHiddenSetting(OperationContext* opCtx, + RecordId catalogId, + StringData idxName, + bool hidden); + bool isEqualToMetadataUUID(OperationContext* opCtx, RecordId catalogId, OptionalCollectionUUID uuid); diff --git a/src/mongo/shell/collection.js b/src/mongo/shell/collection.js index 1333c8dfcd9..274eeaafb6d 100644 --- a/src/mongo/shell/collection.js +++ b/src/mongo/shell/collection.js @@ -60,6 +60,11 @@ DBCollection.prototype.help = function() { print("\tdb." + shortName + ".drop() drop the collection"); print("\tdb." + shortName + ".dropIndex(index) - e.g. db." + shortName + ".dropIndex( \"indexName\" ) or db." + shortName + ".dropIndex( { \"indexKey\" : 1 } )"); + print("\tdb." + shortName + ".hideIndex(index) - e.g. db." + shortName + + ".hideIndex( \"indexName\" ) or db." + shortName + ".hideIndex( { \"indexKey\" : 1 } )"); + print("\tdb." + shortName + ".unhideIndex(index) - e.g. db." + shortName + + ".unhideIndex( \"indexName\" ) or db." + shortName + + ".unhideIndex( { \"indexKey\" : 1 } )"); print("\tdb." + shortName + ".dropIndexes()"); print("\tdb." + shortName + ".ensureIndex(keypattern[,options]) - DEPRECATED, use createIndex() instead"); @@ -874,6 +879,35 @@ DBCollection.prototype.dropIndex = function(index) { return res; }; +/** + * Hide an index from the query planner. + */ +DBCollection.prototype._hiddenIndex = function(index, hidden) { + assert(index, "please specify index to hide"); + + // Need an extra check for array because 'Array' is an 'object', but not every 'object' is an + // 'Array'. + var indexField = {}; + if (typeof index == "string") { + indexField = {name: index, hidden: hidden}; + } else if (typeof index == "object") { + indexField = {keyPattern: index, hidden: hidden}; + } else { + throw new Error("Index must be either the index name or the index specification document"); + } + var cmd = {"collMod": this._shortName, index: indexField}; + var res = this._db.runCommand(cmd); + return res; +}; + +DBCollection.prototype.hideIndex = function(index) { + return this._hiddenIndex(index, true); +}; + +DBCollection.prototype.unhideIndex = function(index) { + return this._hiddenIndex(index, false); +}; + DBCollection.prototype.getCollection = function(subName) { return this._db.getCollection(this._shortName + "." + subName); }; |