diff options
Diffstat (limited to 'src')
22 files changed, 394 insertions, 16 deletions
diff --git a/src/mongo/db/catalog/coll_mod.cpp b/src/mongo/db/catalog/coll_mod.cpp index 9cdb68548cf..ac8ad7e3d94 100644 --- a/src/mongo/db/catalog/coll_mod.cpp +++ b/src/mongo/db/catalog/coll_mod.cpp @@ -300,6 +300,7 @@ Status _collModInternal(OperationContext* opCtx, const NamespaceString& nss, const BSONObj& cmdObj, BSONObjBuilder* result, + bool upgradeUniqueIndexes, OptionalCollectionUUID uuid) { StringData dbName = nss.db(); AutoGetDb autoDb(opCtx, dbName, MODE_X); @@ -424,6 +425,29 @@ Status _collModInternal(OperationContext* opCtx, if (!cmr.noPadding.eoo()) setCollectionOptionFlag(opCtx, coll, cmr.noPadding, result); + // Upgrade unique indexes + if (upgradeUniqueIndexes) { + // A cmdObj with an empty collMod, i.e. nFields = 1, implies that it is a Unique Index + // upgrade collMod. + invariant(cmdObj.nFields() == 1); + std::vector<std::string> indexNames; + coll->getCatalogEntry()->getAllUniqueIndexes(opCtx, &indexNames); + + for (size_t i = 0; i < indexNames.size(); i++) { + const IndexDescriptor* desc = + coll->getIndexCatalog()->findIndexByName(opCtx, indexNames[i]); + invariant(desc); + + // Update index metadata in storage engine. + coll->getCatalogEntry()->updateIndexMetadata(opCtx, desc); + + // Refresh the in-memory instance of the index. + desc = coll->getIndexCatalog()->refreshEntry(opCtx, desc); + + opCtx->recoveryUnit()->onRollback( + [opCtx, desc, coll]() { coll->getIndexCatalog()->refreshEntry(opCtx, desc); }); + } + } // Add collection UUID if it is missing. This returns an error if a collection already has a // different UUID. As we don't assign UUIDs to system.indexes (SERVER-29926), don't implicitly // upgrade them on collMod either. @@ -511,7 +535,36 @@ Status collMod(OperationContext* opCtx, const NamespaceString& nss, const BSONObj& cmdObj, BSONObjBuilder* result) { - return _collModInternal(opCtx, nss, cmdObj, result, /*UUID*/ boost::none); + return _collModInternal(opCtx, + nss, + cmdObj, + result, + /*upgradeUniqueIndexes*/ false, + /*UUID*/ boost::none); +} + +Status collModWithUpgrade(OperationContext* opCtx, + const NamespaceString& nss, + const BSONObj& cmdObj) { + // A cmdObj with an empty collMod, i.e. nFields = 1, implies that it is a Unique Index + // upgrade collMod. + bool upgradeUniqueIndex = createTimestampSafeUniqueIndex && (cmdObj.nFields() == 1); + + // Update all non-replicated unique indexes on upgrade i.e. setFCV=4.2. + if (upgradeUniqueIndex && nss == NamespaceString::kServerConfigurationNamespace) { + auto schemaStatus = updateNonReplicatedUniqueIndexes(opCtx); + if (!schemaStatus.isOK()) { + return schemaStatus; + } + } + + BSONObjBuilder resultWeDontCareAbout; + return _collModInternal(opCtx, + nss, + cmdObj, + &resultWeDontCareAbout, + upgradeUniqueIndex, + /*UUID*/ boost::none); } Status collModForUUIDUpgrade(OperationContext* opCtx, @@ -519,7 +572,12 @@ Status collModForUUIDUpgrade(OperationContext* opCtx, const BSONObj& cmdObj, CollectionUUID uuid) { BSONObjBuilder resultWeDontCareAbout; - return _collModInternal(opCtx, nss, cmdObj, &resultWeDontCareAbout, uuid); + return _collModInternal(opCtx, + nss, + cmdObj, + &resultWeDontCareAbout, + /* upgradeUniqueIndexes */ false, + uuid); } void addCollectionUUIDs(OperationContext* opCtx) { @@ -594,4 +652,131 @@ void addCollectionUUIDs(OperationContext* opCtx) { repl::ReplicationCoordinator::get(opCtx)->awaitReplication(opCtx, awaitOpTime, writeConcern); } +Status _updateNonReplicatedIndexPerCollection(OperationContext* opCtx, Collection* coll) { + BSONObjBuilder collModObjBuilder; + collModObjBuilder.append("collMod", coll->ns().coll()); + BSONObj collModObj = collModObjBuilder.done(); + + BSONObjBuilder resultWeDontCareAbout; + auto collModStatus = _collModInternal(opCtx, + coll->ns(), + collModObj, + &resultWeDontCareAbout, + /*upgradeUniqueIndexes*/ true, + /*UUID*/ boost::none); + return collModStatus; +} + +Status _updateNonReplicatedUniqueIndexesPerDatabase(OperationContext* opCtx, + const std::string& dbName) { + AutoGetDb autoDb(opCtx, dbName, MODE_X); + Database* const db = autoDb.getDb(); + + // Iterate through all collections if we're in the "local" database. + if (dbName == "local") { + for (auto collectionIt = db->begin(); collectionIt != db->end(); ++collectionIt) { + Collection* coll = *collectionIt; + + auto collModStatus = _updateNonReplicatedIndexPerCollection(opCtx, coll); + if (!collModStatus.isOK()) + return collModStatus; + } + } else { + // If we're not in the "local" database, the only non-replicated collection + // could be system.profile. + Collection* coll = + db ? db->getCollection(opCtx, NamespaceString(dbName, "system.profile")) : nullptr; + if (!coll) + return Status::OK(); + + auto collModStatus = _updateNonReplicatedIndexPerCollection(opCtx, coll); + if (!collModStatus.isOK()) + return collModStatus; + } + return Status::OK(); +} + +void _updateUniqueIndexesForDatabase(OperationContext* opCtx, const std::string& dbname) { + // Iterate through all replicated collections of the database, for unique index update. + // Non-replicated unique indexes are updated via the upgrade of admin.system.version + // collection. + { + AutoGetDb autoDb(opCtx, dbname, MODE_X); + Database* const db = autoDb.getDb(); + // If the database no longer exists, nothing more to do. + if (!db) + return; + + for (auto collectionIt = db->begin(); collectionIt != db->end(); ++collectionIt) { + Collection* coll = *collectionIt; + NamespaceString collNSS = coll->ns(); + + // Skip non-replicated collection. + if (collNSS.coll() == "system.profile") + continue; + + BSONObjBuilder collModObjBuilder; + collModObjBuilder.append("collMod", collNSS.coll()); + BSONObj collModObj = collModObjBuilder.done(); + + uassertStatusOK(collModWithUpgrade(opCtx, collNSS, collModObj)); + } + } +} + +void updateUniqueIndexesOnUpgrade(OperationContext* opCtx) { + if (!createTimestampSafeUniqueIndex) + return; + + // Update all unique indexes except the _id index. + std::vector<std::string> dbNames; + StorageEngine* storageEngine = opCtx->getServiceContext()->getStorageEngine(); + { + Lock::GlobalLock lk(opCtx, MODE_IS); + storageEngine->listDatabases(&dbNames); + } + + for (auto it = dbNames.begin(); it != dbNames.end(); ++it) { + auto dbName = *it; + + // Non-replicated unique indexes are updated via the upgrade of admin.system.version + // collection. + if (dbName != "local") + _updateUniqueIndexesForDatabase(opCtx, dbName); + } + + const auto& clientInfo = repl::ReplClientInfo::forClient(opCtx->getClient()); + auto awaitOpTime = clientInfo.getLastOp(); + + log() << "Finished updating version of unique indexes for upgrade, waiting for all" + << " index updates to be committed at optime " << awaitOpTime; + + const WriteConcernOptions writeConcern(WriteConcernOptions::kMajority, + WriteConcernOptions::SyncMode::UNSET, + /*timeout*/ INT_MAX); + repl::ReplicationCoordinator::get(opCtx)->awaitReplication(opCtx, awaitOpTime, writeConcern); +} + +Status updateNonReplicatedUniqueIndexes(OperationContext* opCtx) { + if (!createTimestampSafeUniqueIndex) + return Status::OK(); + + // Update all unique indexes belonging to all non-replicated collections. + // (_id indexes are not updated). + std::vector<std::string> dbNames; + StorageEngine* storageEngine = opCtx->getServiceContext()->getStorageEngine(); + { + Lock::GlobalLock lk(opCtx, MODE_IS); + storageEngine->listDatabases(&dbNames); + } + for (auto it = dbNames.begin(); it != dbNames.end(); ++it) { + auto dbName = *it; + auto schemaStatus = _updateNonReplicatedUniqueIndexesPerDatabase(opCtx, dbName); + if (!schemaStatus.isOK()) { + return schemaStatus; + } + } + return Status::OK(); +} + } // namespace mongo diff --git a/src/mongo/db/catalog/coll_mod.h b/src/mongo/db/catalog/coll_mod.h index 3dceb206981..a0757e2c8de 100644 --- a/src/mongo/db/catalog/coll_mod.h +++ b/src/mongo/db/catalog/coll_mod.h @@ -59,4 +59,23 @@ Status collModForUUIDUpgrade(OperationContext* opCtx, const NamespaceString& ns, const BSONObj& cmdObj, CollectionUUID uuid); + +/** + * Applies the collMod operation and optionally updates formatVersion of unique indexes belonging + * to collection "nss". + */ +Status collModWithUpgrade(OperationContext* opCtx, + const NamespaceString& nss, + const BSONObj& cmdObj); + +/* + * Updates the unique indexes to timestamp safe unique index format on setFCV=4.2. It also updates + * non-replicated unique indexes indirectly by calling updateNonReplicatedUniqueIndexes(). + */ +void updateUniqueIndexesOnUpgrade(OperationContext* opCtx); + +/* + * Updates non-replicated unique indexes to timestamp safe unique index format. + */ +Status updateNonReplicatedUniqueIndexes(OperationContext* opCtx); } // namespace mongo diff --git a/src/mongo/db/catalog/collection_catalog_entry.h b/src/mongo/db/catalog/collection_catalog_entry.h index bd614debc7b..a55280a6724 100644 --- a/src/mongo/db/catalog/collection_catalog_entry.h +++ b/src/mongo/db/catalog/collection_catalog_entry.h @@ -68,6 +68,9 @@ public: virtual void getReadyIndexes(OperationContext* opCtx, std::vector<std::string>* names) const = 0; + virtual void getAllUniqueIndexes(OperationContext* opCtx, + std::vector<std::string>* names) const {} + virtual BSONObj getIndexSpec(OperationContext* opCtx, StringData idxName) const = 0; /** @@ -124,6 +127,8 @@ public: StringData idxName, long long newExpireSeconds) = 0; + virtual void updateIndexMetadata(OperationContext* opCtx, const IndexDescriptor* desc) {} + /** * Sets the flags field of CollectionOptions to newValue. * Subsequent calls to getCollectionOptions should have flags==newValue and flagsSet==true. diff --git a/src/mongo/db/catalog/collection_options.cpp b/src/mongo/db/catalog/collection_options.cpp index 67da1098dd0..91f6ee730a8 100644 --- a/src/mongo/db/catalog/collection_options.cpp +++ b/src/mongo/db/catalog/collection_options.cpp @@ -42,6 +42,13 @@ namespace mongo { +// TODO(SERVER-34489) Remove when upgrade/downgrade is ready. +bool createTimestampSafeUniqueIndex = false; +ExportedServerParameter<bool, ServerParameterType::kStartupOnly> + createTimestampSafeUniqueIndexParameter(ServerParameterSet::getGlobal(), + "createTimestampSafeUniqueIndex", + &createTimestampSafeUniqueIndex); + // static bool CollectionOptions::validMaxCappedDocs(long long* max) { if (*max <= 0 || *max == std::numeric_limits<long long>::max()) { diff --git a/src/mongo/db/catalog/collection_options.h b/src/mongo/db/catalog/collection_options.h index 27a42d5f27e..5d9171ffc79 100644 --- a/src/mongo/db/catalog/collection_options.h +++ b/src/mongo/db/catalog/collection_options.h @@ -38,6 +38,9 @@ namespace mongo { +// TODO(SERVER-34489) Remove when upgrade/downgrade is ready. +extern bool createTimestampSafeUniqueIndex; + class CollatorFactoryInterface; /** diff --git a/src/mongo/db/commands/set_feature_compatibility_version_command.cpp b/src/mongo/db/commands/set_feature_compatibility_version_command.cpp index 8cc02a107b2..892f8f6957a 100644 --- a/src/mongo/db/commands/set_feature_compatibility_version_command.cpp +++ b/src/mongo/db/commands/set_feature_compatibility_version_command.cpp @@ -170,6 +170,8 @@ public: Lock::GlobalLock lk(opCtx, MODE_S); } + updateUniqueIndexesOnUpgrade(opCtx); + // Upgrade shards before config finishes its upgrade. if (serverGlobalParams.clusterRole == ClusterRole::ConfigServer) { auto allDbs = uassertStatusOK(Grid::get(opCtx)->catalogClient()->getAllDBs( diff --git a/src/mongo/db/repl/initial_syncer.cpp b/src/mongo/db/repl/initial_syncer.cpp index 0428e858333..33e5e729a13 100644 --- a/src/mongo/db/repl/initial_syncer.cpp +++ b/src/mongo/db/repl/initial_syncer.cpp @@ -1109,6 +1109,20 @@ void InitialSyncer::_rollbackCheckerCheckForRollbackCallback( return; } + // Update all unique indexes belonging to non-replicated collections on secondaries. See comment + // in ReplicationCoordinatorExternalStateImpl::initializeReplSetStorage() for the explanation of + // why we do this. + // TODO: SERVER-34489 should add a check for latest FCV before making the upgrade call when + // upgrade downgrade is ready. + if (createTimestampSafeUniqueIndex) { + auto opCtx = makeOpCtx(); + auto updateStatus = _storage->upgradeNonReplicatedUniqueIndexes(opCtx.get()); + if (!updateStatus.isOK()) { + onCompletionGuard->setResultAndCancelRemainingWork_inlock(lock, updateStatus); + return; + } + } + // Success! onCompletionGuard->setResultAndCancelRemainingWork_inlock(lock, _lastApplied); } diff --git a/src/mongo/db/repl/initial_syncer_test.cpp b/src/mongo/db/repl/initial_syncer_test.cpp index 97e27678f80..14ffb0c5455 100644 --- a/src/mongo/db/repl/initial_syncer_test.cpp +++ b/src/mongo/db/repl/initial_syncer_test.cpp @@ -236,6 +236,8 @@ protected: bool droppedUserDBs = false; std::vector<std::string> droppedCollections; int documentsInsertedCount = 0; + bool uniqueIndexUpdated = false; + bool upgradeNonReplicatedUniqueIndexesShouldFail = false; }; stdx::mutex _storageInterfaceWorkDoneMutex; // protects _storageInterfaceWorkDone. @@ -248,6 +250,8 @@ protected: const NamespaceString& nss) { LockGuard lock(_storageInterfaceWorkDoneMutex); _storageInterfaceWorkDone.createOplogCalled = true; + _storageInterfaceWorkDone.uniqueIndexUpdated = false; + _storageInterfaceWorkDone.upgradeNonReplicatedUniqueIndexesShouldFail = false; return Status::OK(); }; _storageInterface->truncateCollFn = [this](OperationContext* opCtx, @@ -300,6 +304,19 @@ protected: return StatusWith<std::unique_ptr<CollectionBulkLoader>>( std::unique_ptr<CollectionBulkLoader>(collInfo->loader)); }; + _storageInterface->upgradeNonReplicatedUniqueIndexesFn = [this](OperationContext* opCtx) { + LockGuard lock(_storageInterfaceWorkDoneMutex); + if (_storageInterfaceWorkDone.upgradeNonReplicatedUniqueIndexesShouldFail) { + // One of the status codes a failed upgradeNonReplicatedUniqueIndexes call + // can return is NamespaceNotFound. + return Status(ErrorCodes::NamespaceNotFound, + "upgradeNonReplicatedUniqueIndexes failed because the desired " + "ns was not found."); + } else { + _storageInterfaceWorkDone.uniqueIndexUpdated = true; + return Status::OK(); + } + }; _dbWorkThreadPool = stdx::make_unique<ThreadPool>(ThreadPool::Options()); _dbWorkThreadPool->startup(); @@ -423,8 +440,8 @@ protected: void runInitialSyncWithBadFCVResponse(std::vector<BSONObj> docs, ErrorCodes::Error expectedError); - void doSuccessfulInitialSyncWithOneBatch(); - OplogEntry doInitialSyncWithOneBatch(); + void doSuccessfulInitialSyncWithOneBatch(bool shouldSetFCV); + OplogEntry doInitialSyncWithOneBatch(bool shouldSetFCV); std::unique_ptr<TaskExecutorMock> _executorProxy; @@ -3382,7 +3399,7 @@ TEST_F(InitialSyncerTest, InitialSyncerCancelsGetNextApplierBatchCallbackOnOplog ASSERT_EQUALS(ErrorCodes::OperationFailed, _lastApplied); } -OplogEntry InitialSyncerTest::doInitialSyncWithOneBatch() { +OplogEntry InitialSyncerTest::doInitialSyncWithOneBatch(bool shouldSetFCV) { auto initialSyncer = &getInitialSyncer(); auto opCtx = makeOpCtx(); @@ -3429,7 +3446,12 @@ OplogEntry InitialSyncerTest::doInitialSyncWithOneBatch() { assertRemoteCommandNameEquals("getMore", request); net->blackHole(noi); - // Last rollback ID. + // Last rollback ID check. Before this check, set fCV to 4.2 if required by the test. + // TODO(SERVER-34489) Update below statement to setFCV=4.2 when upgrade/downgrade is ready. + if (shouldSetFCV) { + createTimestampSafeUniqueIndex = true; + } + request = net->scheduleSuccessfulResponse(makeRollbackCheckerResponse(baseRollbackId)); assertRemoteCommandNameEquals("replSetGetRBID", request); net->runReadyNetworkOperations(); @@ -3445,8 +3467,10 @@ OplogEntry InitialSyncerTest::doInitialSyncWithOneBatch() { return lastOp; } -void InitialSyncerTest::doSuccessfulInitialSyncWithOneBatch() { - auto lastOp = doInitialSyncWithOneBatch(); +void InitialSyncerTest::doSuccessfulInitialSyncWithOneBatch(bool shouldSetFCV) { + auto lastOp = doInitialSyncWithOneBatch(shouldSetFCV); + // TODO(SERVER-34489) Replace this by fCV reset when upgrade/downgrade is ready. + createTimestampSafeUniqueIndex = false; ASSERT_EQUALS(lastOp.getOpTime(), unittest::assertGet(_lastApplied).opTime); ASSERT_EQUALS(lastOp.getHash(), unittest::assertGet(_lastApplied).value); @@ -3455,7 +3479,14 @@ void InitialSyncerTest::doSuccessfulInitialSyncWithOneBatch() { TEST_F(InitialSyncerTest, InitialSyncerReturnsLastAppliedOnReachingStopTimestampAfterApplyingOneBatch) { - doSuccessfulInitialSyncWithOneBatch(); + // Tell test to setFCV=4.2 before the last rollback ID check. + // _rollbackCheckerCheckForRollbackCallback() calls upgradeNonReplicatedUniqueIndexes + // only if fCV is 4.2. + doSuccessfulInitialSyncWithOneBatch(true); + + // Ensure that upgradeNonReplicatedUniqueIndexes is called. + LockGuard lock(_storageInterfaceWorkDoneMutex); + ASSERT_TRUE(_storageInterfaceWorkDone.uniqueIndexUpdated); } TEST_F(InitialSyncerTest, @@ -4054,4 +4085,27 @@ TEST_F(InitialSyncerTest, GetInitialSyncProgressOmitsClonerStatsIfClonerStatsExc initialSyncer->join(); } +TEST_F(InitialSyncerTest, InitialSyncerDoesNotCallUpgradeNonReplicatedUniqueIndexesOnFCV40) { + // In MongoDB 4.2, upgradeNonReplicatedUniqueIndexes will only be called if fCV is 4.2. + doSuccessfulInitialSyncWithOneBatch(false); + + // TODO(SERVER-34489) Ensure that upgradeNonReplicatedUniqueIndexes is not called if fCV + // is not 4.2. + LockGuard lock(_storageInterfaceWorkDoneMutex); + ASSERT_FALSE(_storageInterfaceWorkDone.uniqueIndexUpdated); +} + +TEST_F(InitialSyncerTest, InitialSyncerUpgradeNonReplicatedUniqueIndexesError) { + // Ensure upgradeNonReplicatedUniqueIndexes returns a bad status. This should be passed to the + // initial syncer. + { + LockGuard lock(_storageInterfaceWorkDoneMutex); + _storageInterfaceWorkDone.upgradeNonReplicatedUniqueIndexesShouldFail = true; + } + doInitialSyncWithOneBatch(true); + + // Ensure the upgradeNonReplicatedUniqueIndexes status was captured. + ASSERT_EQUALS(ErrorCodes::NamespaceNotFound, _lastApplied); +} + } // namespace diff --git a/src/mongo/db/repl/oplog.cpp b/src/mongo/db/repl/oplog.cpp index 6ce206869a7..7d14dc14d17 100644 --- a/src/mongo/db/repl/oplog.cpp +++ b/src/mongo/db/repl/oplog.cpp @@ -810,9 +810,10 @@ std::map<std::string, ApplyOpMetadata> opsMap = { const OpTime& opTime, OplogApplication::Mode mode) -> Status { NamespaceString nss; - BSONObjBuilder resultWeDontCareAbout; std::tie(std::ignore, nss) = parseCollModUUIDAndNss(opCtx, ui, ns, cmd); - return collMod(opCtx, nss, cmd, &resultWeDontCareAbout); + // The collMod for apply ops could be either a user driven collMod or a collMod triggered + // by an upgrade. + return collModWithUpgrade(opCtx, nss, cmd); }, {ErrorCodes::IndexNotFound, ErrorCodes::NamespaceNotFound}}}, {"dbCheck", {dbCheckOplogCommand, {}}}, diff --git a/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp b/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp index 557a9c67282..fb178001730 100644 --- a/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp +++ b/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp @@ -415,6 +415,30 @@ Status ReplicationCoordinatorExternalStateImpl::initializeReplSetStorage(Operati waitForAllEarlierOplogWritesToBeVisible(opCtx); }); + // Update unique index format version for all non-replicated collections. It is possible + // for MongoDB to have a "clean startup", i.e., no non-local databases, but still have + // unique indexes on collections in the local database. On clean startup, + // setFeatureCompatibilityVersion (which updates the unique index format version of + // collections) is not called, so any pre-existing collections are upgraded here. We exclude + // ShardServers when updating indexes belonging to non-replicated collections on the primary + // because ShardServers are started up by default with featureCompatibilityVersion 4.0, so + // we don't want to update those indexes until the cluster's featureCompatibilityVersion is + // explicitly set to 4.2 by config server. The below unique index update for non-replicated + // collections only occurs on the primary; updates for unique indexes belonging to + // non-replicated collections are done on secondaries during InitialSync. When the config + // server sets the featureCompatibilityVersion to 4.2, the shard primary will update unique + // indexes belonging to all the collections. One special case here is if a shard is already + // in featureCompatibilityVersion 4.2 and a new node is started up with --shardsvr and added + // to that shard, the new node will still start up with featureCompatibilityVersion 4.0 and + // may need to have unique index version updated. Such indexes would be updated during + // InitialSync because the new node is a secondary. + // TODO(SERVER-34489) Add a check for latest FCV when upgrade/downgrade is ready. + if (FeatureCompatibilityVersion::isCleanStartUp() && + serverGlobalParams.clusterRole != ClusterRole::ShardServer) { + auto updateStatus = updateNonReplicatedUniqueIndexes(opCtx); + if (!updateStatus.isOK()) + return updateStatus; + } FeatureCompatibilityVersion::setIfCleanStartup(opCtx, _storageInterface); } catch (const DBException& ex) { return ex.toStatus(); diff --git a/src/mongo/db/repl/storage_interface.h b/src/mongo/db/repl/storage_interface.h index db97cc8c698..5b526456752 100644 --- a/src/mongo/db/repl/storage_interface.h +++ b/src/mongo/db/repl/storage_interface.h @@ -341,6 +341,12 @@ public: const NamespaceString& nss) = 0; /** + * Updates unique indexes belonging to all non-replicated collections. To be called at the + * end of initial sync. + */ + virtual Status upgradeNonReplicatedUniqueIndexes(OperationContext* opCtx) = 0; + + /** * Sets the highest timestamp at which the storage engine is allowed to take a checkpoint. * This timestamp can never decrease, and thus should be a timestamp that can never roll back. */ diff --git a/src/mongo/db/repl/storage_interface_impl.cpp b/src/mongo/db/repl/storage_interface_impl.cpp index fad6858f2e3..c7dc39c8b68 100644 --- a/src/mongo/db/repl/storage_interface_impl.cpp +++ b/src/mongo/db/repl/storage_interface_impl.cpp @@ -1042,6 +1042,10 @@ StatusWith<OptionalCollectionUUID> StorageInterfaceImpl::getCollectionUUID( return collection->uuid(); } +Status StorageInterfaceImpl::upgradeNonReplicatedUniqueIndexes(OperationContext* opCtx) { + return updateNonReplicatedUniqueIndexes(opCtx); +} + void StorageInterfaceImpl::setStableTimestamp(ServiceContext* serviceCtx, Timestamp snapshotName) { serviceCtx->getStorageEngine()->setStableTimestamp(snapshotName); } diff --git a/src/mongo/db/repl/storage_interface_impl.h b/src/mongo/db/repl/storage_interface_impl.h index 8f8fdb8c621..17555669b00 100644 --- a/src/mongo/db/repl/storage_interface_impl.h +++ b/src/mongo/db/repl/storage_interface_impl.h @@ -156,6 +156,8 @@ public: StatusWith<OptionalCollectionUUID> getCollectionUUID(OperationContext* opCtx, const NamespaceString& nss) override; + Status upgradeNonReplicatedUniqueIndexes(OperationContext* opCtx) override; + void setStableTimestamp(ServiceContext* serviceCtx, Timestamp snapshotName) override; void setInitialDataTimestamp(ServiceContext* serviceCtx, Timestamp snapshotName) override; diff --git a/src/mongo/db/repl/storage_interface_mock.h b/src/mongo/db/repl/storage_interface_mock.h index f242c0b6a93..41b3d71c916 100644 --- a/src/mongo/db/repl/storage_interface_mock.h +++ b/src/mongo/db/repl/storage_interface_mock.h @@ -125,6 +125,7 @@ public: using IsAdminDbValidFn = stdx::function<Status(OperationContext* opCtx)>; using GetCollectionUUIDFn = stdx::function<StatusWith<OptionalCollectionUUID>( OperationContext* opCtx, const NamespaceString& nss)>; + using UpgradeNonReplicatedUniqueIndexesFn = stdx::function<Status(OperationContext* opCtx)>; StorageInterfaceMock() = default; @@ -283,6 +284,9 @@ public: return getCollectionUUIDFn(opCtx, nss); } + Status upgradeNonReplicatedUniqueIndexes(OperationContext* opCtx) override { + return upgradeNonReplicatedUniqueIndexesFn(opCtx); + } void setStableTimestamp(ServiceContext* serviceCtx, Timestamp snapshotName) override; void setInitialDataTimestamp(ServiceContext* serviceCtx, Timestamp snapshotName) override; @@ -387,6 +391,11 @@ public: OperationContext* opCtx, const NamespaceString& nss) -> StatusWith<OptionalCollectionUUID> { return Status{ErrorCodes::IllegalOperation, "GetCollectionUUIDFn not implemented."}; }; + UpgradeNonReplicatedUniqueIndexesFn upgradeNonReplicatedUniqueIndexesFn = + [](OperationContext* opCtx) -> Status { + return Status{ErrorCodes::IllegalOperation, + "upgradeNonReplicatedUniqueIndexesFn not implemented."}; + }; bool supportsDocLockingBool = false; Timestamp allCommittedTimestamp = Timestamp::min(); diff --git a/src/mongo/db/storage/bson_collection_catalog_entry.cpp b/src/mongo/db/storage/bson_collection_catalog_entry.cpp index 9703e5f1a89..21c1530e00d 100644 --- a/src/mongo/db/storage/bson_collection_catalog_entry.cpp +++ b/src/mongo/db/storage/bson_collection_catalog_entry.cpp @@ -154,6 +154,18 @@ void BSONCollectionCatalogEntry::getReadyIndexes(OperationContext* opCtx, } } +void BSONCollectionCatalogEntry::getAllUniqueIndexes(OperationContext* opCtx, + std::vector<std::string>* names) const { + MetaData md = _getMetaData(opCtx); + + for (unsigned i = 0; i < md.indexes.size(); i++) { + if (md.indexes[i].spec["unique"]) { + std::string indexName = md.indexes[i].spec["name"].String(); + names->push_back(indexName); + } + } +} + bool BSONCollectionCatalogEntry::isIndexMultikey(OperationContext* opCtx, StringData indexName, MultikeyPaths* multikeyPaths) const { diff --git a/src/mongo/db/storage/bson_collection_catalog_entry.h b/src/mongo/db/storage/bson_collection_catalog_entry.h index dffcecca32c..4401c554775 100644 --- a/src/mongo/db/storage/bson_collection_catalog_entry.h +++ b/src/mongo/db/storage/bson_collection_catalog_entry.h @@ -61,6 +61,9 @@ public: virtual void getReadyIndexes(OperationContext* opCtx, std::vector<std::string>* names) const; + virtual void getAllUniqueIndexes(OperationContext* opCtx, + std::vector<std::string>* names) const; + virtual bool isIndexMultikey(OperationContext* opCtx, StringData indexName, MultikeyPaths* multikeyPaths) const; diff --git a/src/mongo/db/storage/kv/kv_collection_catalog_entry.cpp b/src/mongo/db/storage/kv/kv_collection_catalog_entry.cpp index 1163f89cf3c..4211bc5a6c2 100644 --- a/src/mongo/db/storage/kv/kv_collection_catalog_entry.cpp +++ b/src/mongo/db/storage/kv/kv_collection_catalog_entry.cpp @@ -233,6 +233,12 @@ void KVCollectionCatalogEntry::updateTTLSetting(OperationContext* opCtx, _catalog->putMetaData(opCtx, ns().toString(), md); } +void KVCollectionCatalogEntry::updateIndexMetadata(OperationContext* opCtx, + const IndexDescriptor* desc) { + // Update any metadata Ident has for this index + const string ident = _catalog->getIndexIdent(opCtx, ns().ns(), desc->indexName()); + _engine->alterIdentMetadata(opCtx, ident, desc); +} void KVCollectionCatalogEntry::addUUID(OperationContext* opCtx, CollectionUUID uuid, Collection* coll) { diff --git a/src/mongo/db/storage/kv/kv_collection_catalog_entry.h b/src/mongo/db/storage/kv/kv_collection_catalog_entry.h index 51104c787e7..e21e5b27b15 100644 --- a/src/mongo/db/storage/kv/kv_collection_catalog_entry.h +++ b/src/mongo/db/storage/kv/kv_collection_catalog_entry.h @@ -76,6 +76,8 @@ public: void updateFlags(OperationContext* opCtx, int newValue) final; + void updateIndexMetadata(OperationContext* opCtx, const IndexDescriptor* desc) final; + void updateValidator(OperationContext* opCtx, const BSONObj& validator, StringData validationLevel, diff --git a/src/mongo/db/storage/kv/kv_engine.h b/src/mongo/db/storage/kv/kv_engine.h index 750f3a1d249..f7b79356a04 100644 --- a/src/mongo/db/storage/kv/kv_engine.h +++ b/src/mongo/db/storage/kv/kv_engine.h @@ -163,6 +163,10 @@ public: virtual Status dropIdent(OperationContext* opCtx, StringData ident) = 0; + virtual void alterIdentMetadata(OperationContext* opCtx, + StringData ident, + const IndexDescriptor* desc){}; + // optional virtual int flushAllFiles(OperationContext* opCtx, bool sync) { return 0; diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp index 7abac5b1855..52c6b1b60e8 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp @@ -170,12 +170,10 @@ std::string WiredTigerIndex::generateAppMetadataString(const IndexDescriptor& de int keyStringVersion; - // This gating variable controls the creation between timestamp safe and timestamp unsafe - // unique indexes. The gating condition will be enhanced to check for FCV 4.2 by SERVER-32825 + // The gating variable controls the creation between timestamp safe and timestamp unsafe + // unique indexes. The gating condition will be enhanced to check for FCV 4.2 by SERVER-34489 // and the gating variable will be removed when FCV 4.2 becomes available. - bool createNewStyleUniqueIdx = false; - - if (createNewStyleUniqueIdx && desc.unique() && !desc.isIdIndex()) { + if (createTimestampSafeUniqueIndex && desc.unique() && !desc.isIdIndex()) { keyStringVersion = desc.version() >= IndexDescriptor::IndexVersion::kV2 ? kDataFormatV4KeyStringV1UniqueIndexVersionV2 : kDataFormatV3KeyStringV0UniqueIndexVersionV1; diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp index d90ad9d2e23..2596a4f5cb5 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp @@ -874,6 +874,20 @@ SortedDataInterface* WiredTigerKVEngine::getGroupedSortedDataInterface(Operation return new WiredTigerIndexStandard(opCtx, _uri(ident), desc, prefix, _readOnly); } +void WiredTigerKVEngine::alterIdentMetadata(OperationContext* opCtx, + StringData ident, + const IndexDescriptor* desc) { + WiredTigerSession session(_conn); + std::string uri = _uri(ident); + + // Make the alter call to update metadata without taking exclusive lock to avoid conflicts with + // concurrent operations. + std::string alterString = + WiredTigerIndex::generateAppMetadataString(*desc) + "exclusive_refreshed=false,"; + invariantWTOK( + session.getSession()->alter(session.getSession(), uri.c_str(), alterString.c_str())); +} + Status WiredTigerKVEngine::dropIdent(OperationContext* opCtx, StringData ident) { string uri = _uri(ident); diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h index 6cf7e3c14af..fc7bbc1ed22 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h @@ -136,6 +136,10 @@ public: virtual Status dropIdent(OperationContext* opCtx, StringData ident); + virtual void alterIdentMetadata(OperationContext* opCtx, + StringData ident, + const IndexDescriptor* desc); + virtual Status okToRename(OperationContext* opCtx, StringData fromNS, StringData toNS, |