From 104653ec8eca6865ff938c3e9966139b2cdd340f Mon Sep 17 00:00:00 2001 From: Benety Goh Date: Mon, 10 Apr 2017 13:50:49 -0400 Subject: SERVER-28211 RollbackFixUpInfo handles create, drop and TTL changes to indexes --- src/mongo/db/repl/rollback_fix_up_info.cpp | 105 ++++++ src/mongo/db/repl/rollback_fix_up_info.h | 47 +++ .../db/repl/rollback_fix_up_info_descriptions.cpp | 96 ++++- .../db/repl/rollback_fix_up_info_descriptions.h | 74 ++++ .../rollback_fix_up_info_descriptions_test.cpp | 47 +++ src/mongo/db/repl/rollback_fix_up_info_test.cpp | 386 +++++++++++++++++++++ 6 files changed, 754 insertions(+), 1 deletion(-) diff --git a/src/mongo/db/repl/rollback_fix_up_info.cpp b/src/mongo/db/repl/rollback_fix_up_info.cpp index 791bdccb859..30a449569e6 100644 --- a/src/mongo/db/repl/rollback_fix_up_info.cpp +++ b/src/mongo/db/repl/rollback_fix_up_info.cpp @@ -32,11 +32,13 @@ #include "mongo/db/repl/rollback_fix_up_info.h" +#include "mongo/base/string_data.h" #include "mongo/db/jsobj.h" #include "mongo/db/operation_context.h" #include "mongo/db/repl/rollback_fix_up_info_descriptions.h" #include "mongo/db/repl/storage_interface.h" #include "mongo/util/assert_util.h" +#include "mongo/util/log.h" #include "mongo/util/uuid.h" namespace mongo { @@ -56,6 +58,9 @@ const NamespaceString RollbackFixUpInfo::kRollbackCollectionUuidNamespace(kRollb const NamespaceString RollbackFixUpInfo::kRollbackCollectionOptionsNamespace( kRollbackNamespacePrefix + "collectionOptions"); +const NamespaceString RollbackFixUpInfo::kRollbackIndexNamespace(kRollbackNamespacePrefix + + "indexes"); + RollbackFixUpInfo::RollbackFixUpInfo(StorageInterface* storageInterface) : _storageInterface(storageInterface) { invariant(storageInterface); @@ -122,6 +127,62 @@ Status RollbackFixUpInfo::processCollModOplogEntry(OperationContext* opCtx, return _upsertById(opCtx, kRollbackCollectionOptionsNamespace, desc.toBSON()); } +Status RollbackFixUpInfo::processCreateIndexOplogEntry(OperationContext* opCtx, + const UUID& collectionUuid, + const std::string& indexName) { + IndexDescription desc(collectionUuid, indexName, IndexOpType::kCreate, {}); + + // If the existing document (that may or may not exist in the "kRollbackIndexNamespace" + // collection) has a 'drop' op type, this oplog entry will cancel out the previously processed + // 'dropIndexes" oplog entry. We should remove the existing document from the collection and not + // insert a new document. + BSONObjBuilder bob; + bob.append("_id", desc.makeIdKey()); + auto key = bob.obj(); + auto deleteResult = + _storageInterface->deleteDocuments(opCtx, + kRollbackIndexNamespace, + "_id_"_sd, + StorageInterface::ScanDirection::kForward, + key, + BoundInclusion::kIncludeStartKeyOnly, + 1U); + if (deleteResult.isOK() && !deleteResult.getValue().empty()) { + auto doc = deleteResult.getValue().front(); + auto opTypeResult = IndexDescription::parseOpType(doc); + if (!opTypeResult.isOK()) { + invariant(ErrorCodes::FailedToParse == opTypeResult.getStatus()); + warning() << "While processing createIndex oplog entry for index " << indexName + << " in collection with UUID " << collectionUuid.toString() + << ", found existing entry in rollback collection " << kRollbackIndexNamespace + << " with unrecognized operation type:" << doc + << ". Replacing existing entry."; + return _upsertIndexDescription(opCtx, desc); + } else if (IndexOpType::kDrop == opTypeResult.getValue()) { + return Status::OK(); + } + // Fall through and replace existing document. + } + + return _upsertIndexDescription(opCtx, desc); +} + +Status RollbackFixUpInfo::processUpdateIndexTTLOplogEntry(OperationContext* opCtx, + const UUID& collectionUuid, + const std::string& indexName, + Seconds expireAfterSeconds) { + IndexDescription desc(collectionUuid, indexName, expireAfterSeconds); + return _upsertIndexDescription(opCtx, desc); +} + +Status RollbackFixUpInfo::processDropIndexOplogEntry(OperationContext* opCtx, + const UUID& collectionUuid, + const std::string& indexName, + const BSONObj& infoObj) { + IndexDescription desc(collectionUuid, indexName, IndexOpType::kDrop, infoObj); + return _upsertIndexDescription(opCtx, desc); +} + Status RollbackFixUpInfo::_upsertById(OperationContext* opCtx, const NamespaceString& nss, const BSONObj& update) { @@ -130,5 +191,49 @@ Status RollbackFixUpInfo::_upsertById(OperationContext* opCtx, return _storageInterface->upsertById(opCtx, nss, key, update); } +Status RollbackFixUpInfo::_upsertIndexDescription(OperationContext* opCtx, + const IndexDescription& description) { + switch (description.getOpType()) { + case IndexOpType::kCreate: + case IndexOpType::kDrop: + return _upsertById(opCtx, kRollbackIndexNamespace, description.toBSON()); + case IndexOpType::kUpdateTTL: { + // For updateTTL, if there is an existing document in the collection with a "drop" op + // type, we should update the index info obj in the existing document and leave the op + // type unchanged. Otherwise, we assume that the existing document has an op type of + // "updateTTL"; or is not present in the collection. Therefore, it is safe to overwrite + // any existing data. + // + // It's not possible for the existing document to have a "create" op type while + // processing a collMod (updateTTL) because this implies the follow sequence of + // of operations in the oplog: + // ..., collMod, ..., createIndex, ... + // (createIndex gets processed before collMod) + // This is illegal because there's a missing dropIndex oplog entry between the collMod + // and createIndex oplog entries. + auto expireAfterSeconds = description.getExpireAfterSeconds(); + invariant(expireAfterSeconds); + BSONObjBuilder updateBob; + { + BSONObjBuilder setOnInsertBob(updateBob.subobjStart("$setOnInsert")); + setOnInsertBob.append("operationType", description.getOpTypeAsString()); + } + { + BSONObjBuilder setBob(updateBob.subobjStart("$set")); + setBob.append("infoObj.expireAfterSeconds", + durationCount(*expireAfterSeconds)); + } + auto updateDoc = updateBob.obj(); + + BSONObjBuilder bob; + bob.append("_id", description.makeIdKey()); + auto key = bob.obj(); + return _storageInterface->upsertById( + opCtx, kRollbackIndexNamespace, key.firstElement(), updateDoc); + } + } + MONGO_UNREACHABLE; +} + } // namespace repl } // namespace mongo diff --git a/src/mongo/db/repl/rollback_fix_up_info.h b/src/mongo/db/repl/rollback_fix_up_info.h index 2b392585304..ef273ca9a98 100644 --- a/src/mongo/db/repl/rollback_fix_up_info.h +++ b/src/mongo/db/repl/rollback_fix_up_info.h @@ -35,6 +35,7 @@ #include "mongo/base/status.h" #include "mongo/base/status_with.h" #include "mongo/db/namespace_string.h" +#include "mongo/util/duration.h" #include "mongo/util/uuid.h" namespace mongo { @@ -83,6 +84,12 @@ public: */ static const NamespaceString kRollbackCollectionOptionsNamespace; + /** + * Contains mappings of (collection UUID, index name) -> index info. + * This collection is used to roll back create, drop and TTL changes to indexes. + */ + static const NamespaceString kRollbackIndexNamespace; + /** * Creates an instance of RollbackFixUpInfo. */ @@ -163,12 +170,52 @@ public: const UUID& collectionUuid, const BSONObj& optionsObj); + /** + * Processes an oplog entry representing a createIndex command. Stores information about + * this operation into "kRollbackIndexNamespace" to allow us to roll back this + * operation later by dropping the index from the catalog by UUID/index name. + * + * The mapping in the "kRollbackCollectionUuidNamespace" collection will contain the + * empty namespace. + */ + Status processCreateIndexOplogEntry(OperationContext* opCtx, + const UUID& collectionUuid, + const std::string& indexName); + enum class IndexOpType { kCreate, kDrop, kUpdateTTL }; + class IndexDescription; + + /** + * Processes an oplog entry representing a collMod command that updates the expiration setting + * on a TTL index. Stores information about this operation into "kRollbackIndexNamespace" to + * allow us to roll back this operation later by updating the TTL expiration to the previous + * value. + */ + Status processUpdateIndexTTLOplogEntry(OperationContext* opCtx, + const UUID& collectionUuid, + const std::string& indexName, + Seconds expireAfterSeconds); + + /** + * Processes an oplog entry representing a dropIndexes command with a single index. Stores + * information about this operation into "kRollbackIndexNamespace" to allow us to roll back this + * operation later by recreating the index. + */ + Status processDropIndexOplogEntry(OperationContext* opCtx, + const UUID& collectionUuid, + const std::string& indexName, + const BSONObj& infoObj); + private: /** * Upserts a single document using the _id field of the document in "update". */ Status _upsertById(OperationContext* opCtx, const NamespaceString& nss, const BSONObj& update); + /** + * Upserts an IndexDescription. + */ + Status _upsertIndexDescription(OperationContext* opCtx, const IndexDescription& description); + StorageInterface* const _storageInterface; }; diff --git a/src/mongo/db/repl/rollback_fix_up_info_descriptions.cpp b/src/mongo/db/repl/rollback_fix_up_info_descriptions.cpp index 01c8678d58e..378e42b6071 100644 --- a/src/mongo/db/repl/rollback_fix_up_info_descriptions.cpp +++ b/src/mongo/db/repl/rollback_fix_up_info_descriptions.cpp @@ -32,6 +32,7 @@ #include "mongo/db/repl/rollback_fix_up_info_descriptions.h" +#include "mongo/bson/util/bson_extract.h" #include "mongo/db/jsobj.h" namespace mongo { @@ -40,7 +41,7 @@ namespace repl { namespace { /** - * Appends op type to builder as string element under the field name "op". + * Appends single document op type to builder as string element under the field name "op". */ void appendOpTypeToBuilder(RollbackFixUpInfo::SingleDocumentOpType opType, BSONObjBuilder* builder) { @@ -58,6 +59,27 @@ void appendOpTypeToBuilder(RollbackFixUpInfo::SingleDocumentOpType opType, } } +/** + * Returns string representation of RollbackFixUpInfo::IndexOpType. + */ +std::string toString(RollbackFixUpInfo::IndexOpType opType) { + switch (opType) { + case RollbackFixUpInfo::IndexOpType::kCreate: + return "create"; + case RollbackFixUpInfo::IndexOpType::kDrop: + return "drop"; + case RollbackFixUpInfo::IndexOpType::kUpdateTTL: + return "updateTTL"; + } + MONGO_UNREACHABLE; +} +/** + * Appends index op type to builder as string element under the field name "op". + */ +void appendOpTypeToBuilder(RollbackFixUpInfo::IndexOpType opType, BSONObjBuilder* builder) { + builder->append("operationType", toString(opType)); +} + } // namespace RollbackFixUpInfo::SingleDocumentOperationDescription::SingleDocumentOperationDescription( @@ -111,5 +133,77 @@ BSONObj RollbackFixUpInfo::CollectionOptionsDescription::toBSON() const { return bob.obj(); } +RollbackFixUpInfo::IndexDescription::IndexDescription(const UUID& collectionUuid, + const std::string& indexName, + RollbackFixUpInfo::IndexOpType opType, + const BSONObj& infoObj) + : _collectionUuid(collectionUuid), _indexName(indexName), _opType(opType), _infoObj(infoObj) { + invariant(RollbackFixUpInfo::IndexOpType::kUpdateTTL != _opType); +} + +RollbackFixUpInfo::IndexDescription::IndexDescription(const UUID& collectionUuid, + const std::string& indexName, + Seconds expireAfterSeconds) + : _collectionUuid(collectionUuid), + _indexName(indexName), + _opType(RollbackFixUpInfo::IndexOpType::kUpdateTTL), + _expireAfterSeconds(expireAfterSeconds) { + BSONObjBuilder bob; + bob.append("expireAfterSeconds", durationCount(*_expireAfterSeconds)); + _infoObj = bob.obj(); +} + +RollbackFixUpInfo::IndexOpType RollbackFixUpInfo::IndexDescription::getOpType() const { + return _opType; +} + +std::string RollbackFixUpInfo::IndexDescription::getOpTypeAsString() const { + return toString(_opType); +} + +boost::optional RollbackFixUpInfo::IndexDescription::getExpireAfterSeconds() const { + return _expireAfterSeconds; +} + +// static +StatusWith RollbackFixUpInfo::IndexDescription::parseOpType( + const BSONObj& doc) { + std::string opTypeStr; + auto status = bsonExtractStringField(doc, "operationType"_sd, &opTypeStr); + if (!status.isOK()) { + return status; + } + if ("create" == opTypeStr) { + return RollbackFixUpInfo::IndexOpType::kCreate; + } else if ("drop" == opTypeStr) { + return RollbackFixUpInfo::IndexOpType::kDrop; + } else if ("updateTTL" == opTypeStr) { + return RollbackFixUpInfo::IndexOpType::kUpdateTTL; + } + return Status(ErrorCodes::FailedToParse, + str::stream() << "Unrecognized RollbackFixUpInfo::IndexOpType: " << opTypeStr); +} + +BSONObj RollbackFixUpInfo::IndexDescription::toBSON() const { + BSONObjBuilder bob; + bob.append("_id", makeIdKey()); + appendOpTypeToBuilder(_opType, &bob); + bob.append("infoObj", _infoObj); + + return bob.obj(); +} + +BSONObj RollbackFixUpInfo::IndexDescription::makeIdKey() const { + BSONObjBuilder idBob; + _collectionUuid.appendToBuilder(&idBob, "collectionUuid"); + idBob.append("indexName", _indexName); + return idBob.obj(); +} + } // namespace repl + +std::ostream& operator<<(std::ostream& os, const repl::RollbackFixUpInfo::IndexOpType& opType) { + return os << repl::toString(opType); +} + } // namespace mongo diff --git a/src/mongo/db/repl/rollback_fix_up_info_descriptions.h b/src/mongo/db/repl/rollback_fix_up_info_descriptions.h index 2daed660099..6b663d52cb5 100644 --- a/src/mongo/db/repl/rollback_fix_up_info_descriptions.h +++ b/src/mongo/db/repl/rollback_fix_up_info_descriptions.h @@ -28,10 +28,14 @@ #pragma once +#include +#include + #include "mongo/base/disallow_copying.h" #include "mongo/bson/bsonobj.h" #include "mongo/db/namespace_string.h" #include "mongo/db/repl/rollback_fix_up_info.h" +#include "mongo/util/duration.h" #include "mongo/util/uuid.h" namespace mongo { @@ -99,5 +103,75 @@ private: BSONObj _optionsObj; }; +/** + * Represents a document in the "kRollbackIndexNamespace" namespace. + * Contains information to roll back operations on indexes - creation, drops, and updates to TTL + * expiration settings. + */ +class RollbackFixUpInfo::IndexDescription { + MONGO_DISALLOW_COPYING(IndexDescription); + +public: + /** + * For op types insert and drop. + */ + IndexDescription(const UUID& collectionUuid, + const std::string& indexName, + RollbackFixUpInfo::IndexOpType opType, + const BSONObj& infoObj); + + /** + * For op type update TTL only. + */ + IndexDescription(const UUID& collectionUuid, + const std::string& indexName, + Seconds expireAfterSeconds); + + /** + * Returns op type. + */ + RollbackFixUpInfo::IndexOpType getOpType() const; + + /** + * Returns op type as string. + */ + std::string getOpTypeAsString() const; + + /** + * Returns optional TTL index expiration. + */ + boost::optional getExpireAfterSeconds() const; + + /** + * Parses op type from BSON document representation. + */ + static StatusWith parseOpType(const BSONObj& doc); + + /** + * Returns a BSON representation of this object. + */ + BSONObj toBSON() const; + + /** + * Returns a BSON document containing the _id for the document to be updated. + * For UpdateTTL op type only. + */ + BSONObj makeIdKey() const; + +private: + UUID _collectionUuid; + std::string _indexName; + RollbackFixUpInfo::IndexOpType _opType; + BSONObj _infoObj; + boost::optional _expireAfterSeconds = boost::none; +}; + } // namespace repl + +/** + * Insertion operator for RollbackFixUpInfo::IndexOpType. Formats op type for output stream. + * For testing only. + */ +std::ostream& operator<<(std::ostream& os, const repl::RollbackFixUpInfo::IndexOpType& opType); + } // namespace mongo diff --git a/src/mongo/db/repl/rollback_fix_up_info_descriptions_test.cpp b/src/mongo/db/repl/rollback_fix_up_info_descriptions_test.cpp index 77024868c13..1d5e10e87ef 100644 --- a/src/mongo/db/repl/rollback_fix_up_info_descriptions_test.cpp +++ b/src/mongo/db/repl/rollback_fix_up_info_descriptions_test.cpp @@ -100,4 +100,51 @@ TEST(RollbackFixUpInfoDescriptionsTest, CollectionOptionsDescriptionToBson) { ASSERT_BSONOBJ_EQ(expectedDocument, description.toBSON()); } +TEST(RollbackFixUpInfoDescriptionsTest, IndexDescriptionParseOpType) { + ASSERT_EQUALS( + RollbackFixUpInfo::IndexOpType::kCreate, + unittest::assertGet(RollbackFixUpInfo::IndexDescription::parseOpType(BSON("operationType" + << "create")))); + ASSERT_EQUALS( + RollbackFixUpInfo::IndexOpType::kDrop, + unittest::assertGet(RollbackFixUpInfo::IndexDescription::parseOpType(BSON("operationType" + << "drop")))); + ASSERT_EQUALS(RollbackFixUpInfo::IndexOpType::kUpdateTTL, + unittest::assertGet( + RollbackFixUpInfo::IndexDescription::parseOpType(BSON("operationType" + << "updateTTL")))); + ASSERT_EQUALS(ErrorCodes::NoSuchKey, + RollbackFixUpInfo::IndexDescription::parseOpType(BSON("no_operation_type" << 1)) + .getStatus()); + ASSERT_EQUALS(ErrorCodes::TypeMismatch, + RollbackFixUpInfo::IndexDescription::parseOpType(BSON("operationType" << 12345)) + .getStatus()); + ASSERT_EQUALS(ErrorCodes::FailedToParse, + RollbackFixUpInfo::IndexDescription::parseOpType(BSON("operationType" + << "unknown op type")) + .getStatus()); +} + +TEST(RollbackFixUpInfoDescriptionsTest, IndexDescriptionToBson) { + auto collectionUuid = UUID::gen(); + const std::string indexName = "b_1"; + auto infoObj = BSON("v" << 2 << "key" << BSON("b" << 1) << "name" << indexName << "ns" + << "mydb.mycoll" + << "expireAfterSeconds" + << 60); + + RollbackFixUpInfo::IndexDescription description( + collectionUuid, indexName, RollbackFixUpInfo::IndexOpType::kDrop, infoObj); + + auto expectedDocument = + BSON("_id" << BSON("collectionUuid" << collectionUuid.toBSON().firstElement() << "indexName" + << indexName) + << "operationType" + << "drop" + << "infoObj" + << infoObj); + + ASSERT_BSONOBJ_EQ(expectedDocument, description.toBSON()); +} + } // namespace diff --git a/src/mongo/db/repl/rollback_fix_up_info_test.cpp b/src/mongo/db/repl/rollback_fix_up_info_test.cpp index 3ca7f780dc6..584a72233fb 100644 --- a/src/mongo/db/repl/rollback_fix_up_info_test.cpp +++ b/src/mongo/db/repl/rollback_fix_up_info_test.cpp @@ -97,6 +97,8 @@ void RollbackFixUpInfoTest::setUp() { opCtx.get(), RollbackFixUpInfo::kRollbackCollectionUuidNamespace, {})); ASSERT_OK(_storageInterface->createCollection( opCtx.get(), RollbackFixUpInfo::kRollbackCollectionOptionsNamespace, {})); + ASSERT_OK(_storageInterface->createCollection( + opCtx.get(), RollbackFixUpInfo::kRollbackIndexNamespace, {})); } void RollbackFixUpInfoTest::tearDown() { @@ -429,4 +431,388 @@ TEST_F(RollbackFixUpInfoTest, opCtx.get(), RollbackFixUpInfo::kRollbackCollectionOptionsNamespace, {expectedDocument}); } +TEST_F(RollbackFixUpInfoTest, + ProcessCreateIndexOplogEntryInsertsDocumentIntoRollbackIndexCollectionWithEmptyInfoObj) { + auto operation = + BSON("ts" << Timestamp(Seconds(1), 0) << "h" << 1LL << "op" + << "c" + << "ns" + << "mydb.$cmd" + << "ui" + << UUID::gen().toBSON().firstElement() + << "o" + << BSON("createIndex" << 1 << "v" << 2 << "key" << BSON("b" << 1) << "name" + << "b_1" + << "ns" + << "mydb.mycoll" + << "expireAfterSeconds" + << 60)); + auto collectionUuid = unittest::assertGet(UUID::parse(operation["ui"])); + auto indexName = operation["o"].Obj()["name"].String(); + + ASSERT_TRUE(OplogEntry(operation).isCommand()); + + auto opCtx = makeOpCtx(); + RollbackFixUpInfo rollbackFixUpInfo(_storageInterface.get()); + ASSERT_OK( + rollbackFixUpInfo.processCreateIndexOplogEntry(opCtx.get(), collectionUuid, indexName)); + + auto expectedDocument = + BSON("_id" << BSON("collectionUuid" << collectionUuid.toBSON().firstElement() << "indexName" + << indexName) + << "operationType" + << "create" + << "infoObj" + << BSONObj()); + + _assertDocumentsInCollectionEquals( + opCtx.get(), RollbackFixUpInfo::kRollbackIndexNamespace, {expectedDocument}); +} + +TEST_F(RollbackFixUpInfoTest, + ProcessCreateIndexOplogEntryWhenExistingDocumentHasDropOpTypeRemovesExistingDocument) { + + // State of oplog: + // {createIndex: indexA}, ...., {dropIndexes: indexA}, .... + // (earliest optime) ---> (latest optime) + // + // Oplog entries are processed in reverse optime order. + + // First, process dropIndexes. This should insert a document into the collection with a 'drop' + // op type. + auto collectionUuid = UUID::gen(); + std::string indexName = "b_1"; + auto infoObj = BSON("v" << 2 << "key" << BSON("b" << 1) << "name" << indexName << "ns" + << "mydb.mycoll"); + + auto opCtx = makeOpCtx(); + RollbackFixUpInfo rollbackFixUpInfo(_storageInterface.get()); + ASSERT_OK(rollbackFixUpInfo.processDropIndexOplogEntry( + opCtx.get(), collectionUuid, indexName, infoObj)); + _assertDocumentsInCollectionEquals( + opCtx.get(), + RollbackFixUpInfo::kRollbackIndexNamespace, + {BSON("_id" << BSON("collectionUuid" << collectionUuid.toBSON().firstElement() + << "indexName" + << indexName) + << "operationType" + << "drop" + << "infoObj" + << infoObj)}); + + // Next, process createIndex. This should cancel out the existing 'drop' operation and remove + // existing document from the collection. + ASSERT_OK( + rollbackFixUpInfo.processCreateIndexOplogEntry(opCtx.get(), collectionUuid, indexName)); + _assertDocumentsInCollectionEquals(opCtx.get(), RollbackFixUpInfo::kRollbackIndexNamespace, {}); +} + +TEST_F(RollbackFixUpInfoTest, + ProcessCreateIndexOplogEntryWhenExistingDocumentHasUpdateTTLOpTypeReplacesExistingDocument) { + + // State of oplog: + // {createIndex: indexA}, ...., {collMod: indexA}, .... + // (earliest optime) ---> (latest optime) + // + // Oplog entries are processed in reverse optime order. + + // First, process collMod. This should insert a document into the collection with an 'updateTTL' + // op type. + auto collectionUuid = UUID::gen(); + std::string indexName = "b_1"; + + auto opCtx = makeOpCtx(); + RollbackFixUpInfo rollbackFixUpInfo(_storageInterface.get()); + ASSERT_OK(rollbackFixUpInfo.processUpdateIndexTTLOplogEntry( + opCtx.get(), collectionUuid, indexName, Seconds(60))); + _assertDocumentsInCollectionEquals( + opCtx.get(), + RollbackFixUpInfo::kRollbackIndexNamespace, + {BSON("_id" << BSON("collectionUuid" << collectionUuid.toBSON().firstElement() + << "indexName" + << indexName) + << "operationType" + << "updateTTL" + << "infoObj" + << BSON("expireAfterSeconds" << 60))}); + + // Next, process createIndex. This should replace the existing 'updateTTL' operation so that + // we drop the index when it's time to apply the fix up info. + ASSERT_OK( + rollbackFixUpInfo.processCreateIndexOplogEntry(opCtx.get(), collectionUuid, indexName)); + _assertDocumentsInCollectionEquals( + opCtx.get(), + RollbackFixUpInfo::kRollbackIndexNamespace, + {BSON("_id" << BSON("collectionUuid" << collectionUuid.toBSON().firstElement() + << "indexName" + << indexName) + << "operationType" + << "create" + << "infoObj" + << BSONObj())}); +} + +TEST_F( + RollbackFixUpInfoTest, + ProcessCreateIndexOplogEntryReplacesExistingDocumentAndReturnsFailedToParseErrorWhenExistingDocumentContainsUnrecognizedOperationType) { + + auto collectionUuid = UUID::gen(); + std::string indexName = "b_1"; + + auto opCtx = makeOpCtx(); + RollbackFixUpInfo rollbackFixUpInfo(_storageInterface.get()); + + auto malformedDoc = + BSON("_id" << BSON("collectionUuid" << collectionUuid.toBSON().firstElement() << "indexName" + << indexName) + << "operationType" + << "unknownIndexOpType" + << "infoObj" + << BSON("expireAfterSeconds" << 60)); + ASSERT_OK(_storageInterface->upsertById(opCtx.get(), + RollbackFixUpInfo::kRollbackIndexNamespace, + malformedDoc["_id"], + malformedDoc)); + _assertDocumentsInCollectionEquals( + opCtx.get(), RollbackFixUpInfo::kRollbackIndexNamespace, {malformedDoc}); + + // Process createIndex. This should log an error when checking the operation type on the + // existing document. The malformed document should be replaced. + ASSERT_OK( + rollbackFixUpInfo.processCreateIndexOplogEntry(opCtx.get(), collectionUuid, indexName)); + + auto expectedDocument = + BSON("_id" << BSON("collectionUuid" << collectionUuid.toBSON().firstElement() << "indexName" + << indexName) + << "operationType" + << "create" + << "infoObj" + << BSONObj()); + + _assertDocumentsInCollectionEquals( + opCtx.get(), RollbackFixUpInfo::kRollbackIndexNamespace, {expectedDocument}); +} + +TEST_F( + RollbackFixUpInfoTest, + ProcessUpdateIndexTTLOplogEntryInsertsDocumentIntoRollbackIndexCollectionWithPartialInfoObj) { + auto operation = BSON("ts" << Timestamp(Seconds(1), 0) << "h" << 1LL << "op" + << "c" + << "ns" + << "mydb.$cmd" + << "ui" + << UUID::gen().toBSON().firstElement() + << "o" + << BSON("collMod" + << "mycoll" + << "index" + << BSON("name" + << "b_1" + << "expireAfterSeconds" + << 120)) + << "o2" + << BSON("expireAfterSeconds_before" << 60)); + auto collectionUuid = unittest::assertGet(UUID::parse(operation["ui"])); + auto indexName = operation["o"].Obj().firstElement().String(); + auto expireAfterSeconds = + mongo::Seconds(operation["o2"].Obj()["expireAfterSeconds_before"].numberLong()); + auto infoObj = BSON("expireAfterSeconds" << durationCount(expireAfterSeconds)); + + ASSERT_TRUE(OplogEntry(operation).isCommand()); + + auto opCtx = makeOpCtx(); + RollbackFixUpInfo rollbackFixUpInfo(_storageInterface.get()); + ASSERT_OK(rollbackFixUpInfo.processUpdateIndexTTLOplogEntry( + opCtx.get(), collectionUuid, indexName, expireAfterSeconds)); + + auto expectedDocument = + BSON("_id" << BSON("collectionUuid" << collectionUuid.toBSON().firstElement() << "indexName" + << indexName) + << "operationType" + << "updateTTL" + << "infoObj" + << infoObj); + + _assertDocumentsInCollectionEquals( + opCtx.get(), RollbackFixUpInfo::kRollbackIndexNamespace, {expectedDocument}); +} + +TEST_F( + RollbackFixUpInfoTest, + ProcessUpdateIndexTTLOplogEntryWhenExistingDocumentHasDropOpTypeUpdatesExpirationInExistingDocument) { + auto collectionUuid = UUID::gen(); + NamespaceString nss("mydb.mycoll"); + std::string indexName = "b_1"; + + // First populate collection with document with optype 'drop' and an indexinfo obj + // describing a TTL index with a expiration of 120 seconds. + // This document is the result of processing a dropIndexes oplog entry as we start rollback. + auto infoObj = + BSON("v" << 2 << "key" << BSON("b" << 1) << "name" << indexName << "ns" << nss.ns() + << "expireAfterSeconds" + << 120); + + auto opCtx = makeOpCtx(); + RollbackFixUpInfo rollbackFixUpInfo(_storageInterface.get()); + ASSERT_OK(rollbackFixUpInfo.processDropIndexOplogEntry( + opCtx.get(), collectionUuid, indexName, infoObj)); + + // Process a collMod oplog entry that changes the expiration from 60 seconds to 120 seconds. + // Chronologically, this operation happens before the dropIndexes command but since oplog + // entries are processed in reverse order, we process the collMod operation after dropIndexes. + // We provide the previous 'expireAfterSeconds' value (60 seconds) to + // processUpdateTTLOplogEntry(). + ASSERT_OK(rollbackFixUpInfo.processUpdateIndexTTLOplogEntry( + opCtx.get(), collectionUuid, indexName, Seconds(60))); + + // Expected index info obj is the same as 'infoObj' except for the 'expireAfterSeconds' field + // which should reflect the TTL expiration passed to processUpdateIndexTTLOplogEntry(). + BSONObjBuilder bob; + for (const auto& elt : infoObj) { + if ("expireAfterSeconds"_sd == elt.fieldNameStringData()) { + bob.append("expireAfterSeconds", 60); + } else { + bob.append(elt); + } + } + auto expectedInfoObj = bob.obj(); + + auto expectedDocument = + BSON("_id" << BSON("collectionUuid" << collectionUuid.toBSON().firstElement() << "indexName" + << indexName) + << "operationType" + << "drop" + << "infoObj" + << expectedInfoObj); + + _assertDocumentsInCollectionEquals( + opCtx.get(), RollbackFixUpInfo::kRollbackIndexNamespace, {expectedDocument}); +} + +TEST_F( + RollbackFixUpInfoTest, + ProcessUpdateIndexTTLOplogEntryWhenExistingDocumentHasUpdateTTLOpTypeReplacesExistingDocument) { + auto collectionUuid = UUID::gen(); + std::string indexName = "b_1"; + + // First, process a collMod oplog entry to populate the collection with document with optype + // 'updateTTL' and an expiration of 120 seconds. 120 seconds is the expiration of the TTL index + // BEFORE the oplog entry was applied and is what goes into the rollback fix up info. + auto opCtx = makeOpCtx(); + RollbackFixUpInfo rollbackFixUpInfo(_storageInterface.get()); + ASSERT_OK(rollbackFixUpInfo.processUpdateIndexTTLOplogEntry( + opCtx.get(), collectionUuid, indexName, Seconds(120))); + + // Process a second collMod oplog entry that changes the expiration from 60 seconds to 120 + // seconds. + // This should simply update the expiration in the existing "updateTTL" document in the + // "kRollbackIndexNamespace" collection. We provide the previous 'expireAfterSeconds' value + // (60 seconds) to processUpdateTTLOplogEntry(). + ASSERT_OK(rollbackFixUpInfo.processUpdateIndexTTLOplogEntry( + opCtx.get(), collectionUuid, indexName, Seconds(60))); + + auto expectedDocument = + BSON("_id" << BSON("collectionUuid" << collectionUuid.toBSON().firstElement() << "indexName" + << indexName) + << "operationType" + << "updateTTL" + << "infoObj" + << BSON("expireAfterSeconds" << 60)); + + _assertDocumentsInCollectionEquals( + opCtx.get(), RollbackFixUpInfo::kRollbackIndexNamespace, {expectedDocument}); +} + +TEST_F(RollbackFixUpInfoTest, + ProcessDropIndexOplogEntryInsertsDocumentIntoRollbackIndexCollectionWithCompleteInfoObj) { + auto operation = BSON("ts" << Timestamp(Seconds(1), 0) << "h" << 1LL << "op" + << "c" + << "ns" + << "mydb.$cmd" + << "ui" + << UUID::gen().toBSON().firstElement() + << "o" + << BSON("dropIndexes" + << "mycoll" + << "index" + << "b_1") + << "o2" + << BSON("v" << 2 << "key" << BSON("b" << 1) << "name" + << "b_1" + << "ns" + << "mydb.mycoll" + << "expireAfterSeconds" + << 120)); + auto collectionUuid = unittest::assertGet(UUID::parse(operation["ui"])); + auto indexName = operation["o"].Obj()["index"].String(); + auto infoObj = operation["o2"].Obj(); + + ASSERT_TRUE(OplogEntry(operation).isCommand()); + + auto opCtx = makeOpCtx(); + RollbackFixUpInfo rollbackFixUpInfo(_storageInterface.get()); + ASSERT_OK(rollbackFixUpInfo.processDropIndexOplogEntry( + opCtx.get(), collectionUuid, indexName, infoObj)); + + auto expectedDocument = + BSON("_id" << BSON("collectionUuid" << collectionUuid.toBSON().firstElement() << "indexName" + << indexName) + << "operationType" + << "drop" + << "infoObj" + << infoObj); + + _assertDocumentsInCollectionEquals( + opCtx.get(), RollbackFixUpInfo::kRollbackIndexNamespace, {expectedDocument}); +} + +TEST_F(RollbackFixUpInfoTest, + ProcessDropIndexOplogEntryWhenExistingDocumentHasCreateOpTypeReplacesExistingDocument) { + + // State of oplog: + // {dropIndexes: indexA}, ...., {createIndex: indexA}, .... + // (earliest optime) ---> (latest optime) + // + // Oplog entries are processed in reverse optime order. + + // First, process createIndex. This should insert a document into the collection with a 'create' + // op type. + auto collectionUuid = UUID::gen(); + std::string indexName = "b_1"; + auto infoObj = BSON("v" << 2 << "key" << BSON("b" << 1) << "name" << indexName << "ns" + << "mydb.mycoll"); + + auto opCtx = makeOpCtx(); + RollbackFixUpInfo rollbackFixUpInfo(_storageInterface.get()); + ASSERT_OK( + rollbackFixUpInfo.processCreateIndexOplogEntry(opCtx.get(), collectionUuid, indexName)); + _assertDocumentsInCollectionEquals( + opCtx.get(), + RollbackFixUpInfo::kRollbackIndexNamespace, + {BSON("_id" << BSON("collectionUuid" << collectionUuid.toBSON().firstElement() + << "indexName" + << indexName) + << "operationType" + << "create" + << "infoObj" + << BSONObj())}); + + // Next, process dropIndexes. This should replace the existing 'create' operation with an entry + // with the 'drop' operation type. When fixing up the indexes for the 'drop' (ie. we need to + // re-create the index), we would have to drop any existing indexes in the collection with the + // same name before proceeding with the index creation + ASSERT_OK(rollbackFixUpInfo.processDropIndexOplogEntry( + opCtx.get(), collectionUuid, indexName, infoObj)); + _assertDocumentsInCollectionEquals( + opCtx.get(), + RollbackFixUpInfo::kRollbackIndexNamespace, + {BSON("_id" << BSON("collectionUuid" << collectionUuid.toBSON().firstElement() + << "indexName" + << indexName) + << "operationType" + << "drop" + << "infoObj" + << infoObj)}); +} + } // namespace -- cgit v1.2.1