diff options
author | Daniel Gottlieb <daniel.gottlieb@mongodb.com> | 2020-04-24 09:49:31 -0400 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2020-04-24 21:03:02 +0000 |
commit | 02e0022fe9ce0addbe5add0101e6fd1a555437c3 (patch) | |
tree | 7fae33334aac57da1037dfe53e1894b43cefe29e | |
parent | b89e5a3da73fa8ade2fba3331b833f28fbfde003 (diff) | |
download | mongo-02e0022fe9ce0addbe5add0101e6fd1a555437c3.tar.gz |
SERVER-47658: Improve validate diagnostics, with a focus on multikey failures.
(cherry picked from commit a89edc5d9e2908abce696932462c118d8ecf3345)
-rw-r--r-- | jstests/hooks/run_validate_collections_background.js | 2 | ||||
-rw-r--r-- | src/mongo/db/catalog/collection_validation.cpp | 9 | ||||
-rw-r--r-- | src/mongo/db/catalog/validate_adaptor.cpp | 6 | ||||
-rw-r--r-- | src/mongo/db/catalog/validate_state.cpp | 1 | ||||
-rw-r--r-- | src/mongo/db/catalog/validate_state.h | 6 | ||||
-rw-r--r-- | src/mongo/db/commands/validate.cpp | 3 | ||||
-rw-r--r-- | src/mongo/db/storage/record_store.h | 6 |
7 files changed, 25 insertions, 8 deletions
diff --git a/jstests/hooks/run_validate_collections_background.js b/jstests/hooks/run_validate_collections_background.js index 89c8f2fcc21..9e0ca571d9b 100644 --- a/jstests/hooks/run_validate_collections_background.js +++ b/jstests/hooks/run_validate_collections_background.js @@ -90,6 +90,7 @@ const validateCollectionsBackgroundThread = function validateCollectionsBackgrou return z.name; }); + conn.adminCommand({configureFailPoint: "crashOnMultikeyValidateFailure", mode: "alwaysOn"}); for (let dbName of dbNames) { let db = conn.getDB(dbName); @@ -115,6 +116,7 @@ const validateCollectionsBackgroundThread = function validateCollectionsBackgrou } } } + conn.adminCommand({configureFailPoint: "crashOnMultikeyValidateFailure", mode: "off"}); // If any commands failed, format and return an error. if (failedValidateResults.length) { diff --git a/src/mongo/db/catalog/collection_validation.cpp b/src/mongo/db/catalog/collection_validation.cpp index 76e04d45bd6..bdf944ce6ed 100644 --- a/src/mongo/db/catalog/collection_validation.cpp +++ b/src/mongo/db/catalog/collection_validation.cpp @@ -174,10 +174,9 @@ void _validateIndexes(OperationContext* opCtx, } } - if (curIndexResults.valid) { - keysPerIndex->appendNumber(descriptor->indexName(), - static_cast<long long>(numTraversedKeys)); - } else { + keysPerIndex->appendNumber(descriptor->indexName(), + static_cast<long long>(numTraversedKeys)); + if (!curIndexResults.valid) { results->valid = false; } } @@ -261,6 +260,8 @@ void _reportValidationResults(OperationContext* opCtx, BSONObjBuilder* output) { std::unique_ptr<BSONObjBuilder> indexDetails; + results->readTimestamp = validateState->getValidateTimestamp(); + if (validateState->isFullIndexValidation()) { invariant(opCtx->lockState()->isCollectionLockedForMode(validateState->nss(), MODE_X)); indexDetails = std::make_unique<BSONObjBuilder>(); diff --git a/src/mongo/db/catalog/validate_adaptor.cpp b/src/mongo/db/catalog/validate_adaptor.cpp index fad690d0834..e8cc65d9d5c 100644 --- a/src/mongo/db/catalog/validate_adaptor.cpp +++ b/src/mongo/db/catalog/validate_adaptor.cpp @@ -49,11 +49,14 @@ #include "mongo/db/storage/record_store.h" #include "mongo/logv2/log.h" #include "mongo/rpc/object_check.h" +#include "mongo/util/fail_point.h" namespace mongo { namespace { +MONGO_FAIL_POINT_DEFINE(crashOnMultikeyValidateFailure); + const long long kInterruptIntervalNumRecords = 4096; const long long kInterruptIntervalNumBytes = 50 * 1024 * 1024; // 50MB. @@ -124,6 +127,9 @@ Status ValidateAdaptor::validateRecord(OperationContext* opCtx, ValidateResults& curRecordResults = (*_indexNsResultsMap)[descriptor->indexName()]; curRecordResults.errors.push_back(msg); curRecordResults.valid = false; + if (crashOnMultikeyValidateFailure.shouldFail()) { + invariant(false, msg); + } } if (descriptor->isMultikey()) { diff --git a/src/mongo/db/catalog/validate_state.cpp b/src/mongo/db/catalog/validate_state.cpp index 7eead34d9a5..c2e4d16b73e 100644 --- a/src/mongo/db/catalog/validate_state.cpp +++ b/src/mongo/db/catalog/validate_state.cpp @@ -180,6 +180,7 @@ void ValidateState::initializeCursors(OperationContext* opCtx) { opCtx, _collection->getRecordStore(), &_dataThrottle); _seekRecordStoreCursor = std::make_unique<SeekableRecordThrottleCursor>( opCtx, _collection->getRecordStore(), &_dataThrottle); + _validateTs = opCtx->recoveryUnit()->getPointInTimeReadTimestamp(); const IndexCatalog* indexCatalog = _collection->getIndexCatalog(); // The index iterator for ready indexes is timestamp-aware and will only return indexes that diff --git a/src/mongo/db/catalog/validate_state.h b/src/mongo/db/catalog/validate_state.h index 258fa6ca4c7..da58dc0b51f 100644 --- a/src/mongo/db/catalog/validate_state.h +++ b/src/mongo/db/catalog/validate_state.h @@ -146,6 +146,10 @@ public: return _extraLoggingForTest; } + boost::optional<Timestamp> getValidateTimestamp() { + return _validateTs; + } + private: ValidateState() = delete; @@ -213,6 +217,8 @@ private: // Can be set by unit tests to obtain better insight into what validate sees/does. bool _extraLoggingForTest; + + boost::optional<Timestamp> _validateTs = boost::none; }; } // namespace CollectionValidation diff --git a/src/mongo/db/commands/validate.cpp b/src/mongo/db/commands/validate.cpp index 11c0b2515ad..c60c32076cc 100644 --- a/src/mongo/db/commands/validate.cpp +++ b/src/mongo/db/commands/validate.cpp @@ -183,6 +183,9 @@ public: } result.appendBool("valid", validateResults.valid); + if (validateResults.readTimestamp) { + result.append("readTimestamp", validateResults.readTimestamp.get()); + } result.append("warnings", validateResults.warnings); result.append("errors", validateResults.errors); result.append("extraIndexEntries", validateResults.extraIndexEntries); diff --git a/src/mongo/db/storage/record_store.h b/src/mongo/db/storage/record_store.h index 1b1e17d7aee..5ccf804a584 100644 --- a/src/mongo/db/storage/record_store.h +++ b/src/mongo/db/storage/record_store.h @@ -569,10 +569,8 @@ protected: }; struct ValidateResults { - ValidateResults() { - valid = true; - } - bool valid; + bool valid = true; + boost::optional<Timestamp> readTimestamp = boost::none; std::vector<std::string> errors; std::vector<std::string> warnings; std::vector<BSONObj> extraIndexEntries; |