diff options
24 files changed, 230 insertions, 174 deletions
diff --git a/src/mongo/db/catalog/SConscript b/src/mongo/db/catalog/SConscript index 9b593066c0d..4d8a586789f 100644 --- a/src/mongo/db/catalog/SConscript +++ b/src/mongo/db/catalog/SConscript @@ -373,6 +373,7 @@ env.Library( ], LIBDEPS=[ 'catalog_impl', + 'validate_results', ], LIBDEPS_PRIVATE=[ '$BUILD_DIR/mongo/db/multi_key_path_tracker', @@ -382,6 +383,16 @@ env.Library( ) env.Library( + target='validate_results', + source=[ + 'validate_results.cpp', + ], + LIBDEPS=[ + '$BUILD_DIR/mongo/base', + ] +) + +env.Library( target='throttle_cursor', source=[ 'throttle_cursor.cpp', diff --git a/src/mongo/db/catalog/collection_validation.cpp b/src/mongo/db/catalog/collection_validation.cpp index 808746a2088..a0a40e7152d 100644 --- a/src/mongo/db/catalog/collection_validation.cpp +++ b/src/mongo/db/catalog/collection_validation.cpp @@ -58,8 +58,6 @@ namespace CollectionValidation { namespace { -using ValidateResultsMap = std::map<string, ValidateResults>; - // Indicates whether the failpoint turned on by testing has been reached. AtomicWord<bool> _validationIsPausedForTest{false}; @@ -68,15 +66,10 @@ AtomicWord<bool> _validationIsPausedForTest{false}; * the index files have not been corrupted or compromised. * * May close or invalidate open cursors. - * - * Returns a map from indexName -> number of keys validated. */ -std::map<std::string, int64_t> _validateIndexesInternalStructure( - OperationContext* opCtx, - ValidateState* validateState, - ValidateResultsMap* indexNsResultsMap, - ValidateResults* results) { - std::map<std::string, int64_t> numIndexKeysPerIndex; +void _validateIndexesInternalStructure(OperationContext* opCtx, + ValidateState* validateState, + ValidateResults* results) { // Need to use the IndexCatalog here because the 'validateState->indexes' object hasn't been // constructed yet. It must be initialized to ensure we're validating all indexes. const IndexCatalog* indexCatalog = validateState->getCollection()->getIndexCatalog(); @@ -98,7 +91,7 @@ std::map<std::string, int64_t> _validateIndexesInternalStructure( "index"_attr = descriptor->indexName(), "namespace"_attr = validateState->nss()); - ValidateResults& curIndexResults = (*indexNsResultsMap)[descriptor->indexName()]; + auto& curIndexResults = (results->indexResultsMap)[descriptor->indexName()]; int64_t numValidated; iam->validate(opCtx, &numValidated, &curIndexResults); @@ -107,9 +100,8 @@ std::map<std::string, int64_t> _validateIndexesInternalStructure( results->valid = false; } - numIndexKeysPerIndex[descriptor->indexName()] = numValidated; + curIndexResults.keysTraversedFromFullValidate = numValidated; } - return numIndexKeysPerIndex; } /** @@ -120,10 +112,7 @@ std::map<std::string, int64_t> _validateIndexesInternalStructure( */ void _validateIndexes(OperationContext* opCtx, ValidateState* validateState, - BSONObjBuilder* keysPerIndex, ValidateAdaptor* indexValidator, - const std::map<std::string, int64_t>& numIndexKeysPerIndex, - ValidateResultsMap* indexNsResultsMap, ValidateResults* results) { // Validate Indexes, checking for mismatch between index entries and collection records. for (const auto& index : validateState->getIndexes()) { @@ -137,9 +126,11 @@ void _validateIndexes(OperationContext* opCtx, "index"_attr = descriptor->indexName(), "namespace"_attr = validateState->nss()); - ValidateResults& curIndexResults = (*indexNsResultsMap)[descriptor->indexName()]; int64_t numTraversedKeys; - indexValidator->traverseIndex(opCtx, index.get(), &numTraversedKeys, &curIndexResults); + indexValidator->traverseIndex(opCtx, index.get(), &numTraversedKeys, results); + + auto& curIndexResults = (results->indexResultsMap)[descriptor->indexName()]; + curIndexResults.keysTraversed = numTraversedKeys; // If we are performing a full index validation, we have information on the number of index // keys validated in _validateIndexesInternalStructure (when we validated the internal @@ -148,13 +139,9 @@ void _validateIndexes(OperationContext* opCtx, if (validateState->isFullIndexValidation()) { invariant(opCtx->lockState()->isCollectionLockedForMode(validateState->nss(), MODE_X)); - // Ensure that this index was validated in _validateIndexesInternalStructure. - const auto numIndexKeysIt = numIndexKeysPerIndex.find(descriptor->indexName()); - invariant(numIndexKeysIt != numIndexKeysPerIndex.end()); - // The number of keys counted in _validateIndexesInternalStructure, when checking the // internal structure of the index. - const int64_t numIndexKeys = numIndexKeysIt->second; + const int64_t numIndexKeys = curIndexResults.keysTraversedFromFullValidate; // Check if currIndexResults is valid to ensure that this index is not corrupted or // comprised (which was set in _validateIndexesInternalStructure). If the index is @@ -172,8 +159,6 @@ void _validateIndexes(OperationContext* opCtx, } } - keysPerIndex->appendNumber(descriptor->indexName(), - static_cast<long long>(numTraversedKeys)); if (!curIndexResults.valid) { results->valid = false; } @@ -188,7 +173,6 @@ void _gatherIndexEntryErrors(OperationContext* opCtx, ValidateState* validateState, IndexConsistency* indexConsistency, ValidateAdaptor* indexValidator, - ValidateResultsMap* indexNsResultsMap, ValidateResults* result) { indexConsistency->setSecondPhase(); if (!indexConsistency->limitMemoryUsageForSecondPhase(result)) { @@ -203,10 +187,8 @@ void _gatherIndexEntryErrors(OperationContext* opCtx, { ValidateResults tempValidateResults; BSONObjBuilder tempBuilder; - ValidateResultsMap tempIndexResultsMap; - indexValidator->traverseRecordStore( - opCtx, &tempValidateResults, &tempBuilder, &tempIndexResultsMap); + indexValidator->traverseRecordStore(opCtx, &tempValidateResults, &tempBuilder); } LOGV2_OPTIONS( @@ -243,7 +225,7 @@ void _gatherIndexEntryErrors(OperationContext* opCtx, LOGV2_OPTIONS(20301, {LogComponent::kIndex}, "Finished traversing through all the indexes"); - indexConsistency->addIndexEntryErrors(indexNsResultsMap, result); + indexConsistency->addIndexEntryErrors(result); } void _validateIndexKeyCount(OperationContext* opCtx, @@ -252,7 +234,7 @@ void _validateIndexKeyCount(OperationContext* opCtx, ValidateResultsMap* indexNsResultsMap) { for (const auto& index : validateState->getIndexes()) { const IndexDescriptor* descriptor = index->descriptor(); - ValidateResults& curIndexResults = (*indexNsResultsMap)[descriptor->indexName()]; + auto& curIndexResults = (*indexNsResultsMap)[descriptor->indexName()]; if (curIndexResults.valid) { indexValidator->validateIndexKeyCount(index.get(), curIndexResults); @@ -262,8 +244,6 @@ void _validateIndexKeyCount(OperationContext* opCtx, void _reportValidationResults(OperationContext* opCtx, ValidateState* validateState, - ValidateResultsMap* indexNsResultsMap, - BSONObjBuilder* keysPerIndex, ValidateResults* results, BSONObjBuilder* output) { std::unique_ptr<BSONObjBuilder> indexDetails; @@ -275,15 +255,18 @@ void _reportValidationResults(OperationContext* opCtx, indexDetails = std::make_unique<BSONObjBuilder>(); } + BSONObjBuilder keysPerIndex; + // Report detailed index validation results gathered when using {full: true} for validated // indexes. for (const auto& index : validateState->getIndexes()) { const std::string indexName = index->descriptor()->indexName(); - if (indexNsResultsMap->find(indexName) == indexNsResultsMap->end()) { + auto& indexResultsMap = results->indexResultsMap; + if (indexResultsMap.find(indexName) == indexResultsMap.end()) { continue; } - const ValidateResults& vr = indexNsResultsMap->at(indexName); + auto& vr = indexResultsMap.at(indexName); if (!vr.valid) { results->valid = false; @@ -302,12 +285,14 @@ void _reportValidationResults(OperationContext* opCtx, } } + keysPerIndex.appendNumber(indexName, static_cast<long long>(vr.keysTraversed)); + results->warnings.insert(results->warnings.end(), vr.warnings.begin(), vr.warnings.end()); results->errors.insert(results->errors.end(), vr.errors.begin(), vr.errors.end()); } output->append("nIndexes", static_cast<int>(validateState->getIndexes().size())); - output->append("keysPerIndex", keysPerIndex->done()); + output->append("keysPerIndex", keysPerIndex.done()); if (indexDetails) { output->append("indexDetails", indexDetails->done()); } @@ -315,13 +300,10 @@ void _reportValidationResults(OperationContext* opCtx, void _reportInvalidResults(OperationContext* opCtx, ValidateState* validateState, - ValidateResultsMap* indexNsResultsMap, - BSONObjBuilder* keysPerIndex, ValidateResults* results, BSONObjBuilder* output, const string uuidString) { - _reportValidationResults( - opCtx, validateState, indexNsResultsMap, keysPerIndex, results, output); + _reportValidationResults(opCtx, validateState, results, output); LOGV2_OPTIONS(20302, {LogComponent::kIndex}, "Validation complete -- Corruption found", @@ -453,10 +435,6 @@ Status validate(OperationContext* opCtx, opCtx, nss, ReadPreferenceSetting::get(opCtx).canRunOnSecondary())); try { - std::map<std::string, int64_t> numIndexKeysPerIndex; - ValidateResultsMap indexNsResultsMap; - BSONObjBuilder keysPerIndex; // not using subObjStart to be exception safe. - // Full record store validation code is executed before we open cursors because it may close // and/or invalidate all open cursors. if (validateState.isFullValidation()) { @@ -471,20 +449,13 @@ Status validate(OperationContext* opCtx, // For full index validation, we validate the internal structure of each index and save // the number of keys in the index to compare against _validateIndexes()'s count // results. - numIndexKeysPerIndex = _validateIndexesInternalStructure( - opCtx, &validateState, &indexNsResultsMap, results); + _validateIndexesInternalStructure(opCtx, &validateState, results); } const string uuidString = str::stream() << " (UUID: " << validateState.uuid() << ")"; if (!results->valid) { - _reportInvalidResults(opCtx, - &validateState, - &indexNsResultsMap, - &keysPerIndex, - results, - output, - uuidString); + _reportInvalidResults(opCtx, &validateState, results, output, uuidString); return Status::OK(); } @@ -508,7 +479,7 @@ Status validate(OperationContext* opCtx, // In traverseRecordStore(), the index validator keeps track the records in the record // store so that _validateIndexes() can confirm that the index entries match the records in // the collection. - indexValidator.traverseRecordStore(opCtx, results, output, &indexNsResultsMap); + indexValidator.traverseRecordStore(opCtx, results, output); // Pause collection validation while a lock is held and between collection and index data // validation. @@ -529,24 +500,12 @@ Status validate(OperationContext* opCtx, } if (!results->valid) { - _reportInvalidResults(opCtx, - &validateState, - &indexNsResultsMap, - &keysPerIndex, - results, - output, - uuidString); + _reportInvalidResults(opCtx, &validateState, results, output, uuidString); return Status::OK(); } // Validate indexes and check for mismatches. - _validateIndexes(opCtx, - &validateState, - &keysPerIndex, - &indexValidator, - numIndexKeysPerIndex, - &indexNsResultsMap, - results); + _validateIndexes(opCtx, &validateState, &indexValidator, results); if (indexConsistency.haveEntryMismatch()) { LOGV2_OPTIONS(20305, @@ -554,43 +513,26 @@ Status validate(OperationContext* opCtx, "Index inconsistencies were detected. " "Starting the second phase of index validation to gather concise errors", "namespace"_attr = validateState.nss()); - _gatherIndexEntryErrors(opCtx, - &validateState, - &indexConsistency, - &indexValidator, - &indexNsResultsMap, - results); + _gatherIndexEntryErrors( + opCtx, &validateState, &indexConsistency, &indexValidator, results); } if (!results->valid) { - _reportInvalidResults(opCtx, - &validateState, - &indexNsResultsMap, - &keysPerIndex, - results, - output, - uuidString); + _reportInvalidResults(opCtx, &validateState, results, output, uuidString); return Status::OK(); } // Validate index key count. - _validateIndexKeyCount(opCtx, &validateState, &indexValidator, &indexNsResultsMap); + _validateIndexKeyCount(opCtx, &validateState, &indexValidator, &results->indexResultsMap); if (!results->valid) { - _reportInvalidResults(opCtx, - &validateState, - &indexNsResultsMap, - &keysPerIndex, - results, - output, - uuidString); + _reportInvalidResults(opCtx, &validateState, results, output, uuidString); return Status::OK(); } // At this point, validation is complete and successful. // Report the validation results for the user to see. - _reportValidationResults( - opCtx, &validateState, &indexNsResultsMap, &keysPerIndex, results, output); + _reportValidationResults(opCtx, &validateState, results, output); LOGV2_OPTIONS(20306, {LogComponent::kIndex}, diff --git a/src/mongo/db/catalog/collection_validation.h b/src/mongo/db/catalog/collection_validation.h index 22e7885f949..e5ac3d1329e 100644 --- a/src/mongo/db/catalog/collection_validation.h +++ b/src/mongo/db/catalog/collection_validation.h @@ -29,13 +29,13 @@ #pragma once +#include "mongo/db/catalog/validate_results.h" #include "mongo/db/namespace_string.h" namespace mongo { class OperationContext; class Collection; -struct ValidateResults; class BSONObjBuilder; class Status; diff --git a/src/mongo/db/catalog/index_consistency.cpp b/src/mongo/db/catalog/index_consistency.cpp index e796600e4c5..34b06127b64 100644 --- a/src/mongo/db/catalog/index_consistency.cpp +++ b/src/mongo/db/catalog/index_consistency.cpp @@ -167,6 +167,8 @@ void IndexConsistency::repairMissingIndexEntries(OperationContext* opCtx, // InsertKeys may fail in the scenario where there are missing index entries for duplicate // documents. if (numInserted > 0) { + auto& indexResults = results->indexResultsMap[indexName]; + indexResults.keysTraversed += numInserted; results->numInsertedMissingIndexEntries += numInserted; results->repaired = true; getIndexInfo(indexName).numKeys += numInserted; @@ -183,8 +185,7 @@ void IndexConsistency::repairMissingIndexEntries(OperationContext* opCtx, } } -void IndexConsistency::addIndexEntryErrors(ValidateResultsMap* indexNsResultsMap, - ValidateResults* results) { +void IndexConsistency::addIndexEntryErrors(ValidateResults* results) { invariant(!_firstPhase); // We'll report up to 1MB for extra index entry errors and missing index entry errors. @@ -224,7 +225,7 @@ void IndexConsistency::addIndexEntryErrors(ValidateResultsMap* indexNsResultsMap } std::string indexName = entry["indexName"].String(); - if (!indexNsResultsMap->at(indexName).valid) { + if (!results->indexResultsMap.at(indexName).valid) { continue; } @@ -232,7 +233,7 @@ void IndexConsistency::addIndexEntryErrors(ValidateResultsMap* indexNsResultsMap ss << "Index with name '" << indexName << "' has inconsistencies."; results->errors.push_back(ss.str()); - indexNsResultsMap->at(indexName).valid = false; + results->indexResultsMap.at(indexName).valid = false; } bool extraIndexEntrySizeLimitWarning = false; @@ -252,7 +253,7 @@ void IndexConsistency::addIndexEntryErrors(ValidateResultsMap* indexNsResultsMap } std::string indexName = entry["indexName"].String(); - if (!indexNsResultsMap->at(indexName).valid) { + if (!results->indexResultsMap.at(indexName).valid) { continue; } @@ -260,7 +261,7 @@ void IndexConsistency::addIndexEntryErrors(ValidateResultsMap* indexNsResultsMap ss << "Index with name '" << indexName << "' has inconsistencies."; results->errors.push_back(ss.str()); - indexNsResultsMap->at(indexName).valid = false; + results->indexResultsMap.at(indexName).valid = false; } } @@ -368,6 +369,8 @@ void IndexConsistency::addIndexKey(OperationContext* opCtx, opCtx, {ks}, recordId, options, &numDeleted); wunit.commit(); }); + auto& indexResults = results->indexResultsMap[indexInfo->indexName]; + indexResults.keysTraversed -= numDeleted; results->numRemovedExtraIndexEntries += numDeleted; results->repaired = true; indexInfo->numKeys--; diff --git a/src/mongo/db/catalog/index_consistency.h b/src/mongo/db/catalog/index_consistency.h index c0b6ed48128..19e575f4616 100644 --- a/src/mongo/db/catalog/index_consistency.h +++ b/src/mongo/db/catalog/index_consistency.h @@ -89,7 +89,6 @@ struct IndexEntryInfo { */ class IndexConsistency final { using IndexInfoMap = std::map<std::string, IndexInfo>; - using ValidateResultsMap = std::map<std::string, ValidateResults>; using IndexKey = std::pair<std::string, std::string>; public: @@ -159,7 +158,7 @@ public: * Records the errors gathered from the second phase of index validation into the provided * ValidateResultsMap and ValidateResults. */ - void addIndexEntryErrors(ValidateResultsMap* indexNsResultsMap, ValidateResults* results); + void addIndexEntryErrors(ValidateResults* results); /** * Sets up this IndexConsistency object to limit memory usage in the second phase of index diff --git a/src/mongo/db/catalog/validate_adaptor.cpp b/src/mongo/db/catalog/validate_adaptor.cpp index c70452f0504..d1ac9037276 100644 --- a/src/mongo/db/catalog/validate_adaptor.cpp +++ b/src/mongo/db/catalog/validate_adaptor.cpp @@ -70,8 +70,7 @@ Status ValidateAdaptor::validateRecord(OperationContext* opCtx, const RecordId& recordId, const RecordData& record, size_t* dataSize, - ValidateResults* results, - ValidateResultsMap* indexNsResultsMap) { + ValidateResults* results) { const Status status = validateBSON(record.data(), record.size()); if (!status.isOK()) return status; @@ -132,7 +131,7 @@ Status ValidateAdaptor::validateRecord(OperationContext* opCtx, << " set to multikey."); results->repaired = true; } else { - ValidateResults& curRecordResults = (*indexNsResultsMap)[descriptor->indexName()]; + auto& curRecordResults = (results->indexResultsMap)[descriptor->indexName()]; std::string msg = str::stream() << "Index " << descriptor->indexName() << " is not multikey but has more than one" << " key in document " << recordId; @@ -166,8 +165,7 @@ Status ValidateAdaptor::validateRecord(OperationContext* opCtx, std::string msg = str::stream() << "Index " << descriptor->indexName() << " multikey paths do not cover a document. RecordId: " << recordId; - ValidateResults& curRecordResults = - (*indexNsResultsMap)[descriptor->indexName()]; + auto& curRecordResults = (results->indexResultsMap)[descriptor->indexName()]; curRecordResults.errors.push_back(msg); curRecordResults.valid = false; } @@ -201,7 +199,7 @@ void _validateKeyOrder(OperationContext* opCtx, const IndexCatalogEntry* index, const KeyString::Value& currKey, const KeyString::Value& prevKey, - ValidateResults* results) { + IndexValidateResults* results) { auto descriptor = index->descriptor(); bool unique = descriptor->unique(); @@ -250,6 +248,7 @@ void ValidateAdaptor::traverseIndex(OperationContext* opCtx, ValidateResults* results) { const IndexDescriptor* descriptor = index->descriptor(); auto indexName = descriptor->indexName(); + auto& indexResults = results->indexResultsMap[indexName]; IndexInfo& indexInfo = _indexConsistency->getIndexInfo(indexName); int64_t numKeys = 0; @@ -286,7 +285,7 @@ void ValidateAdaptor::traverseIndex(OperationContext* opCtx, if (!isFirstEntry) { _validateKeyOrder( - opCtx, index, indexEntry->keyString, prevIndexKeyStringValue, results); + opCtx, index, indexEntry->keyString, prevIndexKeyStringValue, &indexResults); } const RecordId kWildcardMultikeyMetadataRecordId{ @@ -327,8 +326,7 @@ void ValidateAdaptor::traverseIndex(OperationContext* opCtx, void ValidateAdaptor::traverseRecordStore(OperationContext* opCtx, ValidateResults* results, - BSONObjBuilder* output, - ValidateResultsMap* indexNsResultsMap) { + BSONObjBuilder* output) { _numRecords = 0; // need to reset it because this function can be called more than once. long long dataSizeTotal = 0; long long interruptIntervalNumBytes = 0; @@ -366,8 +364,7 @@ void ValidateAdaptor::traverseRecordStore(OperationContext* opCtx, interruptIntervalNumBytes += dataSize; dataSizeTotal += dataSize; size_t validatedSize = 0; - Status status = validateRecord( - opCtx, record->id, record->data, &validatedSize, results, indexNsResultsMap); + Status status = validateRecord(opCtx, record->id, record->data, &validatedSize, results); // Checks to ensure isInRecordIdOrder() is being used properly. if (prevRecordId.isValid()) { @@ -463,7 +460,7 @@ void ValidateAdaptor::traverseRecordStore(OperationContext* opCtx, } void ValidateAdaptor::validateIndexKeyCount(const IndexCatalogEntry* index, - ValidateResults& results) { + IndexValidateResults& results) { // Fetch the total number of index entries we previously found traversing the index. const IndexDescriptor* desc = index->descriptor(); const std::string indexName = desc->indexName(); diff --git a/src/mongo/db/catalog/validate_adaptor.h b/src/mongo/db/catalog/validate_adaptor.h index 04d45e66d0f..db3d7d61939 100644 --- a/src/mongo/db/catalog/validate_adaptor.h +++ b/src/mongo/db/catalog/validate_adaptor.h @@ -43,8 +43,6 @@ class OperationContext; * collection validation operation. */ class ValidateAdaptor { - using ValidateResultsMap = std::map<std::string, ValidateResults>; - public: ValidateAdaptor(IndexConsistency* indexConsistency, CollectionValidation::ValidateState* validateState) @@ -59,8 +57,7 @@ public: const RecordId& recordId, const RecordData& record, size_t* dataSize, - ValidateResults* results, - ValidateResultsMap* indexNsResultsMap); + ValidateResults* results); /** * Traverses the index getting index entries to validate them and keep track of the index keys @@ -77,14 +74,13 @@ public: */ void traverseRecordStore(OperationContext* opCtx, ValidateResults* results, - BSONObjBuilder* output, - ValidateResultsMap* indexNsResultsMap); + BSONObjBuilder* output); /** * Validates that the number of document keys matches the number of index keys previously * traversed in traverseIndex(). */ - void validateIndexKeyCount(const IndexCatalogEntry* index, ValidateResults& results); + void validateIndexKeyCount(const IndexCatalogEntry* index, IndexValidateResults& results); private: IndexConsistency* _indexConsistency; diff --git a/src/mongo/db/catalog/validate_results.cpp b/src/mongo/db/catalog/validate_results.cpp new file mode 100644 index 00000000000..2c3c9a4b820 --- /dev/null +++ b/src/mongo/db/catalog/validate_results.cpp @@ -0,0 +1,58 @@ +/** + * Copyright (C) 2020-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/catalog/validate_results.h" + +namespace mongo { + +void ValidateResults::appendToResultObj(BSONObjBuilder& resultObj, bool debugging) const { + resultObj.appendBool("valid", valid); + resultObj.appendBool("repaired", repaired); + if (readTimestamp) { + resultObj.append("readTimestamp", readTimestamp.get()); + } + resultObj.append("warnings", warnings); + resultObj.append("errors", errors); + resultObj.append("extraIndexEntries", extraIndexEntries); + resultObj.append("missingIndexEntries", missingIndexEntries); + + // Need to convert RecordId to int64_t to append to BSONObjBuilder + BSONArrayBuilder builder; + for (RecordId corruptRecord : corruptRecords) { + builder.append(corruptRecord.repr()); + } + resultObj.append("corruptRecords", builder.done()); + + if (repaired || debugging) { + resultObj.appendNumber("numRemovedCorruptRecords", numRemovedCorruptRecords); + resultObj.appendNumber("numRemovedExtraIndexEntries", numRemovedExtraIndexEntries); + resultObj.appendNumber("numInsertedMissingIndexEntries", numInsertedMissingIndexEntries); + } +} +} // namespace mongo diff --git a/src/mongo/db/catalog/validate_results.h b/src/mongo/db/catalog/validate_results.h new file mode 100644 index 00000000000..96752f70877 --- /dev/null +++ b/src/mongo/db/catalog/validate_results.h @@ -0,0 +1,74 @@ +/** + * Copyright (C) 2020-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include <map> +#include <string> +#include <vector> + +#include "mongo/bson/bsonobjbuilder.h" +#include "mongo/db/record_id.h" + +namespace mongo { + +// Per-index validate results. +struct IndexValidateResults { + bool valid = true; + std::vector<std::string> errors; + std::vector<std::string> warnings; + int64_t keysTraversed = 0; + int64_t keysTraversedFromFullValidate = 0; +}; + +using ValidateResultsMap = std::map<std::string, IndexValidateResults>; + +// Validation results for an entire collection. +struct ValidateResults { + bool valid = true; + bool repaired = false; + boost::optional<Timestamp> readTimestamp = boost::none; + std::vector<std::string> errors; + std::vector<std::string> warnings; + std::vector<BSONObj> extraIndexEntries; + std::vector<BSONObj> missingIndexEntries; + std::vector<RecordId> corruptRecords; + long long numRemovedCorruptRecords = 0; + long long numRemovedExtraIndexEntries = 0; + long long numInsertedMissingIndexEntries = 0; + + // Maps index names to index-specific validation results. + ValidateResultsMap indexResultsMap; + + // Takes a bool that indicates the context of the caller and a BSONObjBuilder to append with + // validate results. + void appendToResultObj(BSONObjBuilder& resultObj, bool debugging) const; +}; + +} // namespace mongo diff --git a/src/mongo/db/index/index_access_method.cpp b/src/mongo/db/index/index_access_method.cpp index 94569772cd4..c2a07973796 100644 --- a/src/mongo/db/index/index_access_method.cpp +++ b/src/mongo/db/index/index_access_method.cpp @@ -281,7 +281,7 @@ RecordId AbstractIndexAccessMethod::findSingle(OperationContext* opCtx, void AbstractIndexAccessMethod::validate(OperationContext* opCtx, int64_t* numKeys, - ValidateResults* fullResults) const { + IndexValidateResults* fullResults) const { long long keys = 0; _newInterface->fullValidate(opCtx, &keys, fullResults); *numKeys = keys; diff --git a/src/mongo/db/index/index_access_method.h b/src/mongo/db/index/index_access_method.h index 81e4df363de..c2c3c1e47ca 100644 --- a/src/mongo/db/index/index_access_method.h +++ b/src/mongo/db/index/index_access_method.h @@ -167,7 +167,7 @@ public: */ virtual void validate(OperationContext* opCtx, int64_t* numKeys, - ValidateResults* fullResults) const = 0; + IndexValidateResults* fullResults) const = 0; /** * Add custom statistics about this index to BSON object builder, for display. @@ -486,7 +486,7 @@ public: void validate(OperationContext* opCtx, int64_t* numKeys, - ValidateResults* fullResults) const final; + IndexValidateResults* fullResults) const final; bool appendCustomStats(OperationContext* opCtx, BSONObjBuilder* result, diff --git a/src/mongo/db/repair.cpp b/src/mongo/db/repair.cpp index 16dbf22f548..b892165cd44 100644 --- a/src/mongo/db/repair.cpp +++ b/src/mongo/db/repair.cpp @@ -250,7 +250,14 @@ Status repairCollection(OperationContext* opCtx, return status; } - LOGV2(21028, "Collection validation", "results"_attr = output.done()); + BSONObjBuilder detailedResults; + const bool debug = false; + validateResults.appendToResultObj(detailedResults, debug); + + LOGV2(21028, + "Collection validation", + "results"_attr = output.done(), + "detailedResults"_attr = detailedResults.done()); if (validateResults.repaired) { if (validateResults.valid) { diff --git a/src/mongo/db/repl/storage_interface_impl_test.cpp b/src/mongo/db/repl/storage_interface_impl_test.cpp index 54a2661b70c..3ecdc4e68ed 100644 --- a/src/mongo/db/repl/storage_interface_impl_test.cpp +++ b/src/mongo/db/repl/storage_interface_impl_test.cpp @@ -38,6 +38,7 @@ #include "mongo/db/catalog/database.h" #include "mongo/db/catalog/document_validation.h" #include "mongo/db/catalog/index_catalog.h" +#include "mongo/db/catalog/validate_results.h" #include "mongo/db/client.h" #include "mongo/db/concurrency/d_concurrency.h" #include "mongo/db/concurrency/write_conflict_exception.h" @@ -160,7 +161,7 @@ int64_t getIndexKeyCount(OperationContext* opCtx, const IndexDescriptor* desc) { auto idx = cat->getEntry(desc)->accessMethod(); int64_t numKeys; - ValidateResults fullRes; + IndexValidateResults fullRes; idx->validate(opCtx, &numKeys, &fullRes); return numKeys; } diff --git a/src/mongo/db/storage/SConscript b/src/mongo/db/storage/SConscript index d774f2b8f2a..7c95014a583 100644 --- a/src/mongo/db/storage/SConscript +++ b/src/mongo/db/storage/SConscript @@ -286,6 +286,7 @@ env.Library( 'storage_debug_util.cpp', ], LIBDEPS_PRIVATE=[ + '$BUILD_DIR/mongo/db/catalog/validate_results', '$BUILD_DIR/mongo/db/db_raii', ], ) diff --git a/src/mongo/db/storage/devnull/devnull_kv_engine.cpp b/src/mongo/db/storage/devnull/devnull_kv_engine.cpp index 3ed16efdb3f..6c1c4435b7c 100644 --- a/src/mongo/db/storage/devnull/devnull_kv_engine.cpp +++ b/src/mongo/db/storage/devnull/devnull_kv_engine.cpp @@ -192,7 +192,7 @@ public: virtual void fullValidate(OperationContext* opCtx, long long* numKeysOut, - ValidateResults* fullResults) const {} + IndexValidateResults* fullResults) const {} virtual bool appendCustomStats(OperationContext* opCtx, BSONObjBuilder* output, diff --git a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_sorted_impl.cpp b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_sorted_impl.cpp index 34359e498a8..b9fa6becb24 100644 --- a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_sorted_impl.cpp +++ b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_sorted_impl.cpp @@ -40,6 +40,7 @@ #include "mongo/bson/bsonobjbuilder.h" #include "mongo/db/catalog/collection.h" #include "mongo/db/catalog/index_catalog_entry.h" +#include "mongo/db/catalog/validate_results.h" #include "mongo/db/index/index_descriptor.h" #include "mongo/db/storage/ephemeral_for_test/ephemeral_for_test_radix_store.h" #include "mongo/db/storage/ephemeral_for_test/ephemeral_for_test_recovery_unit.h" @@ -1374,7 +1375,7 @@ Status SortedDataInterfaceUnique::dupKeyCheck(OperationContext* opCtx, void SortedDataInterfaceUnique::fullValidate(OperationContext* opCtx, long long* numKeysOut, - ValidateResults* fullResults) const { + IndexValidateResults* fullResults) const { StringStore* workingCopy(RecoveryUnit::get(opCtx)->getHead()); long long numKeys = 0; auto it = workingCopy->lower_bound(_KSForIdentStart); @@ -1512,7 +1513,7 @@ Status SortedDataInterfaceStandard::dupKeyCheck(OperationContext* opCtx, void SortedDataInterfaceStandard::fullValidate(OperationContext* opCtx, long long* numKeysOut, - ValidateResults* fullResults) const { + IndexValidateResults* fullResults) const { StringStore* workingCopy(RecoveryUnit::get(opCtx)->getHead()); long long numKeys = 0; auto it = workingCopy->lower_bound(_KSForIdentStart); diff --git a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_sorted_impl.h b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_sorted_impl.h index d0dbe23d92d..629f1bedb66 100644 --- a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_sorted_impl.h +++ b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_sorted_impl.h @@ -117,7 +117,7 @@ public: Status dupKeyCheck(OperationContext* opCtx, const KeyString::Value& keyString) override; void fullValidate(OperationContext* opCtx, long long* numKeysOut, - ValidateResults* fullResults) const override; + IndexValidateResults* fullResults) const override; std::unique_ptr<mongo::SortedDataInterface::Cursor> newCursor( OperationContext* opCtx, bool isForward = true) const override; }; @@ -144,7 +144,7 @@ public: Status dupKeyCheck(OperationContext* opCtx, const KeyString::Value& keyString) override; void fullValidate(OperationContext* opCtx, long long* numKeysOut, - ValidateResults* fullResults) const override; + IndexValidateResults* fullResults) const override; std::unique_ptr<mongo::SortedDataInterface::Cursor> newCursor( OperationContext* opCtx, bool isForward = true) const override; }; diff --git a/src/mongo/db/storage/record_store.h b/src/mongo/db/storage/record_store.h index 8cc771f632e..a6a88bc8403 100644 --- a/src/mongo/db/storage/record_store.h +++ b/src/mongo/db/storage/record_store.h @@ -566,45 +566,4 @@ protected: std::string _ns; }; -struct ValidateResults { - bool valid = true; - bool repaired = false; - boost::optional<Timestamp> readTimestamp = boost::none; - std::vector<std::string> errors; - std::vector<std::string> warnings; - std::vector<BSONObj> extraIndexEntries; - std::vector<BSONObj> missingIndexEntries; - std::vector<RecordId> corruptRecords; - long long numRemovedCorruptRecords = 0; - long long numRemovedExtraIndexEntries = 0; - long long numInsertedMissingIndexEntries = 0; - - // Takes a bool that indicates the context of the caller and a BSONObjBuilder to append with - // validate results. - void appendToResultObj(BSONObjBuilder& resultObj, bool debugging) const { - resultObj.appendBool("valid", valid); - resultObj.appendBool("repaired", repaired); - if (readTimestamp) { - resultObj.append("readTimestamp", readTimestamp.get()); - } - resultObj.append("warnings", warnings); - resultObj.append("errors", errors); - resultObj.append("extraIndexEntries", extraIndexEntries); - resultObj.append("missingIndexEntries", missingIndexEntries); - - // Need to convert RecordId to int64_t to append to BSONObjBuilder - BSONArrayBuilder builder; - for (RecordId corruptRecord : corruptRecords) { - builder.append(corruptRecord.repr()); - } - resultObj.append("corruptRecords", builder.done()); - - if (repaired || debugging) { - resultObj.appendNumber("numRemovedCorruptRecords", numRemovedCorruptRecords); - resultObj.appendNumber("numRemovedExtraIndexEntries", numRemovedExtraIndexEntries); - resultObj.appendNumber("numInsertedMissingIndexEntries", - numInsertedMissingIndexEntries); - } - } -}; } // namespace mongo diff --git a/src/mongo/db/storage/sorted_data_interface.h b/src/mongo/db/storage/sorted_data_interface.h index 2ab1b64fc9b..186dd20ea20 100644 --- a/src/mongo/db/storage/sorted_data_interface.h +++ b/src/mongo/db/storage/sorted_data_interface.h @@ -45,7 +45,7 @@ namespace mongo { class BSONObjBuilder; class BucketDeletionNotification; class SortedDataBuilderInterface; -struct ValidateResults; +struct IndexValidateResults; /** * This is the uniform interface for storing indexes and supporting point queries as well as range @@ -133,7 +133,7 @@ public: */ virtual void fullValidate(OperationContext* opCtx, long long* numKeysOut, - ValidateResults* fullResults) const = 0; + IndexValidateResults* fullResults) const = 0; virtual bool appendCustomStats(OperationContext* opCtx, BSONObjBuilder* output, diff --git a/src/mongo/db/storage/storage_debug_util.cpp b/src/mongo/db/storage/storage_debug_util.cpp index c35d3df2645..ca7a1eaed42 100644 --- a/src/mongo/db/storage/storage_debug_util.cpp +++ b/src/mongo/db/storage/storage_debug_util.cpp @@ -33,6 +33,7 @@ #include "mongo/db/storage/storage_debug_util.h" +#include "mongo/db/catalog/validate_results.h" #include "mongo/db/db_raii.h" #include "mongo/db/index/index_access_method.h" #include "mongo/db/storage/key_string.h" diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp index 4b6c449bf1d..4d40231e32e 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp @@ -38,6 +38,7 @@ #include "mongo/base/checked_cast.h" #include "mongo/db/catalog/index_catalog_entry.h" +#include "mongo/db/catalog/validate_results.h" #include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/global_settings.h" #include "mongo/db/index/index_descriptor.h" @@ -296,7 +297,7 @@ void WiredTigerIndex::unindex(OperationContext* opCtx, void WiredTigerIndex::fullValidate(OperationContext* opCtx, long long* numKeysOut, - ValidateResults* fullResults) const { + IndexValidateResults* fullResults) const { dassert(opCtx->lockState()->isReadLocked()); if (fullResults && !WiredTigerRecoveryUnit::get(opCtx)->getSessionCache()->isEphemeral()) { int err = WiredTigerUtil::verifyTable(opCtx, _uri, &(fullResults->errors)); diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_index.h b/src/mongo/db/storage/wiredtiger/wiredtiger_index.h index 6ae2a7b580e..2919183896f 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_index.h +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_index.h @@ -108,7 +108,7 @@ public: virtual void fullValidate(OperationContext* opCtx, long long* numKeysOut, - ValidateResults* fullResults) const; + IndexValidateResults* fullResults) const; virtual bool appendCustomStats(OperationContext* opCtx, BSONObjBuilder* output, double scale) const; diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp index ddc4872119c..a2adafb35ea 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp @@ -43,6 +43,7 @@ #include "mongo/base/checked_cast.h" #include "mongo/base/static_assert.h" #include "mongo/bson/util/builder.h" +#include "mongo/db/catalog/validate_results.h" #include "mongo/db/concurrency/locker.h" #include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/global_settings.h" diff --git a/src/mongo/dbtests/validate_tests.cpp b/src/mongo/dbtests/validate_tests.cpp index 3ee2cdb73e8..be56190e176 100644 --- a/src/mongo/dbtests/validate_tests.cpp +++ b/src/mongo/dbtests/validate_tests.cpp @@ -1568,6 +1568,7 @@ public: StorageDebugUtil::printCollectionAndIndexTableEntries(&_opCtx, coll->ns()); }); + ASSERT_EQ(true, results.valid); ASSERT_EQ(true, results.repaired); ASSERT_EQ(static_cast<size_t>(0), results.errors.size()); @@ -1577,6 +1578,9 @@ public: ASSERT_EQ(3, results.numRemovedExtraIndexEntries); ASSERT_EQ(3, results.numInsertedMissingIndexEntries); + ASSERT_EQ(3, results.indexResultsMap[indexNameA].keysTraversed); + ASSERT_EQ(3, results.indexResultsMap[indexNameB].keysTraversed); + dumpOnErrorGuard.dismiss(); } |