diff options
author | Daniel Solnik <dansolnik@gmail.com> | 2019-07-29 10:13:23 -0400 |
---|---|---|
committer | Daniel Solnik <dansolnik@gmail.com> | 2019-08-16 15:05:05 -0400 |
commit | 65754fc06966fb438c114f113ccae2d742eaa963 (patch) | |
tree | f1fd2b38b41f9d7507cdd4f512c9533098af3e8e /src/mongo | |
parent | 8494ca7d8a88aa3d5df96e89beafacd4caca3801 (diff) | |
download | mongo-65754fc06966fb438c114f113ccae2d742eaa963.tar.gz |
SERVER-42222 Move data cursors for collection validation to the start of collection validation
Diffstat (limited to 'src/mongo')
-rw-r--r-- | src/mongo/db/catalog/SConscript | 2 | ||||
-rw-r--r-- | src/mongo/db/catalog/collection_validation.cpp | 303 | ||||
-rw-r--r-- | src/mongo/db/catalog/collection_validation.h | 5 | ||||
-rw-r--r-- | src/mongo/db/catalog/index_consistency.cpp | 6 | ||||
-rw-r--r-- | src/mongo/db/catalog/index_consistency.h | 3 | ||||
-rw-r--r-- | src/mongo/db/catalog/validate_adaptor.cpp (renamed from src/mongo/db/catalog/record_store_validate_adaptor.cpp) | 49 | ||||
-rw-r--r-- | src/mongo/db/catalog/validate_adaptor.h (renamed from src/mongo/db/catalog/record_store_validate_adaptor.h) | 29 | ||||
-rw-r--r-- | src/mongo/db/concurrency/d_concurrency.cpp | 1 | ||||
-rw-r--r-- | src/mongo/db/storage/record_store.h | 16 |
9 files changed, 283 insertions, 131 deletions
diff --git a/src/mongo/db/catalog/SConscript b/src/mongo/db/catalog/SConscript index 1af2c2b5142..11d92bef9ac 100644 --- a/src/mongo/db/catalog/SConscript +++ b/src/mongo/db/catalog/SConscript @@ -342,7 +342,7 @@ env.Library( target='collection_validation', source=[ "collection_validation.cpp", - "record_store_validate_adaptor.cpp", + "validate_adaptor.cpp", ], LIBDEPS=[ 'catalog_impl', diff --git a/src/mongo/db/catalog/collection_validation.cpp b/src/mongo/db/catalog/collection_validation.cpp index bfeea744c67..082f7964169 100644 --- a/src/mongo/db/catalog/collection_validation.cpp +++ b/src/mongo/db/catalog/collection_validation.cpp @@ -54,24 +54,28 @@ using ValidateResultsMap = std::map<string, ValidateResults>; /** * General validation logic for any RecordStore. Performs sanity checks to confirm that each - * record in the store is valid according to the given RecordStoreValidateAdaptor and updates + * record in the store is valid according to the given ValidateAdaptor and updates * record store stats to match. */ -void _genericRecordStoreValidate(OperationContext* opCtx, - RecordStore* recordStore, - RecordStoreValidateAdaptor* indexValidator, - ValidateResults* results, - BSONObjBuilder* output) { +void _genericRecordStoreValidate( + OperationContext* opCtx, + RecordStore* recordStore, + ValidateAdaptor* indexValidator, + const RecordId& firstRecordId, + const std::unique_ptr<SeekableRecordCursor>& traverseRecordStoreCursor, + const std::unique_ptr<SeekableRecordCursor>& seekRecordStoreCursor, + ValidateResults* results, + BSONObjBuilder* output) { long long nrecords = 0; long long dataSizeTotal = 0; long long nInvalid = 0; results->valid = true; - std::unique_ptr<SeekableRecordCursor> cursor = recordStore->getCursor(opCtx, true); int interruptInterval = 4096; RecordId prevRecordId; - while (auto record = cursor->next()) { + for (auto record = traverseRecordStoreCursor->seekExact(firstRecordId); record; + record = traverseRecordStoreCursor->next()) { if (!(nrecords % interruptInterval)) { opCtx->checkForInterrupt(); } @@ -79,7 +83,8 @@ void _genericRecordStoreValidate(OperationContext* opCtx, auto dataSize = record->data.size(); dataSizeTotal += dataSize; size_t validatedSize; - Status status = indexValidator->validate(record->id, record->data, &validatedSize); + Status status = indexValidator->validateRecord( + record->id, record->data, seekRecordStoreCursor, &validatedSize); // Check to ensure isInRecordIdOrder() is being used properly. if (prevRecordId.isValid()) { @@ -113,68 +118,157 @@ void _validateRecordStore(OperationContext* opCtx, RecordStore* recordStore, ValidateCmdLevel level, bool background, - RecordStoreValidateAdaptor* indexValidator, + ValidateAdaptor* indexValidator, + const RecordId& firstRecordId, + const std::unique_ptr<SeekableRecordCursor>& traverseRecordStoreCursor, + const std::unique_ptr<SeekableRecordCursor>& seekRecordStoreCursor, ValidateResults* results, BSONObjBuilder* output) { if (background) { - indexValidator->traverseRecordStore(recordStore, results, output); + indexValidator->traverseRecordStore(recordStore, + firstRecordId, + traverseRecordStoreCursor, + seekRecordStoreCursor, + results, + output); } else { - // For 'full' validation we use the record store's validation functionality. - if (level == kValidateFull) { - recordStore->validate(opCtx, results, output); - } - _genericRecordStoreValidate(opCtx, recordStore, indexValidator, results, output); + _genericRecordStoreValidate(opCtx, + recordStore, + indexValidator, + firstRecordId, + traverseRecordStoreCursor, + seekRecordStoreCursor, + results, + output); } } -void _validateIndexes(OperationContext* opCtx, - IndexCatalog* indexCatalog, - BSONObjBuilder* keysPerIndex, - RecordStoreValidateAdaptor* indexValidator, - ValidateCmdLevel level, - ValidateResultsMap* indexNsResultsMap, - ValidateResults* results) { +/** + * Opens a cursor on each index in the given 'indexCatalog'. + * + * Returns a map from indexName -> indexCursor. + */ +std::map<std::string, std::unique_ptr<SortedDataInterface::Cursor>> _openIndexCursors( + OperationContext* opCtx, IndexCatalog* indexCatalog) { + std::map<std::string, std::unique_ptr<SortedDataInterface::Cursor>> indexCursors; + const std::unique_ptr<IndexCatalog::IndexIterator> it = + indexCatalog->getIndexIterator(opCtx, false); + while (it->more()) { + const IndexCatalogEntry* entry = it->next(); + indexCursors[entry->descriptor()->indexName()] = + entry->accessMethod()->newCursor(opCtx, true); + } + return indexCursors; +} - std::unique_ptr<IndexCatalog::IndexIterator> it = indexCatalog->getIndexIterator(opCtx, false); +/** + * Validates the internal structure of each index in the Index Catalog 'indexCatalog', ensuring that + * the index files have not been corrupted or compromised. + * + * May close or invalidate open cursors. + * + * Returns a map from indexName -> number of keys validated. + */ +std::map<std::string, int64_t> _validateIndexesInternalStructure( + OperationContext* opCtx, + IndexCatalog* indexCatalog, + ValidateResultsMap* indexNsResultsMap, + ValidateResults* results) { + std::map<std::string, int64_t> numIndexKeysPerIndex; + const std::unique_ptr<IndexCatalog::IndexIterator> it = + indexCatalog->getIndexIterator(opCtx, false); - // Validate Indexes. + // Validate Indexes Internal Structure, checking if index files have been compromised or + // corrupted. while (it->more()) { opCtx->checkForInterrupt(); const IndexCatalogEntry* entry = it->next(); const IndexDescriptor* descriptor = entry->descriptor(); const IndexAccessMethod* iam = entry->accessMethod(); - log(LogComponent::kIndex) << "validating index " << descriptor->indexName() + log(LogComponent::kIndex) << "validating the internal structure of index " + << descriptor->indexName() << " on collection " + << descriptor->parentNS(); + ValidateResults& curIndexResults = (*indexNsResultsMap)[descriptor->indexName()]; + + int64_t numValidated; + iam->validate(opCtx, &numValidated, &curIndexResults); + + numIndexKeysPerIndex[descriptor->indexName()] = numValidated; + } + return numIndexKeysPerIndex; +} + +/** + * Validates each index in the Index Catalog using the cursors in 'indexCursors'. + * + * If 'level' is kValidateFull, then we will compare new index entry counts with a previously taken + * count saved in 'numIndexKeysPerIndex'. + */ +void _validateIndexes( + OperationContext* opCtx, + IndexCatalog* indexCatalog, + BSONObjBuilder* keysPerIndex, + ValidateAdaptor* indexValidator, + ValidateCmdLevel level, + const std::map<std::string, std::unique_ptr<SortedDataInterface::Cursor>>& indexCursors, + const std::map<std::string, int64_t>& numIndexKeysPerIndex, + ValidateResultsMap* indexNsResultsMap, + ValidateResults* results) { + + const std::unique_ptr<IndexCatalog::IndexIterator> it = + indexCatalog->getIndexIterator(opCtx, false); + + // Validate Indexes, checking for mismatch between index entries and collection records. + while (it->more()) { + opCtx->checkForInterrupt(); + const IndexDescriptor* descriptor = it->next()->descriptor(); + + log(LogComponent::kIndex) << "validating index consistency " << descriptor->indexName() << " on collection " << descriptor->parentNS(); + + // Ensure that this index had an index cursor opened in _openIndexCursors. + const auto indexCursorIt = indexCursors.find(descriptor->indexName()); + invariant(indexCursorIt != indexCursors.end()); + ValidateResults& curIndexResults = (*indexNsResultsMap)[descriptor->indexName()]; - bool checkCounts = false; int64_t numTraversedKeys; - int64_t numValidatedKeys; + indexValidator->traverseIndex( + &numTraversedKeys, indexCursorIt->second, descriptor, &curIndexResults); + // If we are performing a full validation, we have information on the number of index keys + // validated in _validateIndexesInternalStructure (when we validated the internal structure + // of the index). Check if this is consistent with 'numTraversedKeys' from traverseIndex + // above. if (level == kValidateFull) { - iam->validate(opCtx, &numValidatedKeys, &curIndexResults); - checkCounts = true; - } + // Ensure that this index was validated in _validateIndexesInternalStructure. + const auto numIndexKeysIt = numIndexKeysPerIndex.find(descriptor->indexName()); + invariant(numIndexKeysIt != numIndexKeysPerIndex.end()); - if (curIndexResults.valid) { - indexValidator->traverseIndex(iam, descriptor, &curIndexResults, &numTraversedKeys); - - if (checkCounts && (numValidatedKeys != numTraversedKeys)) { - curIndexResults.valid = false; - string msg = str::stream() - << "number of traversed index entries (" << numTraversedKeys - << ") does not match the number of expected index entries (" << numValidatedKeys - << ")"; - results->errors.push_back(msg); - results->valid = false; - } + // The number of keys counted in _validateIndexesInternalStructure, when checking the + // internal structure of the index. + const int64_t numIndexKeys = numIndexKeysIt->second; + // Check if currIndexResults is valid to ensure that this index is not corrupted or + // comprised (which was set in _validateIndexesInternalStructure). If the index is + // corrupted, there is no use in checking if the traversal yielded the same key count. if (curIndexResults.valid) { - keysPerIndex->appendNumber(descriptor->indexName(), - static_cast<long long>(numTraversedKeys)); - } else { - results->valid = false; + + if (numIndexKeys != numTraversedKeys) { + curIndexResults.valid = false; + string msg = str::stream() + << "number of traversed index entries (" << numTraversedKeys + << ") does not match the number of expected index entries (" << numIndexKeys + << ")"; + results->errors.push_back(msg); + results->valid = false; + } } + } + + if (curIndexResults.valid) { + keysPerIndex->appendNumber(descriptor->indexName(), + static_cast<long long>(numTraversedKeys)); } else { results->valid = false; } @@ -185,26 +279,32 @@ void _validateIndexes(OperationContext* opCtx, * Executes the second phase of validation for improved error reporting. This is only done if * any index inconsistencies are found during the first phase of validation. */ -void _gatherIndexEntryErrors(OperationContext* opCtx, - RecordStore* recordStore, - IndexCatalog* indexCatalog, - IndexConsistency* indexConsistency, - RecordStoreValidateAdaptor* indexValidator, - ValidateResultsMap* indexNsResultsMap, - ValidateResults* result) { +void _gatherIndexEntryErrors( + OperationContext* opCtx, + IndexCatalog* indexCatalog, + IndexConsistency* indexConsistency, + ValidateAdaptor* indexValidator, + const RecordId& firstRecordId, + const std::unique_ptr<SeekableRecordCursor>& traverseRecordStoreCursor, + const std::unique_ptr<SeekableRecordCursor>& seekRecordStoreCursor, + const std::map<std::string, std::unique_ptr<SortedDataInterface::Cursor>>& indexCursors, + ValidateResultsMap* indexNsResultsMap, + ValidateResults* result) { indexConsistency->setSecondPhase(); log(LogComponent::kIndex) << "Starting to traverse through all the document key sets."; // During the second phase of validation, iterate through each documents key set and only record // the keys that were inconsistent during the first phase of validation. - std::unique_ptr<SeekableRecordCursor> cursor = recordStore->getCursor(opCtx, true); - while (auto record = cursor->next()) { + for (auto record = traverseRecordStoreCursor->seekExact(firstRecordId); record; + record = traverseRecordStoreCursor->next()) { opCtx->checkForInterrupt(); // We can ignore the status of validate as it was already checked during the first phase. size_t validatedSize; - indexValidator->validate(record->id, record->data, &validatedSize).ignore(); + indexValidator + ->validateRecord(record->id, record->data, seekRecordStoreCursor, &validatedSize) + .ignore(); } log(LogComponent::kIndex) << "Finished traversing through all the document key sets."; @@ -216,14 +316,19 @@ void _gatherIndexEntryErrors(OperationContext* opCtx, while (it->more()) { opCtx->checkForInterrupt(); - const IndexCatalogEntry* entry = it->next(); - const IndexDescriptor* descriptor = entry->descriptor(); - const IndexAccessMethod* iam = entry->accessMethod(); + const IndexDescriptor* descriptor = it->next()->descriptor(); log(LogComponent::kIndex) << "Traversing through the index entries for index " << descriptor->indexName() << "."; - indexValidator->traverseIndex( - iam, descriptor, /*ValidateResults=*/nullptr, /*numTraversedKeys=*/nullptr); + + // Ensure that this index had an index cursor opened in _openIndexCursors. + const auto indexCursorIt = indexCursors.find(descriptor->indexName()); + invariant(indexCursorIt != indexCursors.end()); + + indexValidator->traverseIndex(/*numTraversedKeys=*/nullptr, + indexCursorIt->second, + descriptor, + /*ValidateResults=*/nullptr); } log(LogComponent::kIndex) << "Finished traversing through all the indexes."; @@ -234,10 +339,10 @@ void _gatherIndexEntryErrors(OperationContext* opCtx, void _validateIndexKeyCount(OperationContext* opCtx, IndexCatalog* indexCatalog, RecordStore* recordStore, - RecordStoreValidateAdaptor* indexValidator, + ValidateAdaptor* indexValidator, ValidateResultsMap* indexNsResultsMap) { - std::unique_ptr<IndexCatalog::IndexIterator> indexIterator = + const std::unique_ptr<IndexCatalog::IndexIterator> indexIterator = indexCatalog->getIndexIterator(opCtx, false); while (indexIterator->more()) { const IndexDescriptor* descriptor = indexIterator->next()->descriptor(); @@ -352,23 +457,66 @@ Status validate(OperationContext* opCtx, ValidateResults* results, BSONObjBuilder* output) { invariant(opCtx->lockState()->isCollectionLockedForMode(coll->ns(), MODE_IS)); + invariant(!(background && (level == kValidateFull))); + + ValidateResultsMap indexNsResultsMap; + BSONObjBuilder keysPerIndex; // not using subObjStart to be exception safe. + IndexConsistency indexConsistency(opCtx, coll, coll->ns(), background); + ValidateAdaptor indexValidator = ValidateAdaptor( + opCtx, &indexConsistency, level, coll->getIndexCatalog(), &indexNsResultsMap); try { - ValidateResultsMap indexNsResultsMap; - BSONObjBuilder keysPerIndex; // not using subObjStart to be exception safe. - IndexConsistency indexConsistency( - opCtx, coll, coll->ns(), coll->getRecordStore(), background); - RecordStoreValidateAdaptor indexValidator = RecordStoreValidateAdaptor( - opCtx, &indexConsistency, level, coll->getIndexCatalog(), &indexNsResultsMap); + std::map<std::string, int64_t> numIndexKeysPerIndex; + + // Full validation code is executed before we open cursors because it may close + // and/or invalidate all open cursors. + if (level == kValidateFull) { + // For full validation we use the storage engine's validation functionality. + coll->getRecordStore()->validate(opCtx, results, output); + // For full validation, we validate the internal structure of each index and save the + // number of keys in the index to compare against _validateIndexes()'s count results. + numIndexKeysPerIndex = _validateIndexesInternalStructure( + opCtx, coll->getIndexCatalog(), &indexNsResultsMap, results); + } - string uuidString = str::stream() << " (UUID: " << coll->uuid() << ")"; + // Open all cursors at once before running non-full validation code so that all steps of + // validation during background validation use the same view of the data. + const std::map<std::string, std::unique_ptr<SortedDataInterface::Cursor>> indexCursors = + _openIndexCursors(opCtx, coll->getIndexCatalog()); + const std::unique_ptr<SeekableRecordCursor> traverseRecordStoreCursor = + coll->getRecordStore()->getCursor(opCtx, true); + const std::unique_ptr<SeekableRecordCursor> seekRecordStoreCursor = + coll->getRecordStore()->getCursor(opCtx, true); + + // Because SeekableRecordCursors don't have a method to reset to the start, we save and then + // use a seek to the first RecordId to reset the cursor (and reuse it) as needed. When + // iterating through a Record Store cursor, we initialize the loop (and obtain the first + // Record) with a seek to the first Record (using firstRecordId). Subsequent loop iterations + // use cursor->next() to get subsequent Records. However, if the Record Store is empty, + // there is no first record. In this case, we set the first Record Id to an invalid RecordId + // (RecordId()), which will halt iteration at the initialization step. + const boost::optional<Record> record = traverseRecordStoreCursor->next(); + const RecordId firstRecordId = record ? record->id : RecordId(); + + const string uuidString = str::stream() << " (UUID: " << coll->uuid() << ")"; // Validate the record store. log(LogComponent::kIndex) << "validating collection " << coll->ns() << uuidString; - _validateRecordStore( - opCtx, coll->getRecordStore(), level, background, &indexValidator, results, output); + // In _validateRecordStore(), the index validator keeps track the records in the record + // store so that _validateIndexes() can confirm that the index entries match the records in + // the collection. + _validateRecordStore(opCtx, + coll->getRecordStore(), + level, + background, + &indexValidator, + firstRecordId, + traverseRecordStoreCursor, + seekRecordStoreCursor, + results, + output); - // Validate in-memory catalog information with the persisted info. + // Validate in-memory catalog information with persisted info. _validateCatalogEntry(opCtx, coll, coll->getValidatorDoc(), results); // Validate indexes and check for mismatches. @@ -378,6 +526,8 @@ Status validate(OperationContext* opCtx, &keysPerIndex, &indexValidator, level, + indexCursors, + numIndexKeysPerIndex, &indexNsResultsMap, results); @@ -386,10 +536,13 @@ Status validate(OperationContext* opCtx, << "Index inconsistencies were detected on collection " << coll->ns() << ". Starting the second phase of index validation to gather concise errors."; _gatherIndexEntryErrors(opCtx, - coll->getRecordStore(), coll->getIndexCatalog(), &indexConsistency, &indexValidator, + firstRecordId, + traverseRecordStoreCursor, + seekRecordStoreCursor, + indexCursors, &indexNsResultsMap, results); } diff --git a/src/mongo/db/catalog/collection_validation.h b/src/mongo/db/catalog/collection_validation.h index b7993e88c91..fc00269e619 100644 --- a/src/mongo/db/catalog/collection_validation.h +++ b/src/mongo/db/catalog/collection_validation.h @@ -29,7 +29,7 @@ #pragma once -#include "mongo/db/catalog/record_store_validate_adaptor.h" +#include "mongo/db/catalog/validate_adaptor.h" namespace mongo { @@ -44,6 +44,9 @@ namespace CollectionValidation { /** * Expects the caller to hold at least a collection IS lock. * + * Background validation does not support full validation and so the combination of level = + * 'kValidateTrue' and background = 'True' is prohibited. + * * @return OK if the validate run successfully * OK will be returned even if corruption is found * details will be in 'results'. diff --git a/src/mongo/db/catalog/index_consistency.cpp b/src/mongo/db/catalog/index_consistency.cpp index 7491916b87a..6148032137e 100644 --- a/src/mongo/db/catalog/index_consistency.cpp +++ b/src/mongo/db/catalog/index_consistency.cpp @@ -67,12 +67,10 @@ IndexInfo::IndexInfo(const IndexDescriptor* descriptor) IndexConsistency::IndexConsistency(OperationContext* opCtx, Collection* collection, NamespaceString nss, - RecordStore* recordStore, bool background) : _opCtx(opCtx), _collection(collection), _nss(nss), - _recordStore(recordStore), _tracker(opCtx->getServiceContext()->getFastClockSource(), internalQueryExecYieldIterations.load(), Milliseconds(internalQueryExecYieldPeriodMS.load())), @@ -211,6 +209,7 @@ void IndexConsistency::addIndexEntryErrors(ValidateResultsMap* indexNsResultsMap void IndexConsistency::addDocKey(const KeyString::Builder& ks, IndexInfo* indexInfo, RecordId recordId, + const std::unique_ptr<SeekableRecordCursor>& seekRecordStoreCursor, const BSONObj& indexKey) { const uint32_t hash = _hashKeyString(ks, indexInfo->indexNameHash); @@ -223,8 +222,7 @@ void IndexConsistency::addDocKey(const KeyString::Builder& ks, // Found a document key for a hash bucket that had mismatches. // Get the documents _id index key. - auto cursor = _recordStore->getCursor(_opCtx); - auto record = cursor->seekExact(recordId); + auto record = seekRecordStoreCursor->seekExact(recordId); invariant(record); BSONObj data = record->data.toBson(); diff --git a/src/mongo/db/catalog/index_consistency.h b/src/mongo/db/catalog/index_consistency.h index ebfad06f83d..df03c3c129b 100644 --- a/src/mongo/db/catalog/index_consistency.h +++ b/src/mongo/db/catalog/index_consistency.h @@ -74,7 +74,6 @@ public: IndexConsistency(OperationContext* opCtx, Collection* collection, NamespaceString nss, - RecordStore* recordStore, bool background); /** @@ -86,6 +85,7 @@ public: void addDocKey(const KeyString::Builder& ks, IndexInfo* indexInfo, RecordId recordId, + const std::unique_ptr<SeekableRecordCursor>& cursor, const BSONObj& indexKey); /** @@ -143,7 +143,6 @@ private: OperationContext* _opCtx; Collection* _collection; const NamespaceString _nss; - const RecordStore* _recordStore; ElapsedTracker _tracker; // We map the hashed KeyString values to a bucket that contains the count of how many diff --git a/src/mongo/db/catalog/record_store_validate_adaptor.cpp b/src/mongo/db/catalog/validate_adaptor.cpp index 7bfc00e8460..700aa87c360 100644 --- a/src/mongo/db/catalog/record_store_validate_adaptor.cpp +++ b/src/mongo/db/catalog/validate_adaptor.cpp @@ -31,7 +31,7 @@ #include "mongo/platform/basic.h" -#include "mongo/db/catalog/record_store_validate_adaptor.h" +#include "mongo/db/catalog/validate_adaptor.h" #include "mongo/bson/bsonobj.h" #include "mongo/db/catalog/index_catalog.h" @@ -59,9 +59,11 @@ KeyString::Builder makeWildCardMultikeyMetadataKeyString(const BSONObj& indexKey } } // namespace -Status RecordStoreValidateAdaptor::validate(const RecordId& recordId, - const RecordData& record, - size_t* dataSize) { +Status ValidateAdaptor::validateRecord( + const RecordId& recordId, + const RecordData& record, + const std::unique_ptr<SeekableRecordCursor>& seekRecordStoreCursor, + size_t* dataSize) { BSONObj recordBson; try { recordBson = record.toBson(); @@ -136,7 +138,8 @@ Status RecordStoreValidateAdaptor::validate(const RecordId& recordId, indexInfo.ord, keyString.getTypeBits()); indexInfo.ks->resetToKey(key, indexInfo.ord, recordId); - _indexConsistency->addDocKey(*indexInfo.ks, &indexInfo, recordId, key); + _indexConsistency->addDocKey( + *indexInfo.ks, &indexInfo, recordId, seekRecordStoreCursor, key); } catch (...) { return exceptionToStatus(); } @@ -145,10 +148,10 @@ Status RecordStoreValidateAdaptor::validate(const RecordId& recordId, return status; } -void RecordStoreValidateAdaptor::traverseIndex(const IndexAccessMethod* iam, - const IndexDescriptor* descriptor, - ValidateResults* results, - int64_t* numTraversedKeys) { +void ValidateAdaptor::traverseIndex(int64_t* numTraversedKeys, + const std::unique_ptr<SortedDataInterface::Cursor>& indexCursor, + const IndexDescriptor* descriptor, + ValidateResults* results) { auto indexName = descriptor->indexName(); IndexInfo* indexInfo = &_indexConsistency->getIndexInfo(indexName); int64_t numKeys = 0; @@ -157,7 +160,6 @@ void RecordStoreValidateAdaptor::traverseIndex(const IndexAccessMethod* iam, const Ordering ord = Ordering::make(key); bool isFirstEntry = true; - std::unique_ptr<SortedDataInterface::Cursor> cursor = iam->newCursor(_opCtx, true); // We want to use the latest version of KeyString here. const KeyString::Version version = KeyString::Version::kLatestVersion; std::unique_ptr<KeyString::Builder> indexKeyStringBuilder = @@ -166,7 +168,8 @@ void RecordStoreValidateAdaptor::traverseIndex(const IndexAccessMethod* iam, std::make_unique<KeyString::Builder>(version); // Seeking to BSONObj() is equivalent to seeking to the first entry of an index. - for (auto indexEntry = cursor->seek(BSONObj(), true); indexEntry; indexEntry = cursor->next()) { + for (auto indexEntry = indexCursor->seek(BSONObj(), true); indexEntry; + indexEntry = indexCursor->next()) { indexKeyStringBuilder->resetToKey(indexEntry->key, ord, indexEntry->loc); // Ensure that the index entries are in increasing or decreasing order. @@ -211,19 +214,22 @@ void RecordStoreValidateAdaptor::traverseIndex(const IndexAccessMethod* iam, } } -void RecordStoreValidateAdaptor::traverseRecordStore(RecordStore* recordStore, - ValidateResults* results, - BSONObjBuilder* output) { +void ValidateAdaptor::traverseRecordStore( + RecordStore* recordStore, + const RecordId& firstRecordId, + const std::unique_ptr<SeekableRecordCursor>& traverseRecordStoreCursor, + const std::unique_ptr<SeekableRecordCursor>& seekRecordStoreCursor, + ValidateResults* results, + BSONObjBuilder* output) { long long nrecords = 0; long long dataSizeTotal = 0; long long nInvalid = 0; results->valid = true; - std::unique_ptr<SeekableRecordCursor> cursor = recordStore->getCursor(_opCtx, true); int interruptInterval = 4096; RecordId prevRecordId; - - while (auto record = cursor->next()) { + for (auto record = traverseRecordStoreCursor->seekExact(firstRecordId); record; + record = traverseRecordStoreCursor->next()) { ++nrecords; if (!(nrecords % interruptInterval)) { @@ -233,7 +239,8 @@ void RecordStoreValidateAdaptor::traverseRecordStore(RecordStore* recordStore, auto dataSize = record->data.size(); dataSizeTotal += dataSize; size_t validatedSize; - Status status = validate(record->id, record->data, &validatedSize); + Status status = + validateRecord(record->id, record->data, seekRecordStoreCursor, &validatedSize); // Checks to ensure isInRecordIdOrder() is being used properly. if (prevRecordId.isValid()) { @@ -263,9 +270,9 @@ void RecordStoreValidateAdaptor::traverseRecordStore(RecordStore* recordStore, output->appendNumber("nrecords", nrecords); } -void RecordStoreValidateAdaptor::validateIndexKeyCount(const IndexDescriptor* idx, - int64_t numRecs, - ValidateResults& results) { +void ValidateAdaptor::validateIndexKeyCount(const IndexDescriptor* idx, + int64_t numRecs, + ValidateResults& results) { const std::string indexName = idx->indexName(); IndexInfo* indexInfo = &_indexConsistency->getIndexInfo(indexName); auto numTotalKeys = indexInfo->numKeys; diff --git a/src/mongo/db/catalog/record_store_validate_adaptor.h b/src/mongo/db/catalog/validate_adaptor.h index 0e64283bbc0..aefa26dc41b 100644 --- a/src/mongo/db/catalog/record_store_validate_adaptor.h +++ b/src/mongo/db/catalog/validate_adaptor.h @@ -52,13 +52,13 @@ using ValidateResultsMap = std::map<std::string, ValidateResults>; * The record store validate adaptor is used to keep track of the index consistency during * a validation that's running. */ -class RecordStoreValidateAdaptor : public ValidateAdaptor { +class ValidateAdaptor { public: - RecordStoreValidateAdaptor(OperationContext* opCtx, - IndexConsistency* indexConsistency, - ValidateCmdLevel level, - IndexCatalog* ic, - ValidateResultsMap* irm) + ValidateAdaptor(OperationContext* opCtx, + IndexConsistency* indexConsistency, + ValidateCmdLevel level, + IndexCatalog* ic, + ValidateResultsMap* irm) : _opCtx(opCtx), _indexConsistency(indexConsistency), @@ -67,25 +67,32 @@ public: _indexNsResultsMap(irm) {} /** - * Validates the BSON object and traverses through its key set to keep track of the + * Validates the record data and traverses through its key set to keep track of the * index consistency. */ - virtual Status validate(const RecordId& recordId, const RecordData& record, size_t* dataSize); + virtual Status validateRecord( + const RecordId& recordId, + const RecordData& record, + const std::unique_ptr<SeekableRecordCursor>& seekRecordStoreCursor, + size_t* dataSize); /** * Traverses the index getting index entriess to validate them and keep track of the index keys * for index consistency. */ - void traverseIndex(const IndexAccessMethod* iam, + void traverseIndex(int64_t* numTraversedKeys, + const std::unique_ptr<SortedDataInterface::Cursor>& indexCursor, const IndexDescriptor* descriptor, - ValidateResults* results, - int64_t* numTraversedKeys); + ValidateResults* results); /** * Traverses the record store to retrieve every record and go through its document key * set to keep track of the index consistency during a validation. */ void traverseRecordStore(RecordStore* recordStore, + const RecordId& firstRecordId, + const std::unique_ptr<SeekableRecordCursor>& traverseRecordStoreCursor, + const std::unique_ptr<SeekableRecordCursor>& seekRecordStoreCursor, ValidateResults* results, BSONObjBuilder* output); diff --git a/src/mongo/db/concurrency/d_concurrency.cpp b/src/mongo/db/concurrency/d_concurrency.cpp index 0e9f548e773..d574e6c491c 100644 --- a/src/mongo/db/concurrency/d_concurrency.cpp +++ b/src/mongo/db/concurrency/d_concurrency.cpp @@ -283,7 +283,6 @@ Lock::CollectionLock::CollectionLock(OperationContext* opCtx, LockMode mode, Date_t deadline) : _opCtx(opCtx) { - LockMode actualLockMode = mode; if (!supportsDocLocking()) { actualLockMode = isSharedLockMode(mode) ? MODE_S : MODE_X; diff --git a/src/mongo/db/storage/record_store.h b/src/mongo/db/storage/record_store.h index ad6747b2ad1..fd7e1ef3b74 100644 --- a/src/mongo/db/storage/record_store.h +++ b/src/mongo/db/storage/record_store.h @@ -250,7 +250,7 @@ public: virtual long long dataSize(OperationContext* opCtx) const = 0; /** - * Total number of record in the RecordStore. You may need to cache it, so this call + * Total number of records in the RecordStore. You may need to cache it, so this call * takes constant time, as it is called often. */ virtual long long numRecords(OperationContext* opCtx) const = 0; @@ -588,18 +588,4 @@ struct ValidateResults { std::vector<BSONObj> extraIndexEntries; std::vector<BSONObj> missingIndexEntries; }; - -/** - * This is so when a RecordStore is validating all records - * it can call back to someone to check if a record is valid. - * The actual data contained in a Record is totally opaque to the implementation. - */ -class ValidateAdaptor { -public: - virtual ~ValidateAdaptor() {} - - virtual Status validate(const RecordId& recordId, - const RecordData& recordData, - size_t* dataSize) = 0; -}; } // namespace mongo |