SERVER-42180 Move the CollectionImpl::validate logic out of the Collection class and into it's own file + namespace

15 files changed, 554 insertions, 427 deletions

diff --git a/src/mongo/db/catalog/SConscript b/src/mongo/db/catalog/SConscript
index 5ca8a7d62d3..ab110ad1824 100644
--- a/src/mongo/db/catalog/SConscript
+++ b/src/mongo/db/catalog/SConscript
@@ -299,7 +299,6 @@ env.Library(
         "index_catalog_entry_impl.cpp",
         "index_catalog_impl.cpp",
         "index_consistency.cpp",
-        "record_store_validate_adaptor.cpp",
     ],
     LIBDEPS=[
         'collection',
@@ -344,6 +343,17 @@ env.Library(
 )
 
 env.Library(
+    target='collection_validation',
+    source=[
+	"collection_validation.cpp",
+	"record_store_validate_adaptor.cpp",
+    ],
+    LIBDEPS=[
+	'catalog_impl',
+    ]
+)
+
+env.Library(
     target='catalog_helpers',
     source=[
         'capped_utils.cpp',
@@ -423,6 +433,7 @@ env.CppUnitTest(
         'collection_catalog',
         'collection_catalog_helper',
         'collection_options',
+        'collection_validation',
         'commit_quorum_options',
         'database_holder',
         'index_build_block',
diff --git a/src/mongo/db/catalog/collection.h b/src/mongo/db/catalog/collection.h
index 31b3f92b036..3f025bacf25 100644
--- a/src/mongo/db/catalog/collection.h
+++ b/src/mongo/db/catalog/collection.h
@@ -214,6 +214,8 @@ public:
     virtual const RecordStore* getRecordStore() const = 0;
     virtual RecordStore* getRecordStore() = 0;
 
+    virtual const BSONObj getValidatorDoc() const = 0;
+
     virtual bool requiresIdIndex() const = 0;
 
     virtual Snapshotted<BSONObj> docFor(OperationContext* const opCtx, RecordId loc) const = 0;
@@ -338,19 +340,6 @@ public:
     virtual Status truncate(OperationContext* const opCtx) = 0;
 
     /**
-     * Expects the caller to hold at least a collection IS lock.
-     *
-     * @return OK if the validate run successfully
-     *         OK will be returned even if corruption is found
-     *         details will be in 'results'.
-     */
-    virtual Status validate(OperationContext* const opCtx,
-                            const ValidateCmdLevel level,
-                            bool background,
-                            ValidateResults* const results,
-                            BSONObjBuilder* const output) = 0;
-
-    /**
      * forces data into cache.
      */
     virtual Status touch(OperationContext* const opCtx,
diff --git a/src/mongo/db/catalog/collection_impl.cpp b/src/mongo/db/catalog/collection_impl.cpp
index 6a4264d5f8b..de9a80f070d 100644
--- a/src/mongo/db/catalog/collection_impl.cpp
+++ b/src/mongo/db/catalog/collection_impl.cpp
@@ -47,7 +47,6 @@
 #include "mongo/db/catalog/index_catalog_impl.h"
 #include "mongo/db/catalog/index_consistency.h"
 #include "mongo/db/catalog/index_key_validate.h"
-#include "mongo/db/catalog/record_store_validate_adaptor.h"
 #include "mongo/db/clientcursor.h"
 #include "mongo/db/commands/server_status_metric.h"
 #include "mongo/db/concurrency/d_concurrency.h"
@@ -191,8 +190,6 @@ using std::string;
 using std::unique_ptr;
 using std::vector;
 
-using logger::LogComponent;
-
 CollectionImpl::CollectionImpl(OperationContext* opCtx,
                                const NamespaceString& nss,
                                UUID uuid,
@@ -1022,383 +1019,6 @@ StatusWith<std::vector<BSONObj>> CollectionImpl::addCollationDefaultsToIndexSpec
     return newIndexSpecs;
 }
 
-namespace {
-
-using ValidateResultsMap = std::map<std::string, ValidateResults>;
-
-// General validation logic for any RecordStore. Performs sanity checks to confirm that each
-// record in the store is valid according to the given RecordStoreValidateAdaptor and updates
-// record store stats to match.
-void _genericRecordStoreValidate(OperationContext* opCtx,
-                                 RecordStore* recordStore,
-                                 RecordStoreValidateAdaptor* indexValidator,
-                                 ValidateResults* results,
-                                 BSONObjBuilder* output) {
-    long long nrecords = 0;
-    long long dataSizeTotal = 0;
-    long long nInvalid = 0;
-
-    results->valid = true;
-    std::unique_ptr<SeekableRecordCursor> cursor = recordStore->getCursor(opCtx, true);
-    int interruptInterval = 4096;
-    RecordId prevRecordId;
-
-    while (auto record = cursor->next()) {
-        if (!(nrecords % interruptInterval)) {
-            opCtx->checkForInterrupt();
-        }
-        ++nrecords;
-        auto dataSize = record->data.size();
-        dataSizeTotal += dataSize;
-        size_t validatedSize;
-        Status status = indexValidator->validate(record->id, record->data, &validatedSize);
-
-        // Check to ensure isInRecordIdOrder() is being used properly.
-        if (prevRecordId.isValid()) {
-            invariant(prevRecordId < record->id);
-        }
-
-        // ValidatedSize = dataSize is not a general requirement as some storage engines may use
-        // padding, but we still require that they return the unpadded record data.
-        if (!status.isOK() || validatedSize != static_cast<size_t>(dataSize)) {
-            if (results->valid) {
-                // Only log once.
-                results->errors.push_back("detected one or more invalid documents (see logs)");
-            }
-            nInvalid++;
-            results->valid = false;
-            log() << "document at location: " << record->id << " is corrupted";
-        }
-
-        prevRecordId = record->id;
-    }
-
-    if (results->valid) {
-        recordStore->updateStatsAfterRepair(opCtx, nrecords, dataSizeTotal);
-    }
-
-    output->append("nInvalidDocuments", nInvalid);
-    output->appendNumber("nrecords", nrecords);
-}
-
-void _validateRecordStore(OperationContext* opCtx,
-                          RecordStore* recordStore,
-                          ValidateCmdLevel level,
-                          bool background,
-                          RecordStoreValidateAdaptor* indexValidator,
-                          ValidateResults* results,
-                          BSONObjBuilder* output) {
-    if (background) {
-        indexValidator->traverseRecordStore(recordStore, results, output);
-    } else {
-        // For 'full' validation we use the record store's validation functionality.
-        if (level == kValidateFull) {
-            recordStore->validate(opCtx, results, output);
-        }
-        _genericRecordStoreValidate(opCtx, recordStore, indexValidator, results, output);
-    }
-}
-
-void _validateIndexes(OperationContext* opCtx,
-                      IndexCatalog* indexCatalog,
-                      BSONObjBuilder* keysPerIndex,
-                      RecordStoreValidateAdaptor* indexValidator,
-                      ValidateCmdLevel level,
-                      ValidateResultsMap* indexNsResultsMap,
-                      ValidateResults* results) {
-
-    std::unique_ptr<IndexCatalog::IndexIterator> it = indexCatalog->getIndexIterator(opCtx, false);
-
-    // Validate Indexes.
-    while (it->more()) {
-        opCtx->checkForInterrupt();
-        const IndexCatalogEntry* entry = it->next();
-        const IndexDescriptor* descriptor = entry->descriptor();
-        const IndexAccessMethod* iam = entry->accessMethod();
-
-        log(LogComponent::kIndex) << "validating index " << descriptor->indexName()
-                                  << " on collection " << descriptor->parentNS();
-        ValidateResults& curIndexResults = (*indexNsResultsMap)[descriptor->indexName()];
-        bool checkCounts = false;
-        int64_t numTraversedKeys;
-        int64_t numValidatedKeys;
-
-        if (level == kValidateFull) {
-            iam->validate(opCtx, &numValidatedKeys, &curIndexResults);
-            checkCounts = true;
-        }
-
-        if (curIndexResults.valid) {
-            indexValidator->traverseIndex(iam, descriptor, &curIndexResults, &numTraversedKeys);
-
-            if (checkCounts && (numValidatedKeys != numTraversedKeys)) {
-                curIndexResults.valid = false;
-                string msg = str::stream()
-                    << "number of traversed index entries (" << numTraversedKeys
-                    << ") does not match the number of expected index entries (" << numValidatedKeys
-                    << ")";
-                results->errors.push_back(msg);
-                results->valid = false;
-            }
-
-            if (curIndexResults.valid) {
-                keysPerIndex->appendNumber(descriptor->indexName(),
-                                           static_cast<long long>(numTraversedKeys));
-            } else {
-                results->valid = false;
-            }
-        } else {
-            results->valid = false;
-        }
-    }
-}
-
-/**
- * Executes the second phase of validation for improved error reporting. This is only done if
- * any index inconsistencies are found during the first phase of validation.
- */
-void _gatherIndexEntryErrors(OperationContext* opCtx,
-                             RecordStore* recordStore,
-                             IndexCatalog* indexCatalog,
-                             IndexConsistency* indexConsistency,
-                             RecordStoreValidateAdaptor* indexValidator,
-                             ValidateResultsMap* indexNsResultsMap,
-                             ValidateResults* result) {
-    indexConsistency->setSecondPhase();
-
-    log(LogComponent::kIndex) << "Starting to traverse through all the document key sets.";
-
-    // During the second phase of validation, iterate through each documents key set and only record
-    // the keys that were inconsistent during the first phase of validation.
-    std::unique_ptr<SeekableRecordCursor> cursor = recordStore->getCursor(opCtx, true);
-    while (auto record = cursor->next()) {
-        opCtx->checkForInterrupt();
-
-        // We can ignore the status of validate as it was already checked during the first phase.
-        size_t validatedSize;
-        indexValidator->validate(record->id, record->data, &validatedSize).ignore();
-    }
-
-    log(LogComponent::kIndex) << "Finished traversing through all the document key sets.";
-    log(LogComponent::kIndex) << "Starting to traverse through all the indexes.";
-
-    // Iterate through all the indexes in the collection and only record the index entry keys that
-    // had inconsistencies during the first phase.
-    std::unique_ptr<IndexCatalog::IndexIterator> it = indexCatalog->getIndexIterator(opCtx, false);
-    while (it->more()) {
-        opCtx->checkForInterrupt();
-
-        const IndexCatalogEntry* entry = it->next();
-        const IndexDescriptor* descriptor = entry->descriptor();
-        const IndexAccessMethod* iam = entry->accessMethod();
-
-        log(LogComponent::kIndex) << "Traversing through the index entries for index "
-                                  << descriptor->indexName() << ".";
-        indexValidator->traverseIndex(
-            iam, descriptor, /*ValidateResults=*/nullptr, /*numTraversedKeys=*/nullptr);
-    }
-
-    log(LogComponent::kIndex) << "Finished traversing through all the indexes.";
-
-    indexConsistency->addIndexEntryErrors(indexNsResultsMap, result);
-}
-
-void _validateIndexKeyCount(OperationContext* opCtx,
-                            IndexCatalog* indexCatalog,
-                            RecordStore* recordStore,
-                            RecordStoreValidateAdaptor* indexValidator,
-                            ValidateResultsMap* indexNsResultsMap) {
-
-    std::unique_ptr<IndexCatalog::IndexIterator> indexIterator =
-        indexCatalog->getIndexIterator(opCtx, false);
-    while (indexIterator->more()) {
-        const IndexDescriptor* descriptor = indexIterator->next()->descriptor();
-        ValidateResults& curIndexResults = (*indexNsResultsMap)[descriptor->indexName()];
-
-        if (curIndexResults.valid) {
-            indexValidator->validateIndexKeyCount(
-                descriptor, recordStore->numRecords(opCtx), curIndexResults);
-        }
-    }
-}
-
-void _reportValidationResults(OperationContext* opCtx,
-                              IndexCatalog* indexCatalog,
-                              ValidateResultsMap* indexNsResultsMap,
-                              BSONObjBuilder* keysPerIndex,
-                              ValidateCmdLevel level,
-                              ValidateResults* results,
-                              BSONObjBuilder* output) {
-
-    std::unique_ptr<BSONObjBuilder> indexDetails;
-    if (level == kValidateFull) {
-        indexDetails = std::make_unique<BSONObjBuilder>();
-    }
-
-    // Report index validation results.
-    for (const auto& it : *indexNsResultsMap) {
-        const std::string indexNs = it.first;
-        const ValidateResults& vr = it.second;
-
-        if (!vr.valid) {
-            results->valid = false;
-        }
-
-        if (indexDetails) {
-            BSONObjBuilder bob(indexDetails->subobjStart(indexNs));
-            bob.appendBool("valid", vr.valid);
-
-            if (!vr.warnings.empty()) {
-                bob.append("warnings", vr.warnings);
-            }
-
-            if (!vr.errors.empty()) {
-                bob.append("errors", vr.errors);
-            }
-        }
-
-        results->warnings.insert(results->warnings.end(), vr.warnings.begin(), vr.warnings.end());
-        results->errors.insert(results->errors.end(), vr.errors.begin(), vr.errors.end());
-    }
-
-    output->append("nIndexes", indexCatalog->numIndexesReady(opCtx));
-    output->append("keysPerIndex", keysPerIndex->done());
-    if (indexDetails) {
-        output->append("indexDetails", indexDetails->done());
-    }
-}
-
-template <typename T>
-void addErrorIfUnequal(T stored, T cached, StringData name, ValidateResults* results) {
-    if (stored != cached) {
-        results->valid = false;
-        results->errors.push_back(str::stream() << "stored value for " << name
-                                                << " does not match cached value: "
-                                                << stored
-                                                << " != "
-                                                << cached);
-    }
-}
-
-void _validateCatalogEntry(OperationContext* opCtx,
-                           CollectionImpl* coll,
-                           BSONObj validatorDoc,
-                           ValidateResults* results) {
-    CollectionOptions options = DurableCatalog::get(opCtx)->getCollectionOptions(opCtx, coll->ns());
-    invariant(options.uuid);
-    addErrorIfUnequal(*(options.uuid), coll->uuid(), "UUID", results);
-    const CollatorInterface* collation = coll->getDefaultCollator();
-    addErrorIfUnequal(options.collation.isEmpty(), !collation, "simple collation", results);
-    if (!options.collation.isEmpty() && collation)
-        addErrorIfUnequal(options.collation.toString(),
-                          collation->getSpec().toBSON().toString(),
-                          "collation",
-                          results);
-    addErrorIfUnequal(options.capped, coll->isCapped(), "is capped", results);
-
-    addErrorIfUnequal(options.validator.toString(), validatorDoc.toString(), "validator", results);
-    if (!options.validator.isEmpty() && !validatorDoc.isEmpty()) {
-        addErrorIfUnequal(options.validationAction.length() ? options.validationAction : "error",
-                          coll->getValidationAction().toString(),
-                          "validation action",
-                          results);
-        addErrorIfUnequal(options.validationLevel.length() ? options.validationLevel : "strict",
-                          coll->getValidationLevel().toString(),
-                          "validation level",
-                          results);
-    }
-
-    addErrorIfUnequal(options.isView(), false, "is a view", results);
-    auto status = options.validateForStorage();
-    if (!status.isOK()) {
-        results->valid = false;
-        results->errors.push_back(str::stream() << "collection options are not valid for storage: "
-                                                << options.toBSON());
-    }
-}
-
-}  // namespace
-
-Status CollectionImpl::validate(OperationContext* opCtx,
-                                ValidateCmdLevel level,
-                                bool background,
-                                ValidateResults* results,
-                                BSONObjBuilder* output) {
-    invariant(opCtx->lockState()->isCollectionLockedForMode(ns(), MODE_IS));
-
-    try {
-        ValidateResultsMap indexNsResultsMap;
-        BSONObjBuilder keysPerIndex;  // not using subObjStart to be exception safe.
-        IndexConsistency indexConsistency(opCtx, this, ns(), _recordStore.get(), background);
-        RecordStoreValidateAdaptor indexValidator = RecordStoreValidateAdaptor(
-            opCtx, &indexConsistency, level, _indexCatalog.get(), &indexNsResultsMap);
-
-        std::string uuidString = str::stream() << " (UUID: " << _uuid << ")";
-
-        // Validate the record store.
-        log(LogComponent::kIndex) << "validating collection " << ns() << uuidString;
-        _validateRecordStore(
-            opCtx, _recordStore.get(), level, background, &indexValidator, results, output);
-
-        // Validate in-memory catalog information with the persisted info.
-        _validateCatalogEntry(opCtx, this, _validatorDoc, results);
-
-        // Validate indexes and check for mismatches.
-        if (results->valid) {
-            _validateIndexes(opCtx,
-                             _indexCatalog.get(),
-                             &keysPerIndex,
-                             &indexValidator,
-                             level,
-                             &indexNsResultsMap,
-                             results);
-
-            if (indexConsistency.haveEntryMismatch()) {
-                log(LogComponent::kIndex)
-                    << "Index inconsistencies were detected on collection " << ns()
-                    << ". Starting the second phase of index validation to gather concise errors.";
-                _gatherIndexEntryErrors(opCtx,
-                                        _recordStore.get(),
-                                        _indexCatalog.get(),
-                                        &indexConsistency,
-                                        &indexValidator,
-                                        &indexNsResultsMap,
-                                        results);
-            }
-        }
-
-        // Validate index key count.
-        if (results->valid) {
-            _validateIndexKeyCount(opCtx,
-                                   _indexCatalog.get(),
-                                   _recordStore.get(),
-                                   &indexValidator,
-                                   &indexNsResultsMap);
-        }
-
-        // Report the validation results for the user to see.
-        _reportValidationResults(
-            opCtx, _indexCatalog.get(), &indexNsResultsMap, &keysPerIndex, level, results, output);
-
-        if (!results->valid) {
-            log(LogComponent::kIndex) << "Validation complete for collection " << ns() << uuidString
-                                      << ". Corruption found.";
-        } else {
-            log(LogComponent::kIndex) << "Validation complete for collection " << ns() << uuidString
-                                      << ". No corruption found.";
-        }
-    } catch (DBException& e) {
-        if (ErrorCodes::isInterruption(e.code())) {
-            return e.toStatus();
-        }
-        string err = str::stream() << "exception during index validation: " << e.toString();
-        results->errors.push_back(err);
-        results->valid = false;
-    }
-
-    return Status::OK();
-}
-
 Status CollectionImpl::touch(OperationContext* opCtx,
                              bool touchData,
                              bool touchIndexes,
diff --git a/src/mongo/db/catalog/collection_impl.h b/src/mongo/db/catalog/collection_impl.h
index faac2e24fc7..9f561c77693 100644
--- a/src/mongo/db/catalog/collection_impl.h
+++ b/src/mongo/db/catalog/collection_impl.h
@@ -98,6 +98,10 @@ public:
         return _recordStore.get();
     }
 
+    const BSONObj getValidatorDoc() const final {
+        return _validatorDoc.getOwned();
+    }
+
     bool requiresIdIndex() const final;
 
     Snapshotted<BSONObj> docFor(OperationContext* opCtx, RecordId loc) const final {
@@ -226,12 +230,6 @@ public:
      */
     Status truncate(OperationContext* opCtx) final;
 
-    Status validate(OperationContext* opCtx,
-                    ValidateCmdLevel level,
-                    bool background,
-                    ValidateResults* results,
-                    BSONObjBuilder* output) final;
-
     /**
      * forces data into cache
      */
diff --git a/src/mongo/db/catalog/collection_mock.h b/src/mongo/db/catalog/collection_mock.h
index 19eb457bd7a..af1bc78c578 100644
--- a/src/mongo/db/catalog/collection_mock.h
+++ b/src/mongo/db/catalog/collection_mock.h
@@ -80,6 +80,10 @@ public:
         std::abort();
     }
 
+    const BSONObj getValidatorDoc() const {
+        std::abort();
+    }
+
     bool requiresIdIndex() const {
         std::abort();
     }
@@ -160,14 +164,6 @@ public:
         std::abort();
     }
 
-    Status validate(OperationContext* opCtx,
-                    ValidateCmdLevel level,
-                    bool background,
-                    ValidateResults* results,
-                    BSONObjBuilder* output) {
-        std::abort();
-    }
-
     Status touch(OperationContext* opCtx,
                  bool touchData,
                  bool touchIndexes,
diff --git a/src/mongo/db/catalog/collection_test.cpp b/src/mongo/db/catalog/collection_test.cpp
index e55f77b3f7b..3d03f8bf8ef 100644
--- a/src/mongo/db/catalog/collection_test.cpp
+++ b/src/mongo/db/catalog/collection_test.cpp
@@ -35,6 +35,7 @@
 #include "mongo/db/catalog/capped_utils.h"
 #include "mongo/db/catalog/catalog_test_fixture.h"
 #include "mongo/db/catalog/collection.h"
+#include "mongo/db/catalog/collection_validation.h"
 #include "mongo/db/db_raii.h"
 #include "mongo/db/repl/storage_interface_impl.h"
 #include "mongo/stdx/thread.h"
@@ -74,7 +75,7 @@ void CollectionTest::checkValidate(
     for (auto level : levels) {
         ValidateResults results;
         BSONObjBuilder output;
-        auto status = coll->validate(opCtx, level, false, &results, &output);
+        auto status = CollectionValidation::validate(opCtx, coll, level, false, &results, &output);
         ASSERT_OK(status);
         ASSERT_EQ(results.valid, valid);
         ASSERT_EQ(results.errors.size(), (long unsigned int)errors);
diff --git a/src/mongo/db/catalog/collection_validation.cpp b/src/mongo/db/catalog/collection_validation.cpp
new file mode 100644
index 00000000000..4eff8a7f9a4
--- /dev/null
+++ b/src/mongo/db/catalog/collection_validation.cpp
@@ -0,0 +1,436 @@
+/**
+ *    Copyright (C) 2019-present MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kStorage
+
+#include "mongo/platform/basic.h"
+
+#include "mongo/db/catalog/collection_validation.h"
+
+#include "mongo/db/catalog/collection.h"
+#include "mongo/db/operation_context.h"
+#include "mongo/db/storage/durable_catalog.h"
+#include "mongo/db/storage/record_store.h"
+#include "mongo/util/log.h"
+
+namespace mongo {
+
+using std::string;
+
+using logger::LogComponent;
+
+namespace CollectionValidation {
+
+namespace {
+
+using ValidateResultsMap = std::map<string, ValidateResults>;
+
+/**
+ * General validation logic for any RecordStore. Performs sanity checks to confirm that each
+ * record in the store is valid according to the given RecordStoreValidateAdaptor and updates
+ * record store stats to match.
+ */
+void _genericRecordStoreValidate(OperationContext* opCtx,
+                                 RecordStore* recordStore,
+                                 RecordStoreValidateAdaptor* indexValidator,
+                                 ValidateResults* results,
+                                 BSONObjBuilder* output) {
+    long long nrecords = 0;
+    long long dataSizeTotal = 0;
+    long long nInvalid = 0;
+
+    results->valid = true;
+    std::unique_ptr<SeekableRecordCursor> cursor = recordStore->getCursor(opCtx, true);
+    int interruptInterval = 4096;
+    RecordId prevRecordId;
+
+    while (auto record = cursor->next()) {
+        if (!(nrecords % interruptInterval)) {
+            opCtx->checkForInterrupt();
+        }
+        ++nrecords;
+        auto dataSize = record->data.size();
+        dataSizeTotal += dataSize;
+        size_t validatedSize;
+        Status status = indexValidator->validate(record->id, record->data, &validatedSize);
+
+        // Check to ensure isInRecordIdOrder() is being used properly.
+        if (prevRecordId.isValid()) {
+            invariant(prevRecordId < record->id);
+        }
+
+        // ValidatedSize = dataSize is not a general requirement as some storage engines may use
+        // padding, but we still require that they return the unpadded record data.
+        if (!status.isOK() || validatedSize != static_cast<size_t>(dataSize)) {
+            if (results->valid) {
+                // Only log once.
+                results->errors.push_back("detected one or more invalid documents (see logs)");
+            }
+            nInvalid++;
+            results->valid = false;
+            log() << "document at location: " << record->id << " is corrupted";
+        }
+
+        prevRecordId = record->id;
+    }
+
+    if (results->valid) {
+        recordStore->updateStatsAfterRepair(opCtx, nrecords, dataSizeTotal);
+    }
+
+    output->append("nInvalidDocuments", nInvalid);
+    output->appendNumber("nrecords", nrecords);
+}
+
+void _validateRecordStore(OperationContext* opCtx,
+                          RecordStore* recordStore,
+                          ValidateCmdLevel level,
+                          bool background,
+                          RecordStoreValidateAdaptor* indexValidator,
+                          ValidateResults* results,
+                          BSONObjBuilder* output) {
+    if (background) {
+        indexValidator->traverseRecordStore(recordStore, results, output);
+    } else {
+        // For 'full' validation we use the record store's validation functionality.
+        if (level == kValidateFull) {
+            recordStore->validate(opCtx, results, output);
+        }
+        _genericRecordStoreValidate(opCtx, recordStore, indexValidator, results, output);
+    }
+}
+
+void _validateIndexes(OperationContext* opCtx,
+                      IndexCatalog* indexCatalog,
+                      BSONObjBuilder* keysPerIndex,
+                      RecordStoreValidateAdaptor* indexValidator,
+                      ValidateCmdLevel level,
+                      ValidateResultsMap* indexNsResultsMap,
+                      ValidateResults* results) {
+
+    std::unique_ptr<IndexCatalog::IndexIterator> it = indexCatalog->getIndexIterator(opCtx, false);
+
+    // Validate Indexes.
+    while (it->more()) {
+        opCtx->checkForInterrupt();
+        const IndexCatalogEntry* entry = it->next();
+        const IndexDescriptor* descriptor = entry->descriptor();
+        const IndexAccessMethod* iam = entry->accessMethod();
+
+        log(LogComponent::kIndex) << "validating index " << descriptor->indexName()
+                                  << " on collection " << descriptor->parentNS();
+        ValidateResults& curIndexResults = (*indexNsResultsMap)[descriptor->indexName()];
+        bool checkCounts = false;
+        int64_t numTraversedKeys;
+        int64_t numValidatedKeys;
+
+        if (level == kValidateFull) {
+            iam->validate(opCtx, &numValidatedKeys, &curIndexResults);
+            checkCounts = true;
+        }
+
+        if (curIndexResults.valid) {
+            indexValidator->traverseIndex(iam, descriptor, &curIndexResults, &numTraversedKeys);
+
+            if (checkCounts && (numValidatedKeys != numTraversedKeys)) {
+                curIndexResults.valid = false;
+                string msg = str::stream()
+                    << "number of traversed index entries (" << numTraversedKeys
+                    << ") does not match the number of expected index entries (" << numValidatedKeys
+                    << ")";
+                results->errors.push_back(msg);
+                results->valid = false;
+            }
+
+            if (curIndexResults.valid) {
+                keysPerIndex->appendNumber(descriptor->indexName(),
+                                           static_cast<long long>(numTraversedKeys));
+            } else {
+                results->valid = false;
+            }
+        } else {
+            results->valid = false;
+        }
+    }
+}
+
+/**
+ * Executes the second phase of validation for improved error reporting. This is only done if
+ * any index inconsistencies are found during the first phase of validation.
+ */
+void _gatherIndexEntryErrors(OperationContext* opCtx,
+                             RecordStore* recordStore,
+                             IndexCatalog* indexCatalog,
+                             IndexConsistency* indexConsistency,
+                             RecordStoreValidateAdaptor* indexValidator,
+                             ValidateResultsMap* indexNsResultsMap,
+                             ValidateResults* result) {
+    indexConsistency->setSecondPhase();
+
+    log(LogComponent::kIndex) << "Starting to traverse through all the document key sets.";
+
+    // During the second phase of validation, iterate through each documents key set and only record
+    // the keys that were inconsistent during the first phase of validation.
+    std::unique_ptr<SeekableRecordCursor> cursor = recordStore->getCursor(opCtx, true);
+    while (auto record = cursor->next()) {
+        opCtx->checkForInterrupt();
+
+        // We can ignore the status of validate as it was already checked during the first phase.
+        size_t validatedSize;
+        indexValidator->validate(record->id, record->data, &validatedSize).ignore();
+    }
+
+    log(LogComponent::kIndex) << "Finished traversing through all the document key sets.";
+    log(LogComponent::kIndex) << "Starting to traverse through all the indexes.";
+
+    // Iterate through all the indexes in the collection and only record the index entry keys that
+    // had inconsistencies during the first phase.
+    std::unique_ptr<IndexCatalog::IndexIterator> it = indexCatalog->getIndexIterator(opCtx, false);
+    while (it->more()) {
+        opCtx->checkForInterrupt();
+
+        const IndexCatalogEntry* entry = it->next();
+        const IndexDescriptor* descriptor = entry->descriptor();
+        const IndexAccessMethod* iam = entry->accessMethod();
+
+        log(LogComponent::kIndex) << "Traversing through the index entries for index "
+                                  << descriptor->indexName() << ".";
+        indexValidator->traverseIndex(
+            iam, descriptor, /*ValidateResults=*/nullptr, /*numTraversedKeys=*/nullptr);
+    }
+
+    log(LogComponent::kIndex) << "Finished traversing through all the indexes.";
+
+    indexConsistency->addIndexEntryErrors(indexNsResultsMap, result);
+}
+
+void _validateIndexKeyCount(OperationContext* opCtx,
+                            IndexCatalog* indexCatalog,
+                            RecordStore* recordStore,
+                            RecordStoreValidateAdaptor* indexValidator,
+                            ValidateResultsMap* indexNsResultsMap) {
+
+    std::unique_ptr<IndexCatalog::IndexIterator> indexIterator =
+        indexCatalog->getIndexIterator(opCtx, false);
+    while (indexIterator->more()) {
+        const IndexDescriptor* descriptor = indexIterator->next()->descriptor();
+        ValidateResults& curIndexResults = (*indexNsResultsMap)[descriptor->indexName()];
+
+        if (curIndexResults.valid) {
+            indexValidator->validateIndexKeyCount(
+                descriptor, recordStore->numRecords(opCtx), curIndexResults);
+        }
+    }
+}
+
+void _reportValidationResults(OperationContext* opCtx,
+                              IndexCatalog* indexCatalog,
+                              ValidateResultsMap* indexNsResultsMap,
+                              BSONObjBuilder* keysPerIndex,
+                              ValidateCmdLevel level,
+                              ValidateResults* results,
+                              BSONObjBuilder* output) {
+
+    std::unique_ptr<BSONObjBuilder> indexDetails;
+    if (level == kValidateFull) {
+        indexDetails = std::make_unique<BSONObjBuilder>();
+    }
+
+    // Report index validation results.
+    for (const auto& it : *indexNsResultsMap) {
+        const string indexNs = it.first;
+        const ValidateResults& vr = it.second;
+
+        if (!vr.valid) {
+            results->valid = false;
+        }
+
+        if (indexDetails) {
+            BSONObjBuilder bob(indexDetails->subobjStart(indexNs));
+            bob.appendBool("valid", vr.valid);
+
+            if (!vr.warnings.empty()) {
+                bob.append("warnings", vr.warnings);
+            }
+
+            if (!vr.errors.empty()) {
+                bob.append("errors", vr.errors);
+            }
+        }
+
+        results->warnings.insert(results->warnings.end(), vr.warnings.begin(), vr.warnings.end());
+        results->errors.insert(results->errors.end(), vr.errors.begin(), vr.errors.end());
+    }
+
+    output->append("nIndexes", indexCatalog->numIndexesReady(opCtx));
+    output->append("keysPerIndex", keysPerIndex->done());
+    if (indexDetails) {
+        output->append("indexDetails", indexDetails->done());
+    }
+}
+
+template <typename T>
+void addErrorIfUnequal(T stored, T cached, StringData name, ValidateResults* results) {
+    if (stored != cached) {
+        results->valid = false;
+        results->errors.push_back(str::stream() << "stored value for " << name
+                                                << " does not match cached value: "
+                                                << stored
+                                                << " != "
+                                                << cached);
+    }
+}
+
+void _validateCatalogEntry(OperationContext* opCtx,
+                           Collection* coll,
+                           BSONObj validatorDoc,
+                           ValidateResults* results) {
+    CollectionOptions options = DurableCatalog::get(opCtx)->getCollectionOptions(opCtx, coll->ns());
+    invariant(options.uuid);
+    addErrorIfUnequal(*(options.uuid), coll->uuid(), "UUID", results);
+    const CollatorInterface* collation = coll->getDefaultCollator();
+    addErrorIfUnequal(options.collation.isEmpty(), !collation, "simple collation", results);
+    if (!options.collation.isEmpty() && collation)
+        addErrorIfUnequal(options.collation.toString(),
+                          collation->getSpec().toBSON().toString(),
+                          "collation",
+                          results);
+    addErrorIfUnequal(options.capped, coll->isCapped(), "is capped", results);
+
+    addErrorIfUnequal(options.validator.toString(), validatorDoc.toString(), "validator", results);
+    if (!options.validator.isEmpty() && !validatorDoc.isEmpty()) {
+        addErrorIfUnequal(options.validationAction.length() ? options.validationAction : "error",
+                          coll->getValidationAction().toString(),
+                          "validation action",
+                          results);
+        addErrorIfUnequal(options.validationLevel.length() ? options.validationLevel : "strict",
+                          coll->getValidationLevel().toString(),
+                          "validation level",
+                          results);
+    }
+
+    addErrorIfUnequal(options.isView(), false, "is a view", results);
+    auto status = options.validateForStorage();
+    if (!status.isOK()) {
+        results->valid = false;
+        results->errors.push_back(str::stream() << "collection options are not valid for storage: "
+                                                << options.toBSON());
+    }
+}
+
+}  // namespace
+
+Status validate(OperationContext* opCtx,
+                Collection* coll,
+                ValidateCmdLevel level,
+                bool background,
+                ValidateResults* results,
+                BSONObjBuilder* output) {
+    invariant(opCtx->lockState()->isCollectionLockedForMode(coll->ns(), MODE_IS));
+
+    try {
+        ValidateResultsMap indexNsResultsMap;
+        BSONObjBuilder keysPerIndex;  // not using subObjStart to be exception safe.
+        IndexConsistency indexConsistency(
+            opCtx, coll, coll->ns(), coll->getRecordStore(), background);
+        RecordStoreValidateAdaptor indexValidator = RecordStoreValidateAdaptor(
+            opCtx, &indexConsistency, level, coll->getIndexCatalog(), &indexNsResultsMap);
+
+        string uuidString = str::stream() << " (UUID: " << coll->uuid() << ")";
+
+        // Validate the record store.
+        log(LogComponent::kIndex) << "validating collection " << coll->ns() << uuidString;
+        _validateRecordStore(
+            opCtx, coll->getRecordStore(), level, background, &indexValidator, results, output);
+
+        // Validate in-memory catalog information with the persisted info.
+        _validateCatalogEntry(opCtx, coll, coll->getValidatorDoc(), results);
+
+        // Validate indexes and check for mismatches.
+        if (results->valid) {
+            _validateIndexes(opCtx,
+                             coll->getIndexCatalog(),
+                             &keysPerIndex,
+                             &indexValidator,
+                             level,
+                             &indexNsResultsMap,
+                             results);
+
+            if (indexConsistency.haveEntryMismatch()) {
+                log(LogComponent::kIndex)
+                    << "Index inconsistencies were detected on collection " << coll->ns()
+                    << ". Starting the second phase of index validation to gather concise errors.";
+                _gatherIndexEntryErrors(opCtx,
+                                        coll->getRecordStore(),
+                                        coll->getIndexCatalog(),
+                                        &indexConsistency,
+                                        &indexValidator,
+                                        &indexNsResultsMap,
+                                        results);
+            }
+        }
+
+        // Validate index key count.
+        if (results->valid) {
+            _validateIndexKeyCount(opCtx,
+                                   coll->getIndexCatalog(),
+                                   coll->getRecordStore(),
+                                   &indexValidator,
+                                   &indexNsResultsMap);
+        }
+
+        // Report the validation results for the user to see.
+        _reportValidationResults(opCtx,
+                                 coll->getIndexCatalog(),
+                                 &indexNsResultsMap,
+                                 &keysPerIndex,
+                                 level,
+                                 results,
+                                 output);
+
+        if (!results->valid) {
+            log(LogComponent::kIndex) << "Validation complete for collection " << coll->ns()
+                                      << uuidString << ". Corruption found.";
+        } else {
+            log(LogComponent::kIndex) << "Validation complete for collection " << coll->ns()
+                                      << uuidString << ". No corruption found.";
+        }
+    } catch (DBException& e) {
+        if (ErrorCodes::isInterruption(e.code())) {
+            return e.toStatus();
+        }
+        string err = str::stream() << "exception during index validation: " << e.toString();
+        results->errors.push_back(err);
+        results->valid = false;
+    }
+
+    return Status::OK();
+}
+}  // namespace CollectionValidation
+}  // namespace mongo
diff --git a/src/mongo/db/catalog/collection_validation.h b/src/mongo/db/catalog/collection_validation.h
new file mode 100644
index 00000000000..b7993e88c91
--- /dev/null
+++ b/src/mongo/db/catalog/collection_validation.h
@@ -0,0 +1,59 @@
+/**
+ *    Copyright (C) 2019-present MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#pragma once
+
+#include "mongo/db/catalog/record_store_validate_adaptor.h"
+
+namespace mongo {
+
+class OperationContext;
+class Collection;
+struct ValidateResults;
+class BSONObjBuilder;
+class Status;
+
+namespace CollectionValidation {
+
+/**
+ * Expects the caller to hold at least a collection IS lock.
+ *
+ * @return OK if the validate run successfully
+ *         OK will be returned even if corruption is found
+ *         details will be in 'results'.
+ */
+Status validate(OperationContext* opCtx,
+                Collection* coll,
+                ValidateCmdLevel level,
+                bool background,
+                ValidateResults* results,
+                BSONObjBuilder* output);
+
+}  // namespace CollectionValidation
+}  // namespace mongo
diff --git a/src/mongo/db/catalog/record_store_validate_adaptor.h b/src/mongo/db/catalog/record_store_validate_adaptor.h
index a5c043355dc..57fd4852fa1 100644
--- a/src/mongo/db/catalog/record_store_validate_adaptor.h
+++ b/src/mongo/db/catalog/record_store_validate_adaptor.h
@@ -40,6 +40,8 @@ namespace mongo {
 
 class IndexConsistency;
 
+enum ValidateCmdLevel : int { kValidateNormal = 0x01, kValidateFull = 0x02 };
+
 namespace {
 
 using ValidateResultsMap = std::map<std::string, ValidateResults>;
diff --git a/src/mongo/db/commands/SConscript b/src/mongo/db/commands/SConscript
index 3ad95dd53b8..cbfdc088df9 100644
--- a/src/mongo/db/commands/SConscript
+++ b/src/mongo/db/commands/SConscript
@@ -280,6 +280,7 @@ env.Library(
         '$BUILD_DIR/mongo/base',
         '$BUILD_DIR/mongo/db/catalog/catalog_helpers',
         '$BUILD_DIR/mongo/db/catalog/collection_catalog_helper',
+        '$BUILD_DIR/mongo/db/catalog/collection_validation',
         '$BUILD_DIR/mongo/db/catalog/database_holder',
         '$BUILD_DIR/mongo/db/catalog/index_key_validate',
         '$BUILD_DIR/mongo/db/catalog/multi_index_block',
diff --git a/src/mongo/db/commands/validate.cpp b/src/mongo/db/commands/validate.cpp
index ce7721dbfe9..a6505b464d5 100644
--- a/src/mongo/db/commands/validate.cpp
+++ b/src/mongo/db/commands/validate.cpp
@@ -32,7 +32,7 @@
 #include "mongo/platform/basic.h"
 
 #include "mongo/db/catalog/collection.h"
-
+#include "mongo/db/catalog/collection_validation.h"
 #include "mongo/db/client.h"
 #include "mongo/db/commands.h"
 #include "mongo/db/db_raii.h"
@@ -176,7 +176,8 @@ public:
         const bool background = false;
 
         ValidateResults results;
-        Status status = collection->validate(opCtx, level, background, &results, &result);
+        Status status =
+            CollectionValidation::validate(opCtx, collection, level, background, &results, &result);
         if (!status.isOK()) {
             return CommandHelpers::appendCommandStatusNoThrow(result, status);
         }
diff --git a/src/mongo/db/repl/SConscript b/src/mongo/db/repl/SConscript
index 68e102c8925..a5817517955 100644
--- a/src/mongo/db/repl/SConscript
+++ b/src/mongo/db/repl/SConscript
@@ -545,6 +545,7 @@ env.Library(
     LIBDEPS=[
         'sync_tail_test_fixture',
         '$BUILD_DIR/mongo/db/auth/authmocks',
+        '$BUILD_DIR/mongo/db/catalog/collection_validation',
         '$BUILD_DIR/mongo/db/index_builds_coordinator_interface',
     ],
 )
diff --git a/src/mongo/db/repl/idempotency_test_fixture.cpp b/src/mongo/db/repl/idempotency_test_fixture.cpp
index c6221493aa2..97b3f86fa2d 100644
--- a/src/mongo/db/repl/idempotency_test_fixture.cpp
+++ b/src/mongo/db/repl/idempotency_test_fixture.cpp
@@ -37,6 +37,7 @@
 
 #include "mongo/db/catalog/collection.h"
 #include "mongo/db/catalog/collection_catalog.h"
+#include "mongo/db/catalog/collection_validation.h"
 #include "mongo/db/catalog/database.h"
 #include "mongo/db/catalog/database_holder.h"
 #include "mongo/db/catalog/index_catalog.h"
@@ -614,7 +615,8 @@ CollectionState IdempotencyTest::validate(const NamespaceString& nss) {
 
     Lock::DBLock lk(_opCtx.get(), nss.db(), MODE_IX);
     auto lock = std::make_unique<Lock::CollectionLock>(_opCtx.get(), nss, MODE_X);
-    ASSERT_OK(collection->validate(_opCtx.get(), kValidateFull, false, &validateResults, &bob));
+    ASSERT_OK(CollectionValidation::validate(
+        _opCtx.get(), collection, kValidateFull, false, &validateResults, &bob));
     ASSERT_TRUE(validateResults.valid);
 
     std::string dataHash = computeDataHash(collection);
diff --git a/src/mongo/db/storage/record_store.h b/src/mongo/db/storage/record_store.h
index 30176dad464..21e70b95143 100644
--- a/src/mongo/db/storage/record_store.h
+++ b/src/mongo/db/storage/record_store.h
@@ -69,9 +69,6 @@ struct Record {
     RecordData data;
 };
 
-enum ValidateCmdLevel : int { kValidateNormal = 0x01, kValidateFull = 0x02 };
-
-
 /**
  * Retrieves Records from a RecordStore.
  *
diff --git a/src/mongo/dbtests/validate_tests.cpp b/src/mongo/dbtests/validate_tests.cpp
index 4073cbf5db5..925d0a88b9a 100644
--- a/src/mongo/dbtests/validate_tests.cpp
+++ b/src/mongo/dbtests/validate_tests.cpp
@@ -32,6 +32,7 @@
 #include <cstdint>
 
 #include "mongo/db/catalog/collection.h"
+#include "mongo/db/catalog/collection_validation.h"
 #include "mongo/db/catalog/index_catalog.h"
 #include "mongo/db/client.h"
 #include "mongo/db/db_raii.h"
@@ -90,12 +91,12 @@ protected:
         invariant(_opCtx.lockState()->isCollectionLockedForMode(_nss, MODE_X));
 
         Database* db = _autoDb.get()->getDb();
-        ASSERT_OK(db->getCollection(&_opCtx, _nss)
-                      ->validate(&_opCtx,
-                                 _full ? kValidateFull : kValidateNormal,
-                                 _background,
-                                 &results,
-                                 &output));
+        ASSERT_OK(CollectionValidation::validate(&_opCtx,
+                                                 db->getCollection(&_opCtx, _nss),
+                                                 _full ? kValidateFull : kValidateNormal,
+                                                 _background,
+                                                 &results,
+                                                 &output));
 
         //  Check if errors are reported if and only if valid is set to false.
         ASSERT_EQ(results.valid, results.errors.empty());
@@ -1225,8 +1226,12 @@ public:
                 std::make_unique<Lock::CollectionLock>(&_opCtx, _nss, MODE_X);
 
             Database* db = _autoDb.get()->getDb();
-            ASSERT_OK(db->getCollection(&_opCtx, _nss)
-                          ->validate(&_opCtx, kValidateFull, _background, &results, &output));
+            ASSERT_OK(CollectionValidation::validate(&_opCtx,
+                                                     db->getCollection(&_opCtx, _nss),
+                                                     kValidateFull,
+                                                     _background,
+                                                     &results,
+                                                     &output));
 
             ASSERT_EQ(false, results.valid);
             ASSERT_EQ(static_cast<size_t>(1), results.errors.size());
@@ -1329,8 +1334,12 @@ public:
                 std::make_unique<Lock::CollectionLock>(&_opCtx, _nss, MODE_X);
 
             Database* db = _autoDb.get()->getDb();
-            ASSERT_OK(db->getCollection(&_opCtx, _nss)
-                          ->validate(&_opCtx, kValidateFull, _background, &results, &output));
+            ASSERT_OK(CollectionValidation::validate(&_opCtx,
+                                                     db->getCollection(&_opCtx, _nss),
+                                                     kValidateFull,
+                                                     _background,
+                                                     &results,
+                                                     &output));
 
             ASSERT_EQ(false, results.valid);
             ASSERT_EQ(static_cast<size_t>(1), results.errors.size());
@@ -1413,8 +1422,12 @@ public:
                 std::make_unique<Lock::CollectionLock>(&_opCtx, _nss, MODE_X);
 
             Database* db = _autoDb.get()->getDb();
-            ASSERT_OK(db->getCollection(&_opCtx, _nss)
-                          ->validate(&_opCtx, kValidateFull, _background, &results, &output));
+            ASSERT_OK(CollectionValidation::validate(&_opCtx,
+                                                     db->getCollection(&_opCtx, _nss),
+                                                     kValidateFull,
+                                                     _background,
+                                                     &results,
+                                                     &output));
 
             ASSERT_EQ(false, results.valid);
             ASSERT_EQ(static_cast<size_t>(2), results.errors.size());