SERVER-22234 The Validate Command Should Work with failIndexKeyTooLong

9 files changed, 81 insertions, 14 deletions

diff --git a/buildscripts/resmokeconfig/suites/sharded_collections_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/sharded_collections_jscore_passthrough.yml
index 6b51a5f17f5..c8aa4df0857 100644
--- a/buildscripts/resmokeconfig/suites/sharded_collections_jscore_passthrough.yml
+++ b/buildscripts/resmokeconfig/suites/sharded_collections_jscore_passthrough.yml
@@ -25,6 +25,7 @@ selector:
     - jstests/core/geo_s2cursorlimitskip.js  # profiling.
     - jstests/core/geo_update_btree2.js  # notablescan.
     - jstests/core/index_bigkeys_nofail.js  # failIndexKeyTooLong.
+    - jstests/core/index_bigkeys_validation.js  # failIndexKeyTooLong.
     - jstests/core/max_time_ms.js  # sleep, SERVER-2212.
     - jstests/core/mr_replaceIntoDB.js  # MapReduceResult, SERVER-20495.
     - jstests/core/notablescan.js  # notablescan.
diff --git a/buildscripts/resmokeconfig/suites/sharding_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/sharding_jscore_passthrough.yml
index 56a5b581ce8..f217b513534 100644
--- a/buildscripts/resmokeconfig/suites/sharding_jscore_passthrough.yml
+++ b/buildscripts/resmokeconfig/suites/sharding_jscore_passthrough.yml
@@ -24,6 +24,7 @@ selector:
     - jstests/core/geo_s2cursorlimitskip.js  # profiling.
     - jstests/core/geo_update_btree2.js  # notablescan.
     - jstests/core/index_bigkeys_nofail.js  # failIndexKeyTooLong.
+    - jstests/core/index_bigkeys_validation.js  # failIndexKeyTooLong.
     - jstests/core/max_time_ms.js  # sleep, SERVER-2212.
     - jstests/core/notablescan.js  # notablescan.
     - jstests/core/profile*.js  # profiling.
diff --git a/jstests/core/index_bigkeys_nofail.js b/jstests/core/index_bigkeys_nofail.js
index 5a2e407a1a2..417470d7f04 100644
--- a/jstests/core/index_bigkeys_nofail.js
+++ b/jstests/core/index_bigkeys_nofail.js
@@ -46,4 +46,8 @@
     assert.writeError(t.update({_id: 3}, {$set:{name: x}}));
 
     db.getSiblingDB('admin').runCommand( { setParameter: 1, failIndexKeyTooLong: was } );
+
+    // Explicitly drop the collection to avoid failures in post-test hooks that run dbHash and
+    // validate commands.
+    t.drop();
 }());
diff --git a/jstests/core/index_bigkeys_validation.js b/jstests/core/index_bigkeys_validation.js
new file mode 100644
index 00000000000..ef29b07ecc7
--- /dev/null
+++ b/jstests/core/index_bigkeys_validation.js
@@ -0,0 +1,32 @@
+// Tests that index validation succeeds for long keys when failIndexKeyTooLong is set to false.
+// See: SERVER-22234
+'use strict';
+
+(function() {
+    var coll = db.longindex;
+    coll.drop();
+
+    var longVal = new Array(1025).join('x'); // Keys >= 1024 bytes cannot be indexed.
+
+    assert.commandWorked(db.adminCommand({setParameter: 1, failIndexKeyTooLong: false}));
+
+    assert.writeOK(coll.insert({_id: longVal}));
+    // Verify that validation succeeds when the failIndexKeyTooLong parameter is set to false,
+    // even when there are fewer index keys than documents.
+    var res = coll.validate({full: true, scandata: true});
+    assert.commandWorked(res);
+    assert(res.valid, tojson(res));
+
+    // Change failIndexKeyTooLong back to the default value.
+    assert.commandWorked(db.adminCommand({setParameter: 1, failIndexKeyTooLong: true}));
+
+    // Verify that validation fails when the failIndexKeyTooLong parameter is
+    // reverted to its old value and there are mismatched index keys and documents.
+    res = coll.validate({full: true, scandata: true});
+    assert.commandWorked(res);
+    assert.eq(res.valid, false, tojson(res));
+
+    // Explicitly drop the collection to avoid failures in post-test hooks that run dbHash and
+    // validate commands.
+    coll.drop();
+})();
diff --git a/jstests/libs/parallelTester.js b/jstests/libs/parallelTester.js
index 743d15b9825..39035731449 100644
--- a/jstests/libs/parallelTester.js
+++ b/jstests/libs/parallelTester.js
@@ -115,9 +115,10 @@ if (typeof _threadInject != "undefined") {
                                    "extent.js",
                                    "indexb.js",
 
-                                   // sets a failpoint that causes the server to ignore long keys,
-                                   // which makes index_bigkeys.js fail
+                                   // Tests that set a parameter that causes the server to ignore
+                                   // long index keys.
                                    "index_bigkeys_nofail.js",
+                                   "index_bigkeys_validation.js",
 
                                    // tests turn on profiling
                                    "profile1.js",
diff --git a/src/mongo/db/catalog/collection.cpp b/src/mongo/db/catalog/collection.cpp
index 596c80d396b..c3ea2c0ed80 100644
--- a/src/mongo/db/catalog/collection.cpp
+++ b/src/mongo/db/catalog/collection.cpp
@@ -54,6 +54,7 @@
 #include "mongo/db/ops/update_driver.h"
 #include "mongo/db/ops/update_request.h"
 #include "mongo/db/repl/replication_coordinator_global.h"
+#include "mongo/db/server_parameters.h"
 #include "mongo/db/service_context.h"
 #include "mongo/db/storage/mmap_v1/mmap_v1_options.h"
 #include "mongo/db/storage/record_fetcher.h"
@@ -926,11 +927,17 @@ void validateIndexKeyCount(OperationContext* txn,
                            int64_t numRecs,
                            ValidateResults* results) {
     if (idx.isIdIndex() && numIdxKeys != numRecs) {
-        string err = str::stream() << "number of _id index entries (" << numIdxKeys
+        string msg = str::stream() << "number of _id index entries (" << numIdxKeys
                                    << ") does not match the number of documents (" << numRecs
                                    << ")";
-        results->errors.push_back(err);
-        results->valid = false;
+
+        if (!failIndexKeyTooLong && (numIdxKeys < numRecs)) {
+            results->warnings.push_back(msg);
+        } else {
+            results->errors.push_back(msg);
+            results->valid = false;
+        }
+
         return;  // Avoid failing the next two checks, they just add redundant/confusing messages
     }
     if (!idx.isMultikey(txn) && numIdxKeys > numRecs) {
@@ -944,11 +951,16 @@ void validateIndexKeyCount(OperationContext* txn,
     //  index plugin with different semantics.
     if (!idx.isSparse() && !idx.isPartial() && idx.getAccessMethodName() == "" &&
         numIdxKeys < numRecs) {
-        string err = str::stream() << "index " << idx.indexName()
+        string msg = str::stream() << "index " << idx.indexName()
                                    << " is not sparse or partial, but has fewer entries ("
                                    << numIdxKeys << ") than documents (" << numRecs << ")";
-        results->errors.push_back(err);
-        results->valid = false;
+
+        if (!failIndexKeyTooLong) {
+            results->warnings.push_back(msg);
+        } else {
+            results->errors.push_back(msg);
+            results->valid = false;
+        }
     }
 }
 }  // namespace
@@ -966,6 +978,13 @@ Status Collection::validate(OperationContext* txn,
         return status;
 
     {  // indexes
+        if (!failIndexKeyTooLong) {
+            string warning =
+                "The server is configured with the failIndexKeyTooLong parameter set to false. "
+                "Validation failures that result from having fewer index entries than documents "
+                "have been downgraded from errors to warnings";
+            results->warnings.push_back(warning);
+        }
         output->append("nIndexes", _indexCatalog.numIndexesReady(txn));
         int idxn = 0;
         try {
diff --git a/src/mongo/db/commands/validate.cpp b/src/mongo/db/commands/validate.cpp
index 41d5b4afdb8..93693e68c57 100644
--- a/src/mongo/db/commands/validate.cpp
+++ b/src/mongo/db/commands/validate.cpp
@@ -107,17 +107,22 @@ public:
         if (!status.isOK())
             return appendCommandStatus(result, status);
 
-        result.appendBool("valid", results.valid);
-        result.append("errors", results.errors);
-
         if (!full) {
-            result.append(
-                "warning",
+            results.warnings.push_back(
                 "Some checks omitted for speed. use {full:true} option to do more thorough scan.");
         }
 
+        result.appendBool("valid", results.valid);
+        result.append("warnings", results.warnings);
+        result.append("errors", results.errors);
+
         if (!results.valid) {
-            result.append("advice", "ns corrupt. See http://dochub.mongodb.org/core/data-recovery");
+            result.append("advice",
+                          "A corrupt namespace has been detected. See "
+                          "http://dochub.mongodb.org/core/data-recovery for recovery steps. Note "
+                          "that validation failures may also result from running a server with the "
+                          "failIndexKeyTooLong parameter set to false and later disabling the "
+                          "parameter.");
         }
 
         return true;
diff --git a/src/mongo/db/index/index_access_method.h b/src/mongo/db/index/index_access_method.h
index df62c4ae937..ea9ea90c71c 100644
--- a/src/mongo/db/index/index_access_method.h
+++ b/src/mongo/db/index/index_access_method.h
@@ -28,6 +28,7 @@
 
 #pragma once
 
+#include <atomic>
 #include <memory>
 
 #include "mongo/base/disallow_copying.h"
@@ -40,6 +41,8 @@
 
 namespace mongo {
 
+extern std::atomic<bool> failIndexKeyTooLong;  // NOLINT
+
 class BSONObjBuilder;
 class MatchExpression;
 class UpdateTicket;
diff --git a/src/mongo/db/storage/record_store.h b/src/mongo/db/storage/record_store.h
index b6e973a16ab..2beb8368bf4 100644
--- a/src/mongo/db/storage/record_store.h
+++ b/src/mongo/db/storage/record_store.h
@@ -604,6 +604,7 @@ struct ValidateResults {
     }
     bool valid;
     std::vector<std::string> errors;
+    std::vector<std::string> warnings;
 };
 
 /**