SERVER-50761 Crash the shell on invalid BSON errors on server responses from outgoing connections in

all of the jstestfuzz test suites.

Combine a new shell parameter --crashOnInvalidBSONError with --objcheck in the shell for all
jstestfuzz test suites. --objcheck in the shell turns on BSON validation on responses from outgoing
connections to the server. --crashOnInvalidBSONError crashes the shell if that extra BSON validation
encounters an error, and attempts to log the invalid BSON for debugging.

19 files changed, 111 insertions, 8 deletions

diff --git a/buildscripts/resmokeconfig/suites/core.yml b/buildscripts/resmokeconfig/suites/core.yml
index f0b374bc672..23a08a1b47d 100644
--- a/buildscripts/resmokeconfig/suites/core.yml
+++ b/buildscripts/resmokeconfig/suites/core.yml
@@ -14,6 +14,8 @@ executor:
       - ValidateCollections
   config:
     shell_options:
+      crashOnInvalidBSONError: ""
+      objcheck: ""
       eval: load("jstests/libs/override_methods/detect_spawning_own_mongod.js");
   hooks:
   - class: ValidateCollections
diff --git a/buildscripts/resmokeconfig/suites/jstestfuzz.yml b/buildscripts/resmokeconfig/suites/jstestfuzz.yml
index 0db4c0d0f72..88e55cfa817 100644
--- a/buildscripts/resmokeconfig/suites/jstestfuzz.yml
+++ b/buildscripts/resmokeconfig/suites/jstestfuzz.yml
@@ -8,7 +8,10 @@ executor:
   archive:
     hooks:
       - ValidateCollections
-  config: {}
+  config:
+    shell_options:
+      crashOnInvalidBSONError: ""
+      objcheck: ""
   hooks:
   - class: FuzzerRestoreSettings
   - class: ValidateCollections
diff --git a/buildscripts/resmokeconfig/suites/jstestfuzz_interrupt.yml b/buildscripts/resmokeconfig/suites/jstestfuzz_interrupt.yml
index f618ebd99d5..2fc080709e9 100644
--- a/buildscripts/resmokeconfig/suites/jstestfuzz_interrupt.yml
+++ b/buildscripts/resmokeconfig/suites/jstestfuzz_interrupt.yml
@@ -14,6 +14,8 @@ executor:
         TestData:
           checkForInterruptFailpointChance: 0.05
       eval: load('jstests/libs/jstestfuzz/check_for_interrupt_hook.js')
+      crashOnInvalidBSONError: ""
+      objcheck: ""
   hooks:
   - class: FuzzerRestoreSettings
   - class: ValidateCollections
diff --git a/buildscripts/resmokeconfig/suites/jstestfuzz_interrupt_replication.yml b/buildscripts/resmokeconfig/suites/jstestfuzz_interrupt_replication.yml
index 10bc6144747..06acf568150 100644
--- a/buildscripts/resmokeconfig/suites/jstestfuzz_interrupt_replication.yml
+++ b/buildscripts/resmokeconfig/suites/jstestfuzz_interrupt_replication.yml
@@ -16,6 +16,8 @@ executor:
         TestData:
           checkForInterruptFailpointChance: 0.05
       eval: load('jstests/libs/jstestfuzz/check_for_interrupt_hook.js')
+      crashOnInvalidBSONError: ""
+      objcheck: ""
   hooks:
   - class: FuzzerRestoreSettings
   # The CheckReplDBHash hook waits until all operations have replicated to and have been applied
diff --git a/buildscripts/resmokeconfig/suites/jstestfuzz_replication.yml b/buildscripts/resmokeconfig/suites/jstestfuzz_replication.yml
index 7bcfaec532f..dbfb57b6ad0 100644
--- a/buildscripts/resmokeconfig/suites/jstestfuzz_replication.yml
+++ b/buildscripts/resmokeconfig/suites/jstestfuzz_replication.yml
@@ -12,6 +12,8 @@ executor:
       - ValidateCollections
   config:
     shell_options:
+      crashOnInvalidBSONError: ""
+      objcheck: ""
       global_vars:
         TestData:
           # Other fuzzers test commands against replica sets with logical session ids.
diff --git a/buildscripts/resmokeconfig/suites/jstestfuzz_replication_continuous_stepdown.yml b/buildscripts/resmokeconfig/suites/jstestfuzz_replication_continuous_stepdown.yml
index 34b46926943..fd02b4c7989 100644
--- a/buildscripts/resmokeconfig/suites/jstestfuzz_replication_continuous_stepdown.yml
+++ b/buildscripts/resmokeconfig/suites/jstestfuzz_replication_continuous_stepdown.yml
@@ -11,6 +11,8 @@ executor:
       - ValidateCollections
   config:
     shell_options:
+      crashOnInvalidBSONError: ""
+      objcheck: ""
       global_vars:
         TestData:
           ignoreCommandsIncompatibleWithRollback: true
diff --git a/buildscripts/resmokeconfig/suites/jstestfuzz_replication_initsync.yml b/buildscripts/resmokeconfig/suites/jstestfuzz_replication_initsync.yml
index 8eed6daeb99..1223252bfb0 100644
--- a/buildscripts/resmokeconfig/suites/jstestfuzz_replication_initsync.yml
+++ b/buildscripts/resmokeconfig/suites/jstestfuzz_replication_initsync.yml
@@ -10,6 +10,8 @@ executor:
       - BackgroundInitialSync
   config:
     shell_options:
+      crashOnInvalidBSONError: ""
+      objcheck: ""
       global_vars:
         TestData:
           ignoreCommandsIncompatibleWithInitialSync: true
diff --git a/buildscripts/resmokeconfig/suites/jstestfuzz_replication_session.yml b/buildscripts/resmokeconfig/suites/jstestfuzz_replication_session.yml
index 5349d88ffeb..dd80eb713ac 100644
--- a/buildscripts/resmokeconfig/suites/jstestfuzz_replication_session.yml
+++ b/buildscripts/resmokeconfig/suites/jstestfuzz_replication_session.yml
@@ -13,6 +13,8 @@ executor:
   config:
     shell_options:
       eval: load("jstests/libs/override_methods/enable_sessions.js")
+      crashOnInvalidBSONError: ""
+      objcheck: ""
   hooks:
   - class: FuzzerRestoreSettings
   # The CheckReplDBHash hook waits until all operations have replicated to and have been applied
diff --git a/buildscripts/resmokeconfig/suites/jstestfuzz_replication_write_conflicts.yml b/buildscripts/resmokeconfig/suites/jstestfuzz_replication_write_conflicts.yml
index bfb8f6deca3..3346e6236b2 100644
--- a/buildscripts/resmokeconfig/suites/jstestfuzz_replication_write_conflicts.yml
+++ b/buildscripts/resmokeconfig/suites/jstestfuzz_replication_write_conflicts.yml
@@ -8,7 +8,10 @@ executor:
   archive:
     hooks:
       - ValidateCollections
-  config: {}
+  config:
+    shell_options:
+      crashOnInvalidBSONError: ""
+      objcheck: ""
   hooks:
   - class: EnableSpuriousWriteConflicts
     shell_options:
diff --git a/buildscripts/resmokeconfig/suites/jstestfuzz_sharded.yml b/buildscripts/resmokeconfig/suites/jstestfuzz_sharded.yml
index f1c722aa85f..9ae37bd2b47 100644
--- a/buildscripts/resmokeconfig/suites/jstestfuzz_sharded.yml
+++ b/buildscripts/resmokeconfig/suites/jstestfuzz_sharded.yml
@@ -15,6 +15,8 @@ executor:
         TestData:
           # Other fuzzers test commands against sharded clusters with logical session ids.
           disableImplicitSessions: true
+      crashOnInvalidBSONError: ""
+      objcheck: ""
   hooks:
   - class: FuzzerRestoreSettings
   - class: CheckReplDBHash
diff --git a/buildscripts/resmokeconfig/suites/jstestfuzz_sharded_causal_consistency.yml b/buildscripts/resmokeconfig/suites/jstestfuzz_sharded_causal_consistency.yml
index c1451d4dd29..bb5f072fa48 100644
--- a/buildscripts/resmokeconfig/suites/jstestfuzz_sharded_causal_consistency.yml
+++ b/buildscripts/resmokeconfig/suites/jstestfuzz_sharded_causal_consistency.yml
@@ -12,6 +12,8 @@ executor:
   config:
     shell_options:
       eval: load("jstests/libs/override_methods/enable_causal_consistency.js")
+      crashOnInvalidBSONError: ""
+      objcheck: ""
       global_vars:
         TestData:
           runningWithCausalConsistency: true
diff --git a/buildscripts/resmokeconfig/suites/jstestfuzz_sharded_continuous_stepdown.yml b/buildscripts/resmokeconfig/suites/jstestfuzz_sharded_continuous_stepdown.yml
index c2162c21abe..a23d7f76642 100644
--- a/buildscripts/resmokeconfig/suites/jstestfuzz_sharded_continuous_stepdown.yml
+++ b/buildscripts/resmokeconfig/suites/jstestfuzz_sharded_continuous_stepdown.yml
@@ -11,6 +11,8 @@ executor:
       - ValidateCollections
   config:
     shell_options:
+      crashOnInvalidBSONError: ""
+      objcheck: ""
       global_vars:
         TestData:
           runningWithConfigStepdowns: true
diff --git a/buildscripts/resmokeconfig/suites/jstestfuzz_sharded_session.yml b/buildscripts/resmokeconfig/suites/jstestfuzz_sharded_session.yml
index 5701e67acd5..a827c640e07 100644
--- a/buildscripts/resmokeconfig/suites/jstestfuzz_sharded_session.yml
+++ b/buildscripts/resmokeconfig/suites/jstestfuzz_sharded_session.yml
@@ -12,6 +12,8 @@ executor:
   config:
     shell_options:
       eval: load("jstests/libs/override_methods/enable_sessions.js")
+      crashOnInvalidBSONError: ""
+      objcheck: ""
   hooks:
   - class: FuzzerRestoreSettings
   - class: CheckReplDBHash
diff --git a/src/mongo/db/server_options.h b/src/mongo/db/server_options.h
index c8aa3037866..14a4416f0a8 100644
--- a/src/mongo/db/server_options.h
+++ b/src/mongo/db/server_options.h
@@ -72,6 +72,10 @@ struct ServerGlobalParams {
 
     bool objcheck = true;  // --objcheck
 
+    // Shell parameter, used for testing only, to tell the shell to crash on InvalidBSON errors.
+    // Can be paired with --objcheck so that extra BSON validation occurs.
+    bool crashOnInvalidBSONError = false;  // --crashOnInvalidBSONError
+
     int defaultProfile = 0;  // --profile
     boost::optional<BSONObj> defaultProfileFilter;
     int slowMS = 100;                      // --time in ms that is "slow"
diff --git a/src/mongo/db/server_options_general.idl b/src/mongo/db/server_options_general.idl
index 1d595700996..53e303f3aa9 100644
--- a/src/mongo/db/server_options_general.idl
+++ b/src/mongo/db/server_options_general.idl
@@ -164,6 +164,11 @@ configs:
         arg_vartype: Bool
         source: yaml
         hidden: true
+    crashOnInvalidBSONError:
+        description: "Crashes the shell if invalid BSON is returned from a call to the server. Must be paired with objcheck to provoke a BSON validation check."
+        arg_vartype: Switch
+        source: [ cli, ini ]
+        hidden: true
     enableExperimentalStorageDetailsCmd:
         description: 'EXPERIMENTAL (UNSUPPORTED). Enable command computing aggregate statistics on storage.'
         arg_vartype: Switch
diff --git a/src/mongo/rpc/object_check.h b/src/mongo/rpc/object_check.h
index 26bc175539e..ac9ddb899d8 100644
--- a/src/mongo/rpc/object_check.h
+++ b/src/mongo/rpc/object_check.h
@@ -29,10 +29,14 @@
 
 #pragma once
 
+#include <algorithm>
+
 #include "mongo/base/data_type_validated.h"
 #include "mongo/bson/bson_validate.h"
 #include "mongo/bson/bsontypes.h"
 #include "mongo/db/server_options.h"
+#include "mongo/logv2/redaction.h"
+#include "mongo/util/hex.h"
 
 // We do not use the rpc namespace here so we can specialize Validator.
 namespace mongo {
@@ -47,7 +51,24 @@ template <>
 struct Validator<BSONObj> {
 
     inline static Status validateLoad(const char* ptr, size_t length) {
-        return serverGlobalParams.objcheck ? validateBSON(ptr, length) : Status::OK();
+        if (!serverGlobalParams.objcheck) {
+            return Status::OK();
+        }
+
+        auto status = validateBSON(ptr, length);
+        if (serverGlobalParams.crashOnInvalidBSONError && !status.isOK()) {
+            std::string msg = "Invalid BSON was received: " + status.toString() +
+                // Using std::min with length so we do not max anything out in case the corruption
+                // is in the size of the object. The hex dump will be longer if needed.
+                ", beginning 5000 characters: " + std::string(ptr, std::min(length, (size_t)5000)) +
+                ", length: " + std::to_string(length) +
+                // Using std::min with hex dump length, too, to ensure we do not throw in hexdump()
+                // because of exceeded length and miss out on the core dump of the fassert below.
+                ", hex dump: " + hexdump(ptr, std::min(length, (size_t)(1000000 - 1)));
+            Status builtStatus(ErrorCodes::InvalidBSON, redact(msg));
+            fassertFailedWithStatus(50761, builtStatus);
+        }
+        return status;
     }
 
     static Status validateStore(const BSONObj& toStore);
diff --git a/src/mongo/rpc/object_check_test.cpp b/src/mongo/rpc/object_check_test.cpp
index 52010604f53..7ecb0d0ed93 100644
--- a/src/mongo/rpc/object_check_test.cpp
+++ b/src/mongo/rpc/object_check_test.cpp
@@ -35,23 +35,22 @@
 #include "mongo/db/jsobj.h"
 #include "mongo/db/server_options.h"
 #include "mongo/rpc/object_check.h"
+#include "mongo/unittest/death_test.h"
 #include "mongo/unittest/unittest.h"
 #include "mongo/util/scopeguard.h"
 
 namespace {
 
 using namespace mongo;
+using std::begin;
+using std::end;
+using std::swap;
 
 TEST(DataTypeValidated, BSONValidationEnabled) {
-    using std::swap;
-
     bool wasEnabled = serverGlobalParams.objcheck;
     const auto setValidation = [&](bool enabled) { serverGlobalParams.objcheck = enabled; };
     ON_BLOCK_EXIT([=] { setValidation(wasEnabled); });
 
-    using std::begin;
-    using std::end;
-
     BSONObj valid = BSON("baz"
                          << "bar"
                          << "garply"
@@ -88,4 +87,39 @@ TEST(DataTypeValidated, BSONValidationEnabled) {
         ASSERT_OK(cdrc.readAndAdvanceNoThrow(&v));
     }
 }
+
+DEATH_TEST(ObjectCheck, BSONValidationEnabledWithCrashOnError, "50761") {
+    bool objcheckValue = serverGlobalParams.objcheck;
+    serverGlobalParams.objcheck = true;
+    ON_BLOCK_EXIT([=] { serverGlobalParams.objcheck = objcheckValue; });
+
+    bool crashOnErrorValue = serverGlobalParams.crashOnInvalidBSONError;
+    serverGlobalParams.crashOnInvalidBSONError = true;
+    ON_BLOCK_EXIT([&] { serverGlobalParams.crashOnInvalidBSONError = crashOnErrorValue; });
+
+    BSONObj valid = BSON("baz"
+                         << "bar"
+                         << "garply"
+                         << BSON("foo"
+                                 << "bar"));
+    char buf[1024] = {0};
+    std::copy(valid.objdata(), valid.objdata() + valid.objsize(), begin(buf));
+
+    {
+        // mess up the data
+        DataRangeCursor drc(begin(buf), end(buf));
+        // skip past size so we don't trip any sanity checks.
+        drc.advanceNoThrow(4).transitional_ignore();  // skip size
+        while (drc.writeAndAdvanceNoThrow(0xFF).isOK())
+            ;
+    }
+
+    {
+        Validated<BSONObj> v;
+        ConstDataRangeCursor cdrc(begin(buf), end(buf));
+        // Crashes because of invalid BSON
+        cdrc.readAndAdvanceNoThrow(&v).ignore();
+    }
+}
+
 }  // namespace
diff --git a/src/mongo/shell/shell_options.cpp b/src/mongo/shell/shell_options.cpp
index 6f3e6991930..60d6cae2e24 100644
--- a/src/mongo/shell/shell_options.cpp
+++ b/src/mongo/shell/shell_options.cpp
@@ -131,6 +131,12 @@ Status storeMongoShellOptions(const moe::Environment& params,
         serverGlobalParams.objcheck = false;
     }
 
+    // Similar to 'objcheck' above, 'crashOnInvalidBSONError' must be common to both the server
+    // and shell for linking reasons.
+    if (params.count("crashOnInvalidBSONError")) {
+        serverGlobalParams.crashOnInvalidBSONError = true;
+    }
+
     if (params.count("port")) {
         shellGlobalParams.port = params["port"].as<string>();
     }
diff --git a/src/mongo/shell/shell_options.idl b/src/mongo/shell/shell_options.idl
index fe7c9d9121b..dad2a314976 100644
--- a/src/mongo/shell/shell_options.idl
+++ b/src/mongo/shell/shell_options.idl
@@ -102,6 +102,11 @@ configs:
     source: [ cli, ini ]
     conflicts: "objcheck"
     hidden: true
+  "crashOnInvalidBSONError":
+    description: "Crashes the shell if invalid BSON is returned from a call to the server. Must be paired with objcheck to provoke a BSON validation check."
+    arg_vartype: Switch
+    source: [ cli, ini ]
+    hidden: true
 
   "disableJavaScriptJIT":
     description: "disable the Javascript Just In Time compiler"