SERVER-21717 Be more graceful in JS OOM

rather than return null to spidermonkey, just request that we bail out
as soon as possible.

4 files changed, 18 insertions, 14 deletions

diff --git a/src/mongo/scripting/mozjs/implscope.cpp b/src/mongo/scripting/mozjs/implscope.cpp
index 4a90bdc76d7..4d9a85eb13a 100644
--- a/src/mongo/scripting/mozjs/implscope.cpp
+++ b/src/mongo/scripting/mozjs/implscope.cpp
@@ -182,16 +182,18 @@ bool MozJSImplScope::_interruptCallback(JSContext* cx) {
         JS_MaybeGC(cx);
     }
 
-    bool kill = scope->isKillPending();
+    if (scope->_hasOutOfMemoryException) {
+        scope->_status = Status(ErrorCodes::JSInterpreterFailure, "Out of memory");
+    } else if (scope->isKillPending()) {
+        scope->_status = Status(ErrorCodes::JSInterpreterFailure, "Interrupted by the host");
+    }
 
-    if (kill) {
+    if (!scope->_status.isOK()) {
         scope->_engine->getDeadlineMonitor().stopDeadline(scope);
         scope->unregisterOperation();
-
-        scope->_status = Status(ErrorCodes::JSInterpreterFailure, "Interrupted by the host");
     }
 
-    return !kill;
+    return scope->_status.isOK();
 }
 
 void MozJSImplScope::_gcCallback(JSRuntime* rt, JSGCStatus status, void* data) {
@@ -292,6 +294,7 @@ MozJSImplScope::MozJSImplScope(MozJSScriptEngine* engine)
       _status(Status::OK()),
       _quickExit(false),
       _generation(0),
+      _hasOutOfMemoryException(false),
       _binDataProto(_context),
       _bsonProto(_context),
       _countDownLatchProto(_context),
@@ -360,7 +363,7 @@ MozJSImplScope::~MozJSImplScope() {
 }
 
 bool MozJSImplScope::hasOutOfMemoryException() {
-    return false;
+    return _hasOutOfMemoryException;
 }
 
 void MozJSImplScope::init(const BSONObj* data) {
@@ -829,7 +832,8 @@ bool MozJSImplScope::getQuickExit(int* exitCode) {
 }
 
 void MozJSImplScope::setOOM() {
-    _status = Status(ErrorCodes::JSInterpreterFailure, "Out of memory");
+    _hasOutOfMemoryException = true;
+    JS_RequestInterruptCallback(_runtime);
 }
 
 void MozJSImplScope::setParentStack(std::string parentStack) {
diff --git a/src/mongo/scripting/mozjs/implscope.h b/src/mongo/scripting/mozjs/implscope.h
index 34191b5642d..de496546042 100644
--- a/src/mongo/scripting/mozjs/implscope.h
+++ b/src/mongo/scripting/mozjs/implscope.h
@@ -365,6 +365,7 @@ private:
     bool _quickExit;
     std::string _parentStack;
     std::size_t _generation;
+    bool _hasOutOfMemoryException;
 
     WrapType<BinDataInfo> _binDataProto;
     WrapType<BSONInfo> _bsonProto;
diff --git a/src/mongo/scripting/mozjs/jscustomallocator.cpp b/src/mongo/scripting/mozjs/jscustomallocator.cpp
index adba220129d..a817d2a8cad 100644
--- a/src/mongo/scripting/mozjs/jscustomallocator.cpp
+++ b/src/mongo/scripting/mozjs/jscustomallocator.cpp
@@ -118,7 +118,9 @@ void* wrap_alloc(T&& func, void* ptr, size_t bytes) {
 
         scope->setOOM();
 
-        return nullptr;
+        // We fall through here because we want to let spidermonkey continue
+        // with whatever it was doing.  Calling setOOM will fail the top level
+        // operation as soon as possible.
     }
 
 #ifdef MONGO_NO_MALLOC_USABLE_SIZE
diff --git a/src/mongo/scripting/mozjs/proxyscope.cpp b/src/mongo/scripting/mozjs/proxyscope.cpp
index dba29eaee7d..5e48aabf9cf 100644
--- a/src/mongo/scripting/mozjs/proxyscope.cpp
+++ b/src/mongo/scripting/mozjs/proxyscope.cpp
@@ -106,13 +106,10 @@ std::string MozJSProxyScope::getError() {
     return out;
 }
 
-/**
- * This is an artifact of how out of memory errors were communicated in V8.  We
- * just throw out of memory errors from spidermonkey when we get them, rather
- * than setting a flag and having to pick them up here.
- */
 bool MozJSProxyScope::hasOutOfMemoryException() {
-    return false;
+    bool out;
+    run([&] { out = _implScope->hasOutOfMemoryException(); });
+    return out;
 }
 
 void MozJSProxyScope::gc() {