SERVER-12398 bubble error from plan stages to runner

31 files changed, 260 insertions, 68 deletions

diff --git a/src/mongo/db/exec/and_hash.cpp b/src/mongo/db/exec/and_hash.cpp
index 88e988a43a4..410a479e109 100644
--- a/src/mongo/db/exec/and_hash.cpp
+++ b/src/mongo/db/exec/and_hash.cpp
@@ -93,8 +93,7 @@ namespace mongo {
                 for (size_t j = 0; j < kLookAheadWorks; ++j) {
                     StageState childStatus = child->work(&_lookAheadResults[i]);
 
-                    if (PlanStage::IS_EOF == childStatus || PlanStage::DEAD == childStatus ||
-                        PlanStage::FAILURE == childStatus) {
+                    if (PlanStage::IS_EOF == childStatus || PlanStage::DEAD == childStatus) {
 
                         // A child went right to EOF.  Bail out.
                         _hashingChildren = false;
@@ -106,6 +105,14 @@ namespace mongo {
                         // child.
                         break;
                     }
+                    else if (PlanStage::FAILURE == childStatus) {
+                        // Propage error to parent.
+                        *out = _lookAheadResults[i];
+
+                        _hashingChildren = false;
+                        _dataMap.clear();
+                        return PlanStage::FAILURE;
+                    }
                     // We ignore NEED_TIME.  TODO: What do we want to do if the child provides
                     // NEED_FETCH?
                 }
@@ -206,7 +213,7 @@ namespace mongo {
     PlanStage::StageState AndHashStage::readFirstChild(WorkingSetID* out) {
         verify(_currentChild == 0);
 
-        WorkingSetID id;
+        WorkingSetID id = WorkingSet::INVALID_ID;
         StageState childStatus = workChild(0, &id);
 
         if (PlanStage::ADVANCED == childStatus) {
@@ -241,6 +248,10 @@ namespace mongo {
 
             return PlanStage::NEED_TIME;
         }
+        else if (PlanStage::FAILURE == childStatus) {
+            *out = id;
+            return childStatus;
+        }
         else {
             if (PlanStage::NEED_FETCH == childStatus) {
                 *out = id;
@@ -257,7 +268,7 @@ namespace mongo {
     PlanStage::StageState AndHashStage::hashOtherChildren(WorkingSetID* out) {
         verify(_currentChild > 0);
 
-        WorkingSetID id;
+        WorkingSetID id = WorkingSet::INVALID_ID;
         StageState childStatus = workChild(_currentChild, &id);
 
         if (PlanStage::ADVANCED == childStatus) {
@@ -320,6 +331,10 @@ namespace mongo {
             ++_commonStats.needTime;
             return PlanStage::NEED_TIME;
         }
+        else if (PlanStage::FAILURE == childStatus) {
+            *out = id;
+            return childStatus;
+        }
         else {
             if (PlanStage::NEED_FETCH == childStatus) {
                 *out = id;
diff --git a/src/mongo/db/exec/and_sorted.cpp b/src/mongo/db/exec/and_sorted.cpp
index f7c2787400c..2e123455579 100644
--- a/src/mongo/db/exec/and_sorted.cpp
+++ b/src/mongo/db/exec/and_sorted.cpp
@@ -75,7 +75,7 @@ namespace mongo {
         verify(DiskLoc() == _targetLoc);
 
         // Pick one, and get a loc to work toward.
-        WorkingSetID id;
+        WorkingSetID id = WorkingSet::INVALID_ID;
         StageState state = _children[0]->work(&id);
 
         if (PlanStage::ADVANCED == state) {
@@ -103,7 +103,12 @@ namespace mongo {
             ++_commonStats.needTime;
             return PlanStage::NEED_TIME;
         }
-        else if (PlanStage::IS_EOF == state || PlanStage::FAILURE == state) {
+        else if (PlanStage::IS_EOF == state) {
+            _isEOF = true;
+            return state;
+        }
+        else if (PlanStage::FAILURE == state) {
+            *out = id;
             _isEOF = true;
             return state;
         }
@@ -128,7 +133,7 @@ namespace mongo {
         // We have nodes that haven't hit _targetLoc yet.
         size_t workingChildNumber = _workingTowardRep.front();
         PlanStage* next = _children[workingChildNumber];
-        WorkingSetID id;
+        WorkingSetID id = WorkingSet::INVALID_ID;
         StageState state = next->work(&id);
 
         if (PlanStage::ADVANCED == state) {
@@ -206,7 +211,13 @@ namespace mongo {
                 return PlanStage::NEED_TIME;
             }
         }
-        else if (PlanStage::IS_EOF == state || PlanStage::FAILURE == state) {
+        else if (PlanStage::IS_EOF == state) {
+            _isEOF = true;
+            _ws->free(_targetId);
+            return state;
+        }
+        else if (PlanStage::FAILURE == state) {
+            *out = id;
             _isEOF = true;
             _ws->free(_targetId);
             return state;
diff --git a/src/mongo/db/exec/fetch.cpp b/src/mongo/db/exec/fetch.cpp
index 4402a1e6e88..6bd37d3de7f 100644
--- a/src/mongo/db/exec/fetch.cpp
+++ b/src/mongo/db/exec/fetch.cpp
@@ -78,7 +78,7 @@ namespace mongo {
 
         // If we're here, we're not waiting for a DiskLoc to be fetched.  Get another to-be-fetched
         // result from our child.
-        WorkingSetID id;
+        WorkingSetID id = WorkingSet::INVALID_ID;
         StageState status = _child->work(&id);
 
         if (PlanStage::ADVANCED == status) {
@@ -113,6 +113,10 @@ namespace mongo {
                 return returnIfMatches(member, id, out);
             }
         }
+        else if (PlanStage::FAILURE == status) {
+            *out = id;
+            return status;
+        }
         else {
             if (PlanStage::NEED_FETCH == status) {
                 *out = id;
diff --git a/src/mongo/db/exec/limit.cpp b/src/mongo/db/exec/limit.cpp
index cd85d4e2c66..879f0b86e37 100644
--- a/src/mongo/db/exec/limit.cpp
+++ b/src/mongo/db/exec/limit.cpp
@@ -43,7 +43,7 @@ namespace mongo {
         // If we've returned as many results as we're limited to, isEOF will be true.
         if (isEOF()) { return PlanStage::IS_EOF; }
 
-        WorkingSetID id;
+        WorkingSetID id = WorkingSet::INVALID_ID;
         StageState status = _child->work(&id);
 
         if (PlanStage::ADVANCED == status) {
@@ -52,6 +52,10 @@ namespace mongo {
             ++_commonStats.advanced;
             return PlanStage::ADVANCED;
         }
+        else if (PlanStage::FAILURE == status) {
+            *out = id;
+            return status;
+        }
         else {
             if (PlanStage::NEED_FETCH == status) {
                 *out = id;
diff --git a/src/mongo/db/exec/merge_sort.cpp b/src/mongo/db/exec/merge_sort.cpp
index 8e17a1e3e2e..52d1c00bd67 100644
--- a/src/mongo/db/exec/merge_sort.cpp
+++ b/src/mongo/db/exec/merge_sort.cpp
@@ -63,7 +63,7 @@ namespace mongo {
             // We have some child that we don't have a result from.  Each child must have a result
             // in order to pick the minimum result among all our children.  Work a child.
             PlanStage* child = _noResultToMerge.front();
-            WorkingSetID id;
+            WorkingSetID id = WorkingSet::INVALID_ID;
             StageState code = child->work(&id);
 
             if (PlanStage::ADVANCED == code) {
@@ -120,6 +120,10 @@ namespace mongo {
                 ++_commonStats.needTime;
                 return PlanStage::NEED_TIME;
             }
+            else if (PlanStage::FAILURE == code) {
+                *out = id;
+                return code;
+            }
             else {
                 if (PlanStage::NEED_FETCH == code) {
                     *out = id;
diff --git a/src/mongo/db/exec/merge_sort.h b/src/mongo/db/exec/merge_sort.h
index 2c448a0bd60..82540a467ab 100644
--- a/src/mongo/db/exec/merge_sort.h
+++ b/src/mongo/db/exec/merge_sort.h
@@ -103,6 +103,7 @@ namespace mongo {
         // priority_queue to remove the item from the list and quickly.
 
         struct StageWithValue {
+            StageWithValue() : id(WorkingSet::INVALID_ID), stage(NULL) { }
             WorkingSetID id;
             PlanStage* stage;
         };
diff --git a/src/mongo/db/exec/or.cpp b/src/mongo/db/exec/or.cpp
index fad209375c8..8387369c900 100644
--- a/src/mongo/db/exec/or.cpp
+++ b/src/mongo/db/exec/or.cpp
@@ -53,7 +53,7 @@ namespace mongo {
             _specificStats.matchTested = vector<size_t>(_children.size(), 0);
         }
 
-        WorkingSetID id;
+        WorkingSetID id = WorkingSet::INVALID_ID;
         StageState childStatus = _children[_currentChild]->work(&id);
 
         if (PlanStage::ADVANCED == childStatus) {
@@ -106,6 +106,10 @@ namespace mongo {
                 return PlanStage::NEED_TIME;
             }
         }
+        else if (PlanStage::FAILURE == childStatus) {
+            *out = id;
+            return childStatus;
+        }
         else {
             if (PlanStage::NEED_FETCH == childStatus) {
                 *out = id;
diff --git a/src/mongo/db/exec/plan_stage.h b/src/mongo/db/exec/plan_stage.h
index d7d8b52639c..6797841971e 100644
--- a/src/mongo/db/exec/plan_stage.h
+++ b/src/mongo/db/exec/plan_stage.h
@@ -124,8 +124,11 @@ namespace mongo {
             // dropped or state deleted.
             DEAD,
 
-            // Something has gone unrecoverably wrong.  Stop running this query.  There is nothing
-            // output in the out parameter.
+            // Something has gone unrecoverably wrong.  Stop running this query.
+            // If the out parameter does not refer to an invalid working set member,
+            // call WorkingSetCommon::getStatusMemberObject() to get details on the failure.
+            // Any class implementing this interface must set the WSID out parameter to
+            // INVALID_ID or a valid WSM ID if FAILURE is returned.
             FAILURE,
 
             // Something isn't in memory.  Fetch it.
diff --git a/src/mongo/db/exec/projection.cpp b/src/mongo/db/exec/projection.cpp
index 671d5620fb5..d83dd757366 100644
--- a/src/mongo/db/exec/projection.cpp
+++ b/src/mongo/db/exec/projection.cpp
@@ -30,6 +30,7 @@
 
 #include "mongo/db/diskloc.h"
 #include "mongo/db/exec/plan_stage.h"
+#include "mongo/db/exec/working_set_common.h"
 #include "mongo/db/jsobj.h"
 #include "mongo/db/matcher/expression.h"
 #include "mongo/util/mongoutils/str.h"
@@ -51,7 +52,7 @@ namespace mongo {
     PlanStage::StageState ProjectionStage::work(WorkingSetID* out) {
         ++_commonStats.works;
 
-        WorkingSetID id;
+        WorkingSetID id = WorkingSet::INVALID_ID;
         StageState status = _child->work(&id);
 
         // Note that we don't do the normal if isEOF() return EOF thing here.  Our child might be a
@@ -60,15 +61,18 @@ namespace mongo {
             WorkingSetMember* member = _ws->get(id);
             Status projStatus = _exec->transform(member);
             if (!projStatus.isOK()) {
-                // TODO: should this really fail?
                 warning() << "Couldn't execute projection, status = "
                           << projStatus.toString() << endl;
+                *out = WorkingSetCommon::allocateStatusMember(_ws, projStatus);
                 return PlanStage::FAILURE;
             }
 
             *out = id;
             ++_commonStats.advanced;
         }
+        else if (PlanStage::FAILURE == status) {
+            *out = id;
+        }
         else if (PlanStage::NEED_FETCH == status) {
             *out = id;
             ++_commonStats.needFetch;
diff --git a/src/mongo/db/exec/s2near.cpp b/src/mongo/db/exec/s2near.cpp
index 90e85f5dd12..fb7cc66eff9 100644
--- a/src/mongo/db/exec/s2near.cpp
+++ b/src/mongo/db/exec/s2near.cpp
@@ -123,7 +123,13 @@ namespace mongo {
     PlanStage::StageState S2NearStage::work(WorkingSetID* out) {
         if (!_initted) { init(); }
 
-        if (_failed) { return PlanStage::FAILURE; }
+        if (_failed) {
+            mongoutils::str::stream ss;
+            ss << "unable to load geo index " << _params.indexKeyPattern;
+            Status status(ErrorCodes::IndexNotFound, ss);
+            *out = WorkingSetCommon::allocateStatusMember( _ws, status);
+            return PlanStage::FAILURE;
+        }
         if (isEOF()) { return PlanStage::IS_EOF; }
         ++_commonStats.works;
 
diff --git a/src/mongo/db/exec/skip.cpp b/src/mongo/db/exec/skip.cpp
index 0858967d168..09401905d11 100644
--- a/src/mongo/db/exec/skip.cpp
+++ b/src/mongo/db/exec/skip.cpp
@@ -42,7 +42,7 @@ namespace mongo {
 
         if (isEOF()) { return PlanStage::IS_EOF; }
 
-        WorkingSetID id;
+        WorkingSetID id = WorkingSet::INVALID_ID;
         StageState status = _child->work(&id);
 
         if (PlanStage::ADVANCED == status) {
@@ -59,6 +59,10 @@ namespace mongo {
             ++_commonStats.advanced;
             return PlanStage::ADVANCED;
         }
+        else if (PlanStage::FAILURE == status) {
+            *out = id;
+            return status;
+        }
         else {
             if (PlanStage::NEED_FETCH == status) {
                 *out = id;
diff --git a/src/mongo/db/exec/sort.cpp b/src/mongo/db/exec/sort.cpp
index 1a5f609ebb6..3347e25ff39 100644
--- a/src/mongo/db/exec/sort.cpp
+++ b/src/mongo/db/exec/sort.cpp
@@ -307,6 +307,11 @@ namespace mongo {
         }
 
         if (_memUsage > kMaxBytes) {
+            mongoutils::str::stream ss;
+            ss << "sort stage buffered data usage of " << _memUsage
+               << " bytes exceeds internal limit of " << kMaxBytes << " bytes";
+            Status status(ErrorCodes::Overflow, ss);
+            *out = WorkingSetCommon::allocateStatusMember( _ws, status);
             return PlanStage::FAILURE;
         }
 
@@ -314,7 +319,7 @@ namespace mongo {
 
         // Still reading in results to sort.
         if (!_sorted) {
-            WorkingSetID id;
+            WorkingSetID id = WorkingSet::INVALID_ID;
             StageState code = _child->work(&id);
 
             if (PlanStage::ADVANCED == code) {
@@ -354,6 +359,10 @@ namespace mongo {
                 ++_commonStats.needTime;
                 return PlanStage::NEED_TIME;
             }
+            else if (PlanStage::FAILURE == code) {
+                *out = id;
+                return code;
+            }
             else {
                 if (PlanStage::NEED_FETCH == code) {
                     *out = id;
diff --git a/src/mongo/db/exec/sort.h b/src/mongo/db/exec/sort.h
index 9d688b150e6..98054a82e74 100644
--- a/src/mongo/db/exec/sort.h
+++ b/src/mongo/db/exec/sort.h
@@ -245,7 +245,7 @@ namespace mongo {
         CommonStats _commonStats;
         SortStats _specificStats;
 
-        // The usage in bytes of all bufered data that we're sorting.
+        // The usage in bytes of all buffered data that we're sorting.
         size_t _memUsage;
     };
 
diff --git a/src/mongo/db/exec/sort_test.cpp b/src/mongo/db/exec/sort_test.cpp
index 225e2732b09..c890fb4dd15 100644
--- a/src/mongo/db/exec/sort_test.cpp
+++ b/src/mongo/db/exec/sort_test.cpp
@@ -54,7 +54,7 @@ namespace {
         ASSERT_FALSE(sort.isEOF());
 
         // First call to work() initializes sort key generator.
-        WorkingSetID id;
+        WorkingSetID id = WorkingSet::INVALID_ID;
         PlanStage::StageState state = sort.work(&id);
         ASSERT_EQUALS(state, PlanStage::NEED_TIME);
 
@@ -111,7 +111,7 @@ namespace {
 
         SortStage sort(params, &ws, ms);
 
-        WorkingSetID id;
+        WorkingSetID id = WorkingSet::INVALID_ID;
         PlanStage::StageState state = PlanStage::NEED_TIME;
 
         // Keep working sort stage until data is available.
diff --git a/src/mongo/db/exec/text.cpp b/src/mongo/db/exec/text.cpp
index 688f9f549e5..a14e7bc808f 100644
--- a/src/mongo/db/exec/text.cpp
+++ b/src/mongo/db/exec/text.cpp
@@ -31,6 +31,7 @@
 #include "mongo/base/owned_pointer_vector.h"
 #include "mongo/db/exec/filter.h"
 #include "mongo/db/exec/working_set.h"
+#include "mongo/db/exec/working_set_common.h"
 #include "mongo/db/exec/working_set_computed_data.h"
 #include "mongo/db/jsobj.h"
 #include "mongo/db/query/internal_plans.h"
@@ -63,7 +64,7 @@ namespace mongo {
 
         // Fill out our result queue.
         if (!_filledOutResults) {
-            PlanStage::StageState ss = fillOutResults();
+            PlanStage::StageState ss = fillOutResults(out);
             if (ss == PlanStage::IS_EOF || ss == PlanStage::FAILURE) {
                 return ss;
             }
@@ -131,17 +132,25 @@ namespace mongo {
         return ret.release();
     }
 
-    PlanStage::StageState TextStage::fillOutResults() {
+    PlanStage::StageState TextStage::fillOutResults(WorkingSetID* out) {
         Database* db = cc().database();
         Collection* collection = db->getCollection( _params.ns );
         if (NULL == collection) {
-            warning() << "TextStage params namespace error";
+            std::string errmsg = mongoutils::str::stream() << "TextStage params namespace error";
+            warning() << errmsg;
+            Status status(ErrorCodes::NamespaceNotFound, errmsg);
+            *out = WorkingSetCommon::allocateStatusMember( _ws, status);
             return PlanStage::FAILURE;
         }
         vector<IndexDescriptor*> idxMatches;
         collection->getIndexCatalog()->findIndexByType("text", idxMatches);
         if (1 != idxMatches.size()) {
-            warning() << "Expected exactly one text index";
+            std::string errmsg = mongoutils::str::stream() << "Expected exactly one text index";
+            warning() << errmsg;
+            // Using IndexNotFound error code because we are unable to
+            // determine which index to select.
+            Status status(ErrorCodes::IndexNotFound, errmsg);
+            *out = WorkingSetCommon::allocateStatusMember( _ws, status);
             return PlanStage::FAILURE;
         }
 
@@ -171,7 +180,7 @@ namespace mongo {
             BSONObj keyObj;
             DiskLoc loc;
 
-            WorkingSetID id;
+            WorkingSetID id = WorkingSet::INVALID_ID;
             PlanStage::StageState state = scanners.vector()[currentIndexScanner]->work(&id);
 
             if (PlanStage::ADVANCED == state) {
@@ -193,7 +202,19 @@ namespace mongo {
             }
             else {
                 verify(PlanStage::FAILURE == state);
-                warning() << "error from index scan during text stage: invalid FAILURE state";
+                std::string errmsg = mongoutils::str::stream() <<
+                    "error from index scan during text stage: invalid FAILURE state";
+                warning() << errmsg;
+                // Propagate error status from underlying index scan if available.
+                // Otherwise, create a new error status.
+                if (WorkingSet::INVALID_ID == id) {
+                    // Using InternalError error code because this is very uncommon.
+                    // Currently, there are no code paths in IndexScan::work() that return
+                    // PlanStage::FAILURE.
+                    Status status(ErrorCodes::InternalError, errmsg);
+                    id = WorkingSetCommon::allocateStatusMember( _ws, status);
+                }
+                *out = id;
                 return PlanStage::FAILURE;
             }
         }
diff --git a/src/mongo/db/exec/text.h b/src/mongo/db/exec/text.h
index fd600d9fc41..2b27670b067 100644
--- a/src/mongo/db/exec/text.h
+++ b/src/mongo/db/exec/text.h
@@ -95,7 +95,8 @@ namespace mongo {
     private:
         // Helper for buffering results array.  Returns NEED_TIME (if any results were produced),
         // IS_EOF, or FAILURE.
-        StageState fillOutResults();
+        // If the result state is FAILURE, out be set to a valid status member WSID.
+        StageState fillOutResults(WorkingSetID *out);
 
         // Helper to update aggregate score with a new-found (term, score) pair for this document.
         // Also rejects documents that don't match this stage's filter.
diff --git a/src/mongo/db/exec/working_set.h b/src/mongo/db/exec/working_set.h
index 505a7c47f32..2de8f85fa72 100644
--- a/src/mongo/db/exec/working_set.h
+++ b/src/mongo/db/exec/working_set.h
@@ -70,7 +70,7 @@ namespace mongo {
          * Do not delete the returned pointer as the WorkingSet retains ownership. Call free() to
          * release it.
          */
-        WorkingSetMember* get(const WorkingSetID& i) {
+        WorkingSetMember* get(const WorkingSetID& i) const {
             dassert(i < _data.size()); // ID has been allocated.
             dassert(_data[i].nextFreeOrSelf == i); // ID currently in use.
             return _data[i].member;
diff --git a/src/mongo/db/exec/working_set_common.cpp b/src/mongo/db/exec/working_set_common.cpp
index 1406f809c58..8b9e07a974c 100644
--- a/src/mongo/db/exec/working_set_common.cpp
+++ b/src/mongo/db/exec/working_set_common.cpp
@@ -64,4 +64,60 @@ namespace mongo {
         }
     }
 
+    // static
+    WorkingSetID WorkingSetCommon::allocateStatusMember(WorkingSet* ws, const Status& status) {
+        invariant(ws);
+
+        BSONObjBuilder bob;
+        bob.append("ok", status.isOK() ? 1.0 : 0.0);
+        bob.append("code", status.code());
+        bob.append("errmsg", status.reason());
+
+        WorkingSetID wsid = ws->allocate();
+        WorkingSetMember* member = ws->get(wsid);
+        member->state = WorkingSetMember::OWNED_OBJ;
+        member->obj = bob.obj();
+
+        return wsid;
+    }
+
+    // static
+    bool WorkingSetCommon::isValidStatusMemberObject(const BSONObj& obj) {
+        return obj.nFields() == 3 &&
+               obj.hasField("ok") &&
+               obj.hasField("code") &&
+               obj.hasField("errmsg");
+    }
+
+    // static
+    void WorkingSetCommon::getStatusMemberObject(const WorkingSet& ws, WorkingSetID wsid,
+                                                 BSONObj* objOut) {
+        invariant(objOut);
+
+        // Validate ID and working set member.
+        if (WorkingSet::INVALID_ID == wsid) {
+            return;
+        }
+        WorkingSetMember* member = ws.get(wsid);
+        if (!member->hasOwnedObj()) {
+            return;
+        }
+        BSONObj obj = member->obj;
+        if (!isValidStatusMemberObject(obj)) {
+            return;
+        }
+        *objOut = member->obj;
+    }
+
+    // static
+    std::string WorkingSetCommon::toStatusString(const BSONObj& obj) {
+        if (!isValidStatusMemberObject(obj)) {
+            Status unknownStatus(ErrorCodes::UnknownError, "no details available");
+            return unknownStatus.toString();
+        }
+        Status status(ErrorCodes::fromInt(obj.getIntField("code")),
+                      obj.getStringField("errmsg"));
+        return status.toString();
+    }
+
 }  // namespace mongo
diff --git a/src/mongo/db/exec/working_set_common.h b/src/mongo/db/exec/working_set_common.h
index 3038bdf7eac..a983ca055fe 100644
--- a/src/mongo/db/exec/working_set_common.h
+++ b/src/mongo/db/exec/working_set_common.h
@@ -28,9 +28,9 @@
 
 #pragma once
 
-namespace mongo {
+#include "mongo/db/exec/working_set.h"
 
-    class WorkingSetMember;
+namespace mongo {
 
     class WorkingSetCommon {
     public:
@@ -45,6 +45,38 @@ namespace mongo {
          * Initialize the fields in 'dest' from 'src', creating copies of owned objects as needed.
          */
         static void initFrom(WorkingSetMember* dest, const WorkingSetMember& src);
+
+
+        /**
+         * Allocate a new WSM and initialize it with
+         * the code and reason from the status.
+         * Owned BSON object will have the following layout:
+         * {
+         *     ok: <ok>, // 1 for OK; 0 otherwise.
+         *     code: <code>, // Status::code()
+         *     errmsg: <errmsg> // Status::reason()
+         * }
+         */
+        static WorkingSetID allocateStatusMember(WorkingSet* ws, const Status& status);
+
+        /**
+         * Returns true if object was created by allocateStatusMember().
+         */
+        static bool isValidStatusMemberObject(const BSONObj& obj);
+
+        /**
+         * Returns object in working set member created with allocateStatusMember().
+         * Does not assume isValidStatusMemberObject.
+         * If the WSID is invalid or the working set member is created by
+         * allocateStatusMember, objOut will not be updated.
+         */
+        static void getStatusMemberObject(const WorkingSet& ws, WorkingSetID wsid,
+                                          BSONObj* objOut);
+
+        /**
+         * Formats working set member object created with allocateStatusMember().
+         */
+        static std::string toStatusString(const BSONObj& obj);
     };
 
 }  // namespace mongo
diff --git a/src/mongo/db/query/multi_plan_runner.cpp b/src/mongo/db/query/multi_plan_runner.cpp
index fa6d6eb9f56..64c54097a45 100644
--- a/src/mongo/db/query/multi_plan_runner.cpp
+++ b/src/mongo/db/query/multi_plan_runner.cpp
@@ -438,7 +438,7 @@ namespace mongo {
                 restoreState();
             }
 
-            WorkingSetID id;
+            WorkingSetID id = WorkingSet::INVALID_ID;
             PlanStage::StageState state = candidate.root->work(&id);
 
             if (PlanStage::ADVANCED == state) {
diff --git a/src/mongo/db/query/new_find.cpp b/src/mongo/db/query/new_find.cpp
index f6f16fee5e6..e6f27baa4d9 100644
--- a/src/mongo/db/query/new_find.cpp
+++ b/src/mongo/db/query/new_find.cpp
@@ -33,6 +33,7 @@
 #include "mongo/db/commands.h"
 #include "mongo/db/exec/filter.h"
 #include "mongo/db/exec/oplogstart.h"
+#include "mongo/db/exec/working_set_common.h"
 #include "mongo/db/keypattern.h"
 #include "mongo/db/kill_current_op.h"
 #include "mongo/db/query/find_constants.h"
@@ -232,6 +233,11 @@ namespace mongo {
             bool saveClientCursor = false;
 
             if (Runner::RUNNER_DEAD == state || Runner::RUNNER_ERROR == state) {
+                // XXX: Do we need to propagate this error to caller?
+                if (Runner::RUNNER_ERROR == state) {
+                    warning() << "getMore runner error: " << WorkingSetCommon::toStatusString(obj);
+                }
+
                 // If we're dead there's no way to get more results.
                 saveClientCursor = false;
                 // In the old system tailable capped cursors would be killed off at the
@@ -578,10 +584,9 @@ namespace mongo {
         // So, no matter what, deregister the runner.
         safety.reset();
 
-        // Caller expects exceptions thrown in certain cases:
-        // * in-memory sort using too much RAM.
+        // Caller expects exceptions thrown in certain cases.
         if (Runner::RUNNER_ERROR == state) {
-            uasserted(17144, "Runner error, memory limit for sort probably exceeded");
+            uasserted(17144, "Runner error: " + WorkingSetCommon::toStatusString(obj));
         }
 
         // Why save a dead runner?
diff --git a/src/mongo/db/query/plan_executor.cpp b/src/mongo/db/query/plan_executor.cpp
index 6bb69de0bfc..026e98466a5 100644
--- a/src/mongo/db/query/plan_executor.cpp
+++ b/src/mongo/db/query/plan_executor.cpp
@@ -85,7 +85,7 @@ namespace mongo {
                 restoreState();
             }
 
-            WorkingSetID id;
+            WorkingSetID id = WorkingSet::INVALID_ID;
             PlanStage::StageState code = _root->work(&id);
 
             if (PlanStage::ADVANCED == code) {
@@ -179,6 +179,7 @@ namespace mongo {
             }
             else {
                 verify(PlanStage::FAILURE == code);
+                WorkingSetCommon::getStatusMemberObject(*_workingSet, id, objOut);
                 return Runner::RUNNER_ERROR;
             }
         }
diff --git a/src/mongo/db/query/runner.h b/src/mongo/db/query/runner.h
index 842771e2260..88477b96f82 100644
--- a/src/mongo/db/query/runner.h
+++ b/src/mongo/db/query/runner.h
@@ -58,6 +58,9 @@ namespace mongo {
 
             // getNext was asked for data it cannot provide, or the underlying PlanStage had an
             // unrecoverable error.
+            // If the underlying PlanStage has any information on the error, it will be available in
+            // the objOut parameter. Call WorkingSetCommon::toStatusString() to retrieve the error
+            // details from the output BSON object.
             RUNNER_ERROR,
         };
 
@@ -149,6 +152,10 @@ namespace mongo {
          * the object is created from covered index key data, the object is projected or otherwise
          * the result of a computation.
          *
+         * objOut will also be owned when the underlying PlanStage has provided error details in the
+         * event of a RUNNER_ERROR. Call WorkingSetCommon::toStatusString() to convert the object
+         * to a loggable format.
+         *
          * objOut will be unowned if it's the result of a fetch or a collection scan.
          */
         virtual RunnerState getNext(BSONObj* objOut, DiskLoc* dlOut) = 0;
diff --git a/src/mongo/dbtests/oplogstarttests.cpp b/src/mongo/dbtests/oplogstarttests.cpp
index 39780e851ef..85146e574bf 100644
--- a/src/mongo/dbtests/oplogstarttests.cpp
+++ b/src/mongo/dbtests/oplogstarttests.cpp
@@ -106,7 +106,7 @@ namespace OplogStartTests {
 
             setupFromQuery(BSON( "ts" << BSON( "$gte" << 10 )));
 
-            WorkingSetID id;
+            WorkingSetID id = WorkingSet::INVALID_ID;
             // collection scan needs to be initialized
             ASSERT_EQUALS(_stage->work(&id), PlanStage::NEED_TIME);
             // finds starting record
@@ -130,7 +130,7 @@ namespace OplogStartTests {
 
             setupFromQuery(BSON( "ts" << BSON( "$gte" << 1 )));
 
-            WorkingSetID id;
+            WorkingSetID id = WorkingSet::INVALID_ID;
             // collection scan needs to be initialized
             ASSERT_EQUALS(_stage->work(&id), PlanStage::NEED_TIME);
             // full collection scan back to the first oplog record
@@ -157,7 +157,7 @@ namespace OplogStartTests {
 
             setupFromQuery(BSON( "ts" << BSON( "$gte" << 1 )));
 
-            WorkingSetID id;
+            WorkingSetID id = WorkingSet::INVALID_ID;
             // ensure that we go into extent hopping mode immediately
             _stage->setBackwardsScanTime(0);
 
@@ -188,7 +188,7 @@ namespace OplogStartTests {
         void run() {
             buildCollection();
 
-            WorkingSetID id;
+            WorkingSetID id = WorkingSet::INVALID_ID;
             setupFromQuery(BSON( "ts" << BSON( "$gte" << tsGte() )));
 
             // ensure that we go into extent hopping mode immediately
diff --git a/src/mongo/dbtests/query_stage_and.cpp b/src/mongo/dbtests/query_stage_and.cpp
index 9aa863fb98a..4fbfba45378 100644
--- a/src/mongo/dbtests/query_stage_and.cpp
+++ b/src/mongo/dbtests/query_stage_and.cpp
@@ -86,7 +86,7 @@ namespace QueryStageAnd {
         int countResults(PlanStage* stage) {
             int count = 0;
             while (!stage->isEOF()) {
-                WorkingSetID id;
+                WorkingSetID id = WorkingSet::INVALID_ID;
                 PlanStage::StageState status = stage->work(&id);
                 if (PlanStage::ADVANCED != status) { continue; }
                 ++count;
@@ -187,7 +187,7 @@ namespace QueryStageAnd {
             // one because of a mid-plan invalidation, so 10.
             int count = 0;
             while (!ah->isEOF()) {
-                WorkingSetID id;
+                WorkingSetID id = WorkingSet::INVALID_ID;
                 PlanStage::StageState status = ah->work(&id);
                 if (PlanStage::ADVANCED != status) { continue; }
 
@@ -245,7 +245,7 @@ namespace QueryStageAnd {
             // First call to work reads the first result from the children.
             // The first result is for the first scan over foo is {foo: 20, bar: 20, baz: 20}.
             // The first result is for the second scan over bar is {foo: 19, bar: 19, baz: 19}.
-            WorkingSetID id;
+            WorkingSetID id = WorkingSet::INVALID_ID;
             PlanStage::StageState status = ah->work(&id);
             ASSERT_EQUALS(PlanStage::NEED_TIME, status);
 
@@ -271,7 +271,7 @@ namespace QueryStageAnd {
             // And not in our results.
             int count = 0;
             while (!ah->isEOF()) {
-                WorkingSetID id;
+                WorkingSetID id = WorkingSet::INVALID_ID;
                 PlanStage::StageState status = ah->work(&id);
                 if (PlanStage::ADVANCED != status) { continue; }
                 WorkingSetMember* wsm = ws.get(id);
@@ -379,7 +379,7 @@ namespace QueryStageAnd {
             int count = 0;
             int works = 0;
             while (!ah->isEOF()) {
-                WorkingSetID id;
+                WorkingSetID id = WorkingSet::INVALID_ID;
                 ++works;
                 PlanStage::StageState status = ah->work(&id);
                 if (PlanStage::ADVANCED != status) { continue; }
@@ -539,7 +539,7 @@ namespace QueryStageAnd {
             // We're making an assumption here that happens to be true because we clear out the
             // collection before running this: increasing inserts have increasing DiskLocs.
             // This isn't true in general if the collection is not dropped beforehand.
-            WorkingSetID id;
+            WorkingSetID id = WorkingSet::INVALID_ID;
 
             // Sorted AND looks at the first child, which is an index scan over foo==1.
             ah->work(&id);
@@ -567,7 +567,7 @@ namespace QueryStageAnd {
             // Proceed along, AND-ing results.
             int count = 0;
             while (!ah->isEOF() && count < 10) {
-                WorkingSetID id;
+                WorkingSetID id = WorkingSet::INVALID_ID;
                 PlanStage::StageState status = ah->work(&id);
                 if (PlanStage::ADVANCED != status) { continue; }
 
@@ -593,7 +593,7 @@ namespace QueryStageAnd {
 
             // Get all results aside from the two we killed.
             while (!ah->isEOF()) {
-                WorkingSetID id;
+                WorkingSetID id = WorkingSet::INVALID_ID;
                 PlanStage::StageState status = ah->work(&id);
                 if (PlanStage::ADVANCED != status) { continue; }
 
@@ -841,7 +841,7 @@ namespace QueryStageAnd {
 
             int count = 0;
             while (!ah->isEOF()) {
-                WorkingSetID id;
+                WorkingSetID id = WorkingSet::INVALID_ID;
                 PlanStage::StageState status = ah->work(&id);
                 if (PlanStage::ADVANCED != status) { continue; }
                 BSONObj thisObj = ws.get(id)->loc.obj();
diff --git a/src/mongo/dbtests/query_stage_collscan.cpp b/src/mongo/dbtests/query_stage_collscan.cpp
index e3232bebf38..d0adfd442dc 100644
--- a/src/mongo/dbtests/query_stage_collscan.cpp
+++ b/src/mongo/dbtests/query_stage_collscan.cpp
@@ -346,7 +346,7 @@ namespace QueryStageCollectionScan {
 
             scoped_ptr<CollectionScan> scan(new CollectionScan(params, &ws, NULL));
             while (!scan->isEOF()) {
-                WorkingSetID id;
+                WorkingSetID id = WorkingSet::INVALID_ID;
                 PlanStage::StageState state = scan->work(&id);
                 if (PlanStage::ADVANCED == state) {
                     WorkingSetMember* member = ws.get(id);
@@ -495,7 +495,7 @@ namespace QueryStageCollectionScan {
 
             int count = 0;
             while (count < 10) {
-                WorkingSetID id;
+                WorkingSetID id = WorkingSet::INVALID_ID;
                 PlanStage::StageState state = scan->work(&id);
                 if (PlanStage::ADVANCED == state) {
                     WorkingSetMember* member = ws.get(id);
@@ -516,7 +516,7 @@ namespace QueryStageCollectionScan {
 
             // Expect the rest.
             while (!scan->isEOF()) {
-                WorkingSetID id;
+                WorkingSetID id = WorkingSet::INVALID_ID;
                 PlanStage::StageState state = scan->work(&id);
                 if (PlanStage::ADVANCED == state) {
                     WorkingSetMember* member = ws.get(id);
@@ -555,7 +555,7 @@ namespace QueryStageCollectionScan {
 
             int count = 0;
             while (count < 10) {
-                WorkingSetID id;
+                WorkingSetID id = WorkingSet::INVALID_ID;
                 PlanStage::StageState state = scan->work(&id);
                 if (PlanStage::ADVANCED == state) {
                     WorkingSetMember* member = ws.get(id);
@@ -576,7 +576,7 @@ namespace QueryStageCollectionScan {
 
             // Expect the rest.
             while (!scan->isEOF()) {
-                WorkingSetID id;
+                WorkingSetID id = WorkingSet::INVALID_ID;
                 PlanStage::StageState state = scan->work(&id);
                 if (PlanStage::ADVANCED == state) {
                     WorkingSetMember* member = ws.get(id);
diff --git a/src/mongo/dbtests/query_stage_fetch.cpp b/src/mongo/dbtests/query_stage_fetch.cpp
index 8892ddce892..445e6341209 100644
--- a/src/mongo/dbtests/query_stage_fetch.cpp
+++ b/src/mongo/dbtests/query_stage_fetch.cpp
@@ -127,7 +127,7 @@ namespace QueryStageFetch {
             fetchInMemoryFail->setMode(FailPoint::alwaysOn);
 
             // First call should return a fetch request as it's not in memory.
-            WorkingSetID id;
+            WorkingSetID id = WorkingSet::INVALID_ID;
             PlanStage::StageState state;
             state = fetchStage->work(&id);
             ASSERT_EQUALS(PlanStage::NEED_FETCH, state);
@@ -198,7 +198,7 @@ namespace QueryStageFetch {
             fetchInMemorySucceed->setMode(FailPoint::alwaysOn);
 
             // First call fetches as expected.
-            WorkingSetID id;
+            WorkingSetID id = WorkingSet::INVALID_ID;
             PlanStage::StageState state;
             state = fetchStage->work(&id);
             ASSERT_EQUALS(PlanStage::ADVANCED, state);
@@ -263,7 +263,7 @@ namespace QueryStageFetch {
             fetchInMemoryFail->setMode(FailPoint::alwaysOn);
 
             // First call should return a fetch request as it's not in memory.
-            WorkingSetID id;
+            WorkingSetID id = WorkingSet::INVALID_ID;
             PlanStage::StageState state;
             state = fetchStage->work(&id);
             ASSERT_EQUALS(PlanStage::NEED_FETCH, state);
@@ -339,7 +339,7 @@ namespace QueryStageFetch {
             FailPoint* fetchInMemoryFail = reg->getFailPoint("fetchInMemoryFail");
             fetchInMemoryFail->setMode(FailPoint::alwaysOn);
 
-            WorkingSetID id;
+            WorkingSetID id = WorkingSet::INVALID_ID;
             PlanStage::StageState state;
 
             // Don't bother doing any fetching if an obj exists already.
@@ -407,7 +407,7 @@ namespace QueryStageFetch {
             fetchInMemoryFail->setMode(FailPoint::alwaysOn);
 
             // First call should return a fetch request as it's not in memory.
-            WorkingSetID id;
+            WorkingSetID id = WorkingSet::INVALID_ID;
             PlanStage::StageState state;
             state = fetchStage->work(&id);
             ASSERT_EQUALS(PlanStage::NEED_FETCH, state);
diff --git a/src/mongo/dbtests/query_stage_keep.cpp b/src/mongo/dbtests/query_stage_keep.cpp
index 6375aa3b215..3c37af51e95 100644
--- a/src/mongo/dbtests/query_stage_keep.cpp
+++ b/src/mongo/dbtests/query_stage_keep.cpp
@@ -81,7 +81,7 @@ namespace QueryStageKeep {
 
         WorkingSetID getNextResult(PlanStage* stage) {
             while (!stage->isEOF()) {
-                WorkingSetID id;
+                WorkingSetID id = WorkingSet::INVALID_ID;
                 PlanStage::StageState status = stage->work(&id);
                 if (PlanStage::ADVANCED == status) {
                     return id;
diff --git a/src/mongo/dbtests/query_stage_limit_skip.cpp b/src/mongo/dbtests/query_stage_limit_skip.cpp
index 4ba99122e6d..6b9aee95bca 100644
--- a/src/mongo/dbtests/query_stage_limit_skip.cpp
+++ b/src/mongo/dbtests/query_stage_limit_skip.cpp
@@ -66,7 +66,7 @@ namespace {
     int countResults(PlanStage* stage) {
         int count = 0;
         while (!stage->isEOF()) {
-            WorkingSetID id;
+            WorkingSetID id = WorkingSet::INVALID_ID;
             PlanStage::StageState status = stage->work(&id);
             if (PlanStage::ADVANCED != status) { continue; }
             ++count;
diff --git a/src/mongo/dbtests/query_stage_merge_sort.cpp b/src/mongo/dbtests/query_stage_merge_sort.cpp
index 275d7c19044..7449e3e07e2 100644
--- a/src/mongo/dbtests/query_stage_merge_sort.cpp
+++ b/src/mongo/dbtests/query_stage_merge_sort.cpp
@@ -514,7 +514,7 @@ namespace QueryStageMergeSortTests {
             // Get 10 results.  Should be getting results in order of 'locs'.
             int count = 0;
             while (!ms->isEOF() && count < 10) {
-                WorkingSetID id;
+                WorkingSetID id = WorkingSet::INVALID_ID;
                 PlanStage::StageState status = ms->work(&id);
                 if (PlanStage::ADVANCED != status) { continue; }
 
@@ -539,7 +539,7 @@ namespace QueryStageMergeSortTests {
             {
             // TODO: If we have "return upon invalidation" ever triggerable, do the following test.
             /*
-                WorkingSetID id;
+                WorkingSetID id = WorkingSet::INVALID_ID;
                 PlanStage::StageState status;
                 do {
                     status = ms->work(&id);
@@ -562,7 +562,7 @@ namespace QueryStageMergeSortTests {
 
             // And get the rest.
             while (!ms->isEOF()) {
-                WorkingSetID id;
+                WorkingSetID id = WorkingSet::INVALID_ID;
                 PlanStage::StageState status = ms->work(&id);
                 if (PlanStage::ADVANCED != status) { continue; }
 
diff --git a/src/mongo/dbtests/query_stage_sort.cpp b/src/mongo/dbtests/query_stage_sort.cpp
index 2d432215612..7d0b0d8c1a6 100644
--- a/src/mongo/dbtests/query_stage_sort.cpp
+++ b/src/mongo/dbtests/query_stage_sort.cpp
@@ -267,7 +267,7 @@ namespace QueryStageSortTests {
 
             // Have sort read in data from the mock stage.
             for (int i = 0; i < firstRead; ++i) {
-                WorkingSetID id;
+                WorkingSetID id = WorkingSet::INVALID_ID;
                 PlanStage::StageState status = ss->work(&id);
                 ASSERT_NOT_EQUALS(PlanStage::ADVANCED, status);
             }
@@ -280,7 +280,7 @@ namespace QueryStageSortTests {
 
             // Read the rest of the data from the mock stage.
             while (!ms->isEOF()) {
-                WorkingSetID id;
+                WorkingSetID id = WorkingSet::INVALID_ID;
                 ss->work(&id);
             }
 
@@ -297,7 +297,7 @@ namespace QueryStageSortTests {
             // Invalidation of data in the sort stage fetches it but passes it through.
             int count = 0;
             while (!ss->isEOF()) {
-                WorkingSetID id;
+                WorkingSetID id = WorkingSet::INVALID_ID;
                 PlanStage::StageState status = ss->work(&id);
                 if (PlanStage::ADVANCED != status) { continue; }
                 WorkingSetMember* member = ws.get(id);