diff options
author | James Wahlin <james.wahlin@10gen.com> | 2015-04-21 08:50:52 -0400 |
---|---|---|
committer | James Wahlin <james.wahlin@10gen.com> | 2015-06-09 13:41:37 -0400 |
commit | d690653daadef98652e58131ade8b34114f86ab2 (patch) | |
tree | fd38454b9d4bc8d8d64c61334885e40d1644a235 /src/mongo | |
parent | 6f9285ba8e37aee90acb9069cfe477db626281c2 (diff) | |
download | mongo-d690653daadef98652e58131ade8b34114f86ab2.tar.gz |
SERVER-2454 Improve PlanExecutor::DEAD handling
Diffstat (limited to 'src/mongo')
34 files changed, 346 insertions, 140 deletions
diff --git a/src/mongo/db/catalog/capped_utils.cpp b/src/mongo/db/catalog/capped_utils.cpp index eb0265b0877..19455f94e90 100644 --- a/src/mongo/db/catalog/capped_utils.cpp +++ b/src/mongo/db/catalog/capped_utils.cpp @@ -26,6 +26,8 @@ * it in the license file. */ +#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kCommand + #include "mongo/platform/basic.h" #include "mongo/db/catalog/capped_utils.h" @@ -159,17 +161,23 @@ namespace mongo { switch(state) { case PlanExecutor::IS_EOF: return Status::OK(); - case PlanExecutor::DEAD: - db->dropCollection(txn, toNs); - return Status(ErrorCodes::InternalError, "executor died while iterating"); - case PlanExecutor::FAILURE: - return Status(ErrorCodes::InternalError, "executor error while iterating"); case PlanExecutor::ADVANCED: + { if (excessSize > 0) { // 4x is for padding, power of 2, etc... excessSize -= (4 * objToClone.value().objsize()); continue; } + break; + } + default: + // Unreachable as: + // 1) We require a read lock (at a minimum) on the "from" collection + // and won't yield, preventing collection drop and PlanExecutor::DEAD + // 2) PlanExecutor::FAILURE is only returned on PlanStage::FAILURE. The + // CollectionScan PlanStage does not have a FAILURE scenario. + // 3) All other PlanExecutor states are handled above + invariant(false); } try { diff --git a/src/mongo/db/commands/find_and_modify.cpp b/src/mongo/db/commands/find_and_modify.cpp index f911d6520fd..88019cbdc3c 100644 --- a/src/mongo/db/commands/find_and_modify.cpp +++ b/src/mongo/db/commands/find_and_modify.cpp @@ -103,8 +103,12 @@ namespace { return boost::optional<BSONObj>(std::move(value)); } if (PlanExecutor::FAILURE == state || PlanExecutor::DEAD == state) { - if (PlanExecutor::FAILURE == state && - WorkingSetCommon::isValidStatusMemberObject(value)) { + const std::unique_ptr<PlanStageStats> stats(exec->getStats()); + error() << "Plan executor error during findAndModify: " + << PlanExecutor::statestr(state) + << ", stats: " << Explain::statsToBSON(*stats); + + if (WorkingSetCommon::isValidStatusMemberObject(value)) { const Status errorStatus = WorkingSetCommon::getMemberObjectStatus(value); invariant(!errorStatus.isOK()); diff --git a/src/mongo/db/commands/find_cmd.cpp b/src/mongo/db/commands/find_cmd.cpp index 744a8eb88ca..3c8c0fe45ad 100644 --- a/src/mongo/db/commands/find_cmd.cpp +++ b/src/mongo/db/commands/find_cmd.cpp @@ -313,13 +313,17 @@ namespace mongo { } // Throw an assertion if query execution fails for any reason. - if (PlanExecutor::FAILURE == state) { + if (PlanExecutor::FAILURE == state || PlanExecutor::DEAD == state) { const std::unique_ptr<PlanStageStats> stats(exec->getStats()); - error() << "Plan executor error, stats: " << Explain::statsToBSON(*stats); + error() << "Plan executor error during find command: " + << PlanExecutor::statestr(state) + << ", stats: " << Explain::statsToBSON(*stats); + return appendCommandStatus(result, Status(ErrorCodes::OperationFailed, - str::stream() << "Executor error: " - << WorkingSetCommon::toStatusString(obj))); + str::stream() + << "Executor error during find command: " + << WorkingSetCommon::toStatusString(obj))); } // 6) Set up the cursor for getMore. diff --git a/src/mongo/db/commands/getmore_cmd.cpp b/src/mongo/db/commands/getmore_cmd.cpp index bc2bf72328e..e075fbd047e 100644 --- a/src/mongo/db/commands/getmore_cmd.cpp +++ b/src/mongo/db/commands/getmore_cmd.cpp @@ -368,18 +368,15 @@ namespace mongo { } } - if (PlanExecutor::FAILURE == *state) { + if (PlanExecutor::FAILURE == *state || PlanExecutor::DEAD == *state) { const std::unique_ptr<PlanStageStats> stats(exec->getStats()); - error() << "GetMore executor error, stats: " << Explain::statsToBSON(*stats); + error() << "GetMore command executor error: " << PlanExecutor::statestr(*state) + << ", stats: " << Explain::statsToBSON(*stats); + return Status(ErrorCodes::OperationFailed, - str::stream() << "GetMore executor error: " + str::stream() << "GetMore command executor error: " << WorkingSetCommon::toStatusString(obj)); } - else if (PlanExecutor::DEAD == *state) { - return Status(ErrorCodes::OperationFailed, - str::stream() << "Plan executor killed during getMore command, " - << "ns: " << request.nss.ns()); - } return Status::OK(); } diff --git a/src/mongo/db/commands/group.cpp b/src/mongo/db/commands/group.cpp index 4dd4f5a1956..adfb6fae37f 100644 --- a/src/mongo/db/commands/group.cpp +++ b/src/mongo/db/commands/group.cpp @@ -158,8 +158,9 @@ namespace mongo { BSONObj retval; PlanExecutor::ExecState state = planExecutor->getNext(&retval, NULL); if (PlanExecutor::ADVANCED != state) { - if (PlanExecutor::FAILURE == state && - WorkingSetCommon::isValidStatusMemberObject(retval)) { + invariant(PlanExecutor::FAILURE == state || PlanExecutor::DEAD == state); + + if (WorkingSetCommon::isValidStatusMemberObject(retval)) { return appendCommandStatus(out, WorkingSetCommon::getMemberObjectStatus(retval)); } return appendCommandStatus(out, @@ -168,6 +169,7 @@ namespace mongo { << "operation, executor returned " << PlanExecutor::statestr(state))); } + invariant(planExecutor->isEOF()); invariant(STAGE_GROUP == planExecutor->getRootStage()->stageType()); diff --git a/src/mongo/db/dbhelpers.cpp b/src/mongo/db/dbhelpers.cpp index a56fb037464..ce5be7e3697 100644 --- a/src/mongo/db/dbhelpers.cpp +++ b/src/mongo/db/dbhelpers.cpp @@ -381,18 +381,14 @@ namespace mongo { exec.reset(); if (PlanExecutor::IS_EOF == state) { break; } - if (PlanExecutor::DEAD == state) { - warning(LogComponent::kSharding) << "cursor died: aborting deletion for " - << min << " to " << max << " in " << ns - << endl; - break; - } - - if (PlanExecutor::FAILURE == state) { - warning(LogComponent::kSharding) << "cursor error while trying to delete " + if (PlanExecutor::FAILURE == state || PlanExecutor::DEAD == state) { + const std::unique_ptr<PlanStageStats> stats(exec->getStats()); + warning(LogComponent::kSharding) << PlanExecutor::statestr(state) + << " - cursor error while trying to delete " << min << " to " << max << " in " << ns << ": " - << WorkingSetCommon::toStatusString(obj) << endl; + << WorkingSetCommon::toStatusString(obj) << ", stats: " + << Explain::statsToBSON(*stats) << endl; break; } diff --git a/src/mongo/db/exec/and_hash.cpp b/src/mongo/db/exec/and_hash.cpp index 64bac7b3c66..4d5adf6888c 100644 --- a/src/mongo/db/exec/and_hash.cpp +++ b/src/mongo/db/exec/and_hash.cpp @@ -135,7 +135,7 @@ namespace mongo { for (size_t j = 0; j < kLookAheadWorks; ++j) { StageState childStatus = child->work(&_lookAheadResults[i]); - if (PlanStage::IS_EOF == childStatus || PlanStage::DEAD == childStatus) { + if (PlanStage::IS_EOF == childStatus) { // A child went right to EOF. Bail out. _hashingChildren = false; @@ -147,7 +147,7 @@ namespace mongo { // child. break; } - else if (PlanStage::FAILURE == childStatus) { + else if (PlanStage::FAILURE == childStatus || PlanStage::DEAD == childStatus) { // Propage error to parent. *out = _lookAheadResults[i]; // If a stage fails, it may create a status WSM to indicate why it @@ -156,14 +156,15 @@ namespace mongo { if (WorkingSet::INVALID_ID == *out) { mongoutils::str::stream ss; ss << "hashed AND stage failed to read in look ahead results " - << "from child " << i; + << "from child " << i + << ", childStatus: " << PlanStage::stateStr(childStatus); Status status(ErrorCodes::InternalError, ss); *out = WorkingSetCommon::allocateStatusMember( _ws, status); } _hashingChildren = false; _dataMap.clear(); - return PlanStage::FAILURE; + return childStatus; } // We ignore NEED_TIME. TODO: what do we want to do if we get NEED_YIELD here? } @@ -317,7 +318,7 @@ namespace mongo { return PlanStage::NEED_TIME; } - else if (PlanStage::FAILURE == childStatus) { + else if (PlanStage::FAILURE == childStatus || PlanStage::DEAD == childStatus) { *out = id; // If a stage fails, it may create a status WSM to indicate why it // failed, in which case 'id' is valid. If ID is invalid, we @@ -419,7 +420,7 @@ namespace mongo { ++_commonStats.needTime; return PlanStage::NEED_TIME; } - else if (PlanStage::FAILURE == childStatus) { + else if (PlanStage::FAILURE == childStatus || PlanStage::DEAD == childStatus) { *out = id; // If a stage fails, it may create a status WSM to indicate why it // failed, in which case 'id' is valid. If ID is invalid, we diff --git a/src/mongo/db/exec/and_sorted.cpp b/src/mongo/db/exec/and_sorted.cpp index 5262910549e..813895fc619 100644 --- a/src/mongo/db/exec/and_sorted.cpp +++ b/src/mongo/db/exec/and_sorted.cpp @@ -239,7 +239,7 @@ namespace mongo { _ws->free(_targetId); return state; } - else if (PlanStage::FAILURE == state) { + else if (PlanStage::FAILURE == state || PlanStage::DEAD == state) { *out = id; // If a stage fails, it may create a status WSM to indicate why it // failed, in which case 'id' is valid. If ID is invalid, we diff --git a/src/mongo/db/exec/cached_plan.cpp b/src/mongo/db/exec/cached_plan.cpp index 7e3fed36a98..caec538e5cb 100644 --- a/src/mongo/db/exec/cached_plan.cpp +++ b/src/mongo/db/exec/cached_plan.cpp @@ -151,8 +151,18 @@ namespace mongo { return replan(yieldPolicy, shouldCache); } else if (PlanStage::DEAD == state) { - return Status(ErrorCodes::OperationFailed, - "Executor killed during cached plan trial period"); + BSONObj statusObj; + WorkingSetCommon::getStatusMemberObject(*_ws, id, &statusObj); + + LOG(1) << "Execution of cached plan failed: PlanStage died" + << ", query: " + << _canonicalQuery->toStringShort() + << " planSummary: " + << Explain::getPlanSummary(_root.get()) + << " status: " + << statusObj; + + return WorkingSetCommon::getMemberObjectStatus(statusObj); } else { invariant(PlanStage::NEED_TIME == state); diff --git a/src/mongo/db/exec/collection_scan.cpp b/src/mongo/db/exec/collection_scan.cpp index 7815ec85926..e43b881b3c2 100644 --- a/src/mongo/db/exec/collection_scan.cpp +++ b/src/mongo/db/exec/collection_scan.cpp @@ -35,6 +35,7 @@ #include "mongo/db/exec/filter.h" #include "mongo/db/exec/scoped_timer.h" #include "mongo/db/exec/working_set.h" +#include "mongo/db/exec/working_set_common.h" #include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/catalog/collection.h" #include "mongo/db/storage/record_fetcher.h" @@ -78,12 +79,19 @@ namespace mongo { // Adds the amount of time taken by work() to executionTimeMillis. ScopedTimer timer(&_commonStats.executionTimeMillis); - if (_isDead) { return PlanStage::DEAD; } + if (_isDead) { + Status status(ErrorCodes::InternalError, "CollectionScan died"); + *out = WorkingSetCommon::allocateStatusMember(_workingSet, status); + return PlanStage::DEAD; + } // Do some init if we haven't already. if (NULL == _iter) { - if ( _params.collection == NULL ) { + if (_params.collection == NULL) { _isDead = true; + Status status(ErrorCodes::InternalError, + "CollectionScan died: collection pointer was null"); + *out = WorkingSetCommon::allocateStatusMember(_workingSet, status); return PlanStage::DEAD; } @@ -102,10 +110,13 @@ namespace mongo { // Advance _iter past where we were last time. If it returns something else, // mark us as dead since we want to signal an error rather than silently - // dropping data from the stream. This is related to the _lastSeenLock handling + // dropping data from the stream. This is related to the _lastSeenLoc handling // in invalidate. if (_iter->getNext() != _lastSeenLoc) { _isDead = true; + Status status(ErrorCodes::InternalError, + "CollectionScan died: Unexpected RecordId"); + *out = WorkingSetCommon::allocateStatusMember(_workingSet, status); return PlanStage::DEAD; } } @@ -122,14 +133,19 @@ namespace mongo { } // Should we try getNext() on the underlying _iter? - if (isEOF()) + if (isEOF()) { + _commonStats.isEOF = true; return PlanStage::IS_EOF; + } const RecordId curr = _iter->curr(); if (curr.isNull()) { // We just hit EOF if (_params.tailable) _iter.reset(); // pick up where we left off on the next call to work() + else + _commonStats.isEOF = true; + return PlanStage::IS_EOF; } @@ -148,7 +164,7 @@ namespace mongo { member->setFetcher(fetcher.release()); *out = _wsidForFetch; _commonStats.needYield++; - return NEED_YIELD; + return PlanStage::NEED_YIELD; } } @@ -256,8 +272,6 @@ namespace mongo { } PlanStageStats* CollectionScan::getStats() { - _commonStats.isEOF = isEOF(); - // Add a BSON representation of the filter to the stats tree, if there is one. if (NULL != _filter) { BSONObjBuilder bob; diff --git a/src/mongo/db/exec/count.cpp b/src/mongo/db/exec/count.cpp index 5fbcc28a40b..5b3eb904647 100644 --- a/src/mongo/db/exec/count.cpp +++ b/src/mongo/db/exec/count.cpp @@ -128,10 +128,10 @@ namespace mongo { else if (PlanStage::DEAD == state) { return state; } - else if (PlanStage::FAILURE == state) { + else if (PlanStage::FAILURE == state || PlanStage::DEAD == state) { *out = id; - // If a stage fails, it may create a status WSM to indicate why it failed, in which cas - // 'id' is valid. If ID is invalid, we create our own error message. + // If a stage fails, it may create a status WSM to indicate why it failed, in which + // case 'id' is valid. If ID is invalid, we create our own error message. if (WorkingSet::INVALID_ID == id) { const std::string errmsg = "count stage failed to read result from child"; Status status = Status(ErrorCodes::InternalError, errmsg); diff --git a/src/mongo/db/exec/delete.cpp b/src/mongo/db/exec/delete.cpp index 0a8cb0509da..85a847f462c 100644 --- a/src/mongo/db/exec/delete.cpp +++ b/src/mongo/db/exec/delete.cpp @@ -235,7 +235,7 @@ namespace mongo { ++_commonStats.needTime; return PlanStage::NEED_TIME; } - else if (PlanStage::FAILURE == status) { + else if (PlanStage::FAILURE == status || PlanStage::DEAD == status) { *out = id; // If a stage fails, it may create a status WSM to indicate why it failed, in which case // 'id' is valid. If ID is invalid, we create our own error message. @@ -243,7 +243,6 @@ namespace mongo { const std::string errmsg = "delete stage failed to read in results from child"; *out = WorkingSetCommon::allocateStatusMember(_ws, Status(ErrorCodes::InternalError, errmsg)); - return PlanStage::FAILURE; } return status; } diff --git a/src/mongo/db/exec/fetch.cpp b/src/mongo/db/exec/fetch.cpp index 606c7af6662..5fd1a3c1a98 100644 --- a/src/mongo/db/exec/fetch.cpp +++ b/src/mongo/db/exec/fetch.cpp @@ -137,7 +137,7 @@ namespace mongo { return returnIfMatches(member, id, out); } - else if (PlanStage::FAILURE == status) { + else if (PlanStage::FAILURE == status || PlanStage::DEAD == status) { *out = id; // If a stage fails, it may create a status WSM to indicate why it // failed, in which case 'id' is valid. If ID is invalid, we diff --git a/src/mongo/db/exec/limit.cpp b/src/mongo/db/exec/limit.cpp index 13f61532f15..9853cde3e57 100644 --- a/src/mongo/db/exec/limit.cpp +++ b/src/mongo/db/exec/limit.cpp @@ -72,7 +72,7 @@ namespace mongo { ++_commonStats.advanced; return PlanStage::ADVANCED; } - else if (PlanStage::FAILURE == status) { + else if (PlanStage::FAILURE == status || PlanStage::DEAD == status) { *out = id; // If a stage fails, it may create a status WSM to indicate why it // failed, in which case 'id' is valid. If ID is invalid, we diff --git a/src/mongo/db/exec/merge_sort.cpp b/src/mongo/db/exec/merge_sort.cpp index 8fdb3ae96eb..6d394ba00d6 100644 --- a/src/mongo/db/exec/merge_sort.cpp +++ b/src/mongo/db/exec/merge_sort.cpp @@ -139,7 +139,7 @@ namespace mongo { ++_commonStats.needTime; return PlanStage::NEED_TIME; } - else if (PlanStage::FAILURE == code) { + else if (PlanStage::FAILURE == code || PlanStage::DEAD == code) { *out = id; // If a stage fails, it may create a status WSM to indicate why it // failed, in which case 'id' is valid. If ID is invalid, we diff --git a/src/mongo/db/exec/multi_iterator.cpp b/src/mongo/db/exec/multi_iterator.cpp index 9f45d9e4531..559a5a418bd 100644 --- a/src/mongo/db/exec/multi_iterator.cpp +++ b/src/mongo/db/exec/multi_iterator.cpp @@ -38,6 +38,8 @@ namespace mongo { using std::vector; + const char* MultiIteratorStage::kStageType = "MULTI_ITERATOR"; + MultiIteratorStage::MultiIteratorStage(OperationContext* txn, WorkingSet* ws, Collection* collection) @@ -57,8 +59,12 @@ namespace mongo { } PlanStage::StageState MultiIteratorStage::work(WorkingSetID* out) { - if ( _collection == NULL ) + if (_collection == NULL) { + Status status(ErrorCodes::InternalError, + "MultiIteratorStage died on null collection"); + *out = WorkingSetCommon::allocateStatusMember(_ws, status); return PlanStage::DEAD; + } if (_iterators.empty()) return PlanStage::IS_EOF; @@ -151,6 +157,13 @@ namespace mongo { return empty; } + PlanStageStats* MultiIteratorStage::getStats() { + std::unique_ptr<PlanStageStats> ret(new PlanStageStats(CommonStats(kStageType), + STAGE_MULTI_ITERATOR)); + ret->specific.reset(new CollectionScanStats()); + return ret.release(); + } + void MultiIteratorStage::_advance() { if (_iterators.back()->isEOF()) { _iterators.popAndDeleteBack(); diff --git a/src/mongo/db/exec/multi_iterator.h b/src/mongo/db/exec/multi_iterator.h index 0fd88283aa7..cc2256815dd 100644 --- a/src/mongo/db/exec/multi_iterator.h +++ b/src/mongo/db/exec/multi_iterator.h @@ -64,18 +64,23 @@ namespace mongo { virtual void invalidate(OperationContext* txn, const RecordId& dl, InvalidationType type); - // - // These should not be used. - // + // Returns empty PlanStageStats object + virtual PlanStageStats* getStats(); - virtual PlanStageStats* getStats() { return NULL; } + // Not used. virtual CommonStats* getCommonStats() const { return NULL; } + + // Not used. virtual SpecificStats* getSpecificStats() const { return NULL; } + // Not used. virtual std::vector<PlanStage*> getChildren() const; + // Not used. virtual StageType stageType() const { return STAGE_MULTI_ITERATOR; } + static const char* kStageType; + private: void _advance(); diff --git a/src/mongo/db/exec/multi_plan.cpp b/src/mongo/db/exec/multi_plan.cpp index 7f011c02d51..3258ab719b0 100644 --- a/src/mongo/db/exec/multi_plan.cpp +++ b/src/mongo/db/exec/multi_plan.cpp @@ -316,9 +316,7 @@ namespace mongo { if (ix == (size_t)_backupPlanIdx) { continue; } PlanStageStats* stats = _candidates[ix].root->getStats(); - if (stats) { - candidateStats.push_back(stats); - } + candidateStats.push_back(stats); } return candidateStats.release(); diff --git a/src/mongo/db/exec/oplogstart.cpp b/src/mongo/db/exec/oplogstart.cpp index 7fe3257b5cf..866bded1d56 100644 --- a/src/mongo/db/exec/oplogstart.cpp +++ b/src/mongo/db/exec/oplogstart.cpp @@ -37,6 +37,8 @@ namespace mongo { using std::vector; + const char* OplogStart::kStageType = "OPLOG_START"; + // Does not take ownership. OplogStart::OplogStart(OperationContext* txn, const Collection* collection, @@ -197,6 +199,13 @@ namespace mongo { } } + PlanStageStats* OplogStart::getStats() { + std::unique_ptr<PlanStageStats> ret(new PlanStageStats(CommonStats(kStageType), + STAGE_OPLOG_START)); + ret->specific.reset(new CollectionScanStats()); + return ret.release(); + } + vector<PlanStage*> OplogStart::getChildren() const { vector<PlanStage*> empty; return empty; diff --git a/src/mongo/db/exec/oplogstart.h b/src/mongo/db/exec/oplogstart.h index 5a52dc37489..a3592b58f6b 100644 --- a/src/mongo/db/exec/oplogstart.h +++ b/src/mongo/db/exec/oplogstart.h @@ -77,12 +77,12 @@ namespace mongo { virtual std::vector<PlanStage*> getChildren() const; + // Returns empty PlanStageStats object + virtual PlanStageStats* getStats(); + // // Exec stats -- do not call these for the oplog start stage. // - - virtual PlanStageStats* getStats() { return NULL; } - virtual const CommonStats* getCommonStats() const { return NULL; } virtual const SpecificStats* getSpecificStats() const { return NULL; } @@ -93,6 +93,9 @@ namespace mongo { void setBackwardsScanTime(int newTime) { _backwardsScanTime = newTime; } bool isExtentHopping() { return _extentHopping; } bool isBackwardsScanning() { return _backwardsScanning; } + + static const char* kStageType; + private: StageState workBackwardsScan(WorkingSetID* out); diff --git a/src/mongo/db/exec/or.cpp b/src/mongo/db/exec/or.cpp index 7565ea9312a..c27c1a1598f 100644 --- a/src/mongo/db/exec/or.cpp +++ b/src/mongo/db/exec/or.cpp @@ -112,7 +112,7 @@ namespace mongo { return PlanStage::NEED_TIME; } } - else if (PlanStage::FAILURE == childStatus) { + else if (PlanStage::FAILURE == childStatus || PlanStage::DEAD == childStatus) { *out = id; // If a stage fails, it may create a status WSM to indicate why it // failed, in which case 'id' is valid. If ID is invalid, we diff --git a/src/mongo/db/exec/pipeline_proxy.cpp b/src/mongo/db/exec/pipeline_proxy.cpp index 2bcad44d998..a04cb301d9e 100644 --- a/src/mongo/db/exec/pipeline_proxy.cpp +++ b/src/mongo/db/exec/pipeline_proxy.cpp @@ -41,6 +41,8 @@ namespace mongo { using boost::shared_ptr; using std::vector; + const char* PipelineProxyStage::kStageType = "PIPELINE_PROXY"; + PipelineProxyStage::PipelineProxyStage(intrusive_ptr<Pipeline> pipeline, const boost::shared_ptr<PlanExecutor>& child, WorkingSet* ws) @@ -114,6 +116,13 @@ namespace mongo { return empty; } + PlanStageStats* PipelineProxyStage::getStats() { + std::unique_ptr<PlanStageStats> ret(new PlanStageStats(CommonStats(kStageType), + STAGE_PIPELINE_PROXY)); + ret->specific.reset(new CollectionScanStats()); + return ret.release(); + } + boost::optional<BSONObj> PipelineProxyStage::getNextBson() { if (boost::optional<Document> next = _pipeline->output()->getNext()) { if (_includeMetaData) { diff --git a/src/mongo/db/exec/pipeline_proxy.h b/src/mongo/db/exec/pipeline_proxy.h index 23ac96ca365..4de2f947a32 100644 --- a/src/mongo/db/exec/pipeline_proxy.h +++ b/src/mongo/db/exec/pipeline_proxy.h @@ -74,12 +74,13 @@ namespace mongo { */ boost::shared_ptr<PlanExecutor> getChildExecutor(); - // - // These should not be used. - // + // Returns empty PlanStageStats object + virtual PlanStageStats* getStats(); - virtual PlanStageStats* getStats() { return NULL; } + // Not used. virtual CommonStats* getCommonStats() const { return NULL; } + + // Not used. virtual SpecificStats* getSpecificStats() const { return NULL; } // Not used. @@ -88,6 +89,8 @@ namespace mongo { // Not used. virtual StageType stageType() const { return STAGE_PIPELINE_PROXY; } + static const char* kStageType; + private: boost::optional<BSONObj> getNextBson(); diff --git a/src/mongo/db/exec/projection.cpp b/src/mongo/db/exec/projection.cpp index 25616bfb197..7e4f513e4b7 100644 --- a/src/mongo/db/exec/projection.cpp +++ b/src/mongo/db/exec/projection.cpp @@ -227,7 +227,7 @@ namespace mongo { *out = id; ++_commonStats.advanced; } - else if (PlanStage::FAILURE == status) { + else if (PlanStage::FAILURE == status || PlanStage::DEAD == status) { *out = id; // If a stage fails, it may create a status WSM to indicate why it // failed, in which case 'id' is valid. If ID is invalid, we diff --git a/src/mongo/db/exec/skip.cpp b/src/mongo/db/exec/skip.cpp index 1a8f0ab5d57..be2284bca0c 100644 --- a/src/mongo/db/exec/skip.cpp +++ b/src/mongo/db/exec/skip.cpp @@ -69,7 +69,7 @@ namespace mongo { ++_commonStats.advanced; return PlanStage::ADVANCED; } - else if (PlanStage::FAILURE == status) { + else if (PlanStage::FAILURE == status || PlanStage::DEAD == status) { *out = id; // If a stage fails, it may create a status WSM to indicate why it // failed, in which case 'id' is valid. If ID is invalid, we diff --git a/src/mongo/db/exec/sort.cpp b/src/mongo/db/exec/sort.cpp index 50b0b6e23c7..3f2e8be89ad 100644 --- a/src/mongo/db/exec/sort.cpp +++ b/src/mongo/db/exec/sort.cpp @@ -385,7 +385,7 @@ namespace mongo { ++_commonStats.needTime; return PlanStage::NEED_TIME; } - else if (PlanStage::FAILURE == code) { + else if (PlanStage::FAILURE == code || PlanStage::DEAD == code) { *out = id; // If a stage fails, it may create a status WSM to indicate why it // failed, in which case 'id' is valid. If ID is invalid, we diff --git a/src/mongo/db/exec/stagedebug_cmd.cpp b/src/mongo/db/exec/stagedebug_cmd.cpp index 9db17d3d920..8ece46d226d 100644 --- a/src/mongo/db/exec/stagedebug_cmd.cpp +++ b/src/mongo/db/exec/stagedebug_cmd.cpp @@ -26,6 +26,8 @@ * it in the license file. */ +#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kQuery + #include "mongo/platform/basic.h" #include "mongo/base/init.h" @@ -48,10 +50,12 @@ #include "mongo/db/exec/skip.h" #include "mongo/db/exec/sort.h" #include "mongo/db/exec/text.h" +#include "mongo/db/exec/working_set_common.h" #include "mongo/db/index/fts_access_method.h" #include "mongo/db/jsobj.h" #include "mongo/db/matcher/expression_parser.h" #include "mongo/db/query/plan_executor.h" +#include "mongo/util/log.h" namespace mongo { @@ -166,11 +170,28 @@ namespace mongo { BSONArrayBuilder resultBuilder(result.subarrayStart("results")); - for (BSONObj obj; PlanExecutor::ADVANCED == exec->getNext(&obj, NULL); ) { + BSONObj obj; + PlanExecutor::ExecState state; + while (PlanExecutor::ADVANCED == (state = exec->getNext(&obj, NULL))) { resultBuilder.append(obj); } resultBuilder.done(); + + if (PlanExecutor::FAILURE == state || PlanExecutor::DEAD == state) { + const std::unique_ptr<PlanStageStats> stats(exec->getStats()); + error() << "Plan executor error during StageDebug command: " + << PlanExecutor::statestr(state) + << ", stats: " << Explain::statsToBSON(*stats); + + return appendCommandStatus(result, + Status(ErrorCodes::OperationFailed, + str::stream() + << "Executor error during " + << "StageDebug command: " + << WorkingSetCommon::toStatusString(obj))); + } + return true; } diff --git a/src/mongo/db/pipeline/document_source_cursor.cpp b/src/mongo/db/pipeline/document_source_cursor.cpp index 3891008ff1c..415cd63ea64 100644 --- a/src/mongo/db/pipeline/document_source_cursor.cpp +++ b/src/mongo/db/pipeline/document_source_cursor.cpp @@ -122,10 +122,12 @@ namespace mongo { // dispose since we want to keep the _currentBatch. _exec.reset(); - uassert(16028, "collection or index disappeared when cursor yielded", + uassert(16028, str::stream() << "collection or index disappeared when cursor yielded: " + << WorkingSetCommon::toStatusString(obj), state != PlanExecutor::DEAD); - uassert(17285, "cursor encountered an error: " + WorkingSetCommon::toStatusString(obj), + uassert(17285, str::stream() << "cursor encountered an error: " + << WorkingSetCommon::toStatusString(obj), state != PlanExecutor::FAILURE); massert(17286, str::stream() << "Unexpected return from PlanExecutor::getNext: " << state, diff --git a/src/mongo/db/query/find.cpp b/src/mongo/db/query/find.cpp index 274a4761b3e..4ba5a9bcf42 100644 --- a/src/mongo/db/query/find.cpp +++ b/src/mongo/db/query/find.cpp @@ -327,7 +327,7 @@ namespace mongo { // has a valid RecoveryUnit. As such, we use RAII to accomplish this. // // This must be destroyed before the ClientCursor is destroyed. - std::auto_ptr<ScopedRecoveryUnitSwapper> ruSwapper; + std::unique_ptr<ScopedRecoveryUnitSwapper> ruSwapper; // These are set in the QueryResult msg we return. int resultFlags = ResultFlag_AwaitCapable; @@ -432,23 +432,11 @@ namespace mongo { if (PlanExecutor::DEAD == state || PlanExecutor::FAILURE == state) { // Propagate this error to caller. - if (PlanExecutor::FAILURE == state) { - scoped_ptr<PlanStageStats> stats(exec->getStats()); - error() << "Plan executor error, stats: " - << Explain::statsToBSON(*stats); - uasserted(17406, "getMore executor error: " + - WorkingSetCommon::toStatusString(obj)); - } - - // In the old system tailable capped cursors would be killed off at the - // cursorid level. If a tailable capped cursor is nuked the cursorid - // would vanish. - // - // In the new system they die and are cleaned up later (or time out). - // So this is where we get to remove the cursorid. - if (0 == numResults) { - resultFlags = ResultFlag_CursorNotFound; - } + const std::unique_ptr<PlanStageStats> stats(exec->getStats()); + error() << "getMore executor error, stats: " + << Explain::statsToBSON(*stats); + uasserted(17406, "getMore executor error: " + + WorkingSetCommon::toStatusString(obj)); } const bool shouldSaveCursor = @@ -679,10 +667,10 @@ namespace mongo { exec->deregisterExec(); // Caller expects exceptions thrown in certain cases. - if (PlanExecutor::FAILURE == state) { - scoped_ptr<PlanStageStats> stats(exec->getStats()); - error() << "Plan executor error, stats: " - << Explain::statsToBSON(*stats); + if (PlanExecutor::FAILURE == state || PlanExecutor::DEAD == state) { + const std::unique_ptr<PlanStageStats> stats(exec->getStats()); + error() << "Plan executor error during find: " << PlanExecutor::statestr(state) + << ", stats: " << Explain::statsToBSON(*stats); uasserted(17144, "Executor error: " + WorkingSetCommon::toStatusString(obj)); } diff --git a/src/mongo/db/query/plan_executor.cpp b/src/mongo/db/query/plan_executor.cpp index 1a4b38482b6..42ca5164107 100644 --- a/src/mongo/db/query/plan_executor.cpp +++ b/src/mongo/db/query/plan_executor.cpp @@ -446,22 +446,16 @@ namespace mongo { else if (PlanStage::IS_EOF == code) { return PlanExecutor::IS_EOF; } - else if (PlanStage::DEAD == code) { - if (NULL != objOut) { - BSONObj statusObj; - WorkingSetCommon::getStatusMemberObject(*_workingSet, id, &statusObj); - *objOut = Snapshotted<BSONObj>(SnapshotId(), statusObj); - } - return PlanExecutor::DEAD; - } else { - verify(PlanStage::FAILURE == code); + invariant(PlanStage::DEAD == code || PlanStage::FAILURE == code); + if (NULL != objOut) { BSONObj statusObj; WorkingSetCommon::getStatusMemberObject(*_workingSet, id, &statusObj); *objOut = Snapshotted<BSONObj>(SnapshotId(), statusObj); } - return PlanExecutor::FAILURE; + + return (PlanStage::DEAD == code) ? PlanExecutor::DEAD : PlanExecutor::FAILURE; } } } @@ -513,13 +507,10 @@ namespace mongo { state = this->getNext(&obj, NULL); } - if (PlanExecutor::DEAD == state) { - return Status(ErrorCodes::OperationFailed, "Exec error: PlanExecutor killed"); - } - else if (PlanExecutor::FAILURE == state) { + if (PlanExecutor::DEAD == state || PlanExecutor::FAILURE == state) { return Status(ErrorCodes::OperationFailed, - str::stream() << "Exec error: " - << WorkingSetCommon::toStatusString(obj)); + str::stream() << "Exec error: " << WorkingSetCommon::toStatusString(obj) + << ", state: " << PlanExecutor::statestr(state)); } invariant(PlanExecutor::IS_EOF == state); diff --git a/src/mongo/db/query/plan_executor.h b/src/mongo/db/query/plan_executor.h index 8304b5d7e34..69aa7ef8397 100644 --- a/src/mongo/db/query/plan_executor.h +++ b/src/mongo/db/query/plan_executor.h @@ -66,7 +66,11 @@ namespace mongo { // We're EOF. We won't return any more results (edge case exception: capped+tailable). IS_EOF, - // We were killed or had an error. + // We were killed. This is a special failure case in which we cannot rely on the + // collection or database to still be valid. + // If the underlying PlanStage has any information on the error, it will be available in + // the objOut parameter. Call WorkingSetCommon::toStatusString() to retrieve the error + // details from the output BSON object. DEAD, // getNext was asked for data it cannot provide, or the underlying PlanStage had an diff --git a/src/mongo/db/ttl.cpp b/src/mongo/db/ttl.cpp index facbb58082c..9d533c2ea93 100644 --- a/src/mongo/db/ttl.cpp +++ b/src/mongo/db/ttl.cpp @@ -321,9 +321,9 @@ namespace mongo { return true; } } - if (PlanExecutor::IS_EOF != state) { - if (PlanExecutor::FAILURE == state && - WorkingSetCommon::isValidStatusMemberObject(obj)) { + + if (PlanExecutor::FAILURE == state || PlanExecutor::DEAD == state) { + if (WorkingSetCommon::isValidStatusMemberObject(obj)) { error() << "ttl query execution for index " << idx << " failed with: " << WorkingSetCommon::getMemberObjectStatus(obj); return true; @@ -332,6 +332,8 @@ namespace mongo { << PlanExecutor::statestr(state); return true; } + + invariant(PlanExecutor::IS_EOF == state); break; } catch (const WriteConflictException& dle) { diff --git a/src/mongo/dbtests/query_stage_and.cpp b/src/mongo/dbtests/query_stage_and.cpp index 5ce224bf463..2b43feb3b32 100644 --- a/src/mongo/dbtests/query_stage_and.cpp +++ b/src/mongo/dbtests/query_stage_and.cpp @@ -44,10 +44,12 @@ #include "mongo/db/exec/fetch.h" #include "mongo/db/exec/index_scan.h" #include "mongo/db/exec/plan_stage.h" +#include "mongo/db/exec/queued_data_stage.h" #include "mongo/db/json.h" #include "mongo/db/matcher/expression_parser.h" #include "mongo/db/operation_context_impl.h" #include "mongo/dbtests/dbtests.h" +#include "mongo/stdx/memory.h" #include "mongo/util/mongoutils/str.h" namespace QueryStageAnd { @@ -905,6 +907,143 @@ namespace QueryStageAnd { }; + class QueryStageAndHashDeadChild : public QueryStageAndBase { + public: + void run() { + OldClientWriteContext ctx(&_txn, ns()); + Database* db = ctx.db(); + Collection* coll = ctx.getCollection(); + if (!coll) { + WriteUnitOfWork wuow(&_txn); + coll = db->createCollection(&_txn, ns()); + wuow.commit(); + } + + const BSONObj dataObj = fromjson("{'foo': 'bar'}"); + + // Confirm PlanStage::DEAD when children contain the following WorkingSetMembers: + // Child1: Data + // Child2: NEED_TIME, DEAD + { + WorkingSet ws; + const MatchExpression* const matchExp = NULL; + const std::unique_ptr<AndHashStage> andHashStage = + stdx::make_unique<AndHashStage>(&ws, matchExp, coll); + + std::unique_ptr<QueuedDataStage> childStage1 = + stdx::make_unique<QueuedDataStage>(&ws); + { + WorkingSetMember wsm; + wsm.state = WorkingSetMember::LOC_AND_UNOWNED_OBJ; + wsm.loc = RecordId(1); + wsm.obj = Snapshotted<BSONObj>(SnapshotId(), dataObj); + childStage1->pushBack(wsm); + } + + std::unique_ptr<QueuedDataStage> childStage2 = + stdx::make_unique<QueuedDataStage>(&ws); + childStage2->pushBack(PlanStage::NEED_TIME); + childStage2->pushBack(PlanStage::DEAD); + + andHashStage->addChild(childStage1.release()); + andHashStage->addChild(childStage2.release()); + + WorkingSetID id = WorkingSet::INVALID_ID; + PlanStage::StageState state = PlanStage::NEED_TIME; + while (PlanStage::NEED_TIME == state) { + state = andHashStage->work(&id); + } + + ASSERT_EQ(PlanStage::DEAD, state); + } + + // Confirm PlanStage::DEAD when children contain the following WorkingSetMembers: + // Child1: Data, DEAD + // Child2: Data + { + WorkingSet ws; + const MatchExpression* const matchExp = NULL; + const std::unique_ptr<AndHashStage> andHashStage = + stdx::make_unique<AndHashStage>(&ws, matchExp, coll); + + std::unique_ptr<QueuedDataStage> childStage1 = + stdx::make_unique<QueuedDataStage>(&ws); + + { + WorkingSetMember wsm; + wsm.state = WorkingSetMember::LOC_AND_UNOWNED_OBJ; + wsm.loc = RecordId(1); + wsm.obj = Snapshotted<BSONObj>(SnapshotId(), dataObj); + childStage1->pushBack(wsm); + } + childStage1->pushBack(PlanStage::DEAD); + + std::unique_ptr<QueuedDataStage> childStage2 = + stdx::make_unique<QueuedDataStage>(&ws); + { + WorkingSetMember wsm; + wsm.state = WorkingSetMember::LOC_AND_UNOWNED_OBJ; + wsm.loc = RecordId(2); + wsm.obj = Snapshotted<BSONObj>(SnapshotId(), dataObj); + childStage2->pushBack(wsm); + } + + andHashStage->addChild(childStage1.release()); + andHashStage->addChild(childStage2.release()); + + WorkingSetID id = WorkingSet::INVALID_ID; + PlanStage::StageState state = PlanStage::NEED_TIME; + while (PlanStage::NEED_TIME == state) { + state = andHashStage->work(&id); + } + + ASSERT_EQ(PlanStage::DEAD, state); + } + + // Confirm PlanStage::DEAD when children contain the following WorkingSetMembers: + // Child1: Data + // Child2: Data, DEAD + { + WorkingSet ws; + const MatchExpression* const matchExp = NULL; + const std::unique_ptr<AndHashStage> andHashStage = + stdx::make_unique<AndHashStage>(&ws, matchExp, coll); + + std::unique_ptr<QueuedDataStage> childStage1 = + stdx::make_unique<QueuedDataStage>(&ws); + { + WorkingSetMember wsm; + wsm.state = WorkingSetMember::LOC_AND_UNOWNED_OBJ; + wsm.loc = RecordId(1); + wsm.obj = Snapshotted<BSONObj>(SnapshotId(), dataObj); + childStage1->pushBack(wsm); + } + + std::unique_ptr<QueuedDataStage> childStage2 = + stdx::make_unique<QueuedDataStage>(&ws); + { + WorkingSetMember wsm; + wsm.state = WorkingSetMember::LOC_AND_UNOWNED_OBJ; + wsm.loc = RecordId(2); + wsm.obj = Snapshotted<BSONObj>(SnapshotId(), dataObj); + childStage2->pushBack(wsm); + } + childStage2->pushBack(PlanStage::DEAD); + + andHashStage->addChild(childStage1.release()); + andHashStage->addChild(childStage2.release()); + + WorkingSetID id = WorkingSet::INVALID_ID; + PlanStage::StageState state = PlanStage::NEED_TIME; + while (PlanStage::NEED_TIME == state) { + state = andHashStage->work(&id); + } + + ASSERT_EQ(PlanStage::DEAD, state); + } + } + }; + // // Sorted AND tests // @@ -1410,6 +1549,7 @@ namespace QueryStageAnd { add<QueryStageAndHashInvalidateLookahead>(); add<QueryStageAndHashFirstChildFetched>(); add<QueryStageAndHashSecondChildFetched>(); + add<QueryStageAndHashDeadChild>(); add<QueryStageAndSortedInvalidation>(); add<QueryStageAndSortedThreeLeaf>(); add<QueryStageAndSortedWithNothing>(); diff --git a/src/mongo/dbtests/querytests.cpp b/src/mongo/dbtests/querytests.cpp index ffe22a37d2b..d13ad632970 100644 --- a/src/mongo/dbtests/querytests.cpp +++ b/src/mongo/dbtests/querytests.cpp @@ -490,25 +490,10 @@ namespace QueryTests { insert( ns, BSON( "a" << 2 ) ); insert( ns, BSON( "a" << 3 ) ); - // This can either have been killed, or jumped to the right thing. - // Key is that it can't skip. + // We have overwritten the previous cursor position and should encounter a dead cursor. if ( c->more() ) { - BSONObj x = c->next(); - ASSERT_EQUALS( 2, x["a"].numberInt() ); + ASSERT_THROWS(c->nextSafe(), AssertionException); } - - // Inserting a document into a capped collection can force another document out. - // In this case, the capped collection has 2 documents, so inserting two more clobbers - // whatever RecordId that the underlying cursor had as its state. - // - // In the Cursor world, the ClientCursor was responsible for manipulating cursors. It - // would detect that the cursor's "refloc" (translation: diskloc required to maintain - // iteration state) was being clobbered and it would kill the cursor. - // - // In the Runner world there is no notion of a "refloc" and as such the invalidation - // broadcast code doesn't know enough to know that the underlying collection iteration - // can't proceed. - // ASSERT_EQUALS( 0, c->getCursorId() ); } }; @@ -530,11 +515,9 @@ namespace QueryTests { insert( ns, BSON( "a" << 3 ) ); insert( ns, BSON( "a" << 4 ) ); - // This can either have been killed, or jumped to the right thing. - // Key is that it can't skip. + // We have overwritten the previous cursor position and should encounter a dead cursor. if ( c->more() ) { - BSONObj x = c->next(); - ASSERT_EQUALS( 2, x["a"].numberInt() ); + ASSERT_THROWS(c->nextSafe(), AssertionException); } } }; |