diff options
| author | U-tellus\cwestin <cwestin@10gen.com> | 2012-01-30 17:12:51 -0800 |
|---|---|---|
| committer | U-tellus\cwestin <cwestin@10gen.com> | 2012-02-14 18:02:58 -0800 |
| commit | 57ea070eb7c129935d9d96e7655dfd4467eef857 (patch) | |
| tree | eac2b5e232078b2a7e70632272d5f3f13b6a7b0d /src/mongo | |
| parent | 1eaada83e82be3a8c4ef5977f5843f0d7aa3dfc1 (diff) | |
| download | mongo-57ea070eb7c129935d9d96e7655dfd4467eef857.tar.gz | |
checkpoint dependency tracker for SERVER-4644
Diffstat (limited to 'src/mongo')
23 files changed, 577 insertions, 53 deletions
diff --git a/src/mongo/SConscript b/src/mongo/SConscript index cef985fd128..20f41a309e1 100644 --- a/src/mongo/SConscript +++ b/src/mongo/SConscript @@ -117,6 +117,7 @@ coreServerFiles = [ "util/version.cpp", "db/pipeline/accumulator_single_value.cpp", "db/pipeline/accumulator_sum.cpp", "db/pipeline/builder.cpp", + "db/pipeline/dependency_tracker.cpp", "db/pipeline/doc_mem_monitor.cpp", "db/pipeline/document.cpp", "db/pipeline/document_source.cpp", diff --git a/src/mongo/db/commands/document_source_cursor.cpp b/src/mongo/db/commands/document_source_cursor.cpp index e63bca57efc..7beade30234 100755 --- a/src/mongo/db/commands/document_source_cursor.cpp +++ b/src/mongo/db/commands/document_source_cursor.cpp @@ -61,7 +61,8 @@ namespace mongo { /* grab the matching document */
BSONObj documentObj(pCursor->current());
- pCurrent = Document::createFromBsonObj(&documentObj);
+ pCurrent = Document::createFromBsonObj(
+ &documentObj, NULL /* LATER pDependencies.get()*/);
pCursor->advance();
return;
}
@@ -103,4 +104,11 @@ namespace mongo { const shared_ptr<BSONObj> &pBsonObj) {
bsonDependencies.push_back(pBsonObj);
}
+
+ void DocumentSourceCursor::manageDependencies(
+ const intrusive_ptr<DependencyTracker> &pTracker) {
+ /* hang on to the tracker */
+ pDependencies = pTracker;
+ }
+
}
diff --git a/src/mongo/db/commands/pipeline.cpp b/src/mongo/db/commands/pipeline.cpp index eaa4a1f6239..484e9f2a424 100755 --- a/src/mongo/db/commands/pipeline.cpp +++ b/src/mongo/db/commands/pipeline.cpp @@ -161,8 +161,10 @@ namespace mongo { const StageDesc *pDesc = (const StageDesc *)
bsearch(&key, stageDesc, nStageDesc, sizeof(StageDesc),
stageDescCmp);
- if (pDesc)
+ if (pDesc) {
pSource = (*pDesc->pFactory)(&bsonElement, pCtx);
+ pSource->setPipelineStep(iStep);
+ }
else {
ostringstream sb;
sb <<
@@ -345,12 +347,13 @@ namespace mongo { front of it, and finally passes that to the input source before we
execute the pipeline.
*/
- intrusive_ptr<DependencyTracker> pTracker;
+ intrusive_ptr<DependencyTracker> pTracker(new DependencyTracker());
for(SourceVector::reverse_iterator iter(sourceVector.rbegin()),
listBeg(sourceVector.rend()); iter != listBeg; ++iter) {
intrusive_ptr<DocumentSource> pTemp(*iter);
pTemp->manageDependencies(pTracker);
}
+
pInputSource->manageDependencies(pTracker);
/* chain together the sources we found */
diff --git a/src/mongo/db/pipeline/accumulator.cpp b/src/mongo/db/pipeline/accumulator.cpp index b02d6d886d8..13fbfbbe45c 100755 --- a/src/mongo/db/pipeline/accumulator.cpp +++ b/src/mongo/db/pipeline/accumulator.cpp @@ -56,8 +56,6 @@ namespace mongo { }
void agg_framework_reservedErrors() {
- uassert(15984, "reserved error", false);
-
uassert(16023, "reserved error", false);
uassert(16024, "reserved error", false);
uassert(16025, "reserved error", false);
diff --git a/src/mongo/db/pipeline/dependency_tracker.cpp b/src/mongo/db/pipeline/dependency_tracker.cpp new file mode 100755 index 00000000000..c019ceb315a --- /dev/null +++ b/src/mongo/db/pipeline/dependency_tracker.cpp @@ -0,0 +1,59 @@ +/** + * Copyright (c) 2012 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#include "pch.h" + +#include "db/pipeline/dependency_tracker.h" +#include "db/pipeline/document_source.h" + +namespace mongo { + + DependencyTracker::Tracker::Tracker( + const string &fp, const DocumentSource *pS): + fieldPath(fp), + pSource(pS) { + } + + void DependencyTracker::addDependency( + const string &fieldPath, const DocumentSource *pSource) { + Tracker tracker(fieldPath, pSource); + std::pair<MapType::iterator, bool> p( + map.insert(std::pair<string, Tracker>(fieldPath, tracker))); + + /* + If there was already an entry, update the dependency to be the more + recent source. + */ + if (!p.second) + (*p.first).second.pSource = pSource; + } + + void DependencyTracker::removeDependency(const string &fieldPath) { + map.erase(fieldPath); + } + + bool DependencyTracker::getDependency( + intrusive_ptr<const DocumentSource> *ppSource, + const string &fieldPath) const { + MapType::const_iterator i(map.find(fieldPath)); + if (i == map.end()) + return false; + + *ppSource = (*i).second.pSource; + return true; + } + +} diff --git a/src/mongo/db/pipeline/dependency_tracker.h b/src/mongo/db/pipeline/dependency_tracker.h index 69112119beb..81ef34a1ef4 100755 --- a/src/mongo/db/pipeline/dependency_tracker.h +++ b/src/mongo/db/pipeline/dependency_tracker.h @@ -24,17 +24,26 @@ namespace mongo { + class DocumentSource; + class DependencyTracker : public IntrusiveCounterUnsigned { public: - void include(const string &fieldName); - void exclude(const string &fieldName); + void addDependency(const string &fieldPath, + const DocumentSource *pSource); + + void removeDependency(const string &fieldPath); - bool isRequired(const string &fieldName) const; + bool getDependency(intrusive_ptr<const DocumentSource> *ppSource, + const string &fieldPath) const; private: struct Tracker { - string fieldName; + Tracker(const string &fieldPath, + const DocumentSource *pSource); + + string fieldPath; + intrusive_ptr<const DocumentSource> pSource; struct Hash : unary_function<string, size_t> { @@ -42,7 +51,8 @@ namespace mongo { }; }; - boost::unordered_map<string, Tracker, Tracker::Hash> map; + typedef boost::unordered_map<string, Tracker, Tracker::Hash> MapType; + MapType map; }; } diff --git a/src/mongo/db/pipeline/document.cpp b/src/mongo/db/pipeline/document.cpp index 6416a341f41..f7a85459a99 100755 --- a/src/mongo/db/pipeline/document.cpp +++ b/src/mongo/db/pipeline/document.cpp @@ -19,6 +19,7 @@ #undef assert
#define assert MONGO_assert
#include "db/jsobj.h"
+#include "db/pipeline/dependency_tracker.h"
#include "db/pipeline/document.h"
#include "db/pipeline/value.h"
#include "util/mongoutils/str.h"
@@ -28,18 +29,24 @@ namespace mongo { string Document::idName("_id");
- intrusive_ptr<Document> Document::createFromBsonObj(BSONObj *pBsonObj) {
- intrusive_ptr<Document> pDocument(new Document(pBsonObj));
+ intrusive_ptr<Document> Document::createFromBsonObj(
+ BSONObj *pBsonObj, const DependencyTracker *pDependencies) {
+ intrusive_ptr<Document> pDocument(
+ new Document(pBsonObj, pDependencies));
return pDocument;
}
- Document::Document(BSONObj *pBsonObj):
+ Document::Document(BSONObj *pBsonObj,
+ const DependencyTracker *pDependencies):
vFieldName(),
vpValue() {
BSONObjIterator bsonIterator(pBsonObj->begin());
while(bsonIterator.more()) {
BSONElement bsonElement(bsonIterator.next());
string fieldName(bsonElement.fieldName());
+
+ // LATER check pDependencies
+ // LATER grovel through structures???
intrusive_ptr<const Value> pValue(
Value::createFromBsonElement(&bsonElement));
diff --git a/src/mongo/db/pipeline/document.h b/src/mongo/db/pipeline/document.h index b00f7162cef..533f5b2fc30 100755 --- a/src/mongo/db/pipeline/document.h +++ b/src/mongo/db/pipeline/document.h @@ -22,6 +22,7 @@ namespace mongo { class BSONObj; + class DependencyTracker; class FieldIterator; class Value; @@ -36,9 +37,14 @@ namespace mongo { Document field values may be pointed to in the BSONObj, so it must live at least as long as the resulting Document. + LATER - use an abstract class for the dependencies; something like + a "lookup(const string &fieldName)" so there can be other + implementations. + @returns shared pointer to the newly created Document */ - static intrusive_ptr<Document> createFromBsonObj(BSONObj *pBsonObj); + static intrusive_ptr<Document> createFromBsonObj( + BSONObj *pBsonObj, const DependencyTracker *pDependencies = NULL); /* Create a new empty Document. @@ -184,7 +190,7 @@ namespace mongo { friend class FieldIterator; Document(size_t sizeHint); - Document(BSONObj *pBsonObj); + Document(BSONObj *pBsonObj, const DependencyTracker *pDependencies); /* these two vectors parallel each other */ vector<string> vFieldName; diff --git a/src/mongo/db/pipeline/document_source.cpp b/src/mongo/db/pipeline/document_source.cpp index 1ab02880017..89177f58767 100755 --- a/src/mongo/db/pipeline/document_source.cpp +++ b/src/mongo/db/pipeline/document_source.cpp @@ -19,9 +19,19 @@ #include "db/pipeline/document_source.h"
namespace mongo {
+
+ DocumentSource::DocumentSource():
+ step(-1) {
+ }
+
DocumentSource::~DocumentSource() {
}
+ const char *DocumentSource::getSourceName() const {
+ static const char unknown[] = "[UNKNOWN]";
+ return unknown;
+ }
+
void DocumentSource::setSource(
const intrusive_ptr<DocumentSource> &pTheSource) {
assert(!pSource.get());
diff --git a/src/mongo/db/pipeline/document_source.h b/src/mongo/db/pipeline/document_source.h index f7df6170221..6b547032801 100755 --- a/src/mongo/db/pipeline/document_source.h +++ b/src/mongo/db/pipeline/document_source.h @@ -22,6 +22,7 @@ #include "util/intrusive_counter.h"
#include "client/parallel.h"
#include "db/jsobj.h"
+#include "db/pipeline/dependency_tracker.h"
#include "db/pipeline/document.h"
#include "db/pipeline/expression.h"
#include "db/pipeline/value.h"
@@ -45,13 +46,23 @@ namespace mongo { virtual ~DocumentSource();
// virtuals from StringWriter
+ virtual void writeString(stringstream &ss) const;
+
/**
- Write out a string representation of this pipeline operator.
+ Set the step for a user-specified pipeline step.
- @param ss string stream to write the string representation to
- */
- virtual void writeString(stringstream &ss) const;
+ The step is used for diagnostics.
+ @param step step number 0 to n.
+ */
+ void setPipelineStep(int step);
+
+ /**
+ Get the user-specified pipeline step.
+
+ @returns the step number, or -1 if it has never been set
+ */
+ int getPipelineStep() const;
/**
Is the source at EOF?
@@ -78,6 +89,14 @@ namespace mongo { virtual intrusive_ptr<Document> getCurrent() = 0;
/**
+ Get the source's name.
+
+ @returns the string name of the source as a constant string;
+ this is static, and there's no need to worry about adopting it
+ */
+ virtual const char *getSourceName() const;
+
+ /**
Set the underlying source this source should use to get Documents
from.
@@ -139,6 +158,11 @@ namespace mongo { protected:
/**
+ Base constructor.
+ */
+ DocumentSource();
+
+ /**
Create an object that represents the document source. The object
will have a single field whose name is the source's name. This
will be used by the default implementation of addToBsonArray()
@@ -157,6 +181,13 @@ namespace mongo { assert() if this has already been set.
*/
intrusive_ptr<DocumentSource> pSource;
+
+ /*
+ The zero-based user-specified pipeline step. Used for diagnostics.
+ Will be set to -1 for artificial pipeline steps that were not part
+ of the original user specification.
+ */
+ int step;
};
@@ -257,6 +288,8 @@ namespace mongo { virtual bool advance();
virtual intrusive_ptr<Document> getCurrent();
virtual void setSource(const intrusive_ptr<DocumentSource> &pSource);
+ virtual void manageDependencies(
+ const intrusive_ptr<DependencyTracker> &pTracker);
/**
Create a document source based on a cursor.
@@ -306,10 +339,18 @@ namespace mongo { /*
The bsonDependencies must outlive the Cursor wrapped by this
source. Therefore, bsonDependencies must appear before pCursor
- in order its destructor to be called *after* pCursor's.
+ in order cause its destructor to be called *after* pCursor's.
*/
vector<shared_ptr<BSONObj> > bsonDependencies;
shared_ptr<Cursor> pCursor;
+
+ /*
+ This document source hangs on to the dependency tracker when it
+ gets it so that it can be used for selective reification of
+ fields in order to avoid fields that are not required through the
+ pipeline.
+ */
+ intrusive_ptr<DependencyTracker> pDependencies;
};
@@ -370,6 +411,7 @@ namespace mongo { virtual ~DocumentSourceFilter();
virtual bool coalesce(const intrusive_ptr<DocumentSource> &pNextSource);
virtual void optimize();
+ virtual const char *getSourceName() const;
/**
Create a filter.
@@ -426,6 +468,7 @@ namespace mongo { virtual ~DocumentSourceGroup();
virtual bool eof();
virtual bool advance();
+ virtual const char *getSourceName() const;
virtual intrusive_ptr<Document> getCurrent();
/**
@@ -546,6 +589,7 @@ namespace mongo { public:
// virtuals from DocumentSource
virtual ~DocumentSourceMatch();
+ virtual const char *getSourceName() const;
/**
Create a filter.
@@ -593,6 +637,7 @@ namespace mongo { virtual ~DocumentSourceOut();
virtual bool eof();
virtual bool advance();
+ virtual const char *getSourceName() const;
virtual intrusive_ptr<Document> getCurrent();
/**
@@ -618,15 +663,17 @@ namespace mongo { class DocumentSourceProject :
- public DocumentSource,
- public boost::enable_shared_from_this<DocumentSourceProject> {
+ public DocumentSource {
public:
// virtuals from DocumentSource
virtual ~DocumentSourceProject();
virtual bool eof();
virtual bool advance();
+ virtual const char *getSourceName() const;
virtual intrusive_ptr<Document> getCurrent();
virtual void optimize();
+ virtual void manageDependencies(
+ const intrusive_ptr<DependencyTracker> &pTracker);
/**
Create a new DocumentSource that can implement projection.
@@ -686,6 +733,65 @@ namespace mongo { // configuration state
bool excludeId;
intrusive_ptr<ExpressionObject> pEO;
+
+ /*
+ Utility object used by manageDependencies().
+
+ Removes dependencies from a DependencyTracker.
+ */
+ class DependencyRemover :
+ public ExpressionObject::PathSink {
+ public:
+ // virtuals from PathSink
+ virtual void path(const string &path, bool include);
+
+ /*
+ Constructor.
+
+ Captures a reference to the smart pointer to the DependencyTracker
+ that this will remove dependencies from via
+ ExpressionObject::emitPaths().
+
+ @param pTracker reference to the smart pointer to the
+ DependencyTracker
+ */
+ DependencyRemover(const intrusive_ptr<DependencyTracker> &pTracker);
+
+ private:
+ const intrusive_ptr<DependencyTracker> &pTracker;
+ };
+
+ /*
+ Utility object used by manageDependencies().
+
+ Checks dependencies to see if they are present. If not, then
+ throws a user error.
+ */
+ class DependencyChecker :
+ public ExpressionObject::PathSink {
+ public:
+ // virtuals from PathSink
+ virtual void path(const string &path, bool include);
+
+ /*
+ Constructor.
+
+ Captures a reference to the smart pointer to the DependencyTracker
+ that this will check dependencies from from
+ ExpressionObject::emitPaths() to see if they are required.
+
+ @param pTracker reference to the smart pointer to the
+ DependencyTracker
+ @param pThis the projection that is making this request
+ */
+ DependencyChecker(
+ const intrusive_ptr<DependencyTracker> &pTracker,
+ const DocumentSourceProject *pThis);
+
+ private:
+ const intrusive_ptr<DependencyTracker> &pTracker;
+ const DocumentSourceProject *pThis;
+ };
};
@@ -696,7 +802,10 @@ namespace mongo { virtual ~DocumentSourceSort();
virtual bool eof();
virtual bool advance();
+ virtual const char *getSourceName() const;
virtual intrusive_ptr<Document> getCurrent();
+ virtual void manageDependencies(
+ const intrusive_ptr<DependencyTracker> &pTracker);
/*
TODO
Adjacent sorts should reduce to the last sort.
@@ -768,7 +877,8 @@ namespace mongo { long long count;
/* these two parallel each other */
- vector<intrusive_ptr<ExpressionFieldPath> > vSortKey;
+ typedef vector<intrusive_ptr<ExpressionFieldPath> > SortPaths;
+ SortPaths vSortKey;
vector<bool> vAscending;
class Carrier {
@@ -815,6 +925,7 @@ namespace mongo { virtual ~DocumentSourceLimit();
virtual bool eof();
virtual bool advance();
+ virtual const char *getSourceName() const;
virtual intrusive_ptr<Document> getCurrent();
/**
@@ -865,6 +976,7 @@ namespace mongo { virtual ~DocumentSourceSkip();
virtual bool eof();
virtual bool advance();
+ virtual const char *getSourceName() const;
virtual intrusive_ptr<Document> getCurrent();
/**
@@ -915,14 +1027,16 @@ namespace mongo { class DocumentSourceUnwind :
- public DocumentSource,
- public boost::enable_shared_from_this<DocumentSourceUnwind> {
+ public DocumentSource {
public:
// virtuals from DocumentSource
virtual ~DocumentSourceUnwind();
virtual bool eof();
virtual bool advance();
+ virtual const char *getSourceName() const;
virtual intrusive_ptr<Document> getCurrent();
+ virtual void manageDependencies(
+ const intrusive_ptr<DependencyTracker> &pTracker);
/**
Create a new DocumentSource that can implement unwind.
@@ -995,7 +1109,6 @@ namespace mongo { @returns a partial deep clone of pNoUnwindDocument
*/
intrusive_ptr<Document> clonePath() const;
-
};
}
@@ -1005,11 +1118,31 @@ namespace mongo { namespace mongo {
+ inline void DocumentSource::setPipelineStep(int s) {
+ step = s;
+ }
+
+ inline int DocumentSource::getPipelineStep() const {
+ return step;
+ }
+
inline void DocumentSourceGroup::setIdExpression(
const intrusive_ptr<Expression> &pExpression) {
pIdExpression = pExpression;
}
+ inline DocumentSourceProject::DependencyRemover::DependencyRemover(
+ const intrusive_ptr<DependencyTracker> &pT):
+ pTracker(pT) {
+ }
+
+ inline DocumentSourceProject::DependencyChecker::DependencyChecker(
+ const intrusive_ptr<DependencyTracker> &pTrack,
+ const DocumentSourceProject *pT):
+ pTracker(pTrack),
+ pThis(pT) {
+ }
+
inline void DocumentSourceUnwind::resetArray() {
pNoUnwindDocument.reset();
pUnwindArray.reset();
diff --git a/src/mongo/db/pipeline/document_source_filter.cpp b/src/mongo/db/pipeline/document_source_filter.cpp index d4c8b75cf75..8004378e935 100755 --- a/src/mongo/db/pipeline/document_source_filter.cpp +++ b/src/mongo/db/pipeline/document_source_filter.cpp @@ -29,6 +29,10 @@ namespace mongo { DocumentSourceFilter::~DocumentSourceFilter() {
}
+ const char *DocumentSourceFilter::getSourceName() const {
+ return filterName;
+ }
+
bool DocumentSourceFilter::coalesce(
const intrusive_ptr<DocumentSource> &pNextSource) {
diff --git a/src/mongo/db/pipeline/document_source_group.cpp b/src/mongo/db/pipeline/document_source_group.cpp index 0624bbce646..56eccd26218 100755 --- a/src/mongo/db/pipeline/document_source_group.cpp +++ b/src/mongo/db/pipeline/document_source_group.cpp @@ -31,6 +31,10 @@ namespace mongo { DocumentSourceGroup::~DocumentSourceGroup() {
}
+ const char *DocumentSourceGroup::getSourceName() const {
+ return groupName;
+ }
+
bool DocumentSourceGroup::eof() {
if (!populated)
populate();
diff --git a/src/mongo/db/pipeline/document_source_limit.cpp b/src/mongo/db/pipeline/document_source_limit.cpp index fd977cc126a..5dc75122800 100644 --- a/src/mongo/db/pipeline/document_source_limit.cpp +++ b/src/mongo/db/pipeline/document_source_limit.cpp @@ -36,6 +36,10 @@ namespace mongo { DocumentSourceLimit::~DocumentSourceLimit() { } + const char *DocumentSourceLimit::getSourceName() const { + return limitName; + } + bool DocumentSourceLimit::eof() { return pSource->eof() || count >= limit; } diff --git a/src/mongo/db/pipeline/document_source_match.cpp b/src/mongo/db/pipeline/document_source_match.cpp index 76d1305c7f1..c02490e92da 100755 --- a/src/mongo/db/pipeline/document_source_match.cpp +++ b/src/mongo/db/pipeline/document_source_match.cpp @@ -30,6 +30,10 @@ namespace mongo { DocumentSourceMatch::~DocumentSourceMatch() {
}
+ const char *DocumentSourceMatch::getSourceName() const {
+ return matchName;
+ }
+
void DocumentSourceMatch::sourceToBson(BSONObjBuilder *pBuilder) const {
const BSONObj *pQuery = matcher.getQuery();
pBuilder->append(matchName, *pQuery);
@@ -77,4 +81,5 @@ namespace mongo { DocumentSourceFilterBase(),
matcher(query) {
}
+
}
diff --git a/src/mongo/db/pipeline/document_source_out.cpp b/src/mongo/db/pipeline/document_source_out.cpp index c668d60fd3c..c9fdc6888ed 100755 --- a/src/mongo/db/pipeline/document_source_out.cpp +++ b/src/mongo/db/pipeline/document_source_out.cpp @@ -26,6 +26,10 @@ namespace mongo { DocumentSourceOut::~DocumentSourceOut() {
}
+ const char *DocumentSourceOut::getSourceName() const {
+ return outName;
+ }
+
bool DocumentSourceOut::eof() {
return pSource->eof();
}
diff --git a/src/mongo/db/pipeline/document_source_project.cpp b/src/mongo/db/pipeline/document_source_project.cpp index b1d95c20d76..9491e45bf9f 100755 --- a/src/mongo/db/pipeline/document_source_project.cpp +++ b/src/mongo/db/pipeline/document_source_project.cpp @@ -34,6 +34,10 @@ namespace mongo { pEO(ExpressionObject::create()) {
}
+ const char *DocumentSourceProject::getSourceName() const {
+ return projectName;
+ }
+
bool DocumentSourceProject::eof() {
return pSource->eof();
}
@@ -204,4 +208,62 @@ IncludeExclude: return pProject;
}
+
+ void DocumentSourceProject::DependencyRemover::path(
+ const string &path, bool include) {
+ if (include)
+ pTracker->removeDependency(path);
+ }
+
+ void DocumentSourceProject::DependencyChecker::path(
+ const string &path, bool include) {
+ /* if the specified path is included, there's nothing to check */
+ if (include)
+ return;
+
+ /* if the specified path is excluded, see if it is required */
+ intrusive_ptr<const DocumentSource> pSource;
+ if (pTracker->getDependency(&pSource, path)) {
+ uassert(15984, str::stream() <<
+ "unable to satisfy dependency on " <<
+ FieldPath::getPrefix() <<
+ path << " in pipeline step " <<
+ pSource->getPipelineStep() <<
+ " (" << pSource->getSourceName() << "), because step " <<
+ pThis->getPipelineStep() << " ("
+ << pThis->getSourceName() << ") excludes it",
+ false); // printf() is way easier to read than this crap
+ }
+ }
+
+ void DocumentSourceProject::manageDependencies(
+ const intrusive_ptr<DependencyTracker> &pTracker) {
+ /*
+ Look at all the products (inclusions and computed fields) of this
+ projection. For each one that is a dependency, remove it from the
+ list of dependencies, because this product will satisfy that
+ dependency.
+ */
+ pEO->emitPaths(&DependencyRemover(pTracker));
+
+ /*
+ Look at the exclusions of this projection. If any of them are
+ dependencies, inform the user (error/usassert) that the dependency
+ can't be satisfied.
+
+ Note we need to do this after the product examination above because
+ it is possible for there to be an exclusion field name that matches
+ a new computed product field name. The latter would satisfy the
+ dependency.
+ */
+ pEO->emitPaths(&DependencyChecker(pTracker, this));
+
+ /*
+ Look at the products of this projection. For inclusions, add the
+ field names to the list of dependencies. For computed expressions,
+ add their dependencies to the list of dependencies.
+ */
+ pEO->addDependencies(pTracker, this);
+ }
+
}
diff --git a/src/mongo/db/pipeline/document_source_skip.cpp b/src/mongo/db/pipeline/document_source_skip.cpp index f26486e75a2..155f366780b 100644 --- a/src/mongo/db/pipeline/document_source_skip.cpp +++ b/src/mongo/db/pipeline/document_source_skip.cpp @@ -25,6 +25,7 @@ #include "db/pipeline/value.h" namespace mongo { + const char DocumentSourceSkip::skipName[] = "$skip"; DocumentSourceSkip::DocumentSourceSkip(const intrusive_ptr<ExpressionContext> &pTheCtx): @@ -36,6 +37,10 @@ namespace mongo { DocumentSourceSkip::~DocumentSourceSkip() { } + const char *DocumentSourceSkip::getSourceName() const { + return skipName; + } + void DocumentSourceSkip::skipper() { if (count == 0) { while (!pSource->eof() && count++ < skip) { diff --git a/src/mongo/db/pipeline/document_source_sort.cpp b/src/mongo/db/pipeline/document_source_sort.cpp index 8e89617805c..4089bcfe24d 100755 --- a/src/mongo/db/pipeline/document_source_sort.cpp +++ b/src/mongo/db/pipeline/document_source_sort.cpp @@ -19,6 +19,7 @@ #include "db/pipeline/document_source.h"
#include "db/jsobj.h"
+#include "db/pipeline/dependency_tracker.h"
#include "db/pipeline/doc_mem_monitor.h"
#include "db/pipeline/document.h"
#include "db/pipeline/expression.h"
@@ -32,6 +33,10 @@ namespace mongo { DocumentSourceSort::~DocumentSourceSort() {
}
+ const char *DocumentSourceSort::getSourceName() const {
+ return sortName;
+ }
+
bool DocumentSourceSort::eof() {
if (!populated)
populate();
@@ -213,4 +218,14 @@ namespace mongo { /* compare the documents according to the sort key */
return (rL.pSort->compare(rL.pDocument, rR.pDocument) < 0);
}
+
+ void DocumentSourceSort::manageDependencies(
+ const intrusive_ptr<DependencyTracker> &pTracker) {
+ /* get the dependencies out of the matcher */
+ for(SortPaths::iterator i(vSortKey.begin()); i != vSortKey.end(); ++i) {
+ string fieldPath((*i)->getFieldPath(false));
+ pTracker->addDependency(fieldPath, this);
+ }
+ }
+
}
diff --git a/src/mongo/db/pipeline/document_source_unwind.cpp b/src/mongo/db/pipeline/document_source_unwind.cpp index f1d3b4fb420..71a609150a1 100755 --- a/src/mongo/db/pipeline/document_source_unwind.cpp +++ b/src/mongo/db/pipeline/document_source_unwind.cpp @@ -37,6 +37,10 @@ namespace mongo { pUnwindValue() {
}
+ const char *DocumentSourceUnwind::getSourceName() const {
+ return unwindName;
+ }
+
bool DocumentSourceUnwind::eof() {
/*
If we're unwinding an array, and there are more elements, then we
@@ -230,4 +234,10 @@ namespace mongo { return pUnwind;
}
+
+ void DocumentSourceUnwind::manageDependencies(
+ const intrusive_ptr<DependencyTracker> &pTracker) {
+ pTracker->addDependency(unwindPath.getPath(false), this);
+ }
+
}
diff --git a/src/mongo/db/pipeline/expression.cpp b/src/mongo/db/pipeline/expression.cpp index 38d37e9523f..19181e44a73 100755 --- a/src/mongo/db/pipeline/expression.cpp +++ b/src/mongo/db/pipeline/expression.cpp @@ -20,6 +20,7 @@ #include <cstdio>
#include "db/jsobj.h"
#include "db/pipeline/builder.h"
+#include "db/pipeline/dependency_tracker.h"
#include "db/pipeline/document.h"
#include "db/pipeline/expression_context.h"
#include "db/pipeline/value.h"
@@ -295,7 +296,8 @@ namespace mongo { return pExpression;
}
- intrusive_ptr<Expression> Expression::parseOperand(BSONElement *pBsonElement) {
+ intrusive_ptr<Expression> Expression::parseOperand(
+ BSONElement *pBsonElement) {
BSONType type = pBsonElement->type();
switch(type) {
@@ -626,6 +628,12 @@ namespace mongo { return intrusive_ptr<Expression>(this);
}
+ void ExpressionCoerceToBool::addDependencies(
+ const intrusive_ptr<DependencyTracker> &pTracker,
+ const DocumentSource *pSource) const {
+ /* nothing to do */
+ }
+
intrusive_ptr<const Value> ExpressionCoerceToBool::evaluate(
const intrusive_ptr<Document> &pDocument) const {
@@ -919,6 +927,12 @@ namespace mongo { return intrusive_ptr<Expression>(this);
}
+ void ExpressionConstant::addDependencies(
+ const intrusive_ptr<DependencyTracker> &pTracker,
+ const DocumentSource *pSource) const {
+ /* nothing to do */
+ }
+
intrusive_ptr<const Value> ExpressionConstant::evaluate(
const intrusive_ptr<Document> &pDocument) const {
return pValue;
@@ -1127,6 +1141,15 @@ namespace mongo { return intrusive_ptr<Expression>(this);
}
+ void ExpressionObject::addDependencies(
+ const intrusive_ptr<DependencyTracker> &pTracker,
+ const DocumentSource *pSource) const {
+ for(ExpressionVector::const_iterator i(vpExpression.begin());
+ i != vpExpression.end(); ++i) {
+ (*i)->addDependencies(pTracker, pSource);
+ }
+ }
+
void ExpressionObject::addToDocument(
const intrusive_ptr<Document> &pResult,
const intrusive_ptr<Document> &pDocument) const {
@@ -1415,8 +1438,13 @@ namespace mongo { return intrusive_ptr<Expression>();
}
+ void ExpressionObject::emitPaths(PathSink *pPathSink) const {
+ vector<string> vPath;
+ emitPaths(pPathSink, &vPath);
+ }
+
void ExpressionObject::emitPaths(
- BSONObjBuilder *pBuilder, vector<string> *pvPath) const {
+ PathSink *pPathSink, vector<string> *pvPath) const {
if (!path.size())
return;
@@ -1451,7 +1479,7 @@ namespace mongo { ss << (*pvPath)[iPath] << ".";
ss << *iter;
- pBuilder->append(ss.str(), !excludePaths);
+ pPathSink->path(ss.str(), !excludePaths);
}
else {
/*
@@ -1467,7 +1495,7 @@ namespace mongo { then go down into the next level.
*/
PathPusher pathPusher(pvPath, vFieldName[iField]);
- pEO->emitPaths(pBuilder, pvPath);
+ pEO->emitPaths(pPathSink, pvPath);
}
}
}
@@ -1476,8 +1504,8 @@ namespace mongo { BSONObjBuilder *pBuilder, unsigned depth) const {
/* emit any inclusion/exclusion paths */
- vector<string> vPath;
- emitPaths(pBuilder, &vPath);
+ BuilderPathSink builderPathSink(pBuilder);
+ emitPaths(&builderPathSink);
/* then add any expressions */
const size_t nField = vFieldName.size();
@@ -1509,6 +1537,11 @@ namespace mongo { pBuilder->append(objBuilder.done());
}
+ void ExpressionObject::BuilderPathSink::path(
+ const string &path, bool include) {
+ pBuilder->append(path, include);
+ }
+
/* --------------------- ExpressionFieldPath --------------------------- */
ExpressionFieldPath::~ExpressionFieldPath() {
@@ -1531,6 +1564,12 @@ namespace mongo { return intrusive_ptr<Expression>(this);
}
+ void ExpressionFieldPath::addDependencies(
+ const intrusive_ptr<DependencyTracker> &pTracker,
+ const DocumentSource *pSource) const {
+ pTracker->addDependency(fieldPath.getPath(false), pSource);
+ }
+
intrusive_ptr<const Value> ExpressionFieldPath::evaluatePath(
size_t index, const size_t pathLength,
intrusive_ptr<Document> pDocument) const {
@@ -1611,7 +1650,7 @@ namespace mongo { pBuilder->append(getFieldPath(true));
}
- /* --------------------- ExpressionFieldPath --------------------------- */
+ /* --------------------- ExpressionFieldRange -------------------------- */
ExpressionFieldRange::~ExpressionFieldRange() {
}
@@ -1637,6 +1676,12 @@ namespace mongo { return intrusive_ptr<Expression>(this);
}
+ void ExpressionFieldRange::addDependencies(
+ const intrusive_ptr<DependencyTracker> &pTracker,
+ const DocumentSource *pSource) const {
+ pFieldPath->addDependencies(pTracker, pSource);
+ }
+
intrusive_ptr<const Value> ExpressionFieldRange::evaluate(
const intrusive_ptr<Document> &pDocument) const {
/* if there's no range, there can't be a match */
@@ -2287,6 +2332,15 @@ namespace mongo { return pNew;
}
+ void ExpressionNary::addDependencies(
+ const intrusive_ptr<DependencyTracker> &pTracker,
+ const DocumentSource *pSource) const {
+ for(ExpressionVector::const_iterator i(vpOperand.begin());
+ i != vpOperand.end(); ++i) {
+ (*i)->addDependencies(pTracker, pSource);
+ }
+ }
+
void ExpressionNary::addOperand(
const intrusive_ptr<Expression> &pExpression) {
vpOperand.push_back(pExpression);
diff --git a/src/mongo/db/pipeline/expression.h b/src/mongo/db/pipeline/expression.h index dd63df779ee..f5520421112 100755 --- a/src/mongo/db/pipeline/expression.h +++ b/src/mongo/db/pipeline/expression.h @@ -23,14 +23,18 @@ namespace mongo {
+
class BSONArrayBuilder;
class BSONElement;
class BSONObjBuilder;
class Builder;
+ class DependencyTracker;
class Document;
+ class DocumentSource;
class ExpressionContext;
class Value;
+
class Expression :
public IntrusiveCounterUnsigned {
public:
@@ -51,6 +55,17 @@ namespace mongo { */
virtual intrusive_ptr<Expression> optimize() = 0;
+ /**
+ Add this expression's field dependencies to the dependency tracker.
+
+ Expressions are trees, so this is often recursive.
+
+ @params pTracker the tracker to add the dependencies to
+ */
+ virtual void addDependencies(
+ const intrusive_ptr<DependencyTracker> &pTracker,
+ const DocumentSource *pSource) const = 0;
+
/*
Evaluate the Expression using the given document as input.
@@ -199,12 +214,15 @@ namespace mongo { };
static int signum(int i);
+
+ protected:
+ typedef vector<intrusive_ptr<Expression> > ExpressionVector;
+
};
class ExpressionNary :
- public Expression,
- public boost::enable_shared_from_this<ExpressionNary> {
+ public Expression {
public:
// virtuals from Expression
virtual intrusive_ptr<Expression> optimize();
@@ -212,6 +230,9 @@ namespace mongo { BSONObjBuilder *pBuilder, string fieldName, unsigned depth) const;
virtual void addToBsonArray(
BSONArrayBuilder *pBuilder, unsigned depth) const;
+ virtual void addDependencies(
+ const intrusive_ptr<DependencyTracker> &pTracker,
+ const DocumentSource *pSource) const;
/*
Add an operand to the n-ary expression.
@@ -250,7 +271,7 @@ namespace mongo { protected:
ExpressionNary();
- vector<intrusive_ptr<Expression> > vpOperand;
+ ExpressionVector vpOperand;
/*
Add the expression to the builder.
@@ -364,12 +385,14 @@ namespace mongo { class ExpressionCoerceToBool :
- public Expression,
- public boost::enable_shared_from_this<ExpressionCoerceToBool> {
+ public Expression {
public:
// virtuals from ExpressionNary
virtual ~ExpressionCoerceToBool();
virtual intrusive_ptr<Expression> optimize();
+ virtual void addDependencies(
+ const intrusive_ptr<DependencyTracker> &pTracker,
+ const DocumentSource *pSource) const;
virtual intrusive_ptr<const Value> evaluate(
const intrusive_ptr<Document> &pDocument) const;
virtual void addToBsonObj(
@@ -440,12 +463,14 @@ namespace mongo { class ExpressionConstant :
- public Expression,
- public boost::enable_shared_from_this<ExpressionConstant> {
+ public Expression {
public:
// virtuals from Expression
virtual ~ExpressionConstant();
virtual intrusive_ptr<Expression> optimize();
+ virtual void addDependencies(
+ const intrusive_ptr<DependencyTracker> &pTracker,
+ const DocumentSource *pSource) const;
virtual intrusive_ptr<const Value> evaluate(
const intrusive_ptr<Document> &pDocument) const;
virtual const char *getOpName() const;
@@ -543,12 +568,14 @@ namespace mongo { class ExpressionFieldPath :
- public Expression,
- public boost::enable_shared_from_this<ExpressionFieldPath> {
+ public Expression {
public:
// virtuals from Expression
virtual ~ExpressionFieldPath();
virtual intrusive_ptr<Expression> optimize();
+ virtual void addDependencies(
+ const intrusive_ptr<DependencyTracker> &pTracker,
+ const DocumentSource *pSource) const;
virtual intrusive_ptr<const Value> evaluate(
const intrusive_ptr<Document> &pDocument) const;
virtual void addToBsonObj(
@@ -613,12 +640,14 @@ namespace mongo { class ExpressionFieldRange :
- public Expression,
- public boost::enable_shared_from_this<ExpressionFieldRange> {
+ public Expression {
public:
// virtuals from expression
virtual ~ExpressionFieldRange();
virtual intrusive_ptr<Expression> optimize();
+ virtual void addDependencies(
+ const intrusive_ptr<DependencyTracker> &pTracker,
+ const DocumentSource *pSource) const;
virtual intrusive_ptr<const Value> evaluate(
const intrusive_ptr<Document> &pDocument) const;
virtual void addToBsonObj(
@@ -851,12 +880,14 @@ namespace mongo { class ExpressionObject :
- public Expression,
- public boost::enable_shared_from_this<ExpressionObject> {
+ public Expression {
public:
// virtuals from Expression
virtual ~ExpressionObject();
virtual intrusive_ptr<Expression> optimize();
+ virtual void addDependencies(
+ const intrusive_ptr<DependencyTracker> &pTracker,
+ const DocumentSource *pSource) const;
virtual intrusive_ptr<const Value> evaluate(
const intrusive_ptr<Document> &pDocument) const;
virtual void addToBsonObj(
@@ -967,6 +998,35 @@ namespace mongo { */
void documentToBson(BSONObjBuilder *pBuilder, unsigned depth) const;
+ /*
+ Visitor abstraction used by emitPaths(). Each path is recorded by
+ calling path().
+ */
+ class PathSink {
+ public:
+ virtual ~PathSink() {};
+
+ /**
+ Record a path.
+
+ @param path the dotted path string
+ @param include if true, the path is included; if false, the path
+ is excluded
+ */
+ virtual void path(const string &path, bool include) = 0;
+ };
+
+ /**
+ Emit the field paths that have been included or excluded. "Included"
+ includes paths that are referenced in expressions for computed
+ fields.
+
+ @param pSink where to write the paths to
+ @param pvPath pointer to a vector of strings describing the path on
+ descent; the top-level call should pass an empty vector
+ */
+ void emitPaths(PathSink *pPathSink) const;
+
private:
ExpressionObject();
@@ -981,16 +1041,43 @@ namespace mongo { vector<string> vFieldName;
vector<intrusive_ptr<Expression> > vpExpression;
+
/*
Utility function used by documentToBson(). Emits inclusion
and exclusion paths by recursively walking down the nested
ExpressionObject trees these have created.
- @param pBuilder the builder to write boolean valued path "fields" to
+ @param pSink where to write the paths to
@param pvPath pointer to a vector of strings describing the path on
descent; the top-level call should pass an empty vector
*/
- void emitPaths(BSONObjBuilder *pBuilder, vector<string> *pvPath) const;
+ void emitPaths(PathSink *pPathSink, vector<string> *pvPath) const;
+
+ /*
+ Utility object for collecting emitPaths() results in a BSON
+ object.
+ */
+ class BuilderPathSink :
+ public PathSink {
+ public:
+ // virtuals from PathSink
+ virtual void path(const string &path, bool include);
+
+ /*
+ Create a PathSink that writes paths to a BSONObjBuilder,
+ to create an object in the form of { path:is_included,...}
+
+ This object uses a builder pointer that won't guarantee the
+ lifetime of the builder, so make sure it outlasts the use of
+ this for an emitPaths() call.
+
+ @param pBuilder to the builder to write paths to
+ */
+ BuilderPathSink(BSONObjBuilder *pBuilder);
+
+ private:
+ BSONObjBuilder *pBuilder;
+ };
/* utility class used by emitPaths() */
class PathPusher :
@@ -1210,6 +1297,11 @@ namespace mongo { return vFieldName.size();
}
+ inline ExpressionObject::BuilderPathSink::BuilderPathSink(
+ BSONObjBuilder *pB):
+ pBuilder(pB) {
+ }
+
inline ExpressionObject::PathPusher::PathPusher(
vector<string> *pTheVPath, const string &s):
pvPath(pTheVPath) {
diff --git a/src/mongo/db/pipeline/field_path.cpp b/src/mongo/db/pipeline/field_path.cpp index 52baefa560c..e79d2b1e940 100755 --- a/src/mongo/db/pipeline/field_path.cpp +++ b/src/mongo/db/pipeline/field_path.cpp @@ -19,8 +19,11 @@ #include "util/mongoutils/str.h"
namespace mongo {
+
using namespace mongoutils;
+ const char FieldPath::prefix[] = "$";
+
FieldPath::~FieldPath() {
}
@@ -67,7 +70,7 @@ namespace mongo { void FieldPath::writePath(ostream &outStream, bool fieldPrefix) const {
if (fieldPrefix)
- outStream << "$";
+ outStream << prefix;
outStream << vFieldName[0];
diff --git a/src/mongo/db/pipeline/field_path.h b/src/mongo/db/pipeline/field_path.h index d54c2edd1da..618eb283e3e 100755 --- a/src/mongo/db/pipeline/field_path.h +++ b/src/mongo/db/pipeline/field_path.h @@ -24,17 +24,26 @@ namespace mongo { public:
virtual ~FieldPath();
+ /**
+ Constructor.
+
+ @param fieldPath the dotted field path string
+ */
FieldPath(const string &fieldPath);
+
+ /**
+ Constructor.
+ */
FieldPath();
- /*
+ /**
Get the number of path elements in the field path.
@returns the number of path elements
*/
size_t getPathLength() const;
- /*
+ /**
Get a particular path element from the path.
@param i the index of the path element
@@ -42,7 +51,7 @@ namespace mongo { */
string getFieldName(size_t i) const;
- /*
+ /**
Get the full path.
@param fieldPrefix whether or not to include the field prefix
@@ -50,7 +59,7 @@ namespace mongo { */
string getPath(bool fieldPrefix) const;
- /*
+ /**
Write the full path.
@param outStream where to write the path to
@@ -58,8 +67,22 @@ namespace mongo { */
void writePath(ostream &outStream, bool fieldPrefix) const;
+ /**
+ Assignment operator.
+
+ @param rRHS right hand side of the assignment
+ */
FieldPath &operator=(const FieldPath &rRHS);
+ /**
+ Get the prefix string.
+
+ @returns the prefix string
+ */
+ static const char *getPrefix();
+
+ static const char prefix[];
+
private:
vector<string> vFieldName;
};
@@ -78,5 +101,9 @@ namespace mongo { return vFieldName[i];
}
+ inline const char *FieldPath::getPrefix() {
+ return prefix;
+ }
+
}
|