bad memory issue with cursors

11 files changed, 58 insertions, 30 deletions

diff --git a/db/clientcursor.cpp b/db/clientcursor.cpp
index 2e96344895a..6782d893ea1 100644
--- a/db/clientcursor.cpp
+++ b/db/clientcursor.cpp
@@ -62,7 +62,6 @@ void aboutToDelete(const DiskLoc& dl) {
 }
 
 ClientCursor::~ClientCursor() {
-	DEV cout << "~clientcursor " << cursorid << endl;
 	assert( pos != -2 );
 	setLastLoc( DiskLoc() ); // removes us from bylocation multimap
 	clientCursorsById.erase(cursorid);
diff --git a/db/clientcursor.h b/db/clientcursor.h
index 6fd5bb12155..46570c30f22 100644
--- a/db/clientcursor.h
+++ b/db/clientcursor.h
@@ -34,6 +34,7 @@ public:
 	DiskLoc lastLoc() const { return _lastLoc; }
 	void setLastLoc(DiskLoc);
 	auto_ptr< set<string> > filter; // which fields query wants returned
+	Message originalMessage; // this is effectively an auto ptr for data the matcher points to.
 
 	static bool erase(CursorId id) { 
 		ClientCursor *cc = find(id);
diff --git a/db/db.cpp b/db/db.cpp
index 945dd9f0988..67cf5d15dd6 100644
--- a/db/db.cpp
+++ b/db/db.cpp
@@ -152,6 +152,8 @@ void receivedDelete(Message& m) {
 }
 
 void receivedQuery(AbstractMessagingPort& dbMsgPort, Message& m, stringstream& ss) {
+	MSGID responseTo = m.data->id;
+
 	DbMessage d(m);
 	const char *ns = d.getns();
 	setClient(ns);
@@ -166,7 +168,7 @@ void receivedQuery(AbstractMessagingPort& dbMsgPort, Message& m, stringstream& s
 	QueryResult* msgdata;
 
 	try { 
-		msgdata = runQuery(ns, ntoskip, ntoreturn, query, fields, ss);
+		msgdata = runQuery(m, ns, ntoskip, ntoreturn, query, fields, ss);
 	}
 	catch( AssertionException ) { 
 		ss << " exception ";
@@ -196,7 +198,7 @@ void receivedQuery(AbstractMessagingPort& dbMsgPort, Message& m, stringstream& s
 	else { 
 		cout << "ERROR: client is null; ns=" << ns << endl;
 	}
-	dbMsgPort.reply(m, resp);
+	dbMsgPort.reply(m, resp, responseTo);
 }
 
 void receivedGetMore(AbstractMessagingPort& dbMsgPort, Message& m, stringstream& ss) {
@@ -273,8 +275,11 @@ public:
 	}
 };
 
+/* versions
+   114 bad memory bug fixed
+*/
 void listen(int port) { 
-	const char *Version = "db version: 113 10jun2008";
+	const char *Version = "db version: 114 11jun2008";
 	problem() << Version << endl;
 	cout << Version << endl;
 	pdfileInit();
@@ -290,6 +295,9 @@ extern int callDepth;
 class JniMessagingPort : public AbstractMessagingPort { 
 public:
 	JniMessagingPort(Message& _container) : container(_container) { }
+	void reply(Message& received, Message& response, MSGID) {
+		container = response;
+	}
 	void reply(Message& received, Message& response) { 
 		container = response;
 	}
@@ -477,7 +485,7 @@ void connThread()
 				}
 				catch( AssertionException ) { 
 					problem() << " Caught Assertion receviedDelete, continuing" << endl; 
-					cout << "Caught Assertion receviedDelete, continuing" << endl; 
+					cout << "Caught Assertion receivedDelete, continuing" << endl; 
 					ss << " exception ";
 				}
 			}
diff --git a/db/jsobj.cpp b/db/jsobj.cpp
index e55de77d335..e2118bc8b60 100644
--- a/db/jsobj.cpp
+++ b/db/jsobj.cpp
@@ -457,6 +457,8 @@ int JSMatcher::matchesDotted(const char *fieldName, Element& toMatch, JSObj& obj
 	return -1;
 }
 
+extern int dump;
+
 /* deep means we looked into arrays for a match */
 bool JSMatcher::matches(JSObj& jsobj, bool *deep) {
 	if( deep ) 
@@ -493,7 +495,6 @@ bool JSMatcher::matches(JSObj& jsobj, bool *deep) {
 	// check normal non-regex cases:
 	for( int i = 0; i < n; i++ ) {
 		Element& m = toMatch[i]; 
- 
 		int cmp = matchesDotted(toMatch[i].fieldName(), toMatch[i], jsobj, compareOp[i], deep);
 
 		/* missing is ok iff we were looking for null */
diff --git a/db/jsobj.h b/db/jsobj.h
index faa4d9f159a..c0fac57ec36 100644
--- a/db/jsobj.h
+++ b/db/jsobj.h
@@ -4,6 +4,7 @@
 
 #include "../stdafx.h"
 #include "../util/builder.h"
+#include "boost/utility.hpp"
 //#include "javajs.h"
 
 #include <set>
@@ -398,7 +399,7 @@ class Where;
    { a : { $gt
 
 */
-class JSMatcher { 
+class JSMatcher : boost::noncopyable { 
 	int matchesDotted(
 		const char *fieldName, 
 		Element& toMatch, JSObj& obj, 
diff --git a/db/pdfile.h b/db/pdfile.h
index f88f11d704f..ec8b07074c4 100644
--- a/db/pdfile.h
+++ b/db/pdfile.h
@@ -237,7 +237,7 @@ public:
 
 	/* used for multikey index traversal to avoid sending back dups. see JSMatcher::matches() */
 	set<DiskLoc> dups;
-	bool dup(DiskLoc loc) {
+	bool getsetdup(DiskLoc loc) {
 		/* to save mem only call this when there is risk of dups (e.g. when 'deep'/multikey) */
 		if( dups.count(loc) > 0 )
 			return true;
diff --git a/db/query.cpp b/db/query.cpp
index 2446e5da318..47254c97b0b 100644
--- a/db/query.cpp
+++ b/db/query.cpp
@@ -64,6 +64,7 @@ auto_ptr<Cursor> getIndexCursor(const char *ns, JSObj& query, JSObj& order) {
 				/* todo: start with the right key, not just beginning of index, when query is also
 				         specified!
 				*/
+				DEV cout << " using index " << d->indexes[i].indexNamespace() << '\n';
 				return auto_ptr<Cursor>(new BtreeCursor(d->indexes[i].head, reverse ? maxKey : emptyObj, reverse ? -1 : 1, false));
 			}
 		}
@@ -183,7 +184,7 @@ DiskLoc _tempDelLoc;
 				break;
 		}
 		else { 
-			assert( !deep || !c->dup(rloc) ); // can't be a dup, we deleted it!
+			assert( !deep || !c->getsetdup(rloc) ); // can't be a dup, we deleted it!
 //			cout << "  found match to delete" << endl;
 			if( !justOne )
 				c->noteLocation();
@@ -640,7 +641,7 @@ void killCursors(int n, long long *ids) {
 
 auto_ptr<Cursor> findTableScan(const char *ns, JSObj& order);
 
-QueryResult* runQuery(const char *ns, int ntoskip, int _ntoreturn, JSObj jsobj, 
+QueryResult* runQuery(Message& message, const char *ns, int ntoskip, int _ntoreturn, JSObj jsobj, 
 					  auto_ptr< set<string> > filter, stringstream& ss) 
 {
 	bool wantMore = true;
@@ -704,7 +705,7 @@ assert( debug.getN() < 5000 );
 					if( c->tempStopOnMiss() )
 						break;
 				}
-				else if( !deep || !c->dup(c->currLoc()) ) { // i.e., check for dups on deep items only
+				else if( !deep || !c->getsetdup(c->currLoc()) ) { // i.e., check for dups on deep items only
 					// got a match.
 					if( ntoskip > 0 ) {
 						ntoskip--;
@@ -739,6 +740,7 @@ assert( debug.getN() < 5000 );
 											cc->ns = ns;
 											cc->pos = n;
 											cc->filter = filter;
+											cc->originalMessage = message;
 											cc->updateLocation();
 										}
 									}
@@ -782,6 +784,8 @@ assert( debug.getN() < 5000 );
 	return qr;
 }
 
+int dump = 0;
+
 QueryResult* getMore(const char *ns, int ntoreturn, long long cursorid) {
 
 //	cout << "getMore ns:" << ns << " ntoreturn:" << ntoreturn << " cursorid:" << 
@@ -814,30 +818,38 @@ done:
 				break;
 			}
 			JSObj js = c->current();
+
 			bool deep;
+
 			if( !cc->matcher->matches(js, &deep) ) {
 				if( c->tempStopOnMiss() )
 					goto done;
 			} 
-			else if( !deep || !c->dup(c->currLoc()) ) {
-				bool ok = true;
-				if( cc->filter.get() ) {
-					JSObj x;
-					ok = x.addFields(js, *cc->filter) > 0;
-					if( ok ) 
-						b.append((void*) x.objdata(), x.objsize());
+			else { 
+				//cout << "matches " << c->currLoc().toString() << ' ' << deep << '\n';
+				if( deep && c->getsetdup(c->currLoc()) ) { 
+					//cout << "  but it's a dup \n";
 				}
 				else {
-					b.append((void*) js.objdata(), js.objsize());
-				}
-				if( ok ) {
-					n++;
-					if( (ntoreturn>0 && (n >= ntoreturn || b.len() > 16*1024*1024)) ||
-						(ntoreturn==0 && b.len()>1*1024*1024) ) {
-						c->advance();
-						cc->pos += n;
-						cc->updateLocation();
-						break;
+					bool ok = true;
+					if( cc->filter.get() ) {
+						JSObj x;
+						ok = x.addFields(js, *cc->filter) > 0;
+						if( ok ) 
+							b.append((void*) x.objdata(), x.objsize());
+					}
+					else {
+						b.append((void*) js.objdata(), js.objsize());
+					}
+					if( ok ) {
+						n++;
+						if( (ntoreturn>0 && (n >= ntoreturn || b.len() > 16*1024*1024)) ||
+							(ntoreturn==0 && b.len()>1*1024*1024) ) {
+								c->advance();
+								cc->pos += n;
+								cc->updateLocation();
+								break;
+						}
 					}
 				}
 			}
diff --git a/db/query.h b/db/query.h
index 424c6aaa84b..aa06fa04105 100644
--- a/db/query.h
+++ b/db/query.h
@@ -68,7 +68,7 @@ struct QueryResult : public MsgData {
 QueryResult* getMore(const char *ns, int ntoreturn, long long cursorid);
 
 // caller must free() returned QueryResult.
-QueryResult* runQuery(const char *ns, int ntoskip, int ntoreturn, 
+QueryResult* runQuery(Message&, const char *ns, int ntoskip, int ntoreturn, 
 					  JSObj j, auto_ptr< set<string> > fieldFilter,
 					  stringstream&);
 
diff --git a/grid/message.cpp b/grid/message.cpp
index cfa9a02a912..c9f512064bf 100644
--- a/grid/message.cpp
+++ b/grid/message.cpp
@@ -163,6 +163,10 @@ void MessagingPort::reply(Message& received, Message& response) {
 	say(received.from, response, received.data->id);
 }
 
+void MessagingPort::reply(Message& received, Message& response, MSGID responseTo) {
+	say(received.from, response, responseTo);
+}
+
 bool MessagingPort::call(SockAddr& to, Message& toSend, Message& response) {
 	mmm( cout << "*call()" << endl; )
 	MSGID old = toSend.data->id;
diff --git a/grid/message.h b/grid/message.h
index be2749c8bea..3a26909dec9 100644
--- a/grid/message.h
+++ b/grid/message.h
@@ -22,6 +22,7 @@ private:
 
 class AbstractMessagingPort { 
 public:
+	virtual void reply(Message& received, Message& response, MSGID responseTo) = 0; // like the reply below, but doesn't rely on received.data still being available
 	virtual void reply(Message& received, Message& response) = 0;
 };
 
@@ -39,6 +40,7 @@ public:
 	   also, the Message data will go out of scope on the subsequent recv call. 
 	*/
 	bool recv(Message& m);
+	void reply(Message& received, Message& response, MSGID responseTo);
 	void reply(Message& received, Message& response);
 	bool call(SockAddr& to, Message& toSend, Message& response);
 	void say(SockAddr& to, Message& toSend, int responseTo = -1);
diff --git a/stdafx.h b/stdafx.h
index cb6f6da8bc1..8253751833d 100644
--- a/stdafx.h
+++ b/stdafx.h
@@ -111,7 +111,7 @@ inline ostream& problem() {
 #if defined(_WIN32)
 #define DEV if( 1 ) 
 #else
-#define DEV if( 1 ) 
+#define DEV if( 0 ) 
 #endif
 
 #define DEBUGGING if( 0 )