Merge branch 'master' of ssh://git.10gen.com/data/gitroot/p

Conflicts:

	db/makefile

10 files changed, 1024 insertions, 454 deletions

diff --git a/db/commands.cpp b/db/commands.cpp
new file mode 100644
index 00000000000..78c812e8f7d
--- /dev/null
+++ b/db/commands.cpp
@@ -0,0 +1,464 @@
+/* commands.cpp
+   db "commands" (sent via db.$cmd.findOne(...))
+ */
+
+/**
+*    Copyright (C) 2008 10gen Inc.
+*  
+*    This program is free software: you can redistribute it and/or  modify
+*    it under the terms of the GNU Affero General Public License, version 3,
+*    as published by the Free Software Foundation.
+*  
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU Affero General Public License for more details.
+*  
+*    You should have received a copy of the GNU Affero General Public License
+*    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "stdafx.h"
+#include "query.h"
+#include "pdfile.h"
+#include "jsobj.h"
+#include "../util/builder.h"
+#include <time.h>
+#include "introspect.h"
+#include "btree.h"
+#include "../util/lruishmap.h"
+#include "javajs.h"
+#include "json.h"
+#include "repl.h"
+
+extern int queryTraceLevel;
+extern int otherTraceLevel;
+extern int opLogging;
+bool userCreateNS(const char *ns, JSObj& j, string& err);
+void flushOpLog();
+int runCount(const char *ns, JSObj& cmd, string& err);
+
+const int edebug=0;
+
+bool dbEval(JSObj& cmd, JSObjBuilder& result) { 
+	Element e = cmd.firstElement();
+	assert( e.type() == Code );
+	const char *code = e.valuestr();
+	if ( ! JavaJS ) {
+		result.append("errmsg", "db side execution is disabled");
+		return false;
+	}
+
+	jlong f = JavaJS->functionCreate(code);
+	if( f == 0 ) { 
+		result.append("errmsg", "compile failed");
+		return false;
+	}
+
+	Scope s;
+	s.setString("$client", client->name.c_str());
+	Element args = cmd.findElement("args");
+	if( args.type() == Array ) {
+		JSObj eo = args.embeddedObject();
+		if( edebug ) {
+			cout << "args:" << eo.toString() << endl;
+			cout << "code:\n" << code << endl;
+		}
+		s.setObject("args", eo);
+	}
+
+	int res = s.invoke(f);
+	if( res ) {
+		result.append("errno", (double) res);
+		result.append("errmsg", "invoke failed");
+		return false;
+	}
+
+	int type = s.type("return");
+	if( type == Object || type == Array )
+		result.append("retval", s.getObject("return"));
+	else if( type == Number ) 
+		result.append("retval", s.getNumber("return"));
+	else if( type == String )
+		result.append("retval", s.getString("return").c_str());
+	else if( type == Bool ) {
+		result.appendBool("retval", s.getBoolean("return"));
+	}
+
+	return true;
+}
+
+void clean(const char *ns, NamespaceDetails *d) {
+	for( int i = 0; i < Buckets; i++ ) 
+		d->deletedList[i].Null();
+}
+
+string validateNS(const char *ns, NamespaceDetails *d) {
+	bool valid = true;
+	stringstream ss;
+	ss << "\nvalidate\n";
+	ss << "  details: " << hex << d << " ofs:" << nsindex(ns)->detailsOffset(d) << dec << endl;
+	if( d->capped )
+		ss << "  capped:" << d->capped << " max:" << d->max << '\n';
+
+	ss << "  firstExtent:" << d->firstExtent.toString() << " ns:" << d->firstExtent.ext()->ns.buf << '\n';
+	ss << "  lastExtent:" << d->lastExtent.toString()    << " ns:" << d->lastExtent.ext()->ns.buf << '\n';
+	try { 
+		d->firstExtent.ext()->assertOk();
+		d->lastExtent.ext()->assertOk(); 
+	} catch(...) { valid=false; ss << " extent asserted "; }
+
+	ss << "  datasize?:" << d->datasize << " nrecords?:" << d->nrecords << " lastExtentSize:" << d->lastExtentSize << '\n';
+	ss << "  padding:" << d->paddingFactor << '\n';
+	try { 
+
+		try {
+			ss << "  first extent:\n";
+			d->firstExtent.ext()->dump(ss);
+			valid = valid && d->firstExtent.ext()->validates();
+		}
+		catch(...) { 
+		  ss << "\n    exception firstextent\n" << endl;
+		}
+
+		auto_ptr<Cursor> c = theDataFileMgr.findAll(ns);
+		int n = 0;
+		long long len = 0;
+		long long nlen = 0;
+		set<DiskLoc> recs;
+		int outOfOrder = 0;
+		DiskLoc cl_last;
+		while( c->ok() ) { 
+			n++;
+
+			DiskLoc cl = c->currLoc();
+			if( n < 1000000 )
+				recs.insert(cl);
+			if( d->capped ) {
+				if( cl < cl_last )
+					outOfOrder++;
+				cl_last = cl;
+			}
+
+			Record *r = c->_current();
+			len += r->lengthWithHeaders;
+			nlen += r->netLength();
+			c->advance();
+		}
+		if( d->capped ) { 
+			ss << "  capped outOfOrder:" << outOfOrder;
+			if( outOfOrder > 1 ) { 
+				valid = false;
+				ss << " ???";
+			}
+			else ss << " (OK)";
+			ss << '\n';
+		}
+		ss << "  " << n << " objects found, nobj:" << d->nrecords << "\n";
+		ss << "  " << len << " bytes data w/headers\n";
+		ss << "  " << nlen << " bytes data wout/headers\n";
+
+		ss << "  deletedList: ";
+		for( int i = 0; i < Buckets; i++ ) { 
+			ss << (d->deletedList[i].isNull() ? '0' : '1');
+		}
+		ss << endl;
+		int ndel = 0;
+		long long delSize = 0;
+		int incorrect = 0;
+		for( int i = 0; i < Buckets; i++ ) { 
+			DiskLoc loc = d->deletedList[i];
+			try {
+			  int k = 0;
+			  while( !loc.isNull() ) { 
+			    if( recs.count(loc) )
+			      incorrect++;
+			    ndel++;
+
+			    if( loc.questionable() ) { 
+					if( loc.a() <= 0 || strstr(ns, "hudsonSmall") == 0 ) {
+						ss << "    ?bad deleted loc: " << loc.toString() << " bucket:" << i << " k:" << k << endl;
+						valid = false;
+						break;
+					}
+			    }
+
+			    DeletedRecord *d = loc.drec();
+			    delSize += d->lengthWithHeaders;
+			    loc = d->nextDeleted;
+			    k++;
+			  }
+			} catch(...) { ss <<"    ?exception in deleted chain for bucket " << i << endl; valid = false; }
+		}
+		ss << "  deleted: n: " << ndel << " size: " << delSize << '\n';
+		if( incorrect ) { 
+			ss << "    ?corrupt: " << incorrect << " records from datafile are in deleted list\n";
+			valid = false;
+		}
+
+		int idxn = 0;
+		try  {
+			ss << "  nIndexes:" << d->nIndexes << endl;
+			for( ; idxn < d->nIndexes; idxn++ ) {
+				ss << "    " << d->indexes[idxn].indexNamespace() << " keys:" << 
+					d->indexes[idxn].head.btree()->fullValidate(d->indexes[idxn].head) << endl;
+			}
+		} 
+		catch(...) { 
+		  ss << "\n    exception during index validate idxn:" << idxn << endl; valid=false;
+		}
+
+	}
+	catch(AssertionException) {
+		ss << "\n    exception during validate\n" << endl; 
+		valid = false;
+	}
+
+	if( !valid ) 
+		ss << " ns corrupt, requires dbchk\n";
+
+	return ss.str();
+}
+
+// e.g.
+//   system.cmd$.find( { queryTraceLevel: 2 } );
+// 
+// returns true if ran a cmd
+//
+bool _runCommands(const char *ns, JSObj& jsobj, stringstream& ss, BufBuilder &b, JSObjBuilder& anObjBuilder) { 
+
+	const char *p = strchr(ns, '.');
+	if( !p ) return false;
+	if( strcmp(p, ".$cmd") != 0 ) return false;
+
+	bool ok = false;
+	bool valid = false;
+
+	//cout << jsobj.toString() << endl;
+
+	Element e;
+	e = jsobj.firstElement();
+
+	if( e.eoo() ) goto done;
+	if( e.type() == Code ) { 
+		valid = true;
+		ok = dbEval(jsobj, anObjBuilder);
+	}
+	else if( e.type() == Number ) { 
+		if( strcmp(e.fieldName(), "getoptime") == 0 ) { 
+			valid = true;
+			ok = true;
+			anObjBuilder.appendDate("optime", OpTime::now().asDate());
+		}
+		else if( strcmp(e.fieldName(), "dropDatabase") == 0 ) { 
+			if( 1 ) {
+				cout << "dropDatabase " << ns << endl;
+				valid = true;
+				int p = (int) e.number();
+				if( p != 1 ) {
+					ok = false;
+				} else { 
+					dropDatabase(ns);
+					logOp("c", ns, jsobj);
+					ok = true;
+				}
+			}
+			else { 
+				cout << "TEMP CODE: dropdatabase commented out " << endl;
+			}
+		}
+		else if( strcmp(e.fieldName(), "profile") == 0 ) { 
+			anObjBuilder.append("was", (double) client->profile);
+			int p = (int) e.number();
+			valid = true;
+			if( p == -1 )
+				ok = true;
+			else if( p >= 0 && p <= 2 ) { 
+				ok = true;
+				client->profile = p;
+			}
+			else {
+				ok = false;
+			}
+		}
+		else {
+			// admin only commands.
+			if( strncmp(ns, "admin", p-ns) != 0 ) 
+				return false;
+			if( strcmp(e.fieldName(),"opLogging") == 0 ) {
+				valid = ok = true;
+				opLogging = (int) e.number();
+				flushOpLog();
+				log() << "CMD: opLogging set to " << opLogging << endl;
+			} else if( strcmp(e.fieldName(),"queryTraceLevel") == 0 ) {
+				valid = ok = true;
+				queryTraceLevel = (int) e.number();
+			} else if( strcmp(e.fieldName(),"traceAll") == 0 ) { 
+				valid = ok = true;
+				queryTraceLevel = (int) e.number();
+				otherTraceLevel = (int) e.number();
+			}
+		}
+	}
+	else if( e.type() == String ) {
+		/* { count: "collectionname"[, query: <query>] } */
+		string us(ns, p-ns);
+
+		if( strcmp( e.fieldName(), "count" ) == 0 ) { 
+			valid = true;
+			string ns = us + '.' + e.valuestr();
+			string err;
+			int n = runCount(ns.c_str(), jsobj, err);
+			int nn = n;
+			ok = true;
+			if( n < 0 ) { 
+				ok = false;
+				nn = 0;
+				if( !err.empty() )
+					anObjBuilder.append("errmsg", err.c_str());
+			}
+			anObjBuilder.append("n", (double) nn);
+		}
+		else if( strcmp( e.fieldName(), "clone") == 0 ) { 
+			valid = true;
+			string err;
+			ok = cloneFrom(e.valuestr(), err);
+			if( !err.empty() ) 
+				anObjBuilder.append("errmsg", err.c_str());
+		}
+		else if( strcmp( e.fieldName(), "create") == 0 ) { 
+			valid = true;
+			string ns = us + '.' + e.valuestr();
+			string err;
+			ok = userCreateNS(ns.c_str(), jsobj, err);
+			if( ok )
+				logOp("c", ns.c_str(), jsobj);
+			if( !ok && !err.empty() )
+				anObjBuilder.append("errmsg", err.c_str());
+		}
+		else if( strcmp( e.fieldName(), "clean") == 0 ) { 
+			valid = true;
+			string dropNs = us + '.' + e.valuestr();
+			NamespaceDetails *d = nsdetails(dropNs.c_str());
+			log() << "CMD: clean " << dropNs << endl;
+			if( d ) { 
+				ok = true;
+				anObjBuilder.append("ns", dropNs.c_str());
+				clean(dropNs.c_str(), d);
+			}
+			else {
+				anObjBuilder.append("errmsg", "ns not found");
+			}
+		}
+		else if( strcmp( e.fieldName(), "drop") == 0 ) { 
+			valid = true;
+			string nsToDrop = us + '.' + e.valuestr();
+			NamespaceDetails *d = nsdetails(nsToDrop.c_str());
+			log() << "CMD: drop " << nsToDrop << endl;
+			if( d == 0 ) {
+				anObjBuilder.append("errmsg", "ns not found");
+			}
+			else if( d->nIndexes != 0 ) {
+				// client is supposed to drop the indexes first
+				anObjBuilder.append("errmsg", "ns has indexes (not permitted on drop)");
+			}
+			else {
+				ok = true;
+				anObjBuilder.append("ns", nsToDrop.c_str());
+				ClientCursor::invalidate(nsToDrop.c_str());
+				dropNS(nsToDrop);
+				logOp("c", ns, jsobj);
+				/*
+				{
+					JSObjBuilder b;
+					b.append("name", dropNs.c_str());
+					JSObj cond = b.done(); // { name: "colltodropname" }
+					deleteObjects("system.namespaces", cond, false, true);
+				}
+				client->namespaceIndex.kill(dropNs.c_str());
+				*/
+			}
+		}
+		else if( strcmp( e.fieldName(), "validate") == 0 ) { 
+			valid = true;
+			string toValidateNs = us + '.' + e.valuestr();
+			NamespaceDetails *d = nsdetails(toValidateNs.c_str());
+			log() << "CMD: validate " << toValidateNs << endl;
+			if( d ) { 
+				ok = true;
+				anObjBuilder.append("ns", toValidateNs.c_str());
+				string s = validateNS(toValidateNs.c_str(), d);
+				anObjBuilder.append("result", s.c_str());
+			}
+			else {
+				anObjBuilder.append("errmsg", "ns not found");
+			}
+		}
+		else if( strcmp(e.fieldName(),"deleteIndexes") == 0 ) { 
+			valid = true;
+			/* note: temp implementation.  space not reclaimed! */
+			string toDeleteNs = us + '.' + e.valuestr();
+			NamespaceDetails *d = nsdetails(toDeleteNs.c_str());
+			log() << "CMD: deleteIndexes " << toDeleteNs << endl;
+			if( d ) {
+				Element f = jsobj.findElement("index");
+				if( !f.eoo() ) { 
+
+					d->aboutToDeleteAnIndex();
+
+					ClientCursor::invalidate(toDeleteNs.c_str());
+					logOp("c", ns, jsobj);
+
+					// delete a specific index or all?
+					if( f.type() == String ) { 
+						const char *idxName = f.valuestr();
+						if( *idxName == '*' && idxName[1] == 0 ) { 
+							ok = true;
+							log() << "  d->nIndexes was " << d->nIndexes << '\n';
+							anObjBuilder.append("nIndexesWas", (double)d->nIndexes);
+							anObjBuilder.append("msg", "all indexes deleted for collection");
+							for( int i = 0; i < d->nIndexes; i++ )
+								d->indexes[i].kill();
+							d->nIndexes = 0;
+							log() << "  alpha implementation, space not reclaimed" << endl;
+						}
+						else {
+							// delete just one index
+							int x = d->findIndexByName(idxName);
+							if( x >= 0 ) { 
+								cout << "  d->nIndexes was " << d->nIndexes << endl;
+								anObjBuilder.append("nIndexesWas", (double)d->nIndexes);
+
+								/* note it is  important we remove the IndexDetails with this 
+								   call, otherwise, on recreate, the old one would be reused, and its
+								   IndexDetails::info ptr would be bad info.
+								*/
+								d->indexes[x].kill();
+
+								d->nIndexes--;
+								for( int i = x; i < d->nIndexes; i++ )
+									d->indexes[i] = d->indexes[i+1];
+								ok=true;
+								cout << "  alpha implementation, space not reclaimed\n";
+							} else { 
+								cout << "deleteIndexes: " << idxName << " not found" << endl;
+							}
+						}
+					}
+				}
+			}
+			else {
+				anObjBuilder.append("errmsg", "ns not found");
+			}
+		}
+	}
+
+done:
+	if( !valid )
+		anObjBuilder.append("errmsg", "no such cmd");
+	anObjBuilder.append("ok", ok?1.0:0.0);
+	JSObj x = anObjBuilder.done();
+	b.append((void*) x.objdata(), x.objsize());
+	return true;
+}
+
diff --git a/db/db.cpp b/db/db.cpp
index 03f54fa2678..7c3a8e44522 100644
--- a/db/db.cpp
+++ b/db/db.cpp
@@ -29,6 +29,7 @@
 #include "query.h"
 #include "introspect.h"
 #include "repl.h"
+#include "../util/unittest.h"
 
 bool slave = false;
 bool master = false; // true means keep an op log
@@ -740,8 +741,7 @@ void segvhandler(int x) {
 
 void mysighandler(int x) { 
    signal(x, SIG_IGN); 
-   cout << "got kill or ctrl c signal " << x << ", will terminate after current cmd ends" << endl;
-   problem() << "got kill or ctrl c signal " << x << ", will terminate after current cmd ends" << endl;
+   log() << "got kill or ctrl c signal " << x << ", will terminate after current cmd ends" << endl;
    {
 	   dblock lk;
 	   problem() << "  now exiting" << endl;
@@ -760,25 +760,25 @@ void setupSignals() {}
 
 void initAndListen(int listenPort, const char *dbPath, const char *appserverLoc = null) { 
   if( opLogging ) 
-		log() << "opLogging = " << opLogging << endl;
-    _oplog.init();
+    log() << "opLogging = " << opLogging << endl;
+  _oplog.init();
 
 #if !defined(_WIN32)
-	assert( signal(SIGSEGV, segvhandler) != SIG_ERR );
+  assert( signal(SIGSEGV, segvhandler) != SIG_ERR );
 #endif
 
-    /*
-     * ensure that the dbpath ends with a path delim if not supplied
-     * @TODO - the following is embarassing - not sure of there's a clean way to
-     * find the platform delim
-     */
-
-	char endchar = '/';
-	char *endstr = "/";
+  /*
+   * ensure that the dbpath ends with a path delim if not supplied
+   * @TODO - the following is embarassing - not sure of there's a clean way to
+   * find the platform delim
+   */
+  
+  char endchar = '/';
+  const char *endstr = "/";
 
 #if defined(_WIN32)
-    endchar = '\\';
-    endstr = "\\";
+  endchar = '\\';
+  endstr = "\\";
 #endif
     
     if (dbPath && dbPath[strlen(dbPath)-1] != endchar) {
@@ -824,6 +824,8 @@ int main(int argc, char* argv[], char *envp[] )
 #endif
 	srand(curTimeMillis());
 
+	UnitTest::runTests();
+
 	if( argc >= 2 ) {
 		if( strcmp(argv[1], "quicktest") == 0 ) {
 			quicktest();
@@ -948,13 +950,13 @@ int main(int argc, char* argv[], char *envp[] )
 
 #undef exit
 void dbexit(int rc, const char *why) { 
-	cout << "  dbexit: " << why << "; flushing op log and files" << endl;
+	log() << "  dbexit: " << why << "; flushing op log and files" << endl;
 	flushOpLog();
 
 	/* must do this before unmapping mem or you may get a seg fault */
 	closeAllSockets();
 
 	MemoryMappedFile::closeAllFiles();
-	cout << "  dbexit: really exiting now" << endl;
+	log() << "  dbexit: really exiting now" << endl;
 	exit(rc);
 }
diff --git a/db/db.vcproj b/db/db.vcproj
index ece0d5c6bf4..4fdae169214 100644
--- a/db/db.vcproj
+++ b/db/db.vcproj
@@ -191,6 +191,10 @@
 				>

 			</File>

 			<File

+				RelativePath=".\commands.cpp"

+				>

+			</File>

+			<File

 				RelativePath=".\db.cpp"

 				>

 			</File>

@@ -388,6 +392,10 @@
 				RelativePath=".\storage.h"

 				>

 			</File>

+			<File

+				RelativePath="..\util\unittest.h"

+				>

+			</File>

 		</Filter>

 		<Filter

 			Name="Resource Files"

diff --git a/db/makefile b/db/makefile
index abbf77161be..89c6a590197 100644
--- a/db/makefile
+++ b/db/makefile
@@ -10,11 +10,9 @@ DBGRID_LIBS = $(LIB_BOOST) -lstdc++
 
 JVM_LIBS = -L/opt/java/lib/
 
-OBJS=../stdafx.o ../util/sock.o ../grid/message.o ../util/mmap.o pdfile.o query.o jsobj.o introspect.o btree.o clientcursor.o ../util/util.o javajs.o tests.o json.o repl.o dbclient.o btreecursor.o cloner.o namespace.o
+OBJS=../stdafx.o ../util/sock.o ../grid/message.o ../util/mmap.o pdfile.o query.o jsobj.o introspect.o btree.o clientcursor.o ../util/util.o javajs.o tests.o json.o repl.o dbclient.o btreecursor.o cloner.o namespace.o commands.o matcher.o
 
-DBGRID_OBJS=../stdafx.o ../util/sock.o ../grid/message.o ../util/util.o dbclient.o ../dbgrid/dbgrid.o
-
-# ../grid/protorecv.o ../grid/protosend.o
+DBGRID_OBJS=../stdafx.o ../util/sock.o ../grid/message.o ../util/util.o jsobj.o dbclient.o ../dbgrid/dbgrid.o
 
 GPP = g++
 
diff --git a/db/matcher.cpp b/db/matcher.cpp
new file mode 100644
index 00000000000..24c5fdf8ab8
--- /dev/null
+++ b/db/matcher.cpp
@@ -0,0 +1,484 @@
+// matcher.cpp

+

+/* JSMatcher is our boolean expression evaluator for "where" clauses */

+

+/**
+*    Copyright (C) 2008 10gen Inc.
+*  
+*    This program is free software: you can redistribute it and/or  modify
+*    it under the terms of the GNU Affero General Public License, version 3,
+*    as published by the Free Software Foundation.
+*  
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU Affero General Public License for more details.
+*  
+*    You should have received a copy of the GNU Affero General Public License
+*    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "stdafx.h"
+#include "jsobj.h"
+#include "../util/goodies.h"
+#include "javajs.h"
+#include "../util/unittest.h"
+
+#if defined(_WIN32)
+
+#include <hash_map>
+using namespace stdext;
+
+typedef const char * MyStr;
+struct less_str {
+   bool operator()(const MyStr & x, const MyStr & y) const {
+      if ( strcmp(x, y) > 0)
+         return true;
+
+      return false;
+   }
+};
+
+typedef hash_map<const char*, int, hash_compare<const char *, less_str> > strhashmap;
+
+#else
+
+#include <ext/hash_map>
+using namespace __gnu_cxx;
+
+typedef const char * MyStr;
+struct eq_str {
+   bool operator()(const MyStr & x, const MyStr & y) const {
+      if ( strcmp(x, y) == 0)
+         return true;
+
+      return false;
+   }
+};
+
+typedef hash_map<const char*, int, hash<const char *>, eq_str > strhashmap;
+
+#endif
+
+#include "minilex.h"
+
+MiniLex minilex;
+
+class Where { 
+public:
+	Where() { codeCopy = 0; }
+	~Where() {
+		JavaJS->scopeFree(scope);
+		delete codeCopy;
+		scope = 0; func = 0; codeCopy = 0;
+	}
+	jlong scope, func;
+	strhashmap fields;
+//	map<string,int> fields;
+	bool fullObject;
+	int nFields;
+	char *codeCopy;
+
+	void setFunc(const char *code) {
+		codeCopy = new char[strlen(code)+1];
+		strcpy(codeCopy,code);
+		func = JavaJS->functionCreate( code );
+		minilex.grabVariables(codeCopy, fields);
+		// if user references db, eg db.foo.save(obj), 
+		// we make sure we have the whole thing.
+		fullObject = fields.count("fullObject") +
+			fields.count("db") > 0;
+		nFields = fields.size();
+	}
+
+	void buildSubset(JSObj& src, JSObjBuilder& dst) { 
+		JSElemIter it(src);
+		int n = 0;
+		if( !it.more() ) return;
+		while( 1 ) {
+			Element e = it.next();
+			if( e.eoo() )
+				break;
+			if( //n == 0 && 
+				fields.find(e.fieldName()) != fields.end()
+				//fields.count(e.fieldName())
+				) {
+				dst.append(e);
+				if( ++n >= nFields )
+					break;
+			}
+		}
+	}
+};
+
+JSMatcher::~JSMatcher() { 
+	for( int i = 0; i < nBuilders; i++ )
+		delete builders[i];
+	delete in;
+	delete where;
+}
+
+#include "pdfile.h"
+
+JSMatcher::JSMatcher(JSObj &_jsobj) : 
+   in(0), where(0), jsobj(_jsobj), nRegex(0)
+{
+	nBuilders = 0;
+
+	JSElemIter i(jsobj);
+	n = 0;
+	while( i.more() ) {
+		Element e = i.next();
+		if( e.eoo() )
+			break;
+
+		if( e.type() == Code && strcmp(e.fieldName(), "$where")==0 ) { 
+			// $where: function()...
+			assert( where == 0 );
+			where = new Where();
+			const char *code = e.valuestr();
+			massert( "$where query, but jni is disabled", JavaJS );
+			where->scope = JavaJS->scopeCreate();
+			JavaJS->scopeSetString(where->scope, "$client", client->name.c_str());
+			where->setFunc(code);
+			continue;
+		}
+
+		if( e.type() == RegEx ) {
+			if( nRegex >= 4 ) {
+				cout << "ERROR: too many regexes in query" << endl;
+			}
+			else {
+				pcrecpp::RE_Options options;
+				options.set_utf8(true);
+				const char *flags = e.regexFlags();
+				while( flags && *flags ) { 
+					if( *flags == 'i' )
+						options.set_caseless(true);
+					else if( *flags == 'm' )
+						options.set_multiline(true);
+					else if( *flags == 'x' )
+						options.set_extended(true);
+					flags++;
+				}
+				regexs[nRegex].re = new pcrecpp::RE(e.regex(), options);
+				regexs[nRegex].fieldName = e.fieldName();
+				nRegex++;
+			}
+			continue;
+		}
+
+		// greater than / less than...
+		// e.g., e == { a : { $gt : 3 } }
+		//       or 
+		//            { a : { $in : [1,2,3] } }
+		if( e.type() == Object ) {
+			// e.g., fe == { $gt : 3 }
+			JSElemIter j(e.embeddedObject());
+			bool ok = false;
+			while( j.more() ) {
+				Element fe = j.next();
+				if( fe.eoo() ) 
+					break;
+				// Element fe = e.embeddedObject().firstElement();
+				const char *fn = fe.fieldName();
+				if( fn[0] == '$' && fn[1] ) { 
+					if( fn[2] == 't' ) { 
+						int op = Equality;
+						if( fn[1] == 'g' ) { 
+							if( fn[3] == 0 ) op = GT;
+							else if( fn[3] == 'e' && fn[4] == 0 ) op = GTE;
+						}
+						else if( fn[1] == 'l' ) { 
+							if( fn[3] == 0 ) op = LT;
+							else if( fn[3] == 'e' && fn[4] == 0 ) op = LTE;
+						}
+						if( op && nBuilders < 8) { 
+							JSObjBuilder *b = new JSObjBuilder();
+							builders[nBuilders++] = b;
+							b->appendAs(fe, e.fieldName());
+							toMatch.push_back( b->done().firstElement() );
+							compareOp.push_back(op);
+							n++;
+							ok = true;
+						}
+					}
+					else if( fn[1] == 'i' && fn[2] == 'n' && fn[3] == 0 && fe.type() == Array ) {
+						// $in
+						assert( in == 0 ); // only one per query supported so far.  finish...
+						in = new set<Element,element_lt>();
+						JSElemIter i(fe.embeddedObject());
+                        if( i.more() ) {
+                            while( 1 ) {
+                                Element ie = i.next();
+                                if( ie.eoo() ) 
+                                    break;
+                                in->insert(ie);
+                            }
+                        }
+						toMatch.push_back(e); // not actually used at the moment
+						compareOp.push_back(opIN);
+						n++;
+						ok = true;
+					}
+				}
+				else {
+					ok = false;
+					break;
+				}
+			}
+			if( ok ) 
+				continue;
+		}
+
+		{
+			// normal, simple case e.g. { a : "foo" }
+			toMatch.push_back(e);
+			compareOp.push_back(Equality);
+			n++;
+		}
+	}
+}
+
+inline int JSMatcher::valuesMatch(Element& l, Element& r, int op) { 
+	if( op == 0 ) 
+		return l.valuesEqual(r);
+
+	if( op == opIN ) {
+		// { $in : [1,2,3] }
+        int c = in->count(l);
+        return c;
+	}
+
+	/* check LT, GTE, ... */
+	if( l.type() != r.type() )
+		return false;
+	int c = compareElementValues(l, r);
+	int z = 1 << (c+1); 
+	return (op & z);
+}
+
+/* Check if a particular field matches.
+
+   fieldName - field to match "a.b" if we are reaching into an embedded object.
+   toMatch   - element we want to match.
+   obj       - database object to check against
+   compareOp - Equality, LT, GT, etc.
+   deep      - out param.  set to true/false if we scanned an array
+   isArr     - 
+
+   Special forms:
+
+     { "a.b" : 3 }             means       obj.a.b == 3
+     { a : { $lt : 3 } }       means       obj.a < 3
+	 { a : { $in : [1,2] } }   means       [1,2].contains(obj.a)
+
+   return value
+   -1 mismatch
+    0 missing element
+    1 match
+*/
+int JSMatcher::matchesDotted(const char *fieldName, Element& toMatch, JSObj& obj, int compareOp, bool *deep, bool isArr) { 
+	{
+		const char *p = strchr(fieldName, '.');
+		if( p ) { 
+			string left(fieldName, p-fieldName);
+
+			Element e = obj.getField(left.c_str());
+			if( e.eoo() )
+				return 0;
+			if( e.type() != Object && e.type() != Array )
+				return -1;
+
+			JSObj eo = e.embeddedObject();
+			return matchesDotted(p+1, toMatch, eo, compareOp, deep, e.type() == Array);
+		}
+	}
+
+	Element e = obj.getField(fieldName);
+
+	if( valuesMatch(e, toMatch, compareOp) ) {
+		return 1;
+	}
+	else if( e.type() == Array ) {
+		JSElemIter ai(e.embeddedObject());
+		while( ai.more() ) { 
+			Element z = ai.next();
+			if( valuesMatch( z, toMatch, compareOp) ) {
+				if( deep )
+					*deep = true;
+				return 1;
+			}
+		}
+	}
+	else if( isArr ) { 
+		JSElemIter ai(obj);
+		while( ai.more() ) { 
+			Element z = ai.next();
+			if( z.type() == Object ) {
+				JSObj eo = z.embeddedObject();
+				int cmp = matchesDotted(fieldName, toMatch, eo, compareOp, deep);
+				if( cmp > 0 ) { 
+					if( deep ) *deep = true;
+					return 1;
+				}
+			}
+		}
+	}
+	else if( e.eoo() ) { 
+		return 0;
+	}
+	return -1;
+}
+
+extern int dump;
+
+inline bool _regexMatches(RegexMatcher& rm, Element& e) { 
+	char buf[64];
+	const char *p = buf;
+	if( e.type() == String || e.type() == Symbol )
+		p = e.valuestr();
+	else if( e.type() == Number ) { 
+		sprintf(buf, "%f", e.number());
+	}
+	else if( e.type() == Date ) { 
+		unsigned long long d = e.date();
+		time_t t = (d/1000);
+		time_t_to_String(t, buf);
+	}
+	else
+		return false;
+	return rm.re->PartialMatch(p);
+}
+/* todo: internal dotted notation scans -- not done yet here. */
+inline bool regexMatches(RegexMatcher& rm, Element& e, bool *deep) { 
+	if( e.type() != Array ) 
+		return _regexMatches(rm, e);
+
+	JSElemIter ai(e.embeddedObject());
+	while( ai.more() ) { 
+		Element z = ai.next();
+		if( _regexMatches(rm, z) ) {
+			if( deep )
+				*deep = true;
+			return true;
+		}
+	}
+	return false;
+}
+
+/* See if an object matches the query.
+   deep - return true when meanswe looked into arrays for a match 
+*/
+bool JSMatcher::matches(JSObj& jsobj, bool *deep) {
+	if( deep ) 
+		*deep = false;
+
+	/* assuming there is usually only one thing to match.  if more this
+	could be slow sometimes. */
+
+	// check normal non-regex cases:
+	for( int i = 0; i < n; i++ ) {
+		Element& m = toMatch[i]; 
+		int cmp = matchesDotted(toMatch[i].fieldName(), toMatch[i], jsobj, compareOp[i], deep);
+
+		/* missing is ok iff we were looking for null */
+		if( cmp < 0 ) 
+			return false;
+		if( cmp == 0 && (m.type() != jstNULL && m.type() != Undefined ) )
+			return false;
+	}
+
+	for( int r = 0; r < nRegex; r++ ) { 
+		RegexMatcher& rm = regexs[r];
+		Element e = jsobj.getFieldDotted(rm.fieldName);
+		if( e.eoo() )
+			return false;
+		if( !regexMatches(rm, e, deep) )
+			return false;
+	}
+
+	if( where ) { 
+		if( where->func == 0 )
+			return false; // didn't compile
+		if( jsobj.objsize() < 200 || where->fullObject ) {
+			JavaJS->scopeSetObject(where->scope, "obj", &jsobj);
+		} else {
+			JSObjBuilder b;
+			where->buildSubset(jsobj, b);
+			JSObj temp = b.done();
+			JavaJS->scopeSetObject(where->scope, "obj", &temp);
+		}
+		if( JavaJS->invoke(where->scope, where->func) )
+			return false;
+		return JavaJS->scopeGetBoolean(where->scope, "return") != 0;
+	}
+
+	return true;
+}
+
+struct JSObj1 js1;
+
+#pragma pack(push,1)
+struct JSObj2 {
+	JSObj2() {
+		totsize=sizeof(JSObj2);
+		s = String; strcpy_s(sname, 7, "abcdef"); slen = 10; 
+		strcpy_s(sval, 10, "123456789"); eoo = EOO;
+	}
+	unsigned totsize;
+	char s;
+	char sname[7];
+	unsigned slen;
+	char sval[10];
+	char eoo;
+} js2;
+
+struct JSUnitTest : public UnitTest {
+	void run() {
+
+		JSObj j1((const char *) &js1);
+		JSObj j2((const char *) &js2);
+		cout << "j1:" << j1.toString() << endl;
+		cout << "j2:" << j2.toString() << endl;
+		JSMatcher m(j2);
+		assert( m.matches(j1) );
+		js2.sval[0] = 'z';
+		assert( !m.matches(j1) );
+		JSMatcher n(j1);
+		assert( n.matches(j1) );
+		cout << "j2:" << j2.toString() << endl;
+		assert( !n.matches(j2) );
+
+cout << "temp1" << endl;
+		JSObj j0 = emptyObj;
+//		JSObj j0((const char *) &js0);
+		JSMatcher p(j0);
+		assert( p.matches(j1) );
+		assert( p.matches(j2) );
+	}
+} jsunittest;
+
+#pragma pack(pop)
+
+struct RXTest : public UnitTest { 
+	void run() { 
+		/*
+		static const boost::regex e("(\\d{4}[- ]){3}\\d{4}");
+		static const boost::regex b(".....");
+		cout << "regex result: " << regex_match("hello", e) << endl;
+		cout << "regex result: " << regex_match("abcoo", b) << endl;
+		*/
+		pcrecpp::RE re1(")({a}h.*o");
+		pcrecpp::RE re("h.llo");
+		assert( re.FullMatch("hello") );
+		assert( !re1.FullMatch("hello") );
+
+
+		pcrecpp::RE_Options options;
+		options.set_utf8(true);
+		pcrecpp::RE part("dwi", options);
+		assert( part.PartialMatch("dwight") );
+	}
+} rxtest;
+
diff --git a/db/query.cpp b/db/query.cpp
index 3e29977f461..7f5a575dc3c 100644
--- a/db/query.cpp
+++ b/db/query.cpp
@@ -43,7 +43,6 @@ extern bool useCursors;
 
 int getGtLtOp(Element& e);
 void appendElementHandlingGtLt(JSObjBuilder& b, Element& e);
-int runCount(const char *ns, JSObj& cmd, string& err);
 
 /* todo: _ cache query plans 
          _ use index on partial match with the query
@@ -413,434 +412,7 @@ int otherTraceLevel = 0;
 
 int initialExtentSize(int len);
 
-void clean(const char *ns, NamespaceDetails *d) {
-	for( int i = 0; i < Buckets; i++ ) 
-		d->deletedList[i].Null();
-}
-
-string validateNS(const char *ns, NamespaceDetails *d) {
-	bool valid = true;
-	stringstream ss;
-	ss << "\nvalidate\n";
-	ss << "  details: " << hex << d << " ofs:" << nsindex(ns)->detailsOffset(d) << dec << endl;
-	if( d->capped )
-		ss << "  capped:" << d->capped << " max:" << d->max << '\n';
-
-	ss << "  firstExtent:" << d->firstExtent.toString() << " ns:" << d->firstExtent.ext()->ns.buf << '\n';
-	ss << "  lastExtent:" << d->lastExtent.toString()    << " ns:" << d->lastExtent.ext()->ns.buf << '\n';
-	try { 
-		d->firstExtent.ext()->assertOk();
-		d->lastExtent.ext()->assertOk(); 
-	} catch(...) { valid=false; ss << " extent asserted "; }
-
-	ss << "  datasize?:" << d->datasize << " nrecords?:" << d->nrecords << " lastExtentSize:" << d->lastExtentSize << '\n';
-	ss << "  padding:" << d->paddingFactor << '\n';
-	try { 
-
-		try {
-			ss << "  first extent:\n";
-			d->firstExtent.ext()->dump(ss);
-			valid = valid && d->firstExtent.ext()->validates();
-		}
-		catch(...) { 
-		  ss << "\n    exception firstextent\n" << endl;
-		}
-
-		auto_ptr<Cursor> c = theDataFileMgr.findAll(ns);
-		int n = 0;
-		long long len = 0;
-		long long nlen = 0;
-		set<DiskLoc> recs;
-		int outOfOrder = 0;
-		DiskLoc cl_last;
-		while( c->ok() ) { 
-			n++;
-
-			DiskLoc cl = c->currLoc();
-			if( n < 1000000 )
-				recs.insert(cl);
-			if( d->capped ) {
-				if( cl < cl_last )
-					outOfOrder++;
-				cl_last = cl;
-			}
-
-			Record *r = c->_current();
-			len += r->lengthWithHeaders;
-			nlen += r->netLength();
-			c->advance();
-		}
-		if( d->capped ) { 
-			ss << "  capped outOfOrder:" << outOfOrder;
-			if( outOfOrder > 1 ) { 
-				valid = false;
-				ss << " ???";
-			}
-			else ss << " (OK)";
-			ss << '\n';
-		}
-		ss << "  " << n << " objects found, nobj:" << d->nrecords << "\n";
-		ss << "  " << len << " bytes data w/headers\n";
-		ss << "  " << nlen << " bytes data wout/headers\n";
-
-		ss << "  deletedList: ";
-		for( int i = 0; i < Buckets; i++ ) { 
-			ss << (d->deletedList[i].isNull() ? '0' : '1');
-		}
-		ss << endl;
-		int ndel = 0;
-		long long delSize = 0;
-		int incorrect = 0;
-		for( int i = 0; i < Buckets; i++ ) { 
-			DiskLoc loc = d->deletedList[i];
-			try {
-			  int k = 0;
-			  while( !loc.isNull() ) { 
-			    if( recs.count(loc) )
-			      incorrect++;
-			    ndel++;
-
-			    if( loc.questionable() ) { 
-					if( loc.a() <= 0 || strstr(ns, "hudsonSmall") == 0 ) {
-						ss << "    ?bad deleted loc: " << loc.toString() << " bucket:" << i << " k:" << k << endl;
-						valid = false;
-						break;
-					}
-			    }
-
-			    DeletedRecord *d = loc.drec();
-			    delSize += d->lengthWithHeaders;
-			    loc = d->nextDeleted;
-			    k++;
-			  }
-			} catch(...) { ss <<"    ?exception in deleted chain for bucket " << i << endl; valid = false; }
-		}
-		ss << "  deleted: n: " << ndel << " size: " << delSize << '\n';
-		if( incorrect ) { 
-			ss << "    ?corrupt: " << incorrect << " records from datafile are in deleted list\n";
-			valid = false;
-		}
-
-		int idxn = 0;
-		try  {
-			ss << "  nIndexes:" << d->nIndexes << endl;
-			for( ; idxn < d->nIndexes; idxn++ ) {
-				ss << "    " << d->indexes[idxn].indexNamespace() << " keys:" << 
-					d->indexes[idxn].head.btree()->fullValidate(d->indexes[idxn].head) << endl;
-			}
-		} 
-		catch(...) { 
-		  ss << "\n    exception during index validate idxn:" << idxn << endl; valid=false;
-		}
-
-	}
-	catch(AssertionException) {
-		ss << "\n    exception during validate\n" << endl; 
-		valid = false;
-	}
-
-	if( !valid ) 
-		ss << " ns corrupt, requires dbchk\n";
-
-	return ss.str();
-}
-
-bool userCreateNS(const char *ns, JSObj& j, string& err);
-
-const int edebug=0;
-
-bool dbEval(JSObj& cmd, JSObjBuilder& result) { 
-	Element e = cmd.firstElement();
-	assert( e.type() == Code );
-	const char *code = e.valuestr();
-	if ( ! JavaJS ) {
-		result.append("errmsg", "db side execution is disabled");
-		return false;
-	}
-
-	jlong f = JavaJS->functionCreate(code);
-	if( f == 0 ) { 
-		result.append("errmsg", "compile failed");
-		return false;
-	}
-
-	Scope s;
-	s.setString("$client", client->name.c_str());
-	Element args = cmd.findElement("args");
-	if( args.type() == Array ) {
-		JSObj eo = args.embeddedObject();
-		if( edebug ) {
-			cout << "args:" << eo.toString() << endl;
-			cout << "code:\n" << code << endl;
-		}
-		s.setObject("args", eo);
-	}
-
-	int res = s.invoke(f);
-	if( res ) {
-		result.append("errno", (double) res);
-		result.append("errmsg", "invoke failed");
-		return false;
-	}
-
-	int type = s.type("return");
-	if( type == Object || type == Array )
-		result.append("retval", s.getObject("return"));
-	else if( type == Number ) 
-		result.append("retval", s.getNumber("return"));
-	else if( type == String )
-		result.append("retval", s.getString("return").c_str());
-	else if( type == Bool ) {
-		result.appendBool("retval", s.getBoolean("return"));
-	}
-
-	return true;
-}
-
-extern int opLogging;
-void flushOpLog();
-
-// e.g.
-//   system.cmd$.find( { queryTraceLevel: 2 } );
-// 
-// returns true if ran a cmd
-//
-bool _runCommands(const char *ns, JSObj& jsobj, stringstream& ss, BufBuilder &b, JSObjBuilder& anObjBuilder) { 
-
-	const char *p = strchr(ns, '.');
-	if( !p ) return false;
-	if( strcmp(p, ".$cmd") != 0 ) return false;
-
-	bool ok = false;
-	bool valid = false;
-
-	//cout << jsobj.toString() << endl;
-
-	Element e;
-	e = jsobj.firstElement();
-
-	if( e.eoo() ) goto done;
-	if( e.type() == Code ) { 
-		valid = true;
-		ok = dbEval(jsobj, anObjBuilder);
-	}
-	else if( e.type() == Number ) { 
-		if( strcmp(e.fieldName(), "getoptime") == 0 ) { 
-			valid = true;
-			ok = true;
-			anObjBuilder.appendDate("optime", OpTime::now().asDate());
-		}
-		else if( strcmp(e.fieldName(), "dropDatabase") == 0 ) { 
-			if( 1 ) {
-				cout << "dropDatabase " << ns << endl;
-				valid = true;
-				int p = (int) e.number();
-				if( p != 1 ) {
-					ok = false;
-				} else { 
-					dropDatabase(ns);
-					logOp("c", ns, jsobj);
-					ok = true;
-				}
-			}
-			else { 
-				cout << "TEMP CODE: dropdatabase commented out " << endl;
-			}
-		}
-		else if( strcmp(e.fieldName(), "profile") == 0 ) { 
-			anObjBuilder.append("was", (double) client->profile);
-			int p = (int) e.number();
-			valid = true;
-			if( p == -1 )
-				ok = true;
-			else if( p >= 0 && p <= 2 ) { 
-				ok = true;
-				client->profile = p;
-			}
-			else {
-				ok = false;
-			}
-		}
-		else {
-			// admin only commands.
-			if( strncmp(ns, "admin", p-ns) != 0 ) 
-				return false;
-			if( strcmp(e.fieldName(),"opLogging") == 0 ) {
-				valid = ok = true;
-				opLogging = (int) e.number();
-				flushOpLog();
-				log() << "CMD: opLogging set to " << opLogging << endl;
-			} else if( strcmp(e.fieldName(),"queryTraceLevel") == 0 ) {
-				valid = ok = true;
-				queryTraceLevel = (int) e.number();
-			} else if( strcmp(e.fieldName(),"traceAll") == 0 ) { 
-				valid = ok = true;
-				queryTraceLevel = (int) e.number();
-				otherTraceLevel = (int) e.number();
-			}
-		}
-	}
-	else if( e.type() == String ) {
-		/* { count: "collectionname"[, query: <query>] } */
-		string us(ns, p-ns);
-
-		if( strcmp( e.fieldName(), "count" ) == 0 ) { 
-			valid = true;
-			string ns = us + '.' + e.valuestr();
-			string err;
-			int n = runCount(ns.c_str(), jsobj, err);
-			int nn = n;
-			ok = true;
-			if( n < 0 ) { 
-				ok = false;
-				nn = 0;
-				if( !err.empty() )
-					anObjBuilder.append("errmsg", err.c_str());
-			}
-			anObjBuilder.append("n", (double) nn);
-		}
-		else if( strcmp( e.fieldName(), "clone") == 0 ) { 
-			valid = true;
-			string err;
-			ok = cloneFrom(e.valuestr(), err);
-			if( !err.empty() ) 
-				anObjBuilder.append("errmsg", err.c_str());
-		}
-		else if( strcmp( e.fieldName(), "create") == 0 ) { 
-			valid = true;
-			string ns = us + '.' + e.valuestr();
-			string err;
-			ok = userCreateNS(ns.c_str(), jsobj, err);
-			if( ok )
-				logOp("c", ns.c_str(), jsobj);
-			if( !ok && !err.empty() )
-				anObjBuilder.append("errmsg", err.c_str());
-		}
-		else if( strcmp( e.fieldName(), "clean") == 0 ) { 
-			valid = true;
-			string dropNs = us + '.' + e.valuestr();
-			NamespaceDetails *d = nsdetails(dropNs.c_str());
-			log() << "CMD: clean " << dropNs << endl;
-			if( d ) { 
-				ok = true;
-				anObjBuilder.append("ns", dropNs.c_str());
-				clean(dropNs.c_str(), d);
-			}
-			else {
-				anObjBuilder.append("errmsg", "ns not found");
-			}
-		}
-		else if( strcmp( e.fieldName(), "drop") == 0 ) { 
-			valid = true;
-			string nsToDrop = us + '.' + e.valuestr();
-			NamespaceDetails *d = nsdetails(nsToDrop.c_str());
-			log() << "CMD: drop " << nsToDrop << endl;
-			if( d == 0 ) {
-				anObjBuilder.append("errmsg", "ns not found");
-			}
-			else if( d->nIndexes != 0 ) {
-				// client is supposed to drop the indexes first
-				anObjBuilder.append("errmsg", "ns has indexes (not permitted on drop)");
-			}
-			else {
-				ok = true;
-				anObjBuilder.append("ns", nsToDrop.c_str());
-				ClientCursor::invalidate(nsToDrop.c_str());
-				dropNS(nsToDrop);
-				logOp("c", ns, jsobj);
-				/*
-				{
-					JSObjBuilder b;
-					b.append("name", dropNs.c_str());
-					JSObj cond = b.done(); // { name: "colltodropname" }
-					deleteObjects("system.namespaces", cond, false, true);
-				}
-				client->namespaceIndex.kill(dropNs.c_str());
-				*/
-			}
-		}
-		else if( strcmp( e.fieldName(), "validate") == 0 ) { 
-			valid = true;
-			string toValidateNs = us + '.' + e.valuestr();
-			NamespaceDetails *d = nsdetails(toValidateNs.c_str());
-			log() << "CMD: validate " << toValidateNs << endl;
-			if( d ) { 
-				ok = true;
-				anObjBuilder.append("ns", toValidateNs.c_str());
-				string s = validateNS(toValidateNs.c_str(), d);
-				anObjBuilder.append("result", s.c_str());
-			}
-			else {
-				anObjBuilder.append("errmsg", "ns not found");
-			}
-		}
-		else if( strcmp(e.fieldName(),"deleteIndexes") == 0 ) { 
-			valid = true;
-			/* note: temp implementation.  space not reclaimed! */
-			string toDeleteNs = us + '.' + e.valuestr();
-			NamespaceDetails *d = nsdetails(toDeleteNs.c_str());
-			log() << "CMD: deleteIndexes " << toDeleteNs << endl;
-			if( d ) {
-				Element f = jsobj.findElement("index");
-				if( !f.eoo() ) { 
-
-					d->aboutToDeleteAnIndex();
-
-					ClientCursor::invalidate(toDeleteNs.c_str());
-					logOp("c", ns, jsobj);
-
-					// delete a specific index or all?
-					if( f.type() == String ) { 
-						const char *idxName = f.valuestr();
-						if( *idxName == '*' && idxName[1] == 0 ) { 
-							ok = true;
-							cout << "  d->nIndexes was " << d->nIndexes << endl;
-							anObjBuilder.append("nIndexesWas", (double)d->nIndexes);
-							anObjBuilder.append("msg", "all indexes deleted for collection");
-							cout << "  alpha implementation, space not reclaimed" << endl;
-							for( int i = 0; i < d->nIndexes; i++ )
-								d->indexes[i].kill();
-							d->nIndexes = 0;
-						}
-						else {
-							// delete just one index
-							int x = d->findIndexByName(idxName);
-							if( x >= 0 ) { 
-								cout << "  d->nIndexes was " << d->nIndexes << endl;
-								anObjBuilder.append("nIndexesWas", (double)d->nIndexes);
-
-								/* note it is  important we remove the IndexDetails with this 
-								   call, otherwise, on recreate, the old one would be reused, and its
-								   IndexDetails::info ptr would be bad info.
-								*/
-								d->indexes[x].kill();
-
-								d->nIndexes--;
-								for( int i = x; i < d->nIndexes; i++ )
-									d->indexes[i] = d->indexes[i+1];
-								ok=true;
-								cout << "  alpha implementation, space not reclaimed\n";
-							} else { 
-								cout << "deleteIndexes: " << idxName << " not found" << endl;
-							}
-						}
-					}
-				}
-			}
-			else {
-				anObjBuilder.append("errmsg", "ns not found");
-			}
-		}
-	}
-
-done:
-	if( !valid )
-		anObjBuilder.append("errmsg", "no such cmd");
-	anObjBuilder.append("ok", ok?1.0:0.0);
-	JSObj x = anObjBuilder.done();
-	b.append((void*) x.objdata(), x.objsize());
-	return true;
-}
+bool _runCommands(const char *ns, JSObj& jsobj, stringstream& ss, BufBuilder &b, JSObjBuilder& anObjBuilder);
 
 bool runCommands(const char *ns, JSObj& jsobj, stringstream& ss, BufBuilder &b, JSObjBuilder& anObjBuilder) { 
 	try {
diff --git a/db/repl.cpp b/db/repl.cpp
index 1973afad48d..e41473b0e5c 100644
--- a/db/repl.cpp
+++ b/db/repl.cpp
@@ -3,7 +3,6 @@
 /* TODO
 
    PAIRING
-    _ better concurrency; ie don't block if connection to slave blocks
     _ on a syncexception, don't allow going back to master state?
 
 */
diff --git a/util/mmap.cpp b/util/mmap.cpp
index a3296cd2412..e67fe4560b0 100644
--- a/util/mmap.cpp
+++ b/util/mmap.cpp
@@ -36,7 +36,7 @@ void MemoryMappedFile::closeAllFiles() {
 	++closingAllFiles;
 	for( set<MemoryMappedFile*>::iterator i = mmfiles.begin(); i != mmfiles.end(); i++ )
 		(*i)->close();
-	cout << "  closeAllFiles() finished" << endl;
+	log() << "  closeAllFiles() finished" << endl;
 	--closingAllFiles;
 }
 
diff --git a/util/unittest.h b/util/unittest.h
new file mode 100644
index 00000000000..db80517c463
--- /dev/null
+++ b/util/unittest.h
@@ -0,0 +1,40 @@
+// unittest.h
+
+/**
+*    Copyright (C) 2008 10gen Inc.
+*  
+*    This program is free software: you can redistribute it and/or  modify
+*    it under the terms of the GNU Affero General Public License, version 3,
+*    as published by the Free Software Foundation.
+*  
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU Affero General Public License for more details.
+*  
+*    You should have received a copy of the GNU Affero General Public License
+*    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#pragma once
+
+/* the idea here is to let all initialization of global variables (classes inheriting from UnitTest)
+   complete before we run the tests -- otherwise order of initilization being arbitrary may mess 
+   us up.  The app's main() function should call runTests.
+*/
+struct UnitTest { 
+	UnitTest() { registerTest(this); }
+
+	// assert if fails
+	virtual void run() = 0;
+
+	static vector<UnitTest*> tests;
+
+	static void registerTest(UnitTest *t) { tests.push_back(t); }
+
+	static void runTests() { 
+		for( vector<UnitTest*>::iterator i = tests.begin(); i != tests.end(); i++ ) { 
+		}
+	}
+};
+
diff --git a/util/util.cpp b/util/util.cpp
index c78767f3a69..0879a27b50b 100644
--- a/util/util.cpp
+++ b/util/util.cpp
@@ -16,6 +16,9 @@
 
 #include "stdafx.h"
 #include "goodies.h"
+#include "unittest.h"
+
+vector<UnitTest*> UnitTest::tests;
 
 Nullstream nullstream;
 Logstream logstream;
@@ -43,8 +46,8 @@ int nextPrime(int n) {
 	return n;
 }
 
-struct UtilTest { 
-	UtilTest() { 
+struct UtilTest : public UnitTest { 
+	void run() { 
 		assert( WrappingInt(0) <= WrappingInt(0) );
 		assert( WrappingInt(0) <= WrappingInt(1) );
 		assert( !(WrappingInt(1) <= WrappingInt(0)) );