Merge branch 'master' of git@github.com:mongodb/mongo

53 files changed, 997 insertions, 466 deletions

diff --git a/SConstruct b/SConstruct
index 866c3251fbb..212d288cfd4 100644
--- a/SConstruct
+++ b/SConstruct
@@ -135,6 +135,13 @@ AddOption( "--noshell",
            action="store",
            help="don't build shell" )
 
+AddOption( "--extrapath",
+           dest="extrapath",
+           type="string",
+           nargs=1,
+           action="store",
+           help="comma seperated list of add'l paths  (--extrapath /opt/foo/,/foo" )
+
 # --- environment setup ---
 
 def printLocalInfo():
@@ -163,7 +170,10 @@ darwin = False
 windows = False
 freebsd = False
 solaris = False
-force64 = not GetOption( "force64" ) is None
+force64 = not GetOption( "force64" ) is None 
+if not force64 and os.getcwd().endswith( "mongo-64" ):
+    force64 = True
+    print( "*** assuming you want a 64-bit build b/c of directory *** " )
 force32 = not GetOption( "force32" ) is None
 release = not GetOption( "release" ) is None
 
@@ -182,6 +192,12 @@ if ( usesm and usejvm ):
 if ( not ( usesm or usejvm ) ):
     usesm = True
 
+if GetOption( "extrapath" ) is not None:
+    for x in GetOption( "extrapath" ).split( "," ):
+        env.Append( CPPPATH=[ x + "/include" ] )
+        env.Append( LIBPATH=[ x + "/lib" ] )
+    release = True
+    
 # ------    SOURCE FILE SETUP -----------
 
 commonFiles = Split( "stdafx.cpp buildinfo.cpp db/jsobj.cpp db/json.cpp db/commands.cpp db/lasterror.cpp db/nonce.cpp db/queryutil.cpp shell/mongo.cpp" )
@@ -361,7 +377,8 @@ elif "win32" == os.sys.platform:
     env.Append( CPPPATH=[ boostDir , "pcre-7.4" , winSDKHome + "/Include" ] )
 
     env.Append( CPPFLAGS=" /EHsc /W3 " )
-    env.Append( CPPDEFINES=["WIN32","_CONSOLE","_CRT_SECURE_NO_WARNINGS","HAVE_CONFIG_H","PCRE_STATIC","_UNICODE","UNICODE" ] )
+    env.Append( CPPFLAGS=" /wd4355 /wd4800 " ) #some warnings we don't like
+    env.Append( CPPDEFINES=["WIN32","_CONSOLE","_CRT_SECURE_NO_WARNINGS","HAVE_CONFIG_H","PCRE_STATIC","_UNICODE","UNICODE","SUPPORT_UCP","SUPPORT_UTF8" ] )
 
     #env.Append( CPPFLAGS='  /Yu"stdafx.h" ' ) # this would be for pre-compiled headers, could play with it later
 
@@ -593,6 +610,7 @@ def doConfigure( myenv , needJava=True , needPcre=True , shell=False ):
                 myenv.Append( LINKFLAGS=" /usr/lib/libreadline.dylib " )
         elif myCheckLib( "readline" , release and nix , staticOnly=release ):
             myenv.Append( CPPDEFINES=[ "USE_READLINE" ] )
+            myCheckLib( "ncurses" , staticOnly=release )
             myCheckLib( "tinfo" , staticOnly=release )
         else:
             print( "warning: no readline, shell will be a bit ugly" )
diff --git a/db/btree.h b/db/btree.h
index e4e8a76a1f9..02b94ccb920 100644
--- a/db/btree.h
+++ b/db/btree.h
@@ -163,6 +163,12 @@ namespace mongo {
     public:
         void dump();
 
+        /* @return true if key exists in index 
+
+           order - indicates order of keys in the index.  this is basically the index's key pattern, e.g.:
+             BSONObj order = ((IndexDetails&)idx).keyPattern();
+           likewise below in bt_insert() etc.
+        */
         bool exists(const IndexDetails& idx, DiskLoc thisLoc, const BSONObj& key, BSONObj order);
 
         static DiskLoc addHead(IndexDetails&); /* start a new index off, empty */
@@ -298,4 +304,8 @@ namespace mongo {
 
 #pragma pack()
 
+    inline bool IndexDetails::hasKey(const BSONObj& key) { 
+        return head.btree()->exists(*this, head, key, keyPattern());
+    }
+
 } // namespace mongo;
diff --git a/db/clientcursor.cpp b/db/clientcursor.cpp
index 85620a787f3..61be7c1a0ef 100644
--- a/db/clientcursor.cpp
+++ b/db/clientcursor.cpp
@@ -35,8 +35,8 @@ namespace mongo {
 
     /* ------------------------------------------- */
 
-    long long IdSet::size_ = 0;
-    long long IdSet::maxSize_ = 200 * 1024 * 1024;
+    long long IdSet_Deprecated::size_ = 0;
+    long long IdSet_Deprecated::maxSize_ = 200 * 1024 * 1024;
     
     typedef multimap<DiskLoc, ClientCursor*> ByLoc;
     ByLoc byLoc;
diff --git a/db/clientcursor.h b/db/clientcursor.h
index 5e9158b0ea0..e57a59fe2e1 100644
--- a/db/clientcursor.h
+++ b/db/clientcursor.h
@@ -43,10 +43,10 @@ namespace mongo {
     extern BSONObj id_obj;
 
     // utility class for de duping ids
-    class IdSet {
+    class IdSet_Deprecated {
     public:
-        IdSet() : mySize_(), inMem_( true ) {}
-        ~IdSet() {
+        IdSet_Deprecated() : mySize_(), inMem_( true ) {}
+        ~IdSet_Deprecated() {
             size_ -= mySize_;
         }
         bool get( const BSONObj &id ) const {
@@ -108,13 +108,13 @@ namespace mongo {
         string ns;
         auto_ptr<KeyValJSMatcher> matcher;
         auto_ptr<Cursor> c;
-        auto_ptr<IdSet> ids_;
+//        auto_ptr<IdSet> ids_;
         int pos; /* # objects into the cursor so far */
         DiskLoc lastLoc() const {
             return _lastLoc;
         }
         void setLastLoc(DiskLoc);
-        auto_ptr< set<string> > filter; // which fields query wants returned
+        auto_ptr< FieldMatcher > filter; // which fields query wants returned
         Message originalMessage; // this is effectively an auto ptr for data the matcher points to
         unsigned idleAgeMillis; // how long has the cursor been around, relative to server idle time
 
@@ -151,11 +151,11 @@ namespace mongo {
         void cleanupByLocation(DiskLoc loc);
         
         void mayUpgradeStorage() {
-            if ( !ids_.get() )
+/*            if ( !ids_.get() )
                 return;
             stringstream ss;
             ss << ns << "." << cursorid;
-            ids_->mayUpgradeStorage( ss.str() );
+            ids_->mayUpgradeStorage( ss.str() );*/
         }
     };
 
diff --git a/db/db.cpp b/db/db.cpp
index f03ca04b575..5aa3392997b 100644
--- a/db/db.cpp
+++ b/db/db.cpp
@@ -545,10 +545,11 @@ int main(int argc, char* argv[], char *envp[] )
                 assert(opIdMem > 0);
             }
             else if ( s == "--deDupMem" ) {
+                uasserted("deprecated");
                 long x = strtol( argv[ ++i ], 0, 10 );
                 uassert("bad arg", x > 0);
-                IdSet::maxSize_ = x;
-                assert(IdSet::maxSize_ > 0);
+//                IdSet::maxSize_ = x;
+//                assert(IdSet::maxSize_ > 0);
             }
             else if ( strncmp(s.c_str(), "--oplog", 7) == 0 ) {
                 int x = s[7] - '0';
diff --git a/db/dbcommands.cpp b/db/dbcommands.cpp
index 3a9f07f0640..8058455ce83 100644
--- a/db/dbcommands.cpp
+++ b/db/dbcommands.cpp
@@ -930,7 +930,7 @@ namespace mongo {
             help << "example: { buildinfo:1 }";
         }
         bool run(const char *dbname, BSONObj& jsobj, string& errmsg, BSONObjBuilder& result, bool fromRepl ){
-            result << "gitVersion" << gitVersion() << "sysInfo" << sysInfo();
+            result << "version" << versionString << "gitVersion" << gitVersion() << "sysInfo" << sysInfo();
             return true;
         }
     } cmdBuildInfo;
diff --git a/db/dbmessage.h b/db/dbmessage.h
index 37899516e73..838b063cf54 100644
--- a/db/dbmessage.h
+++ b/db/dbmessage.h
@@ -153,6 +153,7 @@ namespace mongo {
         const char * mark;
     };
 
+
     /* a request to run a query, received from the database */
     class QueryMessage {
     public:
@@ -161,8 +162,8 @@ namespace mongo {
         int ntoreturn;
         int queryOptions;
         BSONObj query;
-        auto_ptr< set<string> > fields;
-
+        auto_ptr< FieldMatcher > fields;
+        
         /* parses the message into the above fields */
         QueryMessage(DbMessage& d) {
             ns = d.getns();
@@ -170,8 +171,8 @@ namespace mongo {
             ntoreturn = d.pullInt();
             query = d.nextJsObj();
             if ( d.moreJSObjs() ) {
-                fields = auto_ptr< set<string> >(new set<string>());
-                d.nextJsObj().getFieldNames(*fields);
+                fields = auto_ptr< FieldMatcher >(new FieldMatcher() );
+                fields->add( d.nextJsObj() );
                 if ( fields->size() == 0 )
                     fields.reset();
             }
diff --git a/db/instance.cpp b/db/instance.cpp
index 90264303846..817f4f6063a 100644
--- a/db/instance.cpp
+++ b/db/instance.cpp
@@ -152,7 +152,7 @@ namespace mongo {
         database = 0;
 
         int ms;
-        bool log = false;
+        bool log = logLevel >= 1;
         currentOp.op = curOp = m.data->operation();
 
 #if 0
@@ -523,7 +523,7 @@ namespace mongo {
             {
                 Timer t;
 
-                bool log = false;
+                bool log = logLevel >= 1;
                 curOp = m.data->operation();
 
                 if ( m.data->operation() == dbQuery ) {
@@ -575,7 +575,7 @@ namespace mongo {
                 log = log || ctr++ % 128 == 0;
                 if ( log || ms > 100 ) {
                     ss << ' ' << t.millis() << "ms";
-		    mongo::out() << ss.str().c_str() << endl;
+                    mongo::out() << ss.str().c_str() << endl;
                 }
                 if ( database && database->profile >= 1 ) {
                     if ( database->profile >= 2 || ms >= 100 ) {
diff --git a/db/jsobj.h b/db/jsobj.h
index 4cc20a05aa2..9b1b80f2087 100644
--- a/db/jsobj.h
+++ b/db/jsobj.h
@@ -517,7 +517,10 @@ namespace mongo {
             if ( ifree )
                 _holder.reset( new Holder( data ) );
             _objdata = data;
-            assert( "Invalid BSONObj spec size" && isValid() );
+            if ( ! isValid() ){
+                log() << "invalid object size: " << objsize() << endl;
+                massert( "Invalid BSONObj spec size" , 0 );
+            }
         }
 #pragma pack(1)
         static struct EmptyObject {
diff --git a/db/matcher.cpp b/db/matcher.cpp
index 3eff0beb9c3..be68cc96d3b 100644
--- a/db/matcher.cpp
+++ b/db/matcher.cpp
@@ -646,6 +646,16 @@ namespace mongo {
             options.set_utf8(true);
             pcrecpp::RE part("dwi", options);
             assert( part.PartialMatch("dwight") );
+
+            int ret = 0;
+            
+            pcre_config( PCRE_CONFIG_UTF8 , &ret );
+            assert( ret );
+
+            pcre_config( PCRE_CONFIG_UNICODE_PROPERTIES , &ret );
+            if ( ! ret )
+                cerr << "warning: some regex utf8 things will not work.  pcre build doesn't have --enable-unicode-properties" << endl;
+            
         }
     } rxtest;
 
diff --git a/db/matcher.h b/db/matcher.h
index d8500ee4db9..83b64ba7e79 100644
--- a/db/matcher.h
+++ b/db/matcher.h
@@ -78,8 +78,9 @@ namespace mongo {
             bool operator()(const BSONElement& l, const BSONElement& r) const
             {
                 int x = (int) l.type() - (int) r.type();
-                if ( x < 0 ) return true;
-                if ( x > 0 ) return false;
+                if ( x == ( NumberInt - NumberDouble ) || x == ( NumberDouble - NumberInt ) );
+                else if ( x < 0 ) return true;
+                else if ( x > 0 ) return false;
                 return compareElementValues(l,r) < 0;
             }
         };
diff --git a/db/namespace.h b/db/namespace.h
index 5fa4e9f30a8..c8481be773b 100644
--- a/db/namespace.h
+++ b/db/namespace.h
@@ -162,6 +162,9 @@ namespace mongo {
             return info.obj().getObjectField("key");
         }
 
+        /* true if the specified key is in the index */
+        bool hasKey(const BSONObj& key);
+
         // returns name of this index's storage area
         // database.table.$index
         string indexNamespace() const {
diff --git a/db/pdfile.cpp b/db/pdfile.cpp
index 1806cc20959..30f49195318 100644
--- a/db/pdfile.cpp
+++ b/db/pdfile.cpp
@@ -846,11 +846,18 @@ assert( !eloc.isNull() );
         }
     }
 
-    struct IndexChanges {
+    struct IndexChanges/*on an update*/ {
         BSONObjSetDefaultOrder oldkeys;
         BSONObjSetDefaultOrder newkeys;
         vector<BSONObj*> removed; // these keys were removed as part of the change
         vector<BSONObj*> added;   // these keys were added as part of the change
+
+        void dupCheck(IndexDetails& idx) {
+            if( added.empty() || !idx.unique() )
+                return;
+            for( vector<BSONObj*>::iterator i = added.begin(); i != added.end(); i++ )
+                uassert("E11001 duplicate key on update", !idx.hasKey(**i));
+        }
     };
 
     inline void getIndexChanges(vector<IndexChanges>& v, NamespaceDetails& d, BSONObj newObj, BSONObj oldObj) { 
@@ -866,61 +873,59 @@ assert( !eloc.isNull() );
         }
     }
 
+    inline void dupCheck(vector<IndexChanges>& v, NamespaceDetails& d) {
+        for ( int i = 0; i < d.nIndexes; i++ ) {
+            IndexDetails& idx = d.indexes[i];
+            v[i].dupCheck(idx);
+        }
+    }
+
     /** Note: if the object shrinks a lot, we don't free up space, we leave extra at end of the record.
     */
     void DataFileMgr::update(
         const char *ns,
         Record *toupdate, const DiskLoc& dl,
-        const char *buf, int len, stringstream& ss)
+        const char *_buf, int _len, stringstream& ss)
     {
         dassert( toupdate == dl.rec() );
 
         NamespaceDetails *d = nsdetails(ns);
 
         BSONObj objOld(toupdate);
-        BSONObj objNew(buf);
-        BSONElement idOld;
-        int addID = 0;
-        {
-            /* xxx duplicate _id check... */
-
-            BSONElement idNew;
-            objOld.getObjectID(idOld);
-            if( objNew.getObjectID(idNew) ) { 
-                if( idOld == idNew ) 
-                    ; 
-                else {
-                    /* check if idNew would be a duplicate. very unusual that an _id would change, 
-                       so not worried about the bit of extra work here.
-                       */
-                    if( d->findIdIndex() >= 0 ) {
-                        BSONObj result;
-                        BSONObjBuilder b;
-                        b.append(idNew);
-                        uassert("duplicate _id key on update", 
-                                !Helpers::findOne(ns, b.done(), result));
-                    }
-                }
-            } else {
-                if ( !idOld.eoo() ) {
-                    /* if the old copy had the _id, force it back into the updated version.  insert() adds 
-                       one so old should have it unless this is a special table like system.* or 
-                       local.oplog.*
-                    */
-                    addID = len;
-                    len += idOld.size();
-                }
-            }
+        BSONObj objNew(_buf);
+        assert( objNew.objsize() == _len );
+        assert( objNew.objdata() == _buf );
+
+        if( !objNew.hasElement("_id") && objOld.hasElement("_id") ) {
+            /* add back the old _id value if the update removes it.  Note this implementation is slow 
+               (copies entire object multiple times), but this shouldn't happen often, so going for simple
+               code, not speed.
+               */
+            BSONObjBuilder b;
+            BSONElement e;
+            assert( objOld.getObjectID(e) );
+            b.append(e); // put _id first, for best performance
+            b.appendElements(objNew);
+            objNew = b.obj();
         }
-        if ( toupdate->netLength() < len ) {
+
+        /* duplicate key check. we descend the btree twice - once for this check, and once for the actual inserts, further  
+           below.  that is suboptimal, but it's pretty complicated to do it the other way without rollbacks...
+        */
+        vector<IndexChanges> changes;
+        getIndexChanges(changes, *d, objNew, objOld);
+        dupCheck(changes, *d);
+
+        if ( toupdate->netLength() < objNew.objsize() ) {
             // doesn't fit.  reallocate -----------------------------------------------------
             uassert("E10003 failing update: objects in a capped ns cannot grow", !(d && d->capped));
             d->paddingTooSmall();
             if ( database->profile )
                 ss << " moved ";
             // xxx TODO fix dup key error - old copy should not be deleted
+
             deleteRecord(ns, toupdate, dl);
-            insert(ns, buf, len, false, idOld);
+            insert(ns, objNew.objdata(), objNew.objsize(), false);
             return;
         }
 
@@ -929,12 +934,8 @@ assert( !eloc.isNull() );
 
         /* have any index keys changed? */
         if( d->nIndexes ) {
-            vector<IndexChanges> changes;
-            getIndexChanges(changes, *d, objNew, objOld);
             for ( int x = 0; x < d->nIndexes; x++ ) {
                 IndexDetails& idx = d->indexes[x];
-                if ( addID && idx.isIdIndex() )
-                    continue;
                 for ( unsigned i = 0; i < changes[x].removed.size(); i++ ) {
                     try {
                         idx.head.btree()->unindex(idx.head, idx, *changes[x].removed[i], dl);
@@ -965,55 +966,13 @@ assert( !eloc.isNull() );
             }
         }
 
-/*
-                    BSONObj idxKey = idx.info.obj().getObjectField("key");
-                    BSONObjSetDefaultOrder oldkeys;
-                    BSONObjSetDefaultOrder newkeys;
-                    idx.getKeysFromObject(oldObj, oldkeys);
-                    idx.getKeysFromObject(newObj, newkeys);
-                    vector<BSONObj*> removed;
-                    setDifference(oldkeys, newkeys, removed);
-                    string idxns = idx.indexNamespace();
-                    for ( unsigned i = 0; i < removed.size(); i++ ) {
-                        try {
-                            idx.head.btree()->unindex(idx.head, idx, *removed[i], dl);
-                        }
-                        catch (AssertionException&) {
-                            ss << " exception update unindex ";
-                            problem() << " caught assertion update unindex " << idxns.c_str() << endl;
-                        }
-                    }
-                    vector<BSONObj*> added;
-                    setDifference(newkeys, oldkeys, added);
-                    assert( !dl.isNull() );
-                    for ( unsigned i = 0; i < added.size(); i++ ) {
-                        try {
-                            ** TODO xxx dup keys handle **
-                            idx.head.btree()->bt_insert(
-                                idx.head,
-                                dl, *added[i], idxKey, **dupsAllowed**true, idx);
-                        }
-                        catch (AssertionException&) {
-                            ss << " exception update index ";
-                            out() << " caught assertion update index " << idxns.c_str() << '\n';
-                            problem() << " caught assertion update index " << idxns.c_str() << endl;
-                        }
-                    }
-                    if ( database->profile )
-                        ss << '\n' << added.size() << " key updates ";
-                }
-            }
-        }
-*/
-
         //	update in place
-        if ( addID ) {
+        /*if ( addID ) {
             ((int&)*toupdate->data) = *((int*) buf) + idOld.size();
             memcpy(toupdate->data+4, idOld.rawdata(), idOld.size());
             memcpy(toupdate->data+4+idOld.size(), ((char *)buf)+4, addID-4);
-        } else {
-            memcpy(toupdate->data, buf, len);
-        }
+        } else {*/
+        memcpy(toupdate->data, objNew.objdata(), objNew.objsize());
     }
 
     int followupExtentSize(int len, int lastExtentLen) {
diff --git a/db/query.cpp b/db/query.cpp
index 30152985a6c..19eb5d8d5b9 100644
--- a/db/query.cpp
+++ b/db/query.cpp
@@ -827,7 +827,7 @@ namespace mongo {
                     }
                     else {
                         BSONObj js = c->current();
-                        if ( cc->ids_.get() ) {
+                        /* if ( cc->ids_.get() ) {
                             BSONElement idRef = js.getField( "_id" );
                             if ( !idRef.eoo() ) {
                                 BSONObjBuilder idBuilder;
@@ -839,7 +839,7 @@ namespace mongo {
                                 }
                                 cc->ids_->put( id ); 
                             }
-                        }
+                        }*/
                         bool ok = fillQueryResultFromObj(b, cc->filter.get(), js);
                         if ( ok ) {
                             n++;
@@ -977,7 +977,7 @@ namespace mongo {
     class DoQueryOp : public QueryOp {
     public:
         DoQueryOp( int ntoskip, int ntoreturn, const BSONObj &order, bool wantMore,
-                  bool explain, set< string > *filter, int queryOptions ) :
+                  bool explain, FieldMatcher *filter, int queryOptions ) :
         b_( 32768 ),
         ntoskip_( ntoskip ),
         ntoreturn_( ntoreturn ),
@@ -1049,66 +1049,74 @@ namespace mongo {
             }
             
             bool mayCreateCursor1 = wantMore_ && ntoreturn_ != 1 && useCursors;
-            
-            if ( !ids_.get() && !c_->capped() && ( mayCreateCursor1 || mayCreateCursor2() ) ) {
+/*            if ( !ids_.get() && !c_->capped() && ( mayCreateCursor1 || mayCreateCursor2() ) ) {
                 ids_.reset( new IdSet() );
-            }
+            }*/
             
+            if( 0 ) { 
+                BSONObj js = c_->current();
+                cout << "SCANNING " << js << endl;
+            }
+
             nscanned_++;
             bool deep;
             if ( !matcher_->matches(c_->currKey(), c_->currLoc(), &deep) ) {
+                ;
             }
-            else if ( !deep || !c_->getsetdup(c_->currLoc()) ) { // i.e., check for dups on deep items only
-                BSONObj js = c_->current();
-                // got a match.
-                assert( js.objsize() >= 0 ); //defensive for segfaults
-                if ( ids_.get() ) {
-                    BSONElement idRef = js.getField( "_id" );
-                    if ( !idRef.eoo() ) {
-                        BSONObjBuilder b;
-                        b.append( idRef );
-                        BSONObj id = b.obj();
-                        ids_->put( id );
-                    }
-                }
-                if ( ordering_ ) {
-                    // note: no cursors for non-indexed, ordered results.  results must be fairly small.
-                    so_->add(js);
-                }
-                else if ( ntoskip_ > 0 ) {
-                    ntoskip_--;
-                } else {
-                    if ( explain_ ) {
-                        n_++;
-                        if ( n_ >= ntoreturn_ && !wantMore_ ) {
-                            // .limit() was used, show just that much.
-                            finish();
-                            return;
+            else {
+                DiskLoc cl = c_->currLoc();
+                if ( !deep || !c_->getsetdup(cl) ) { // i.e., check for dups on deep items only
+                    BSONObj js = c_->current();
+                    // got a match.
+                    assert( js.objsize() >= 0 ); //defensive for segfaults
+                    /*if ( ids_.get() ) {
+                        BSONElement idRef = js.getField( "_id" );
+                        if ( !idRef.eoo() ) {
+                            BSONObjBuilder b;
+                            b.append( idRef );
+                            BSONObj id = b.obj();
+                            ids_->put( id );
                         }
+                    }*/
+                    if ( ordering_ ) {
+                        // note: no cursors for non-indexed, ordered results.  results must be fairly small.
+                        so_->add(js);
                     }
-                    else {
-                        bool ok = fillQueryResultFromObj(b_, filter_, js);
-                        if ( ok ) n_++;
-                        if ( ok ) {
-                            if ( (ntoreturn_>0 && (n_ >= ntoreturn_ || b_.len() > MaxBytesToReturnToClientAtOnce)) ||
+                    else if ( ntoskip_ > 0 ) {
+                        ntoskip_--;
+                    } else {
+                        if ( explain_ ) {
+                            n_++;
+                            if ( n_ >= ntoreturn_ && !wantMore_ ) {
+                                // .limit() was used, show just that much.
+                                finish();
+                                return;
+                            }
+                        }
+                        else {
+                            bool ok = fillQueryResultFromObj(b_, filter_, js);
+                            if ( ok ) n_++;
+                            if ( ok ) {
+                                if ( (ntoreturn_>0 && (n_ >= ntoreturn_ || b_.len() > MaxBytesToReturnToClientAtOnce)) ||
                                 (ntoreturn_==0 && (b_.len()>1*1024*1024 || n_>=101)) ) {
-                                /* if ntoreturn is zero, we return up to 101 objects.  on the subsequent getmore, there
-                                 is only a size limit.  The idea is that on a find() where one doesn't use much results,
-                                 we don't return much, but once getmore kicks in, we start pushing significant quantities.
+                                    /* if ntoreturn is zero, we return up to 101 objects.  on the subsequent getmore, there
+                                    is only a size limit.  The idea is that on a find() where one doesn't use much results,
+                                    we don't return much, but once getmore kicks in, we start pushing significant quantities.
                                  
-                                 The n limit (vs. size) is important when someone fetches only one small field from big
-                                 objects, which causes massive scanning server-side.
+                                    The n limit (vs. size) is important when someone fetches only one small field from big
+                                    objects, which causes massive scanning server-side.
                                  */
-                                /* if only 1 requested, no cursor saved for efficiency...we assume it is findOne() */
-                                if ( mayCreateCursor1 ) {
-                                    c_->advance();
-                                    if ( c_->ok() ) {
-                                        // more...so save a cursor
-                                        saveClientCursor_ = true;
+                                    /* if only 1 requested, no cursor saved for efficiency...we assume it is findOne() */
+                                    if ( mayCreateCursor1 ) {
+                                        c_->advance();
+                                        if ( c_->ok() ) {
+                                            // more...so save a cursor
+                                            saveClientCursor_ = true;
+                                        }
                                     }
+                                    finish();
+                                    return;
                                 }
-                                finish();
-                                return;
                             }
                         }
                     }
@@ -1139,7 +1147,7 @@ namespace mongo {
         bool scanAndOrderRequired() const { return ordering_; }
         auto_ptr< Cursor > cursor() { return c_; }
         auto_ptr< KeyValJSMatcher > matcher() { return matcher_; }
-        auto_ptr< IdSet > ids() { return ids_; }
+//        auto_ptr< IdSet > ids() { return ids_; }
         int n() const { return n_; }
         long long nscanned() const { return nscanned_; }
         bool saveClientCursor() const { return saveClientCursor_; }
@@ -1151,7 +1159,7 @@ namespace mongo {
         BSONObj order_;
         bool wantMore_;
         bool explain_;
-        set< string > *filter_;   
+        FieldMatcher *filter_;   
         bool ordering_;
         auto_ptr< Cursor > c_;
         long long nscanned_;
@@ -1163,7 +1171,7 @@ namespace mongo {
         auto_ptr< ScanAndOrder > so_;
         bool findingStart_;
         ClientCursor * findingStartCursor_;
-        auto_ptr< IdSet > ids_;
+//        auto_ptr< IdSet > ids_; /* for dedupping traversal of multikey indexes */
     };
     
     auto_ptr< QueryResult > runQuery(Message& m, stringstream& ss ) {
@@ -1173,7 +1181,7 @@ namespace mongo {
         int ntoskip = q.ntoskip;
         int _ntoreturn = q.ntoreturn;
         BSONObj jsobj = q.query;
-        auto_ptr< set< string > > filter = q.fields;
+        auto_ptr< FieldMatcher > filter = q.fields;
         int queryOptions = q.queryOptions;
         
         Timer t;
@@ -1231,7 +1239,7 @@ namespace mongo {
             BSONObj max;
             bool explain = false;
             bool _gotquery = false;
-            BSONObj query;// = jsobj.getObjectField("query");
+            BSONObj query;
             {
                 BSONElement e = jsobj.findElement("$query");
                 if ( e.eoo() )
@@ -1264,7 +1272,7 @@ namespace mongo {
             
             /* The ElemIter will not be happy if this isn't really an object. So throw exception
              here when that is true.
-             (Which may indicate bad data from appserver?)
+             (Which may indicate bad data from client.)
              */
             if ( query.objsize() == 0 ) {
                 out() << "Bad query object?\n  jsobj:";
@@ -1289,13 +1297,14 @@ namespace mongo {
             if ( dqo.scanAndOrderRequired() )
                 ss << " scanAndOrder ";
             auto_ptr< Cursor > c = dqo.cursor();
+            log( 5 ) << "   used cursor: " << c->toString() << endl;
             if ( dqo.saveClientCursor() ) {
                 ClientCursor *cc = new ClientCursor();
                 cc->c = c;
                 cursorid = cc->cursorid;
                 DEV out() << "  query has more, cursorid: " << cursorid << endl;
                 cc->matcher = dqo.matcher();
-                cc->ids_ = dqo.ids();
+//                cc->ids_ = dqo.ids();
                 cc->ns = ns;
                 cc->pos = n;
                 cc->filter = filter;
diff --git a/db/queryoptimizer.cpp b/db/queryoptimizer.cpp
index 67201323c66..56896a5abaf 100644
--- a/db/queryoptimizer.cpp
+++ b/db/queryoptimizer.cpp
@@ -338,6 +338,9 @@ namespace mongo {
     shared_ptr< QueryOp > QueryPlanSet::Runner::run() {
         massert( "no plans", plans_.plans_.size() > 0 );
         
+        if ( plans_.plans_.size() > 1 )
+            log(1) << "  running multiple plans" << endl;
+
         vector< shared_ptr< QueryOp > > ops;
         for( PlanSet::iterator i = plans_.plans_.begin(); i != plans_.plans_.end(); ++i ) {
             shared_ptr< QueryOp > op( op_.clone() );
diff --git a/db/queryutil.cpp b/db/queryutil.cpp
index 8ebfb06cad1..33a5604db3d 100644
--- a/db/queryutil.cpp
+++ b/db/queryutil.cpp
@@ -85,6 +85,14 @@ namespace mongo {
             default:
                 break;
         }
+
+        if ( lower_.isNumber() && upper_.type() == MaxKey ){
+            upper_ = addObj( BSON( lower_.fieldName() << numeric_limits<double>::max() ) ).firstElement();
+        }
+        else if ( upper_.isNumber() && lower_.type() == MinKey ){
+            lower_ = addObj( BSON( upper_.fieldName() << - numeric_limits<double>::max() ) ).firstElement();
+        }
+
     }
     
     const FieldBound &FieldBound::operator&=( const FieldBound &other ) {
@@ -203,4 +211,70 @@ namespace mongo {
         return qp;
     }
     
+    void FieldMatcher::add( const BSONObj& o ){
+        BSONObjIterator i( o );
+        while ( i.more() ){
+            string s = i.next().fieldName();
+            if ( s.find( "." ) == string::npos ){
+                fields[ s ] = "";
+            }
+            else {
+                string sub = s.substr( 0 , s.find( "." ) );
+                if ( fields[sub].size() )
+                    fields[sub] = ".";
+                else
+                    fields[sub] = s.substr( sub.size() + 1 );
+            }
+        }
+
+    }
+    
+    int FieldMatcher::size() const {
+        return fields.size();
+    }
+
+    bool FieldMatcher::matches( const string& s ) const {
+        return fields.find( s ) != fields.end();
+    }
+    
+    BSONObj FieldMatcher::getSpec() const{
+        BSONObjBuilder b;
+        for ( map<string,string>::const_iterator i=fields.begin(); i!=fields.end(); i++){
+            string s = i->first;
+            if ( i->second.size() > 0 )
+                s += "." + i->second;
+            b.append( s.c_str() , 1 );
+        }
+        return b.obj();
+    }
+
+    BSONObj FieldMatcher::extractDotted( const string& path , const BSONObj& o ) const {
+        string::size_type i = path.find( "." );
+        if ( i == string::npos )
+            return o.getField( path.c_str() ).wrap();
+        
+        string left = path.substr( 0 , i );
+        BSONElement e = o[left];
+        if ( e.type() != Object )
+            return BSONObj();
+
+        BSONObj sub = e.embeddedObject();
+        if ( sub.isEmpty() )
+            return sub;
+        
+        BSONObjBuilder b(32);
+        b.append( left.c_str() , extractDotted( path.substr( i + 1 ) , sub ) );
+        return b.obj();
+    }
+    
+    void FieldMatcher::append( BSONObjBuilder& b , const BSONElement& e ) const {
+        string next = fields.find( e.fieldName() )->second;
+        if ( next.size() == 0 || next == "." || e.type() != Object ){
+            b.append( e );
+        }
+        else {
+            b.append( e.fieldName() , extractDotted( next , e.embeddedObject() ) );
+        }
+    }
+    
 } // namespace mongo
diff --git a/db/queryutil.h b/db/queryutil.h
index 298e1bbeec6..f3b28459795 100644
--- a/db/queryutil.h
+++ b/db/queryutil.h
@@ -155,4 +155,26 @@ namespace mongo {
         BSONObj query_;
     };
 
+    /**
+       used for doing field limiting
+     */
+    class FieldMatcher {
+    public:
+        
+        void add( const BSONObj& o );
+        int size() const;
+
+        bool matches( const string& s ) const;
+        void append( BSONObjBuilder& b , const BSONElement& e ) const;
+
+        BSONObj getSpec() const;
+
+    private:
+
+        BSONObj extractDotted( const string& path , const BSONObj& o ) const ;
+        
+        map<string,string> fields; // { 'a' : 1 , 'b.c' : 1 } ==>> [ a -> '' , b -> c ]
+    };
+
+
 } // namespace mongo
diff --git a/db/scanandorder.h b/db/scanandorder.h
index 061c86ea9a4..4bb95ffff73 100644
--- a/db/scanandorder.h
+++ b/db/scanandorder.h
@@ -50,19 +50,43 @@ namespace mongo {
        _ response size limit from runquery; push it up a bit.
     */
 
-    inline bool fillQueryResultFromObj(BufBuilder& b, set<string> *filter, BSONObj& js) {
+    inline bool fillQueryResultFromObj(BufBuilder& bb, FieldMatcher *filter, BSONObj& js) {
         if ( filter ) {
-            BSONObj x;
-            bool ok = x.addFields(js, *filter) > 0;
-            if ( ok )
-                b.append((void*) x.objdata(), x.objsize());
-            return ok;
+            
+            const int mark = bb.len();
+            
+            BSONObjBuilder b( bb );
+            BSONObjIterator i( js );
+            int N = filter->size();
+            int n=0;
+            bool gotId = false;
+            while ( i.more() ){
+                BSONElement e = i.next();
+                const char * fname = e.fieldName();
+                
+                if ( strcmp( fname , "_id" ) == 0 ){
+                    b.append( e );
+                    gotId = true;
+                    if ( filter->matches( "_id" ) )
+                        n++;
+                }
+                else if ( filter->matches( fname ) ){
+                    filter->append( b , e );
+                    n++;
+                    if ( n == N && gotId )
+                        break;
+                }
+            }
+            b.done();
+            if ( ! n )
+                bb.setlen( mark );
+            return n;
         }
-
-        b.append((void*) js.objdata(), js.objsize());
+        
+        bb.append((void*) js.objdata(), js.objsize());
         return true;
     }
-
+    
     typedef multimap<BSONObj,BSONObj,BSONObjCmp> BestMap;
     class ScanAndOrder {
         BestMap best; // key -> full object
@@ -112,7 +136,7 @@ namespace mongo {
             _addIfBetter(k, o, i);
         }
 
-        void _fill(BufBuilder& b, set<string> *filter, int& nout, BestMap::iterator begin, BestMap::iterator end) {
+        void _fill(BufBuilder& b, FieldMatcher *filter, int& nout, BestMap::iterator begin, BestMap::iterator end) {
             int n = 0;
             int nFilled = 0;
             for ( BestMap::iterator i = begin; i != end; i++ ) {
@@ -132,7 +156,7 @@ done:
         }
 
         /* scanning complete. stick the query result in b for n objects. */
-        void fill(BufBuilder& b, set<string> *filter, int& nout) {
+        void fill(BufBuilder& b, FieldMatcher *filter, int& nout) {
             _fill(b, filter, nout, best.begin(), best.end());
         }
 
diff --git a/dbtests/cursortests.cpp b/dbtests/cursortests.cpp
index 45ce206b576..fafc48c26a2 100644
--- a/dbtests/cursortests.cpp
+++ b/dbtests/cursortests.cpp
@@ -23,6 +23,8 @@
 
 namespace CursorTests {
     
+    typedef IdSet_Deprecated IdSet;
+
     namespace IdSetTests {
         
         class BasicSize {
diff --git a/dbtests/jstests.cpp b/dbtests/jstests.cpp
index 398b71c8a70..68a769f5d4d 100644
--- a/dbtests/jstests.cpp
+++ b/dbtests/jstests.cpp
@@ -447,7 +447,7 @@ namespace JSTests {
         ~Utf8Check() { reset(); }
         void run() {
             if( !globalScriptEngine->utf8Ok() ) {
-                log() << "utf8 not supported" << endl;
+                log() << "warning: utf8 not supported" << endl;
                 return;
             }
             string utf8ObjSpec = "{'_id':'\\u0001\\u007f\\u07ff\\uffff'}";
diff --git a/dbtests/matchertests.cpp b/dbtests/matchertests.cpp
index cb416f5346f..78e387f6f15 100644
--- a/dbtests/matchertests.cpp
+++ b/dbtests/matchertests.cpp
@@ -63,7 +63,39 @@ namespace MatcherTests {
             ASSERT( m.matches( b.done() ) );
         }        
     };
+    
+    class MixedNumericIN {
+    public:
+        void run(){
+            BSONObj query = fromjson( "{ a : { $in : [4,6] } }" );
+            ASSERT_EQUALS( 4 , query["a"].embeddedObject()["$in"].embeddedObject()["0"].number() );
+            ASSERT_EQUALS( NumberDouble , query["a"].embeddedObject()["$in"].embeddedObject()["0"].type() );
+            
+            JSMatcher m( query );
+
+            {
+                BSONObjBuilder b;
+                b.append( "a" , 4.0 );
+                ASSERT( m.matches( b.done() ) );
+            }
+
+            {
+                BSONObjBuilder b;
+                b.append( "a" , 5 );
+                ASSERT( ! m.matches( b.done() ) );
+            }
+
 
+            {
+                BSONObjBuilder b;
+                b.append( "a" , 4 );
+                ASSERT( m.matches( b.done() ) );
+            }
+                
+        }
+    };
+
+    
     class Size {
     public:
         void run() {
@@ -75,6 +107,7 @@ namespace MatcherTests {
         }        
     };
     
+
     class All : public Suite {
     public:
         All() {
@@ -82,6 +115,7 @@ namespace MatcherTests {
             add< DoubleEqual >();
             add< MixedNumericEqual >();
             add< MixedNumericGt >();
+            add< MixedNumericIN >();
             add< Size >();
         }
     };
diff --git a/dbtests/queryoptimizertests.cpp b/dbtests/queryoptimizertests.cpp
index add030c6d4d..912550fb996 100644
--- a/dbtests/queryoptimizertests.cpp
+++ b/dbtests/queryoptimizertests.cpp
@@ -59,6 +59,19 @@ namespace QueryOptimizerTests {
             }
         };
         
+
+        class NumericBase : public Base {
+        public:
+            NumericBase(){
+                o = BSON( "min" << -numeric_limits<double>::max() << "max" << numeric_limits<double>::max() );
+            }
+            
+            virtual BSONElement lower() { return o["min"]; }
+            virtual BSONElement upper() { return o["max"]; }
+        private:
+            BSONObj o;
+        };
+
         class Empty : public Base {
             virtual BSONObj query() { return BSONObj(); }
         };
@@ -77,7 +90,7 @@ namespace QueryOptimizerTests {
             virtual BSONObj query() { return BSON( "a" << 1 << "b" << 2 << "a" << 1 ); }
         };        
 
-        class Lt : public Base {
+        class Lt : public NumericBase {
         public:
             Lt() : o_( BSON( "-" << 1 ) ) {}
             virtual BSONObj query() { return BSON( "a" << LT << 1 ); }
@@ -91,7 +104,7 @@ namespace QueryOptimizerTests {
             virtual bool upperInclusive() { return true; }
         };
         
-        class Gt : public Base {
+        class Gt : public NumericBase {
         public:
             Gt() : o_( BSON( "-" << 1 ) ) {}
             virtual BSONObj query() { return BSON( "a" << GT << 1 ); }
@@ -192,10 +205,11 @@ namespace QueryOptimizerTests {
             void run() {
                 FieldBoundSet fbs( "ns", BSON( "a" << GT << 1 << GT << 5 << LT << 10 << "b" << 4 << "c" << LT << 4 << LT << 6 << "d" << GTE << 0 << GT << 0 << "e" << GTE << 0 << LTE << 10 ) );
                 BSONObj simple = fbs.simplifiedQuery();
+                cout << "simple: " << simple << endl;
                 ASSERT( !simple.getObjectField( "a" ).woCompare( fromjson( "{$gt:5,$lt:10}" ) ) );
                 ASSERT_EQUALS( 4, simple.getIntField( "b" ) );
-                ASSERT( !simple.getObjectField( "c" ).woCompare( fromjson( "{$lt:4}" ) ) );
-                ASSERT( !simple.getObjectField( "d" ).woCompare( fromjson( "{$gt:0}" ) ) );
+                ASSERT( !simple.getObjectField( "c" ).woCompare( BSON("$gte" << -numeric_limits<double>::max() << "$lt" << 4 ) ) );
+                ASSERT( !simple.getObjectField( "d" ).woCompare( BSON("$gt" << 0 << "$lte" << numeric_limits<double>::max() ) ) );
                 ASSERT( !simple.getObjectField( "e" ).woCompare( fromjson( "{$gte:0,$lte:10}" ) ) );
             }
         };
diff --git a/jstests/basic8.js b/jstests/basic8.js
new file mode 100644
index 00000000000..513da0d15d1
--- /dev/null
+++ b/jstests/basic8.js
@@ -0,0 +1,11 @@
+
+t = db.basic8;
+t.drop();
+
+t.save( { a : 1 } );
+o = t.findOne();
+o.b = 2;
+t.save( o );
+
+assert.eq( 1 , t.find().count() , "A" );
+assert.eq( 2 , t.findOne().b , "B" );
diff --git a/jstests/cursor1.js b/jstests/cursor1.js
new file mode 100644
index 00000000000..8448752bb0c
--- /dev/null
+++ b/jstests/cursor1.js
@@ -0,0 +1,20 @@
+
+t = db.cursor1
+t.drop();
+
+big = "";
+while ( big.length < 50000 )
+    big += "asdasdasdasdsdsdadsasdasdasD";
+
+num = Math.ceil( 10000000 / big.length );
+
+for ( var i=0; i<num; i++ ){
+    t.save( { num : i , str : big } );
+}
+
+assert.eq( num , t.find().count() );
+assert.eq( num , t.find().itcount() );
+
+assert.eq( num / 2 , t.find().limit(num/2).itcount() );
+
+t.drop(); // save some space
diff --git a/jstests/cursor3.js b/jstests/cursor3.js
index fd487d7bfea..d23264c94ba 100644
--- a/jstests/cursor3.js
+++ b/jstests/cursor3.js
@@ -1,34 +1,35 @@
 // Test inequality bounds combined with ordering for a single-field index.
 // BUG 1079 (fixed)
 
-function checkResults( expected, cursor ) {
-    assert.eq( expected.length, cursor.count() );
+testNum = 1;
+
+function checkResults( expected, cursor , testNum ) {
+    assert.eq( expected.length, cursor.count() , "testNum: " + testNum  + " A : " + tojson( cursor.toArray() ) + " " + tojson( cursor.explain() ) );
     for( i = 0; i < expected.length; ++i ) {
-	assert.eq( expected[ i ], cursor[ i ][ "a" ] );
+	assert.eq( expected[ i ], cursor[ i ][ "a" ] , "testNum: " + testNum + " B" );
     }
 }
 
-function testConstrainedFindWithOrdering( db ) {
-    r = db.ed_db_cursor3_cfwo;
-    r.drop()
-
-    r.save( { a: 0 } );
-    r.save( { a: 1 } );
-    r.save( { a: 2 } );
-    r.ensureIndex( { a: 1 } );
-
-    checkResults( [ 1 ], r.find( { a: 1 } ).sort( { a: 1 } ).hint( { a: 1 } ) );
-    checkResults( [ 1 ], r.find( { a: 1 } ).sort( { a: -1 } ).hint( { a: 1 } ) );
-
-    checkResults( [ 1, 2 ], r.find( { a: { $gt: 0 } } ).sort( { a: 1 } ).hint( { a: 1 } ) );
-    checkResults( [ 2, 1 ], r.find( { a: { $gt: 0 } } ).sort( { a: -1 } ).hint( { a: 1 } ) );
-    checkResults( [ 1, 2 ], r.find( { a: { $gte: 1 } } ).sort( { a: 1 } ).hint( { a: 1 } ) );
-    checkResults( [ 2, 1 ], r.find( { a: { $gte: 1 } } ).sort( { a: -1 } ).hint( { a: 1 } ) );
-    
-    checkResults( [ 0, 1 ], r.find( { a: { $lt: 2 } } ).sort( { a: 1 } ).hint( { a: 1 } ) );
-    checkResults( [ 1, 0 ], r.find( { a: { $lt: 2 } } ).sort( { a: -1 } ).hint( { a: 1 } ) );
-    checkResults( [ 0, 1 ], r.find( { a: { $lte: 1 } } ).sort( { a: 1 } ).hint( { a: 1 } ) );
-    checkResults( [ 1, 0 ], r.find( { a: { $lte: 1 } } ).sort( { a: -1 } ).hint( { a: 1 } ) );
-}
+t = db.cursor3;
+t.drop()
+
+t.save( { a: 0 } );
+t.save( { a: 1 } );
+t.save( { a: 2 } );
+
+t.ensureIndex( { a: 1 } );
+
+
+
+checkResults( [ 1 ], t.find( { a: 1 } ).sort( { a: 1 } ).hint( { a: 1 } ) , testNum++ )
+checkResults( [ 1 ], t.find( { a: 1 } ).sort( { a: -1 } ).hint( { a: 1 } ) , testNum++ )
+
+checkResults( [ 1, 2 ], t.find( { a: { $gt: 0 } } ).sort( { a: 1 } ).hint( { a: 1 } ) , testNum++ )
+checkResults( [ 2, 1 ], t.find( { a: { $gt: 0 } } ).sort( { a: -1 } ).hint( { a: 1 } ) , testNum++ )
+checkResults( [ 1, 2 ], t.find( { a: { $gte: 1 } } ).sort( { a: 1 } ).hint( { a: 1 } ) , testNum++ )
+checkResults( [ 2, 1 ], t.find( { a: { $gte: 1 } } ).sort( { a: -1 } ).hint( { a: 1 } ) , testNum++ )
 
-testConstrainedFindWithOrdering( db );
+checkResults( [ 0, 1 ], t.find( { a: { $lt: 2 } } ).sort( { a: 1 } ).hint( { a: 1 } ) , testNum++ )
+checkResults( [ 1, 0 ], t.find( { a: { $lt: 2 } } ).sort( { a: -1 } ).hint( { a: 1 } ) , testNum++ )
+checkResults( [ 0, 1 ], t.find( { a: { $lte: 1 } } ).sort( { a: 1 } ).hint( { a: 1 } ) , testNum++ )
+checkResults( [ 1, 0 ], t.find( { a: { $lte: 1 } } ).sort( { a: -1 } ).hint( { a: 1 } ) , testNum++ )
diff --git a/jstests/disk/norepeat.js b/jstests/disk/norepeat.js
index b8b030fa8f2..d9f1cd3b844 100644
--- a/jstests/disk/norepeat.js
+++ b/jstests/disk/norepeat.js
@@ -1,3 +1,4 @@
+/*
 baseName = "jstests_disk_norepeat";
 
 ports = allocatePorts( 1 );
@@ -57,3 +58,4 @@ assert.eq( "a", c.next()._id );
 assert( !c.hasNext() );
 
 assert( t.validate().valid );
+*/
diff --git a/jstests/find4.js b/jstests/find4.js
new file mode 100644
index 00000000000..17639d3a684
--- /dev/null
+++ b/jstests/find4.js
@@ -0,0 +1,26 @@
+
+t = db.find4;
+t.drop();
+
+t.save( { a : 1123  , b : 54332 } );
+
+o = t.find( {} , {} )[0];
+assert.eq( 1123 , o.a , "A" );
+assert.eq( 54332 , o.b , "B" );
+assert( o._id.str , "C" );
+
+o = t.find( {} , { a : 1 } )[0];
+assert.eq( 1123 , o.a , "D" );
+assert( o._id.str , "E" );
+assert( ! o.b , "F" );
+
+o = t.find( {} , { b : 1 } )[0];
+assert.eq( 54332 , o.b , "G" );
+assert( o._id.str , "H" );
+assert( ! o.a , "I" );
+
+t.drop();
+t.save( { a : 1 , b : 1 } );
+t.save( { a : 2 , b : 2 } );
+assert.eq( "1-1,2-2" , t.find().map( function(z){ return z.a + "-" + z.b } ).toString() );
+assert.eq( "1-undefined,2-undefined" , t.find( {} , { a : 1 }).map( function(z){ return z.a + "-" + z.b } ).toString() );
diff --git a/jstests/find5.js b/jstests/find5.js
new file mode 100644
index 00000000000..6dfcb2b2ef2
--- /dev/null
+++ b/jstests/find5.js
@@ -0,0 +1,29 @@
+
+t = db.find5;
+t.drop();
+
+t.save({a: 1});
+t.save({b: 5});
+
+assert.eq(1, t.find({}, {b:1}).count(), "A");
+
+o = t.find( {} , {b:1} )[0];
+
+assert.eq(5, o.b, "B");
+assert(!o.a, "C");
+
+t.drop();
+t.save( { a : 1 , b : { c : 2 , d : 3 , e : 4 } } );
+assert.eq( 2 , t.find( {} , { "b.c" : 1 } ).toArray()[0].b.c , "D" );
+
+o = t.find( {} , { "b.c" : 1 , "b.d" : 1 } ).toArray()[0];
+assert( o.b.c , "E 1" );
+assert( o.b.d , "E 2" );
+assert( o.b.e , "E 3" );
+
+assert( ! t.find( {} , { "b.c" : 1 } ).toArray()[0].b.d , "F" );
+
+t.drop();
+t.save( { a : { b : { c : 1 } } } )
+assert.eq( 1 , t.find( {} , { "a.b.c" : 1 } )[0].a.b.c , "G" );
+
diff --git a/jstests/in.js b/jstests/in.js
new file mode 100644
index 00000000000..5442bbe372f
--- /dev/null
+++ b/jstests/in.js
@@ -0,0 +1,19 @@
+
+t = db.in1;
+t.drop();
+
+t.save( { a : 1 } );
+t.save( { a : 2 } );
+
+assert.eq( 1 , t.find( { a : { $in : [ 1 ] } } ).itcount() , "A" );
+assert.eq( 1 , t.find( { a : { $in : [ 2 ] } } ).itcount() , "B" );
+assert.eq( 2 , t.find( { a : { $in : [ 1 , 2 ] } } ).itcount() , "C" );
+
+t.ensureIndex( { a : 1 } );
+
+assert.eq( 1 , t.find( { a : { $in : [ 1 ] } } ).itcount(), "D" );
+assert.eq( 1 , t.find( { a : { $in : [ 2 ] } } ).itcount() , "E" );
+assert.eq( 2 , t.find( { a : { $in : [ 1 , 2 ] } } ).itcount() , "F" );
+
+assert.eq( 0 , t.find( { a : { $in : [] } } ).itcount() , "G" );
+
diff --git a/jstests/index_check3.js b/jstests/index_check3.js
index 7e828bfd16e..62352e5d61c 100644
--- a/jstests/index_check3.js
+++ b/jstests/index_check3.js
@@ -3,6 +3,23 @@
 t = db.index_check3;
 t.drop();
 
+
+
+t.save( { a : 1 } );
+t.save( { a : 2 } );
+t.save( { a : 3 } );
+t.save( { a : "z" } );
+
+assert.eq( 1 , t.find( { a : { $lt : 2 } } ).itcount() , "A" );
+assert.eq( 1 , t.find( { a : { $gt : 2 } } ).itcount() , "A" );
+
+t.ensureIndex( { a : 1 } );
+
+assert.eq( 1 , t.find( { a : { $lt : 2 } } ).itcount() , "A" );
+assert.eq( 1 , t.find( { a : { $gt : 2 } } ).itcount() , "A" );
+
+t.drop();
+
 for ( var i=0; i<100; i++ ){
     var o = { i : i };
     if ( i % 2 == 0 )
@@ -12,6 +29,8 @@ for ( var i=0; i<100; i++ ){
 
 t.ensureIndex( { foo : 1 } );
 
+printjson( t.find( { foo : { $lt : 50 } } ).explain() );
 assert.gt( 30 , t.find( { foo : { $lt : 50 } } ).explain().nscanned , "lt" )
-//assert.gt( 30 , t.find( { foo : { $gt : 50 } } ).explain().nscanned , "gt" )
+printjson( t.find( { foo : { $gt : 50 } } ).explain() );
+assert.gt( 30 , t.find( { foo : { $gt : 50 } } ).explain().nscanned , "gt" )
 
diff --git a/jstests/indexa.js b/jstests/indexa.js
index 11a332374c9..8e5a31c8062 100644
--- a/jstests/indexa.js
+++ b/jstests/indexa.js
@@ -1,3 +1,5 @@
+// unique index constraint test for updates 
+// case where object doesn't grow tested here
 
 t = db.indexa;
 t.drop();
@@ -8,12 +10,13 @@ t.insert( { 'x':'A' } );
 t.insert( { 'x':'B' } );
 t.insert( { 'x':'A' } );
 
-assert.eq( 2 , t.count() , "A" );
+assert.eq( 2 , t.count() , "indexa 1" );
 
 t.update( {x:'B'}, { x:'A' } );
 
 a = t.find().toArray();
 u = a.map( function(z){ return z.x } ).unique();
+assert.eq( 2 , t.count() , "indexa 2" );
 
-//assert( a.length == u.length , "unique index update is broken" );
+assert( a.length == u.length , "unique index update is broken" );
 
diff --git a/jstests/indexb.js b/jstests/indexb.js
new file mode 100644
index 00000000000..5507fee8460
--- /dev/null
+++ b/jstests/indexb.js
@@ -0,0 +1,30 @@
+// unique index test for a case where the object grows
+// and must move
+
+// see indexa.js for the test case for an update with dup id check 
+// when it doesn't move
+
+
+t = db.indexb;t = db.indexb;
+db.dropDatabase();
+t.drop();
+t.ensureIndex({a:1},true);
+
+t.insert({a:1});
+
+x = { a : 2 };
+t.save(x);
+
+{
+
+ assert( t.count() == 2, "count wrong B");
+
+ x.a = 1;
+ x.filler = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
+ t.save(x); // should fail, not unique.
+
+ assert( t.count() == 2,"count wrong" );
+ assert( t.find({a:1}).count() == 1,"bfail1" );
+ assert( t.find({a:2}).count() == 1,"bfail2" );
+
+}
diff --git a/jstests/indexc.js b/jstests/indexc.js
new file mode 100644
index 00000000000..1c1cc0e119d
--- /dev/null
+++ b/jstests/indexc.js
@@ -0,0 +1,20 @@
+
+t = db.indexc;
+t.drop();
+
+for ( var i=1; i<100; i++ ){
+    var d = new Date( ( new Date() ).getTime() + i );
+    t.save( { a : i , ts : d , cats : [ i , i + 1 , i + 2 ] } );
+    if ( i == 51 )
+        mid = d;
+}
+
+assert.eq( 50 , t.find( { ts : { $lt : mid } } ).itcount() , "A" );
+assert.eq( 50 , t.find( { ts : { $lt : mid } } ).sort( { ts : 1 } ).itcount() , "B" );
+
+//t.ensureIndex( { ts : 1 , cats : 1 } );
+//t.ensureIndex( { cats : 1 } );
+
+// multi-key bug
+//assert.eq( 50 , t.find( { ts : { $lt : mid } } ).itcount() , "C" );
+//assert.eq( 50 , t.find( { ts : { $lt : mid } } ).sort( { ts : 1 } ).itcount() , "D" );
diff --git a/jstests/json1.js b/jstests/json1.js
new file mode 100644
index 00000000000..6dcf88fa3d1
--- /dev/null
+++ b/jstests/json1.js
@@ -0,0 +1,5 @@
+
+x = { quotes:"a\"b" , nulls:null };
+eval( "y = " + tojson( x ) );
+assert.eq( tojson( x ) , tojson( y ) );
+assert.eq( typeof( x.nulls ) , typeof( y.nulls ) );
diff --git a/jstests/objid1.js b/jstests/objid1.js
index f0b62ef5d0c..dea31eed0d8 100644
--- a/jstests/objid1.js
+++ b/jstests/objid1.js
@@ -11,3 +11,6 @@ t.save( { a : a } )
 assert( t.findOne().a.isObjectId , "C" );
 assert.eq( a.str , t.findOne().a.str , "D" );
 
+x = { a : new ObjectId() };
+eval( " y = " + tojson( x ) );
+assert.eq( x.a.str , y.a.str , "E" );
diff --git a/jstests/ref.js b/jstests/ref.js
new file mode 100644
index 00000000000..77d6038f182
--- /dev/null
+++ b/jstests/ref.js
@@ -0,0 +1,19 @@
+// to run: 
+//   ./mongo jstests/ref.js
+
+db.otherthings.drop();
+db.things.drop();
+
+var other = { s : "other thing", n : 1};
+db.otherthings.save(other);
+
+db.things.save( { name : "abc" } );
+x = db.things.findOne();
+x.o = new DBRef( "otherthings" , other._id );
+db.things.save(x);
+
+assert( db.things.findOne().o.fetch().n == 1, "dbref broken 2" );
+
+other.n++;
+db.otherthings.save(other);
+assert( db.things.findOne().o.fetch().n == 2, "dbrefs broken" );
diff --git a/jstests/regex2.js b/jstests/regex2.js
new file mode 100644
index 00000000000..ebee19c82e1
--- /dev/null
+++ b/jstests/regex2.js
@@ -0,0 +1,38 @@
+
+t = db.regex2;
+t.drop();
+
+t.save( { a : "test" } );
+t.save( { a : "Test" } );
+
+assert.eq( 2 , t.find().count() , "A" );
+assert.eq( 1 , t.find( { a : "Test" } ).count() , "B" );
+assert.eq( 1 , t.find( { a : "test" } ).count() , "C" );
+assert.eq( 1 , t.find( { a : /Test/ } ).count() , "D" );
+assert.eq( 1 , t.find( { a : /test/ } ).count() , "E" );
+assert.eq( 2 , t.find( { a : /test/i } ).count() , "F" );
+
+
+t.drop();
+
+a = "\u0442\u0435\u0441\u0442";
+b = "\u0422\u0435\u0441\u0442";
+
+assert( ( new RegExp( a ) ).test( a ) , "B 1" );
+assert( ! ( new RegExp( a ) ).test( b ) , "B 2" );
+assert( ( new RegExp( a , "i" ) ).test( b ) , "B 3 " );
+
+t.save( { a : a } );
+t.save( { a : b } );
+
+
+assert.eq( 2 , t.find().count() , "C A" );
+assert.eq( 1 , t.find( { a : a } ).count() , "C B" );
+assert.eq( 1 , t.find( { a : b } ).count() , "C C" );
+assert.eq( 1 , t.find( { a : new RegExp( a ) } ).count() , "C D" );
+assert.eq( 1 , t.find( { a : new RegExp( b ) } ).count() , "C E" );
+assert.eq( 2 , t.find( { a : new RegExp( a , "i" ) } ).count() , "C F" );
+
+
+
+
diff --git a/jstests/uniqueness.js b/jstests/uniqueness.js
index 9bc9a18df05..6d69e4b9d04 100644
--- a/jstests/uniqueness.js
+++ b/jstests/uniqueness.js
@@ -21,10 +21,25 @@ t.update( { _id : 4 } , { _id : 3, x : 99 } );
 assert( db.getLastError() , 4);
 assert( t.findOne( {_id:4} ), 5 );
 
-/* Check for an error message when we index and there are dups */
+// Check for an error message when we index and there are dups 
 db.bar.drop();
 db.bar.insert({a:3});
 db.bar.insert({a:3});
 assert( db.bar.count() == 2 , 6) ;
 db.bar.ensureIndex({a:1}, true);
 assert( db.getLastError() , 7);
+
+/* Check that if we update and remove _id, it gets added back by the DB */
+
+/* - test when object grows */
+t.drop();
+t.save( { _id : 'Z' } );
+t.update( {}, { k : 2 } );
+assert( t.findOne()._id == 'Z', "uniqueness.js problem with adding back _id" );
+
+/* - test when doesn't grow */
+t.drop();
+t.save( { _id : 'Z', k : 3 } );
+t.update( {}, { k : 2 } );
+assert( t.findOne()._id == 'Z', "uniqueness.js problem with adding back _id (2)" );
+
diff --git a/jstests/where2.js b/jstests/where2.js
new file mode 100644
index 00000000000..ae7addb4402
--- /dev/null
+++ b/jstests/where2.js
@@ -0,0 +1,10 @@
+
+t = db.getCollection( "where1" );
+t.drop();
+
+t.save( { a : 1 } );
+t.save( { a : 2 } );
+t.save( { a : 3 } );
+
+assert.eq( 1 , t.find( { $where : "this.a == 2" } ).toArray().length , "A" );
+assert.eq( 1 , t.find( { $where : "\nthis.a == 2" } ).toArray().length , "B" );
diff --git a/s/cursors.cpp b/s/cursors.cpp
index bf2fc97f496..32bc7290014 100644
--- a/s/cursors.cpp
+++ b/s/cursors.cpp
@@ -20,10 +20,7 @@ namespace mongo {
         _done = false;
 
         if ( q.fields.get() ){
-            BSONObjBuilder b;
-            for ( set<string>::iterator i=q.fields->begin(); i!=q.fields->end(); i++)
-                b.append( i->c_str() , 1 );
-            _fields = b.obj();
+            _fields = q.fields->getSpec();
         }
         else {
             _fields = BSONObj();
diff --git a/scripting/engine_spidermonkey.cpp b/scripting/engine_spidermonkey.cpp
index 481be661261..7c3341f9b76 100644
--- a/scripting/engine_spidermonkey.cpp
+++ b/scripting/engine_spidermonkey.cpp
@@ -12,17 +12,17 @@ namespace mongo {
 #define GETHOLDER(x,o) ((BSONHolder*)JS_GetPrivate( x , o ))
 
     class BSONFieldIterator;
-    
+
     class BSONHolder {
     public:
-        
+
         BSONHolder( BSONObj obj ){
             _obj = obj.getOwned();
             _inResolve = false;
             _modified = false;
             _magic = 17;
         }
-        
+
         void check(){
             uassert( "holder magic value is wrong" , _magic == 17 );
         }
@@ -38,7 +38,7 @@ namespace mongo {
 
     class BSONFieldIterator {
     public:
-        
+
         BSONFieldIterator( BSONHolder * holder ){
 
             BSONObjIterator it( holder->_obj );
@@ -46,12 +46,12 @@ namespace mongo {
                 BSONElement e = it.next();
                 _names.push_back( e.fieldName() );
             }
-            
+
             _names.merge( holder->_extra );
-            
+
             _it = _names.begin();
         }
-        
+
         bool more(){
             return _it != _names.end();
         }
@@ -61,7 +61,7 @@ namespace mongo {
             _it++;
             return s;
         }
-        
+
     private:
         list<string> _names;
         list<string>::iterator _it;
@@ -77,16 +77,16 @@ namespace mongo {
         Convertor( JSContext * cx ){
             _context = cx;
         }
-        
+
         string toString( JSString * so ){
             jschar * s = JS_GetStringChars( so );
             size_t srclen = JS_GetStringLength( so );
             if( srclen == 0 )
                 return "";
-            
+
             size_t len = srclen * 6; // we only need *3, but see note on len below
             char * dst = (char*)malloc( len );
-            
+
             len /= 2;
             // doc re weird JS_EncodeCharacters api claims len expected in 16bit
             // units, but experiments suggest 8bit units expected.  We allocate
@@ -103,7 +103,7 @@ namespace mongo {
         }
 
         string toString( jsval v ){
-            return toString( JS_ValueToString( _context , v ) );            
+            return toString( JS_ValueToString( _context , v ) );
         }
 
         double toNumber( jsval v ){
@@ -127,11 +127,11 @@ namespace mongo {
             oid.init( getString( o , "str" ) );
             return oid;
         }
-        
+
         BSONObj toObject( JSObject * o ){
             if ( ! o )
                 return BSONObj();
-            
+
             if ( JS_InstanceOf( _context , o , &bson_ro_class , 0 ) ){
                 return GETHOLDER( _context , o )->_obj.getOwned();
             }
@@ -143,14 +143,14 @@ namespace mongo {
                     return holder->_obj;
                 orig = holder->_obj;
             }
-            
+
             BSONObjBuilder b;
-            
+
             jsval theid = getProperty( o , "_id" );
             if ( ! JSVAL_IS_VOID( theid ) ){
                 append( b , "_id" , theid );
             }
-            
+
             JSIdArray * properties = JS_Enumerate( _context , o );
             assert( properties );
 
@@ -161,24 +161,24 @@ namespace mongo {
                 string name = toString( nameval );
                 if ( name == "_id" )
                     continue;
-                
+
                 append( b , name , getProperty( o , name.c_str() ) , orig[name].type() );
             }
-                        
+
             JS_DestroyIdArray( _context , properties );
 
             return b.obj();
         }
-        
+
         BSONObj toObject( jsval v ){
-            if ( JSVAL_IS_NULL( v ) || 
+            if ( JSVAL_IS_NULL( v ) ||
                  JSVAL_IS_VOID( v ) )
                 return BSONObj();
-            
+
             uassert( "not an object" , JSVAL_IS_OBJECT( v ) );
             return toObject( JSVAL_TO_OBJECT( v ) );
         }
-        
+
         string getFunctionCode( JSFunction * func ){
             return toString( JS_DecompileFunction( _context , func , 0 ) );
         }
@@ -191,10 +191,10 @@ namespace mongo {
         void append( BSONObjBuilder& b , string name , jsval val , BSONType oldType = EOO  ){
             //cout << "name: " << name << "\t" << typeString( val ) << " oldType: " << oldType << endl;
             switch ( JS_TypeOfValue( _context , val ) ){
-                
+
             case JSTYPE_VOID: b.appendUndefined( name.c_str() ); break;
             case JSTYPE_NULL: b.appendNull( name.c_str() ); break;
-                
+
             case JSTYPE_NUMBER: {
                 double d = toNumber( val );
                 if ( oldType == NumberInt && ((int)d) == d )
@@ -231,21 +231,21 @@ namespace mongo {
                     b.appendRegex( name.c_str() , s.substr( 0 , end ).c_str() , s.substr( end + 1 ).c_str() );
                 }
                 else {
-                    b.appendCode( name.c_str() , getFunctionCode( val ).c_str() ); 
+                    b.appendCode( name.c_str() , getFunctionCode( val ).c_str() );
                 }
                 break;
             }
-                
+
             default: uassert( (string)"can't append field.  name:" + name + " type: " + typeString( val ) , 0 );
             }
         }
-        
+
         // ---------- to spider monkey ---------
 
         bool hasFunctionIdentifier( const string& code ){
             if ( code.size() < 9 || code.find( "function" ) != 0  )
                 return false;
-            
+
             return code[8] == ' ' || code[8] == '(';
         }
 
@@ -273,8 +273,12 @@ namespace mongo {
             addRoot( f );
             return f;
         }
-        
+
         JSFunction * _compileFunction( const char * code, JSObject * assoc ){
+            while (isspace(*code)) {
+                code++;
+            }
+
             if ( ! hasFunctionIdentifier( code ) ){
                 string s = code;
                 if ( isSimpleStatement( s ) ){
@@ -282,7 +286,7 @@ namespace mongo {
                 }
                 return JS_CompileFunction( _context , assoc , "anonymous" , 0 , 0 , s.c_str() , strlen( s.c_str() ) , "nofile_a" , 0 );
             }
-            
+
             // TODO: there must be a way in spider monkey to do this - this is a total hack
 
             string s = "return ";
@@ -294,7 +298,7 @@ namespace mongo {
                 cerr << "compile for hack failed" << endl;
                 return 0;
             }
-            
+
             jsval ret;
             if ( ! JS_CallFunction( _context , 0 , func , 0 , 0 , &ret ) ){
                 cerr << "call function for hack failed" << endl;
@@ -307,7 +311,7 @@ namespace mongo {
             return JS_ValueToFunction( _context , ret );
         }
 
-        
+
         jsval toval( double d ){
             jsval val;
             assert( JS_NewNumberValue( _context, d , &val ) );
@@ -326,14 +330,14 @@ namespace mongo {
             assert( JS_SetPrivate( _context , o , (void*)(new BSONHolder( obj->getOwned() ) ) ) );
             return o;
         }
-        
+
         jsval toval( const BSONObj* obj , bool readOnly=false ){
             JSObject * o = toJSObject( obj , readOnly );
             return OBJECT_TO_JSVAL( o );
         }
 
         jsval toval( const BSONElement& e ){
-            
+
             switch( e.type() ){
             case EOO:
             case jstNULL:
@@ -352,9 +356,9 @@ namespace mongo {
                 return toval( &embed );
             }
             case Array:{
-            
+
                 BSONObj embed = e.embeddedObject().getOwned();
-                
+
                 if ( embed.isEmpty() ){
                     return OBJECT_TO_JSVAL( JS_NewArrayObject( _context , 0 , 0 ) );
                 }
@@ -364,14 +368,14 @@ namespace mongo {
 
                 JSObject * array = JS_NewArrayObject( _context , embed.nFields() , 0 );
                 assert( array );
-                
+
                 jsval myarray = OBJECT_TO_JSVAL( array );
-                
+
                 for ( int i=0; i<n; i++ ){
                     jsval v = toval( embed[i] );
                     assert( JS_SetElement( _context , array , i , &v ) );
                 }
-                
+
                 return myarray;
             }
             case jstOID:{
@@ -393,7 +397,7 @@ namespace mongo {
                     }
                     flags++;
                 }
-                
+
                 JSObject * r = JS_NewRegExpObject( _context , (char*)e.regex() , strlen( e.regex() ) , flagNumber );
                 assert( r );
                 return OBJECT_TO_JSVAL( r );
@@ -404,7 +408,7 @@ namespace mongo {
             }
             case CodeWScope:{
                 JSFunction * func = compileFunction( e.codeWScopeCode() );
-                
+
                 BSONObj extraScope = e.codeWScopeObject();
                 if ( ! extraScope.isEmpty() ){
                     log() << "warning: CodeWScope doesn't transfer to db.eval" << endl;
@@ -412,9 +416,9 @@ namespace mongo {
 
                 return OBJECT_TO_JSVAL( JS_GetFunctionObject( func ) );
             }
-            case Date: 
+            case Date:
                 return OBJECT_TO_JSVAL( js_NewDateObjectMsec( _context , (jsdouble) e.date() ) );
-                
+
             case MinKey:
                 return OBJECT_TO_JSVAL( JS_NewObject( _context , &minkey_class , 0 , 0 ) );
 
@@ -434,7 +438,7 @@ namespace mongo {
 
                 JSObject * oid = JS_NewObject( _context , &object_id_class , 0 , 0 );
                 setProperty( oid , "str" , toval( e.dbrefOID().str().c_str() ) );
-                
+
                 setProperty( o , "id" , OBJECT_TO_JSVAL( oid ) );
                 return OBJECT_TO_JSVAL( o );
             }
@@ -443,17 +447,17 @@ namespace mongo {
                 int len;
                 void * data = (void*)e.binData( len );
                 assert( JS_SetPrivate( _context , o , data ) );
-                
+
                 setProperty( o , "len" , toval( len ) );
                 setProperty( o , "type" , toval( (int)e.binDataType() ) );
                 return OBJECT_TO_JSVAL( o );
             }
             }
-            
+
             uassert( "not done: toval" , 0 );
             return 0;
         }
-        
+
         // ------- object helpers ------
 
         JSObject * getJSObject( JSObject * o , const char * name ){
@@ -461,7 +465,7 @@ namespace mongo {
             assert( JS_GetProperty( _context , o , name , &v ) );
             return JSVAL_TO_OBJECT( v );
         }
-        
+
         JSObject * getGlobalObject( const char * name ){
             return getJSObject( JS_GetGlobalObject( _context ) , name );
         }
@@ -469,7 +473,7 @@ namespace mongo {
         JSObject * getGlobalPrototype( const char * name ){
             return getJSObject( getGlobalObject( name ) , "prototype" );
         }
-        
+
         bool hasProperty( JSObject * o , const char * name ){
             JSBool res;
             assert( JS_HasProperty( _context , o , name , & res ) );
@@ -486,12 +490,12 @@ namespace mongo {
         void setProperty( JSObject * o , const char * field , jsval v ){
             assert( JS_SetProperty( _context , o , field , &v ) );
         }
-        
+
         string typeString( jsval v ){
             JSType t = JS_TypeOfValue( _context , v );
             return JS_GetTypeName( _context , t );
         }
-        
+
         bool getBoolean( JSObject * o , const char * field ){
             return toBoolean( getProperty( o , field ) );
         }
@@ -499,7 +503,7 @@ namespace mongo {
         double getNumber( JSObject * o , const char * field ){
             return toNumber( getProperty( o , field ) );
         }
-        
+
         string getString( JSObject * o , const char * field ){
             return toString( getProperty( o , field ) );
         }
@@ -525,9 +529,9 @@ namespace mongo {
                 *idp = JSVAL_ZERO;
             return JS_TRUE;
         }
-        
+
         BSONFieldIterator * it = (BSONFieldIterator*)JSVAL_TO_PRIVATE( *statep );
-        
+
         if ( enum_op == JSENUMERATE_NEXT ){
             if ( it->more() ){
                 string name = it->next();
@@ -540,17 +544,17 @@ namespace mongo {
             }
             return JS_TRUE;
         }
-        
+
         if ( enum_op == JSENUMERATE_DESTROY ){
-            if ( it ) 
+            if ( it )
                 delete it;
             return JS_TRUE;
         }
-        
+
         uassert( "don't know what to do with this op" , 0 );
         return JS_FALSE;
     }
-    
+
     JSBool noaccess( JSContext *cx, JSObject *obj, jsval idval, jsval *vp){
         BSONHolder * holder = GETHOLDER( cx , obj );
         if ( holder->_inResolve )
@@ -558,9 +562,9 @@ namespace mongo {
         JS_ReportError( cx , "doing write op on read only operation" );
         return JS_FALSE;
     }
-    
+
     JSClass bson_ro_class = {
-        "bson_ro_object" , JSCLASS_HAS_PRIVATE | JSCLASS_NEW_RESOLVE | JSCLASS_NEW_ENUMERATE , 
+        "bson_ro_object" , JSCLASS_HAS_PRIVATE | JSCLASS_NEW_RESOLVE | JSCLASS_NEW_ENUMERATE ,
         noaccess, noaccess, JS_PropertyStub, noaccess,
         (JSEnumerateOp)bson_enumerate, (JSResolveOp)(&resolveBSONField) , JS_ConvertStub, bson_finalize ,
         JSCLASS_NO_OPTIONAL_MEMBERS
@@ -576,7 +580,7 @@ namespace mongo {
         return JS_TRUE;
     }
 
-    
+
     JSBool mark_modified( JSContext *cx, JSObject *obj, jsval idval, jsval *vp){
         BSONHolder * holder = GETHOLDER( cx , obj );
         if ( holder->_inResolve )
@@ -586,7 +590,7 @@ namespace mongo {
     }
 
     JSClass bson_class = {
-        "bson_object" , JSCLASS_HAS_PRIVATE | JSCLASS_NEW_RESOLVE | JSCLASS_NEW_ENUMERATE , 
+        "bson_object" , JSCLASS_HAS_PRIVATE | JSCLASS_NEW_RESOLVE | JSCLASS_NEW_ENUMERATE ,
         bson_add_prop, mark_modified, JS_PropertyStub, mark_modified,
         (JSEnumerateOp)bson_enumerate, (JSResolveOp)(&resolveBSONField) , JS_ConvertStub, bson_finalize ,
         JSCLASS_NO_OPTIONAL_MEMBERS
@@ -597,7 +601,7 @@ namespace mongo {
         JS_PropertyStub, JS_PropertyStub, JS_PropertyStub, JS_PropertyStub,
         JS_EnumerateStub, JS_ResolveStub, JS_ConvertStub, JS_FinalizeStub,
         JSCLASS_NO_OPTIONAL_MEMBERS
-    };    
+    };
 
     // --- global helpers ---
 
@@ -624,7 +628,7 @@ namespace mongo {
         }
 
         BSONObj out = func( args.obj() );
-        
+
         if ( out.isEmpty() ){
             *rval = JSVAL_VOID;
         }
@@ -642,27 +646,27 @@ namespace mongo {
         return JS_TRUE;
     }
 
-    JSFunctionSpec globalHelpers[] = { 
-        { "print" , &native_print , 0 , 0 , 0 } , 
-        { "nativeHelper" , &native_helper , 1 , 0 , 0 } , 
-        { "load" , &native_load , 1 , 0 , 0 } , 
-        { "gc" , &native_gc , 1 , 0 , 0 } , 
-        
-        { 0 , 0 , 0 , 0 , 0 } 
+    JSFunctionSpec globalHelpers[] = {
+        { "print" , &native_print , 0 , 0 , 0 } ,
+        { "nativeHelper" , &native_helper , 1 , 0 , 0 } ,
+        { "load" , &native_load , 1 , 0 , 0 } ,
+        { "gc" , &native_gc , 1 , 0 , 0 } ,
+
+        { 0 , 0 , 0 , 0 , 0 }
     };
 
     // ----END global helpers ----
 
-    
+
     JSBool resolveBSONField( JSContext *cx, JSObject *obj, jsval id, uintN flags, JSObject **objp ){
         assert( JS_EnterLocalRootScope( cx ) );
         Convertor c( cx );
-        
+
         BSONHolder * holder = GETHOLDER( cx , obj );
         holder->check();
-        
+
         string s = c.toString( id );
-        
+
         BSONElement e = holder->_obj[ s.c_str() ];
 
         if ( e.type() == EOO ){
@@ -670,7 +674,7 @@ namespace mongo {
             JS_LeaveLocalRootScope( cx );
             return JS_TRUE;
         }
-        
+
         jsval val = c.toval( e );
 
         assert( ! holder->_inResolve );
@@ -682,13 +686,13 @@ namespace mongo {
         JS_LeaveLocalRootScope( cx );
         return JS_TRUE;
     }
-    
+
 
     class SMScope;
-    
+
     class SMEngine : public ScriptEngine {
     public:
-        
+
         SMEngine(){
             _runtime = JS_NewRuntime(8L * 1024L * 1024L);
             uassert( "JS_NewRuntime failed" , _runtime );
@@ -703,16 +707,16 @@ namespace mongo {
         }
 
         Scope * createScope();
-        
+
         void runTest();
-        
+
         virtual bool utf8Ok() const { return JS_CStringsAreUTF8(); }
-        
+
     private:
         JSRuntime * _runtime;
         friend class SMScope;
     };
-    
+
     SMEngine * globalSMEngine;
 
 
@@ -723,58 +727,58 @@ namespace mongo {
 
 
     // ------ special helpers -------
-    
+
     JSBool object_keyset(JSContext *cx, JSObject *obj, uintN argc, jsval *argv, jsval *rval){
-        
+
         JSIdArray * properties = JS_Enumerate( cx , obj );
         assert( properties );
 
         JSObject * array = JS_NewArrayObject( cx , properties->length , 0 );
         assert( array );
-        
+
         for ( jsint i=0; i<properties->length; i++ ){
             jsid id = properties->vector[i];
             jsval idval;
             assert( JS_IdToValue( cx , id , &idval ) );
             assert( JS_SetElement( cx , array , i ,  &idval ) );
         }
-        
+
         JS_DestroyIdArray( cx , properties );
 
         *rval = OBJECT_TO_JSVAL( array );
         return JS_TRUE;
     }
-    
+
     // ------ scope ------
 
 
     JSBool no_gc(JSContext *cx, JSGCStatus status){
         return JS_FALSE;
     }
-        
+
     class SMScope : public Scope {
     public:
         SMScope(){
             _context = JS_NewContext( globalSMEngine->_runtime , 8192 );
             _convertor = new Convertor( _context );
             massert( "JS_NewContext failed" , _context );
-            
+
             JS_SetOptions( _context , JSOPTION_VAROBJFIX);
             //JS_SetVersion( _context , JSVERSION_LATEST); TODO
             JS_SetErrorReporter( _context , errorReporter );
-            
+
             _global = JS_NewObject( _context , &global_class, NULL, NULL);
             massert( "JS_NewObject failed for global" , _global );
             JS_SetGlobalObject( _context , _global );
             massert( "js init failed" , JS_InitStandardClasses( _context , _global ) );
-            
+
             JS_SetOptions( _context , JS_GetOptions( _context ) | JSOPTION_VAROBJFIX );
 
             JS_DefineFunctions( _context , _global , globalHelpers );
-            
+
             // install my special helpers
-            
-            assert( JS_DefineFunction( _context , _convertor->getGlobalPrototype( "Object" ) , 
+
+            assert( JS_DefineFunction( _context , _convertor->getGlobalPrototype( "Object" ) ,
                                        "keySet" , object_keyset , 0 , JSPROP_READONLY ) );
 
             _this = 0;
@@ -787,7 +791,7 @@ namespace mongo {
             for ( list<void*>::iterator i=_roots.begin(); i != _roots.end(); i++ ){
                 JS_RemoveRoot( _context , *i );
             }
-            
+
             if ( _this )
                 JS_RemoveRoot( _context , &_this );
 
@@ -801,20 +805,20 @@ namespace mongo {
                 _context = 0;
             }
         }
-        
+
         void reset(){
             massert( "SMScope::reset() not implemented yet" , 0 );
         }
-        
+
         void addRoot( void * root , const char * name ){
             JS_AddNamedRoot( _context , root , name );
             _roots.push_back( root );
         }
-        
+
         void init( BSONObj * data ){
             if ( ! data )
                 return;
-                
+
             BSONObjIterator i( *data );
             while ( i.more() ){
                 BSONElement e = i.next();
@@ -829,18 +833,18 @@ namespace mongo {
 
         void localConnect( const char * dbName ){
             initMongoJS( this , _context , _global , true );
-            
+
             exec( "_mongo = new Mongo();" );
             exec( ((string)"db = _mongo.getDB( \"" + dbName + "\" ); ").c_str() );
         }
-        
+
         // ----- getters ------
         double getNumber( const char *field ){
             jsval val;
             assert( JS_GetProperty( _context , _global , field , &val ) );
             return _convertor->toNumber( val );
         }
-        
+
         string getString( const char *field ){
             jsval val;
             assert( JS_GetProperty( _context , _global , field , &val ) );
@@ -851,7 +855,7 @@ namespace mongo {
         bool getBoolean( const char *field ){
             return _convertor->getBoolean( _global , field );
         }
-        
+
         BSONObj getObject( const char *field ){
             return _convertor->toObject( _convertor->getProperty( _global , field ) );
         }
@@ -863,11 +867,11 @@ namespace mongo {
         int type( const char *field ){
             jsval val;
             assert( JS_GetProperty( _context , _global , field , &val ) );
-            
+
             switch ( JS_TypeOfValue( _context , val ) ){
             case JSTYPE_VOID: return Undefined;
             case JSTYPE_NULL: return jstNULL;
-            case JSTYPE_OBJECT:	{
+            case JSTYPE_OBJECT: {
                 JSObject * o = JSVAL_TO_OBJECT( val );
                 if ( JS_IsArrayObject( _context , o ) )
                     return Array;
@@ -884,7 +888,7 @@ namespace mongo {
         }
 
         // ----- setters ------
-        
+
         void setNumber( const char *field , double val ){
             jsval v = _convertor->toval( val );
             assert( JS_SetProperty( _context , _global , field , &v ) );
@@ -902,20 +906,20 @@ namespace mongo {
 
         void setBoolean( const char *field , bool val ){
             jsval v = BOOLEAN_TO_JSVAL( val );
-            assert( JS_SetProperty( _context , _global , field , &v ) );            
+            assert( JS_SetProperty( _context , _global , field , &v ) );
         }
-        
+
         void setThis( const BSONObj * obj ){
             if ( _this )
                 JS_RemoveRoot( _context , &_this );
-            
+
             _this = _convertor->toJSObject( obj );
-            
+
             JS_AddNamedRoot( _context , &_this , "scope this" );
         }
 
         // ---- functions -----
-        
+
         ScriptingFunction createFunction( const char * code ){
             precall();
             return (ScriptingFunction)_convertor->compileFunction( code );
@@ -926,7 +930,7 @@ namespace mongo {
             boost::posix_time::time_duration timeout;
             int count;
         };
-        
+
         static JSBool checkTimeout( JSContext *cx, JSScript *script ) {
             TimeoutSpec &spec = *(TimeoutSpec *)( JS_GetContextPrivate( cx ) );
             if ( ++spec.count % 1000 != 0 )
@@ -947,7 +951,7 @@ namespace mongo {
                 spec->count = 0;
                 JS_SetContextPrivate( _context, (void*)spec );
                 JS_SetBranchCallback( _context, checkTimeout );
-            }            
+            }
         }
 
         void uninstallCheckTimeout( int timeoutMs ) {
@@ -957,32 +961,32 @@ namespace mongo {
                 JS_SetContextPrivate( _context, 0 );
             }
         }
-        
+
         void precall(){
             _error = "";
             currentScope.reset( this );
         }
-        
+
         bool exec( const string& code , const string& name = "(anon)" , bool printResult = false , bool reportError = true , bool assertOnError = true, int timeoutMs = 0 ){
             precall();
-            
+
             jsval ret = JSVAL_VOID;
-            
+
             installCheckTimeout( timeoutMs );
             JSBool worked = JS_EvaluateScript( _context , _global , code.c_str() , strlen( code.c_str() ) , name.c_str() , 0 , &ret );
             uninstallCheckTimeout( timeoutMs );
-            
+
             if ( assertOnError )
                 uassert( name + " exec failed" , worked );
-            
+
             if ( reportError && ! _error.empty() ){
                 // cout << "exec error: " << _error << endl;
                 // already printed in reportError, so... TODO
             }
-            
+
             if ( worked )
                 _convertor->setProperty( _global , "__lastres__" , ret );
-            
+
             if ( worked && printResult && ! JSVAL_IS_VOID( ret ) )
                 cout << _convertor->toString( ret ) << endl;
 
@@ -992,7 +996,7 @@ namespace mongo {
         int invoke( JSFunction * func , const BSONObj& args, int timeoutMs ){
             precall();
             jsval rval;
-            
+
             int nargs = args.nFields();
             auto_ptr<jsval> smargsPtr( new jsval[nargs] );
             jsval* smargs = smargsPtr.get();
@@ -1000,13 +1004,13 @@ namespace mongo {
             BSONObjIterator it( args );
             for ( int i=0; i<nargs; i++ )
                 smargs[i] = _convertor->toval( it.next() );
-            
+
             setObject( "args" , args , true ); // this is for backwards compatability
 
             installCheckTimeout( timeoutMs );
             JSBool ret = JS_CallFunction( _context , _this , func , nargs , smargs , &rval );
             uninstallCheckTimeout( timeoutMs );
-            
+
             if ( !ret ) {
                 return -3;
             }
@@ -1022,7 +1026,7 @@ namespace mongo {
         void gotError( string s ){
             _error = s;
         }
-        
+
         string getError(){
             return _error;
         }
@@ -1030,11 +1034,11 @@ namespace mongo {
         void injectNative( const char *field, NativeFunction func ){
             string name = field;
             _convertor->setProperty( _global , (name + "_").c_str() , PRIVATE_TO_JSVAL( func ) );
-            
+
             stringstream code;
             code << field << " = function(){ var a = [ " << field << "_ ]; for ( var i=0; i<arguments.length; i++ ){ a.push( arguments[i] ); } return nativeHelper.apply( null , a ); }";
             exec( code.str().c_str() );
-            
+
         }
 
         virtual void gc(){
@@ -1042,7 +1046,7 @@ namespace mongo {
         }
 
         JSContext *context() const { return _context; }
-        
+
     private:
         JSContext * _context;
         Convertor * _convertor;
@@ -1057,11 +1061,11 @@ namespace mongo {
     void errorReporter( JSContext *cx, const char *message, JSErrorReport *report ){
         stringstream ss;
         ss << "JS Error: " << message;
-        
+
         if ( report ){
             ss << " " << report->filename << ":" << report->lineno;
         }
-        
+
         log() << ss.str() << endl;
 
         if ( currentScope.get() ){
@@ -1077,21 +1081,21 @@ namespace mongo {
         for ( uintN i=0; i<argc; i++ ){
             string filename = c.toString( argv[i] );
             cout << "should load [" << filename << "]" << endl;
-            
+
             if ( ! s->execFile( filename , false , true , false ) ){
                 JS_ReportError( cx , ((string)"error loading file: " + filename ).c_str() );
                 return JS_FALSE;
             }
         }
-        
+
         return JS_TRUE;
     }
-    
+
 
 
     void SMEngine::runTest(){
         SMScope s;
-        
+
         s.localConnect( "foo" );
 
         s.exec( "assert( db.getMongo() )" );
@@ -1120,13 +1124,13 @@ namespace mongo {
     void Convertor::addRoot( JSFunction * f ){
         if ( ! f )
             return;
-        
+
         SMScope * scope = currentScope.get();
         uassert( "need a scope" , scope );
-        
+
         scope->addRoot( f , "cf" );
-    }    
-    
+    }
+
 }
 
 #include "sm_db.cpp"
diff --git a/shell/collection.js b/shell/collection.js
index 88d25c9f269..1601bc0f640 100644
--- a/shell/collection.js
+++ b/shell/collection.js
@@ -86,13 +86,21 @@ DBCollection.prototype._massageObject = function( q ){
         
 }
 
+
+DBCollection.prototype._validateObject = function( o ){
+    if ( o._ensureSpecial && o._checkModify )
+        throw "can't save a DBQuery object";
+}
+
 DBCollection.prototype._validateForStorage = function( o ){
+    this._validateObject( o );
     for ( var k in o ){
         if ( k.indexOf( "." ) >= 0 )
             throw "can't have . in field names [" + k + "]" ;
     }
 }
 
+
 DBCollection.prototype.find = function( query , fields , limit , skip ){
     return new DBQuery( this._mongo , this._db , this , 
                         this._fullName , this._massageObject( query ) , fields , limit , skip );
@@ -123,6 +131,7 @@ DBCollection.prototype.remove = function( t ){
 DBCollection.prototype.update = function( query , obj , upsert ){
     assert( query , "need a query" );
     assert( obj , "need an object" );
+    this._validateObject( obj );
     return this._mongo.update( this._fullName , query , obj , upsert ? true : false );
 }
 
diff --git a/shell/dbshell.cpp b/shell/dbshell.cpp
index 8db5bde907c..519e79b44b2 100644
--- a/shell/dbshell.cpp
+++ b/shell/dbshell.cpp
@@ -240,6 +240,8 @@ int main(int argc, char* argv[]) {
     scope->externalSetup();
     mongo::shellUtils::installShellUtils( *scope );
 
+    cout << "MongoDB shell version: " << mongo::versionString << endl;
+
     if ( !nodb ) { // connect to db
         cout << "url: " << url << endl;
         string setup = (string)"db = connect( \"" + fixHost( url , dbhost , port ) + "\")";
@@ -313,10 +315,12 @@ int main(int argc, char* argv[]) {
                 if ( cmd.find( " " ) > 0 )
                     cmd = cmd.substr( 0 , cmd.find( " " ) );
                 
-                scope->exec( (string)"__iscmd__ = shellHelper[\"" + cmd + "\"];" , "(shellhelp1)" , false , true , true );
-                if ( scope->getBoolean( "__iscmd__" )  ){
-                    scope->exec( (string)"shellHelper( \"" + cmd + "\" , \"" + code.substr( cmd.size() ) + "\");" , "(shellhelp2)" , false , true , false );
-                    wascmd = true;
+                if ( cmd.find( "\"" ) == string::npos ){
+                    scope->exec( (string)"__iscmd__ = shellHelper[\"" + cmd + "\"];" , "(shellhelp1)" , false , true , true );
+                    if ( scope->getBoolean( "__iscmd__" )  ){
+                        scope->exec( (string)"shellHelper( \"" + cmd + "\" , \"" + code.substr( cmd.size() ) + "\");" , "(shellhelp2)" , false , true , false );
+                        wascmd = true;
+                    }
                 }
                 
             }
diff --git a/shell/query.js b/shell/query.js
index 3ba17467f8b..1933c68ef98 100644
--- a/shell/query.js
+++ b/shell/query.js
@@ -173,6 +173,13 @@ DBQuery.prototype.forEach = function( func ){
         func( this.next() );
 }
 
+DBQuery.prototype.map = function( func ){
+    var a = [];
+    while ( this.hasNext() )
+        a.push( func( this.next() ) );
+    return a;
+}
+
 DBQuery.prototype.arrayAccess = function( idx ){
     return this.toArray()[idx];
 }
diff --git a/shell/utils.js b/shell/utils.js
index 241d2e2bfd3..ef2889fc867 100644
--- a/shell/utils.js
+++ b/shell/utils.js
@@ -188,7 +188,7 @@ ObjectId.prototype.toString = function(){
 }
 
 ObjectId.prototype.tojson = function(){
-    return "\"" + this.str + "\"";
+    return " ObjectId( \"" + this.str + "\") ";
 }
 
 ObjectId.prototype.isObjectId = true;
@@ -213,13 +213,25 @@ BinData.prototype.tojson = function(){
 }
 
 tojson = function( x ){
-    if ( x == null || x == undefined )
+    if ( x == null )
+        return "null";
+    
+    if ( x == undefined )
         return "";
     
     switch ( typeof x ){
         
-    case "string": 
-        return "\"" + x + "\"";
+    case "string": {
+        var s = "\"";
+        for ( var i=0; i<x.length; i++ ){
+            if ( x[i] == '"' ){
+                s += "\\\"";
+            }
+            else
+                s += x[i];
+        }
+        return s + "\"";
+    }
         
     case "number": 
     case "boolean":
diff --git a/stdafx.cpp b/stdafx.cpp
index 0be2fa55322..9c6213be448 100644
--- a/stdafx.cpp
+++ b/stdafx.cpp
@@ -18,8 +18,21 @@
 
 #include "stdafx.h"
 
+#if defined( __MSVC__ )
+// should probably check VS version here
+#elif defined( __GNUC__ )
+
+#if __GNUC__ < 4
+#error gcc < 4 not supported
+#endif
+
+#else
+// unknown compiler
+#endif 
+
+
 namespace mongo {
 
-    const char versionString[] = "0.9.4+";
+    const char versionString[] = "0.9.5+";
 
 } // namespace mongo
diff --git a/tools/Tool.cpp b/tools/Tool.cpp
index b4056a5058d..82ed79686ea 100644
--- a/tools/Tool.cpp
+++ b/tools/Tool.cpp
@@ -13,9 +13,9 @@ using namespace mongo;
 
 namespace po = boost::program_options;
 
-mongo::Tool::Tool( string name , string defaultDB , string defaultCollection ) : 
+mongo::Tool::Tool( string name , string defaultDB , string defaultCollection ) :
     _name( name ) , _db( defaultDB ) , _coll( defaultCollection ), _useDirect() {
-    
+
     _options = new po::options_description( name + " options" );
     _options->add_options()
         ("help","produce help message")
@@ -34,9 +34,14 @@ mongo::Tool::~Tool(){
 void mongo::Tool::printExtraHelp( ostream & out ){
 }
 
+void mongo::Tool::printHelp(ostream &out) {
+    _options->print(out);
+    printExtraHelp(out);
+}
+
 int mongo::Tool::main( int argc , char ** argv ){
     boost::filesystem::path::default_name_check( boost::filesystem::no_check );
-    
+
     po::store( po::command_line_parser( argc , argv ).
                options( *_options ).
                positional( _positonalOptions ).run() , _params );
@@ -44,8 +49,7 @@ int mongo::Tool::main( int argc , char ** argv ){
     po::notify( _params );
 
     if ( _params.count( "help" ) ){
-        _options->print( cerr );
-        printExtraHelp( cerr );
+        printHelp(cerr);
         return 0;
     }
 
@@ -53,13 +57,13 @@ int mongo::Tool::main( int argc , char ** argv ){
         const char * host = "127.0.0.1";
         if ( _params.count( "host" ) )
             host = _params["host"].as<string>().c_str();
-        
+
         string errmsg;
         if ( ! _conn.connect( host , errmsg ) ){
             cerr << "couldn't connect to [" << host << "] " << errmsg << endl;
             return -1;
         }
-        
+
         cerr << "connected to: " << host << endl;
     } else {
         _useDirect = true;
@@ -68,10 +72,10 @@ int mongo::Tool::main( int argc , char ** argv ){
         mongo::acquirePathLock();
         theFileAllocator().start();
     }
-    
+
     if ( _params.count( "db" ) )
         _db = _params["db"].as<string>();
-    
+
     if ( _params.count( "collection" ) )
         _coll = _params["collection"].as<string>();
 
diff --git a/tools/Tool.h b/tools/Tool.h
index a579ad9afd6..7f23e6554dd 100644
--- a/tools/Tool.h
+++ b/tools/Tool.h
@@ -16,21 +16,21 @@
 using std::string;
 
 namespace mongo {
-    
+
     class Tool {
     public:
         Tool( string name , string defaultDB="test" , string defaultCollection="");
         virtual ~Tool();
 
         int main( int argc , char ** argv );
-        
+
         boost::program_options::options_description_easy_init add_options(){
             return _options->add_options();
         }
         void addPositionArg( const char * name , int pos ){
             _positonalOptions.add( name , pos );
         }
-        
+
         string getParam( string name , string def="" ){
             if ( _params.count( name ) )
                 return _params[name.c_str()].as<string>();
@@ -49,9 +49,11 @@ namespace mongo {
         }
 
         virtual int run() = 0;
-        
+
+        virtual void printHelp(ostream &out);
+
         virtual void printExtraHelp( ostream & out );
-        
+
     protected:
         string _name;
 
@@ -59,7 +61,7 @@ namespace mongo {
         string _coll;
 
         mongo::DBClientBase &conn() { return _useDirect ? (mongo::DBClientBase&)_direct : (mongo::DBClientBase&)_conn; };
-        
+
     private:
         mongo::DBClientConnection _conn;
         mongo::DBDirectClient _direct;
@@ -68,7 +70,7 @@ namespace mongo {
         boost::program_options::positional_options_description _positonalOptions;
 
         boost::program_options::variables_map _params;
-        
+
     };
-    
+
 }
diff --git a/tools/dump.cpp b/tools/dump.cpp
index 03bcb2c7b0e..6bfe4b14377 100644
--- a/tools/dump.cpp
+++ b/tools/dump.cpp
@@ -69,7 +69,10 @@ public:
             
             const string name = obj.getField( "name" ).valuestr();
             const string filename = name.substr( db.size() + 1 );
-            
+        
+            if ( _coll.length() > 0 && db + "." + _coll != name && _coll != name )
+                continue;
+    
             doCollection( name.c_str() , outdir / ( filename + ".bson" ) );
             
         }
diff --git a/tools/export.cpp b/tools/export.cpp
index 49e402d5149..c0ae691a0f0 100644
--- a/tools/export.cpp
+++ b/tools/export.cpp
@@ -36,52 +36,59 @@ class Export : public Tool {
 public:
     Export() : Tool( "export" ){
         add_options()
-            ("query,q" , po::value<string>() , "query filter" )
+            ("query,q" , po::value<string>() , "query filter, as a JSON string" )
             ("fields,f" , po::value<string>() , "comma seperated list of field names e.g. -f=name,age " )
             ("csv","export to csv instead of json")
             ("out,o", po::value<string>(), "output file; if not specified, stdout is used")
             ;
     }
-    
+
     int run(){
-        const string ns = getNS();
+        string ns;
         const bool csv = hasParam( "csv" );
         ostream *outPtr = &cout;
         string outfile = getParam( "out" );
         if ( hasParam( "out" ) )
             outPtr = new ofstream( outfile.c_str() );
         ostream &out = *outPtr;
-     
+
         BSONObj * fieldsToReturn = 0;
         BSONObj realFieldsToReturn;
-        
+
         vector<string> fields;
-        
+
+        try {
+            ns = getNS();
+        } catch (...) {
+            printHelp(cerr);
+            return 1;
+        }
+
         if ( hasParam( "fields" ) ){
-            
+
             BSONObjBuilder b;
-            
+
             pcrecpp::StringPiece input( getParam( "fields" ) );
-            
+
             string f;
             pcrecpp::RE re("(\\w+),?" );
             while ( re.Consume( &input, &f ) ){
                 fields.push_back( f );
                 b.append( f.c_str() , 1 );
             }
-            
+
             realFieldsToReturn = b.obj();
             fieldsToReturn = &realFieldsToReturn;
         }
-        
+
         if ( csv && fields.size() == 0 ){
             cerr << "csv mode requires a field list" << endl;
             return -1;
         }
-            
+
 
         auto_ptr<DBClientCursor> cursor = conn().query( ns.c_str() , getParam( "query" , "" ) , 0 , 0 , fieldsToReturn , Option_SlaveOk );
-        
+
         if ( csv ){
             for ( vector<string>::iterator i=fields.begin(); i != fields.end(); i++ ){
                 if ( i != fields.begin() )
@@ -90,7 +97,7 @@ public:
             }
             out << endl;
         }
-        
+
         while ( cursor->more() ) {
             BSONObj obj = cursor->next();
             if ( csv ){
@@ -100,7 +107,7 @@ public:
                     const BSONElement & e = obj[i->c_str()];
                     if ( ! e.eoo() )
                         out << e.jsonString( TenGen , false );
-                }              
+                }
                 out << endl;
             }
             else {
diff --git a/tools/importJSON.cpp b/tools/importJSON.cpp
index 28bd48b72ba..69b20c209df 100644
--- a/tools/importJSON.cpp
+++ b/tools/importJSON.cpp
@@ -35,7 +35,7 @@ class ImportJSON : public Tool {
 public:
     ImportJSON() : Tool( "importjson" ){
         add_options()
-            ("file",po::value<string>() , "file to import from" )
+            ("file",po::value<string>() , "file to import from; if not specified stdin is used" )
             ("idbefore", "create id index before importing " )
             ("id", "create id index after importing (recommended) " )
             ("drop", "drop collection first " )
@@ -45,21 +45,23 @@ public:
 
     int run(){
         string filename = getParam( "file" );
-        if ( filename.size() == 0 ){
-            cerr << "need to specify a file!" << endl;
-            return -1;
-
-        }
 
         istream * in = &cin;
 
         ifstream file( filename.c_str() , ios_base::in | ios_base::binary);
 
-        if ( filename != "-" ){
+        if ( filename.size() > 0 && filename != "-" ){
             in = &file;
         }
 
-        string ns = getNS();
+        string ns;
+
+        try {
+            ns = getNS();
+        } catch (...) {
+            printHelp(cerr);
+            return -1;
+        }
 
         if ( hasParam( "drop" ) ){
             cout << "dropping: " << ns << endl;
diff --git a/tools/sniffer.cpp b/tools/sniffer.cpp
index c0379f4093a..2f988adb6b4 100644
--- a/tools/sniffer.cpp
+++ b/tools/sniffer.cpp
@@ -4,7 +4,7 @@
   TODO:
     large messages - need to track what's left and ingore
     single object over packet size - can only display begging of object
-    
+
     getmore
     delete
     killcursors
@@ -87,7 +87,11 @@ struct sniff_tcp {
 #define TH_URG  0x20
 #define TH_ECE  0x40
 #define TH_CWR  0x80
+
+#ifndef TH_FLAGS
 #define TH_FLAGS        (TH_FIN|TH_SYN|TH_RST|TH_ACK|TH_URG|TH_ECE|TH_CWR)
+#endif
+
     u_short th_win;                 /* window */
     u_short th_sum;                 /* checksum */
     u_short th_urp;                 /* urgent pointer */
@@ -122,43 +126,43 @@ map< Connection, long long > lastCursor;
 map< Connection, map< long long, long long > > mapCursor;
 
 void got_packet(u_char *args, const struct pcap_pkthdr *header, const u_char *packet){
-	
-	const struct sniff_ip* ip = (struct sniff_ip*)(packet + captureHeaderSize);
-	int size_ip = IP_HL(ip)*4;
-	if ( size_ip < 20 ){
-		cerr << "*** Invalid IP header length: " << size_ip << " bytes" << endl;
-		return;
-	}
-    
+
+    const struct sniff_ip* ip = (struct sniff_ip*)(packet + captureHeaderSize);
+    int size_ip = IP_HL(ip)*4;
+    if ( size_ip < 20 ){
+        cerr << "*** Invalid IP header length: " << size_ip << " bytes" << endl;
+        return;
+    }
+
     assert( ip->ip_p == IPPROTO_TCP );
 
-	const struct sniff_tcp* tcp = (struct sniff_tcp*)(packet + captureHeaderSize + size_ip);
-	int size_tcp = TH_OFF(tcp)*4;
-	if (size_tcp < 20){
-		cerr << "*** Invalid TCP header length: " << size_tcp << " bytes" << endl;
-		return;
-	}
+    const struct sniff_tcp* tcp = (struct sniff_tcp*)(packet + captureHeaderSize + size_ip);
+    int size_tcp = TH_OFF(tcp)*4;
+    if (size_tcp < 20){
+        cerr << "*** Invalid TCP header length: " << size_tcp << " bytes" << endl;
+        return;
+    }
 
     if ( ! ( serverPorts.count( ntohs( tcp->th_sport ) ) ||
              serverPorts.count( ntohs( tcp->th_dport ) ) ) ){
         return;
     }
-	
-	const u_char * payload = (const u_char*)(packet + captureHeaderSize + size_ip + size_tcp);
 
-	unsigned totalSize = ntohs(ip->ip_len);
+    const u_char * payload = (const u_char*)(packet + captureHeaderSize + size_ip + size_tcp);
+
+    unsigned totalSize = ntohs(ip->ip_len);
     assert( totalSize <= header->caplen );
 
     int size_payload = totalSize - (size_ip + size_tcp);
-	if (size_payload <= 0 )
+    if (size_payload <= 0 )
         return;
-        
+
     Connection c;
     c.srcAddr = ip->ip_src;
     c.srcPort = tcp->th_sport;
     c.dstAddr = ip->ip_dst;
     c.dstPort = tcp->th_dport;
-    
+
     if ( seen[ c ] ) {
         if ( expectedSeq[ c ] != ntohl( tcp->th_seq ) ) {
             cerr << "Warning: sequence # mismatch, there may be dropped packets" << endl;
@@ -166,11 +170,11 @@ void got_packet(u_char *args, const struct pcap_pkthdr *header, const u_char *pa
     } else {
         seen[ c ] = true;
     }
-    
+
     expectedSeq[ c ] = ntohl( tcp->th_seq ) + size_payload;
-    
+
     Message m;
-    
+
     if ( bytesRemainingInMessage[ c ] == 0 ) {
         m.setData( (MsgData*)payload , false );
         if ( !m.data->valid() ) {
@@ -204,14 +208,14 @@ void got_packet(u_char *args, const struct pcap_pkthdr *header, const u_char *pa
     }
 
     DbMessage d( m );
-    
-    cout << inet_ntoa(ip->ip_src) << ":" << ntohs( tcp->th_sport ) 
+
+    cout << inet_ntoa(ip->ip_src) << ":" << ntohs( tcp->th_sport )
          << ( serverPorts.count( ntohs( tcp->th_dport ) ) ? "  -->> " : "  <<--  " )
-         << inet_ntoa(ip->ip_dst) << ":" << ntohs( tcp->th_dport ) 
-         << " " << d.getns() 
+         << inet_ntoa(ip->ip_dst) << ":" << ntohs( tcp->th_dport )
+         << " " << d.getns()
          << "  " << m.data->len << " bytes "
          << m.data->id;
-    
+
     if ( m.data->operation() == mongo::opReply )
         cout << " - " << m.data->responseTo;
     cout << endl;
@@ -248,13 +252,13 @@ void got_packet(u_char *args, const struct pcap_pkthdr *header, const u_char *pa
         int nToReturn = d.pullInt();
         long long cursorId = d.pullInt64();
         cout << "\tgetMore nToReturn: " << nToReturn << " cursorId: " << cursorId << endl;
-        break;        
+        break;
     }
     case mongo::dbDelete:{
         int flags = d.pullInt();
         BSONObj q = d.nextJsObj();
         cout << "\tdelete flags: " << flags << " q: " << q << endl;
-        break;                
+        break;
     }
     case mongo::dbKillCursors:{
         int *x = (int *) m.data->_data;
@@ -265,8 +269,8 @@ void got_packet(u_char *args, const struct pcap_pkthdr *header, const u_char *pa
     }
     default:
         cerr << "*** CANNOT HANDLE TYPE: " << m.data->operation() << endl;
-    }    
-    
+    }
+
     if ( !forwardAddress.empty() ) {
         if ( m.data->operation() != mongo::opReply ) {
             DBClientConnection *conn = forwarder[ c ];
@@ -333,22 +337,22 @@ void usage() {
 
 int main(int argc, char **argv){
 
-	const char *dev = NULL;
-	char errbuf[PCAP_ERRBUF_SIZE];
-	pcap_t *handle;
+    const char *dev = NULL;
+    char errbuf[PCAP_ERRBUF_SIZE];
+    pcap_t *handle;
+
+    struct bpf_program fp;
+    bpf_u_int32 mask;
+    bpf_u_int32 net;
 
-	struct bpf_program fp;
-	bpf_u_int32 mask;
-	bpf_u_int32 net;
-    
     bool source = false;
     bool replay = false;
-    const char *file = 0;    
+    const char *file = 0;
 
     vector< const char * > args;
     for( int i = 1; i < argc; ++i )
         args.push_back( argv[ i ] );
-    
+
     try {
         for( unsigned i = 0; i < args.size(); ++i ) {
             const char *arg = args[ i ];
@@ -359,6 +363,7 @@ int main(int argc, char **argv){
                 forwardAddress = args[ ++i ];
             } else if ( arg == string( "--source" ) ) {
                 assert( source == false );
+                assert(args.size() > i + 2);
                 source = true;
                 replay = ( args[ ++i ] == string( "FILE" ) );
                 if ( replay )
@@ -373,26 +378,26 @@ int main(int argc, char **argv){
         usage();
         return -1;
     }
-    
+
     if ( !serverPorts.size() )
         serverPorts.insert( 27017 );
-    
+
     if ( !replay ) {
         if ( !dev ) {
             dev = pcap_lookupdev(errbuf);
             if ( ! dev ) {
-                cerr << "error finding device" << endl;
+                cerr << "error finding device: " << errbuf << endl;
                 return -1;
             }
             cout << "found device: " << dev << endl;
         }
         if (pcap_lookupnet(dev, &net, &mask, errbuf) == -1){
-            cerr << "can't get netmask!" << endl;
+            cerr << "can't get netmask: " << errbuf << endl;
             return -1;
         }
         handle = pcap_open_live(dev, SNAP_LEN, 1, 1000, errbuf);
         if ( ! handle ){
-            cerr << "error opening device!" << endl;
+            cerr << "error opening device: " << errbuf << endl;
             return -1;
         }
     } else {
@@ -400,11 +405,11 @@ int main(int argc, char **argv){
         if ( ! handle ){
             cerr << "error opening capture file!" << endl;
             return -1;
-        }        
+        }
     }
-    
+
     switch ( pcap_datalink( handle ) ){
-    case DLT_EN10MB: 
+    case DLT_EN10MB:
         captureHeaderSize = 14;
         break;
     case DLT_NULL:
@@ -413,23 +418,23 @@ int main(int argc, char **argv){
     default:
         cerr << "don't know how to handle datalink type: " << pcap_datalink( handle ) << endl;
     }
-    
-	assert( pcap_compile(handle, &fp, const_cast< char * >( "tcp" ) , 0, net) != -1 );
-	assert( pcap_setfilter(handle, &fp) != -1 );
-    
+
+    assert( pcap_compile(handle, &fp, const_cast< char * >( "tcp" ) , 0, net) != -1 );
+    assert( pcap_setfilter(handle, &fp) != -1 );
+
     cout << "sniffing... ";
     for ( set<int>::iterator i = serverPorts.begin(); i != serverPorts.end(); i++ )
         cout << *i << " ";
     cout << endl;
-    
-	pcap_loop(handle, 0 , got_packet, NULL);
 
-	pcap_freecode(&fp);
-	pcap_close(handle);
+    pcap_loop(handle, 0 , got_packet, NULL);
+
+    pcap_freecode(&fp);
+    pcap_close(handle);
 
     for( map< Connection, DBClientConnection* >::iterator i = forwarder.begin(); i != forwarder.end(); ++i )
         free( i->second );
-    
+
     return 0;
 }
 
diff --git a/util/builder.h b/util/builder.h
index 08b3237c42c..3265ab1b95e 100644
--- a/util/builder.h
+++ b/util/builder.h
@@ -96,6 +96,10 @@ namespace mongo {
             return l;
         }
 
+        void setlen( int newLen ){
+            l = newLen;
+        }
+
     private:
         /* returns the pre-grow write position */
         char* grow(int by) {