Merge branch 'master' of github.com:mongodb/mongo

12 files changed, 218 insertions, 20 deletions

diff --git a/db/clientcursor.cpp b/db/clientcursor.cpp
index 6d058cbeb91..cac55506263 100644
--- a/db/clientcursor.cpp
+++ b/db/clientcursor.cpp
@@ -258,8 +258,7 @@ namespace mongo {
         _cursorid = allocCursorId_inlock();
         clientCursorsById.insert( make_pair(_cursorid, this) );
         
-#if 0  
-        { 
+        if ( ! _c->modifiedKeys() ) { 
             // store index information so we can decide if we can 
             // get something out of the index key rather than full object
             
@@ -274,7 +273,7 @@ namespace mongo {
                 x++;
             }
         }
-#endif
+
     }
     
 
@@ -292,6 +291,24 @@ namespace mongo {
         }
     }
 
+    bool ClientCursor::getFieldsDotted( const string& name, BSONElementSet &ret ) {
+        
+        map<string,int>::const_iterator i = _indexedFields.find( name );
+        if ( i == _indexedFields.end() ){
+            current().getFieldsDotted( name , ret );
+            return false;
+        }
+
+        int x = i->second;
+        
+        BSONObjIterator it( currKey() );
+        while ( x && it.more() )
+            it.next();
+        assert( x == 0 );
+        ret.insert( it.next() );
+        return true;
+    }
+
     /* call when cursor's location changes so that we can update the
        cursorsbylocation map.  if you are locked and internally iterating, only
        need to call when you are ready to "unlock".
diff --git a/db/clientcursor.h b/db/clientcursor.h
index 0f08a97134d..ddd39a18b25 100644
--- a/db/clientcursor.h
+++ b/db/clientcursor.h
@@ -221,7 +221,15 @@ namespace mongo {
         bool advance(){ return _c->advance(); }
         BSONObj current() { return _c->current(); }
         DiskLoc currLoc() { return _c->currLoc(); }
-        BSONObj currKey() { return _c->currKey(); }
+        BSONObj currKey() const { return _c->currKey(); }
+        
+
+        /**
+         * same as BSONObj::getFieldsDotted
+         * if it can be retrieved from key, it is
+         * @return if this was retrieved from key
+         */
+        bool getFieldsDotted( const string& name, BSONElementSet &ret );
 
         bool currentIsDup() { return _c->getsetdup( _c->currLoc() ); }
         
diff --git a/db/commands/distinct.cpp b/db/commands/distinct.cpp
index 19bd493e5d4..162b4cbfa85 100644
--- a/db/commands/distinct.cpp
+++ b/db/commands/distinct.cpp
@@ -34,7 +34,7 @@ namespace mongo {
 
         bool run(const string& dbname, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl ){
             string ns = dbname + '.' + cmdObj.firstElement().valuestr();
-            
+
             string key = cmdObj["key"].valuestrsafe();
             BSONObj keyPattern = BSON( key << 1 );
 
@@ -47,15 +47,59 @@ namespace mongo {
             BSONArrayBuilder arr( bb );
             BSONElementSet values;
             
-            shared_ptr<Cursor> cursor = bestGuessCursor(ns.c_str() , query , BSONObj() );
-            scoped_ptr<ClientCursor> cc (new ClientCursor(QueryOption_NoCursorTimeout, cursor, ns));
+            long long nscanned = 0; // locations looked at
+            long long nscannedObjects = 0; // full objects looked at
+            long long n = 0; // matches
+            MatchDetails md;
+            
+            NamespaceDetails * d = nsdetails( ns.c_str() );
+
+            if ( ! d ){
+                result.appendArray( "values" , BSONObj() );
+                result.append( "stats" , BSON( "n" << 0 << "nscanned" << 0 << "nscannedObjects" << 0 ) );
+                return true;
+            }
+
+            shared_ptr<Cursor> cursor;
+            if ( ! query.isEmpty() ) {
+                cursor = bestGuessCursor(ns.c_str() , query , BSONObj() );
+            }
+            else {
+
+                // query is empty, so lets see if we can find an index
+                // with the key so we don't have to hit the raw data
+                NamespaceDetails::IndexIterator ii = d->ii();
+                while ( ii.more() ){
+                    IndexDetails& idx = ii.next();
 
+                    if ( d->isMultikey( ii.pos() - 1 ) )
+                        continue;
+
+                    if ( idx.inKeyPattern( key ) ) {
+                        cursor = bestGuessCursor( ns.c_str() , BSONObj() , idx.keyPattern() );
+                        break;
+                    }
+                        
+                }
+                
+                if ( ! cursor.get() )
+                    cursor = bestGuessCursor(ns.c_str() , query , BSONObj() );
+                
+            }
+            
+
+
+            scoped_ptr<ClientCursor> cc (new ClientCursor(QueryOption_NoCursorTimeout, cursor, ns));
+            
             while ( cursor->ok() ){
-                if ( !cursor->matcher() || cursor->matcher()->matchesCurrent( cursor.get() ) ){
-                    BSONObj o = cursor->current();
+                nscanned++;
+                bool loadedObject = false;
+                
+                if ( !cursor->matcher() || cursor->matcher()->matchesCurrent( cursor.get() , &md ) ){
+                    n++;
 
                     BSONElementSet temp;
-                    o.getFieldsDotted( key, temp );
+                    loadedObject = ! cc->getFieldsDotted( key , temp );
                     
                     for ( BSONElementSet::iterator i=temp.begin(); i!=temp.end(); ++i ){
                         BSONElement e = *i;
@@ -73,6 +117,9 @@ namespace mongo {
                     }
                 }
 
+                if ( loadedObject || md.loadedObject ) 
+                    nscannedObjects++;
+
                 cursor->advance();
 
                 if (!cc->yieldSometimes())
@@ -84,7 +131,15 @@ namespace mongo {
             assert( start == bb.buf() );
             
             result.appendArray( "values" , arr.done() );
-
+            
+            {
+                BSONObjBuilder b;
+                b.appendNumber( "n" , n );
+                b.appendNumber( "nscanned" , nscanned );
+                b.appendNumber( "nscannedObjects" , nscannedObjects );
+                result.append( "stats" , b.obj() );
+            }
+            
             return true;
         }
 
diff --git a/db/commands/isself.cpp b/db/commands/isself.cpp
index 7525f68ea25..626728cc0a2 100644
--- a/db/commands/isself.cpp
+++ b/db/commands/isself.cpp
@@ -73,7 +73,7 @@ namespace mongo {
         vector<string> out;
 
         int ret = getaddrinfo(iporhost.data(), portNum.c_str(), &hints, &addrs);
-        if ( ret == 0 ){
+        if ( ret ){
             warning() << "getaddrinfo(\"" << iporhost.data() << "\") failed: " << gai_strerror(ret) << endl;
             return out;
         }
diff --git a/db/index.cpp b/db/index.cpp
index f2aad60de2e..7326430096d 100644
--- a/db/index.cpp
+++ b/db/index.cpp
@@ -53,6 +53,18 @@ namespace mongo {
         }
     }
 
+    int IndexDetails::keyPatternOffset( const string& key ) const {
+        BSONObjIterator i( keyPattern() );
+        int n = 0;
+        while ( i.more() ){
+            BSONElement e = i.next();
+            if ( key == e.fieldName() )
+                return n;
+            n++;
+        }
+        return -1;
+    }
+
     const IndexSpec& IndexDetails::getSpec() const {
         scoped_lock lk(NamespaceDetailsTransient::_qcMutex);
         return NamespaceDetailsTransient::get_inlock( info.obj()["ns"].valuestr() ).getIndexSpec( this );
diff --git a/db/index.h b/db/index.h
index 52e88dd2f56..f7c900947de 100644
--- a/db/index.h
+++ b/db/index.h
@@ -77,6 +77,13 @@ namespace mongo {
             return info.obj().getObjectField("key");
         }
 
+        /**
+         * @return offset into keyPattern for key
+                   -1 if doesn't exist
+         */
+        int keyPatternOffset( const string& key ) const;
+        bool inKeyPattern( const string& key ) const { return keyPatternOffset( key ) >= 0; }
+        
         /* true if the specified key is in the index */
         bool hasKey(const BSONObj& key);
         bool wouldCreateDup(const BSONObj& key, DiskLoc self);
diff --git a/jstests/distinct1.js b/jstests/distinct1.js
index 433e0515e23..5e47400bfd4 100644
--- a/jstests/distinct1.js
+++ b/jstests/distinct1.js
@@ -2,6 +2,8 @@
 t = db.distinct1;
 t.drop();
 
+assert.eq( 0 , t.distinct( "a" ).length , "test empty" );
+
 t.save( { a : 1 } )
 t.save( { a : 2 } )
 t.save( { a : 2 } )
diff --git a/jstests/distinct_index1.js b/jstests/distinct_index1.js
new file mode 100644
index 00000000000..8677457a48a
--- /dev/null
+++ b/jstests/distinct_index1.js
@@ -0,0 +1,50 @@
+
+t = db.distinct_index1
+t.drop();
+
+function r( x ){
+    return Math.floor( Math.sqrt( x * 123123 ) ) % 10;
+}
+
+function d( k , q ){
+    return t.runCommand( "distinct" , { key : k , query : q || {} } )
+}
+
+for ( i=0; i<1000; i++ ){
+    o = { a : r(i*5) , b : r(i) };
+    t.insert( o );
+}
+
+x = d( "a" );
+assert.eq( 1000 , x.stats.n , "AA1" )
+assert.eq( 1000 , x.stats.nscanned , "AA2" )
+assert.eq( 1000 , x.stats.nscannedObjects , "AA3" )
+
+x = d( "a" , { a : { $gt : 5 } } );
+assert.eq( 398 , x.stats.n , "AB1" )
+assert.eq( 1000 , x.stats.nscanned , "AB2" )
+assert.eq( 1000 , x.stats.nscannedObjects , "AB3" )
+
+x = d( "b" , { a : { $gt : 5 } } );
+assert.eq( 398 , x.stats.n , "AC1" )
+assert.eq( 1000 , x.stats.nscanned , "AC2" )
+assert.eq( 1000 , x.stats.nscannedObjects , "AC3" )
+
+
+
+t.ensureIndex( { a : 1 } )
+
+x = d( "a" );
+assert.eq( 1000 , x.stats.n , "BA1" )
+assert.eq( 1000 , x.stats.nscanned , "BA2" )
+assert.eq( 0 , x.stats.nscannedObjects , "BA3" )
+
+x = d( "a" , { a : { $gt : 5 } } );
+assert.eq( 398 , x.stats.n , "BB1" )
+assert.eq( 398 , x.stats.nscanned , "BB2" )
+assert.eq( 0 , x.stats.nscannedObjects , "BB3" )
+
+x = d( "b" , { a : { $gt : 5 } } );
+assert.eq( 398 , x.stats.n , "BC1" )
+assert.eq( 398 , x.stats.nscanned , "BC2" )
+assert.eq( 398 , x.stats.nscannedObjects , "BC3" )
diff --git a/jstests/replsets/buildindexes.js b/jstests/replsets/buildindexes.js
index bf16b923ddb..76de7974f39 100644
--- a/jstests/replsets/buildindexes.js
+++ b/jstests/replsets/buildindexes.js
@@ -20,8 +20,11 @@ doTest = function( signal ) {
     slaveConns[i].setSlaveOk();
     slave.push(slaveConns[i].getDB(name));
   }
-      
+  replTest.awaitReplication();
+
+  print("creating an index on x");
   master.x.ensureIndex({y : 1});
+  printjson(master.x.stats());
 
   for (var i=0; i<100; i++) {
     master.x.insert({x:1,y:"abc",c:1});
@@ -34,11 +37,14 @@ doTest = function( signal ) {
   print("namespace: "+ns);
 
   // can't query system.indexes from slave, so we'll look at coll.stats()
-  printjson(slave[0].runCommand({replSetGetStatus:1}));
+  printjson(slave[0].adminCommand({replSetGetStatus:1}));
   printjson(slave[0].getSisterDB("local").system.replset.findOne());
+  printjson(master.stats());
   printjson(slave[0].stats());
   printjson(slave[1].stats());
-  printjson(master.stats());
+  printjson(master.x.stats());
+  printjson(slave[0].x.stats());
+  printjson(slave[1].x.stats());
   print("sleeping");  
   sleep(20000);
   var indexes = slave[0].stats().indexes;
diff --git a/jstests/slowWeekly/disk_reuse1.js b/jstests/slowWeekly/disk_reuse1.js
new file mode 100644
index 00000000000..4e504c08b9f
--- /dev/null
+++ b/jstests/slowWeekly/disk_reuse1.js
@@ -0,0 +1,41 @@
+
+t = db.disk_reuse1;
+t.drop();
+
+N = 10000;
+
+function k(){
+    return Math.floor( Math.random() * N );
+}
+
+s = "";
+while ( s.length < 1024 )
+    s += "abc";
+
+state = {}
+
+for ( i=0; i<N; i++ )
+    t.insert( { _id : i , s : s } );
+
+orig = t.stats();
+
+t.remove();
+
+for ( i=0; i<N; i++ )
+    t.insert( { _id : i , s : s } );
+
+assert.eq( orig.storageSize , t.stats().storageSize , "A" )
+
+for ( j=0; j<100; j++ ){
+    for ( i=0; i<N; i++ ){
+        var r = Math.random();
+        if ( r > .5 )
+            t.remove( { _id : i } )
+        else
+            t.insert( { _id : i , s : s } )
+    }
+
+    //printjson( t.stats() );    
+
+    assert.eq( orig.storageSize , t.stats().storageSize , "B" + j  )
+}
diff --git a/jstests/slowWeekly/query_yield1.js b/jstests/slowWeekly/query_yield1.js
index e996b53ecf1..1a95b877199 100644
--- a/jstests/slowWeekly/query_yield1.js
+++ b/jstests/slowWeekly/query_yield1.js
@@ -2,10 +2,10 @@
 t = db.query_yield1;
 t.drop()
 
-N = 10000;
+N = 20000;
 i = 0;
 
-q = function(){ var x=this.n; for ( var i=0; i<500; i++ ){ x = x * 2; } return false; }
+q = function(){ var x=this.n; for ( var i=0; i<250; i++ ){ x = x * 2; } return false; }
 
 while ( true ){
     function fill(){
@@ -59,7 +59,7 @@ while ( ( (new Date()).getTime() - start ) < ( time * 2 ) ){
         assert.eq( 1 , x.inprog.length , "nothing in prog" );
     }
     
-    assert.gt( 50 , me );
+    assert.gt( 200 , me , "took too long for me to run" );
 
     if ( x.inprog.length == 0 )
         break;
diff --git a/jstests/slowWeekly/query_yield2.js b/jstests/slowWeekly/query_yield2.js
index e13fabea0f2..f0e87bbed08 100644
--- a/jstests/slowWeekly/query_yield2.js
+++ b/jstests/slowWeekly/query_yield2.js
@@ -2,10 +2,10 @@
 t = db.query_yield2;
 t.drop()
 
-N = 100;
+N = 200;
 i = 0;
 
-q = function(){ var x=this.n; for ( var i=0; i<50000; i++ ){ x = x * 2; } return false; }
+q = function(){ var x=this.n; for ( var i=0; i<25000; i++ ){ x = x * 2; } return false; }
 
 while ( true ){
     function fill(){