merge

12 files changed, 109 insertions, 42 deletions

diff --git a/buildscripts/makedist.py b/buildscripts/makedist.py
index 5f92a30871a..e088033564f 100644
--- a/buildscripts/makedist.py
+++ b/buildscripts/makedist.py
@@ -365,7 +365,9 @@ sed -i 's/dh_installinit/dh_installinit --name=mongodb/' debian/rules) || exit 1
     # If we're just packaging up nightlies, do this:
     nightly_build_mangle_files="""
 ( cd  "{pkg_name}{pkg_name_suffix}-{pkg_version}" && sed -i '/scons[[:space:]]*$/d; s^scons.*install^mkdir -p debian/{pkg_name}{pkg_name_suffix} \&\& wget http://downloads.mongodb.org/linux/mongodb-linux-{mongo_arch}-{mongo_pub_version}.tgz \&\& tar xzvf mongodb-linux-{mongo_arch}-{mongo_pub_version}.tgz \&\& find `tar tzf mongodb-linux-{mongo_arch}-{mongo_pub_version}.tgz | sed "s|/.*||" | sort -u | head -n1` -mindepth 1 -maxdepth 1 -type d | xargs -n1 -IARG mv -v ARG debian/{pkg_name}{pkg_name_suffix}/usr \&\& (rm debian/{pkg_name}{pkg_name_suffix}/usr/bin/mongosniff || true)^' debian/rules)
-( cd "{pkg_name}{pkg_name_suffix}-{pkg_version}" && sed -i 's/^BuildRequires:.*//; s/scons.*\ -c//; s/scons.*\ all//; s^scons.*install^(mkdir -p $RPM_BUILD_ROOT/usr ; cd /tmp \&\& curl http://downloads.mongodb.org/linux/mongodb-linux-{mongo_arch}-{mongo_pub_version}.tgz > mongodb-linux-{mongo_arch}-{mongo_pub_version}.tgz \&\& tar xzvf mongodb-linux-{mongo_arch}-{mongo_pub_version}.tgz \&\& find `tar tzf mongodb-linux-{mongo_arch}-{mongo_pub_version}.tgz | sed "s|/.*||" | sort -u | head -n1` -mindepth 1 -maxdepth 1 -type d | xargs -n1 -IARG cp -pRv ARG $RPM_BUILD_ROOT/usr \&\& (rm $RPM_BUILD_ROOT/usr/bin/mongosniff || true))^' rpm/mongo.spec)
+( cd "{pkg_name}{pkg_name_suffix}-{pkg_version}" && sed -i 's/^BuildRequires:.*//; s/scons.*\ -c//; s/scons.*\ all//; s^scons.*install^(mkdir -p $RPM_BUILD_ROOT/usr ; cd /tmp \&\& curl http://downloads.mongodb.org/linux/mongodb-linux-{mongo_arch}-{mongo_pub_version}.tgz > mongodb-linux-{mongo_arch}-{mongo_pub_version}.tgz \&\& tar xzvf mongodb-linux-{mongo_arch}-{mongo_pub_version}.tgz \&\& find `tar tzf mongodb-linux-{mongo_arch}-{mongo_pub_version}.tgz | sed "s|/.*||" | sort -u | head -n1` -mindepth 1 -maxdepth 1 -type d | xargs -n1 -IARG cp -pRv ARG $RPM_BUILD_ROOT/usr \&\& (rm -r $RPM_BUILD_ROOT/usr/bin/mongosniff $RPM_BUILD_ROOT/usr/lib64/libmongoclient.a $RPM_BUILD_ROOT/usr/include/mongo || true))^' rpm/mongo.spec)
+# Upstream nightlies no longer contain libmongoclient.
+( cd "{pkg_name}{pkg_name_suffix}-{pkg_version}" && sed -i '/%package devel/{{N;N;d;}}; /%description devel/{{N;N;N;N;N;d;}}; /%files devel/{{N;N;N;d;}};' rpm/mongo.spec )
 ( cd  "{pkg_name}{pkg_name_suffix}-{pkg_version}" && cat debian/rules)
 ( cd  "{pkg_name}{pkg_name_suffix}-{pkg_version}" && cat rpm/mongo.spec)
 """
diff --git a/db/btree.h b/db/btree.h
index dbe6a14272a..d487042daa8 100644
--- a/db/btree.h
+++ b/db/btree.h
@@ -356,6 +356,12 @@ namespace mongo {
         
         void forgetEndKey() { endKey = BSONObj(); }
 
+        virtual CoveredIndexMatcher *matcher() const { return _matcher.get(); }
+        
+        virtual void setMatcher( auto_ptr< CoveredIndexMatcher > matcher ) {
+            _matcher = matcher;
+        }
+        
     private:
         /* Our btrees may (rarely) have "unused" keys when items are deleted.
            Skip past them.
@@ -393,7 +399,7 @@ namespace mongo {
         BoundList bounds_;
         unsigned boundIndex_;
         const IndexSpec& _spec;
-
+        auto_ptr< CoveredIndexMatcher > _matcher;
     };
 
 
diff --git a/db/clientcursor.h b/db/clientcursor.h
index e12f34a67c9..1bce1a3b463 100644
--- a/db/clientcursor.h
+++ b/db/clientcursor.h
@@ -103,7 +103,6 @@ namespace mongo {
 
         /*const*/ CursorId cursorid;
         string ns;
-        auto_ptr<CoveredIndexMatcher> matcher;
         shared_ptr<Cursor> c;
         int pos;                                 // # objects into the cursor so far 
         BSONObj query;
@@ -198,9 +197,9 @@ namespace mongo {
         }
 
         bool currentMatches(){
-            if ( ! matcher.get() )
+            if ( ! c->matcher() )
                 return true;
-            return matcher->matchesCurrent( c.get() );
+            return c->matcher()->matchesCurrent( c.get() );
         }
 
         BSONObj current(){
diff --git a/db/cursor.h b/db/cursor.h
index 5fca8745dad..3c636bec305 100644
--- a/db/cursor.h
+++ b/db/cursor.h
@@ -20,10 +20,12 @@
 
 #include "jsobj.h"
 #include "diskloc.h"
+#include "matcher.h"
 
 namespace mongo {
     
     class Record;
+    class CoveredIndexMatcher;
 
     /* Query cursors, base class.  This is for our internal cursors.  "ClientCursor" is a separate
        concept and is for the user's cursor.
@@ -93,6 +95,17 @@ namespace mongo {
 
         virtual bool capped() const { return false; }
 
+        // The implementation may return different matchers depending on the
+        // position of the cursor.  If matcher() is nonzero at the start,
+        // matcher() should be checked each time advance() is called.
+        virtual CoveredIndexMatcher *matcher() const { return 0; }
+        
+        // A convenience function for setting the value of matcher() manually
+        // so it may accessed later.  Implementations which must generate
+        // their own matcher() should assert here.
+        virtual void setMatcher( auto_ptr< CoveredIndexMatcher > matcher ) {
+            massert( 13285, "manual matcher config not allowed", false );
+        }
     };
 
     // strategy object implementing direction of traversal.
@@ -113,6 +126,7 @@ namespace mongo {
 
     private:
         bool tailable_;
+        auto_ptr< CoveredIndexMatcher > _matcher;
         void init() {
             tailable_ = false;
         }
@@ -157,6 +171,13 @@ namespace mongo {
         virtual bool getsetdup(DiskLoc loc) { return false; }
 
         virtual bool supportGetMore() { return true; }
+
+        virtual CoveredIndexMatcher *matcher() const { return _matcher.get(); }
+        
+        virtual void setMatcher( auto_ptr< CoveredIndexMatcher > matcher ) {
+            _matcher = matcher;
+        }
+        
     };
 
     /* used for order { $natural: -1 } */
diff --git a/db/dbcommands.cpp b/db/dbcommands.cpp
index 0e23abc71e4..fc3f55b3cce 100644
--- a/db/dbcommands.cpp
+++ b/db/dbcommands.cpp
@@ -882,13 +882,12 @@ namespace mongo {
             BSONObj query = BSON( "files_id" << jsobj["filemd5"] );
             BSONObj sort = BSON( "files_id" << 1 << "n" << 1 );
 
-            shared_ptr<Cursor> cursor = MultiPlanScanner(ns.c_str(), query, sort).getBestGuess()->newCursor();
-            scoped_ptr<CoveredIndexMatcher> matcher (new CoveredIndexMatcher(query, cursor->indexKeyPattern()));
+            shared_ptr<Cursor> cursor = bestGuessCursor(ns.c_str(), query, sort);
             scoped_ptr<ClientCursor> cc (new ClientCursor(QueryOption_NoCursorTimeout, cursor, ns.c_str()));
 
             int n = 0;
             while ( cursor->ok() ){
-                if ( ! matcher->matchesCurrent( cursor.get() ) ){
+                if ( ! cursor->matcher()->matchesCurrent( cursor.get() ) ){
                     log() << "**** NOT MATCHING ****" << endl;
                     PRINT(cursor->current());
                     cursor->advance();
@@ -1226,7 +1225,6 @@ namespace mongo {
             {
                 shared_ptr<Cursor> c = theDataFileMgr.findAll( fromNs.c_str(), startLoc );
                 ClientCursor *cc = new ClientCursor(0, c, fromNs.c_str());
-                cc->matcher.reset( new CoveredIndexMatcher( BSONObj(), fromjson( "{$natural:1}" ) ) );
                 id = cc->cursorid;
             }
 
@@ -1363,13 +1361,10 @@ namespace mongo {
             map<BSONObj,int,BSONObjCmp> map;
             list<BSONObj> blah;
 
-            shared_ptr<Cursor> cursor = MultiPlanScanner(ns.c_str() , query , BSONObj() ).getBestGuess()->newCursor();
-            auto_ptr<CoveredIndexMatcher> matcher;
-            if ( ! query.isEmpty() )
-                matcher.reset( new CoveredIndexMatcher( query , cursor->indexKeyPattern() ) );
+            shared_ptr<Cursor> cursor = bestGuessCursor(ns.c_str() , query , BSONObj() );
 
             while ( cursor->ok() ){
-                if ( matcher.get() && ! matcher->matchesCurrent( cursor.get() ) ){
+                if ( cursor->matcher() && ! cursor->matcher()->matchesCurrent( cursor.get() ) ){
                     cursor->advance();
                     continue;
                 }
@@ -1498,13 +1493,10 @@ namespace mongo {
             BSONObj query = getQuery( cmdObj );
             
             BSONElementSet values;
-            shared_ptr<Cursor> cursor = MultiPlanScanner(ns.c_str() , query , BSONObj() ).getBestGuess()->newCursor();
-            auto_ptr<CoveredIndexMatcher> matcher;
-            if ( ! query.isEmpty() )
-                matcher.reset( new CoveredIndexMatcher( query , cursor->indexKeyPattern() ) );
+            shared_ptr<Cursor> cursor = bestGuessCursor(ns.c_str() , query , BSONObj() );
 
             while ( cursor->ok() ){
-                if ( matcher.get() && ! matcher->matchesCurrent( cursor.get() ) ){
+                if ( cursor->matcher() && ! cursor->matcher()->matchesCurrent( cursor.get() ) ){
                     cursor->advance();
                     continue;
                 }
diff --git a/db/json.cpp b/db/json.cpp
index 86db26c4ced..8af7292ebdd 100644
--- a/db/json.cpp
+++ b/db/json.cpp
@@ -17,11 +17,27 @@
 */
 
 #include "pch.h"
+
+#define BOOST_SPIRIT_THREADSAFE
+#if BOOST_VERSION >= 103800
+#define BOOST_SPIRIT_USE_OLD_NAMESPACE
+#include <boost/spirit/include/classic_core.hpp>
+#include <boost/spirit/include/classic_loops.hpp>
+#include <boost/spirit/include/classic_lists.hpp>
+#else
+#include <boost/spirit/core.hpp>
+#include <boost/spirit/utility/loops.hpp>
+#include <boost/spirit/utility/lists.hpp>
+#endif
+#undef assert
+#define assert MONGO_assert
+
 #include "json.h"
 #include "../bson/util/builder.h"
 #include "../util/base64.h"
 #include "../util/hex.h"
 
+
 using namespace boost::spirit;
 
 namespace mongo {
diff --git a/db/mr.cpp b/db/mr.cpp
index d3944c8e056..46f5d1fefae 100644
--- a/db/mr.cpp
+++ b/db/mr.cpp
@@ -452,12 +452,9 @@ namespace mongo {
                         readlock lock( mr.ns );
                         Client::Context ctx( mr.ns );
                         
-                        shared_ptr<Cursor> temp = MultiPlanScanner(mr.ns.c_str() , mr.filter , BSONObj() ).getBestGuess()->newCursor();
+                        shared_ptr<Cursor> temp = bestGuessCursor( mr.ns.c_str(), mr.filter, BSONObj() );
                         auto_ptr<ClientCursor> cursor( new ClientCursor( QueryOption_NoCursorTimeout , temp , mr.ns.c_str() ) );
 
-                        if ( ! mr.filter.isEmpty() )
-                            cursor->matcher.reset( new CoveredIndexMatcher( mr.filter , cursor->indexKeyPattern() ) );
-                        
                         Timer mt;
                         while ( cursor->ok() ){
                             
@@ -533,7 +530,7 @@ namespace mongo {
                         
                         assert( pm == op->setMessage( "m/r: (3/3) final reduce to collection" , db.count( mr.incLong ) ) );
 
-                        shared_ptr<Cursor> temp = MultiPlanScanner(mr.incLong.c_str() , BSONObj() , sortKey ).getBestGuess()->newCursor();
+                        shared_ptr<Cursor> temp = bestGuessCursor( mr.incLong.c_str() , BSONObj() , sortKey );
                         auto_ptr<ClientCursor> cursor( new ClientCursor( QueryOption_NoCursorTimeout , temp , mr.incLong.c_str() ) );
                         
                         while ( cursor->ok() ){
diff --git a/db/query.cpp b/db/query.cpp
index f90e2c3da06..a4d19debd76 100644
--- a/db/query.cpp
+++ b/db/query.cpp
@@ -148,7 +148,7 @@ namespace mongo {
         unsigned long long nScanned = 0;
         bool justOne = justOneOrig;
         do {
-            if ( ++nScanned % 128 == 0 && !god && !creal->matcher().docMatcher().atomic() ) {
+            if ( ++nScanned % 128 == 0 && !god && !creal->matcher()->docMatcher().atomic() ) {
                 if ( ! cc->yield() ){
                     cc.release(); // has already been deleted elsewhere
                     break;
@@ -164,7 +164,7 @@ namespace mongo {
 
             // NOTE Calling advance() may change the matcher, so it's important 
             // to try to match first.
-            bool match = creal->matcher().matches( key , rloc );
+            bool match = creal->matcher()->matches( key , rloc );
             
             if ( ! cc->c->advance() )
                 justOne = true;
@@ -299,8 +299,6 @@ namespace mongo {
             
             start = cc->pos;
             Cursor *c = cc->c.get();
-            // TODO clean up
-            MultiCursor *mc = dynamic_cast< MultiCursor * >( c );
             c->checkLocation();
             DiskLoc last;
 
@@ -327,8 +325,8 @@ namespace mongo {
                     cc = 0;
                     break;
                 }
-                CoveredIndexMatcher *matcher = mc ? &mc->matcher() : cc->matcher.get();
-                if ( !matcher->matches(c->currKey(), c->currLoc() ) ) {
+                // in some cases (clone collection) there won't be a matcher
+                if ( c->matcher() && !c->matcher()->matches(c->currKey(), c->currLoc() ) ) {
                 }
                 else {
                     //out() << "matches " << c->currLoc().toString() << '\n';
@@ -674,6 +672,7 @@ namespace mongo {
         virtual QueryOp *clone() const {
             UserQueryOp *ret = new UserQueryOp( _pq, _response, _explainSuffix, _curop );
             ret->_oldN = _n;
+            ret->_ntoskip = _ntoskip;
             // do these when implement explain - store total or per or clause?
 //            ret->_nscanned = _nscanned;
 //            ret->_nscannedObjects = _nscannedObjects;
@@ -893,14 +892,12 @@ namespace mongo {
                 shared_ptr< Cursor > multi( new MultiCursor( mps, cursor, dqo.matcher() ) );
                 cc = new ClientCursor(queryOptions, multi, ns);
             } else {
+                cursor->setMatcher( dqo.matcher() );
                 cc = new ClientCursor( queryOptions, cursor, ns );
             }
             cursorid = cc->cursorid;
             cc->query = jsobj.getOwned();
             DEV tlog() << "  query has more, cursorid: " << cursorid << endl;
-            if ( !moreClauses ) {
-                cc->matcher = dqo.matcher();
-            }
             cc->pos = n;
             cc->pq = pq_shared;
             cc->fields = pq.getFieldPtr();
diff --git a/db/queryoptimizer.cpp b/db/queryoptimizer.cpp
index 33041e702d1..0f4b209c0c9 100644
--- a/db/queryoptimizer.cpp
+++ b/db/queryoptimizer.cpp
@@ -455,6 +455,7 @@ namespace mongo {
 
     QueryPlanSet::PlanPtr QueryPlanSet::getBestGuess() const {
         assert( plans_.size() );
+        massert( 13284, "best guess plan requested, but scan and order required", !plans_[ 0 ]->scanAndOrderRequired() );
         return plans_[0];
     }
     
diff --git a/db/queryoptimizer.h b/db/queryoptimizer.h
index 92316e7a47c..61b54648975 100644
--- a/db/queryoptimizer.h
+++ b/db/queryoptimizer.h
@@ -224,7 +224,6 @@ namespace mongo {
         bool mayRunMore() const { return _i < _n; }
         BSONObj explain() const { assertNotOr(); return _currentQps->explain(); }
         bool usingPrerecordedPlan() const { assertNotOr(); return _currentQps->usingPrerecordedPlan(); }
-        QueryPlanSet::PlanPtr getBestGuess() const { assertNotOr(); return _currentQps->getBestGuess(); }
         void setBestGuessOnly() { _bestGuessOnly = true; }
     private:
         //temp
@@ -283,9 +282,14 @@ namespace mongo {
             virtual auto_ptr< CoveredIndexMatcher > newMatcher() const = 0;
         };
         // takes ownership of 'op'
-        MultiCursor( const char *ns, const BSONObj &pattern, const BSONObj &order, auto_ptr< CursorOp > op )
-        : _op( op ),
-        _mps( new MultiPlanScanner( ns, pattern, order ) ) {
+        MultiCursor( const char *ns, const BSONObj &pattern, const BSONObj &order, auto_ptr< CursorOp > op = auto_ptr< CursorOp >( 0 ) )
+        : _mps( new MultiPlanScanner( ns, pattern, order ) ) {
+            if ( op.get() ) {
+                _op = op;
+            } else {
+                _op.reset( new NoOp() );
+                _mps->setBestGuessOnly();
+            }
             if ( _mps->mayRunMore() ) {
                 nextClause();
                 if ( !ok() ) {
@@ -320,14 +324,14 @@ namespace mongo {
                 advance();
             }
         }        
-        virtual bool supportGetMore() { return false; }
+        virtual bool supportGetMore() { return true; }
         // with update we could potentially get the same document on multiple
         // indexes, but update appears to already handle this with seenObjects
         // so we don't have to do anything special here.
         virtual bool getsetdup(DiskLoc loc) {
             return _c->getsetdup( loc );   
         }
-        CoveredIndexMatcher &matcher() const { return *_matcher; }
+        virtual CoveredIndexMatcher *matcher() const { return _matcher.get(); }
     private:
         class NoOp : public CursorOp {
             virtual void init() { setComplete(); }
@@ -360,5 +364,19 @@ namespace mongo {
             query.nFields() == 1 && 
             query.firstElement().isSimpleType();
     }
+    
+    // matcher() will always work on the returned cursor
+    inline shared_ptr< Cursor > bestGuessCursor( const char *ns, const BSONObj &query, const BSONObj &sort ) {
+        if( !query.getField( "$or" ).eoo() ) {
+            return shared_ptr< Cursor >( new MultiCursor( ns, query, sort ) );
+        } else {
+            shared_ptr< Cursor > ret = QueryPlanSet( ns, query, sort ).getBestGuess()->newCursor();
+            if ( !query.isEmpty() ) {
+                auto_ptr< CoveredIndexMatcher > matcher( new CoveredIndexMatcher( query, ret->indexKeyPattern() ) );
+                ret->setMatcher( matcher );
+            }
+            return ret;
+        }
+    }
         
 } // namespace mongo
diff --git a/db/update.cpp b/db/update.cpp
index 20946a1f517..5193fad20f8 100644
--- a/db/update.cpp
+++ b/db/update.cpp
@@ -760,10 +760,10 @@ namespace mongo {
         while ( c->ok() ) {
             nscanned++;
 
-            bool atomic = c->matcher().docMatcher().atomic();
+            bool atomic = c->matcher()->docMatcher().atomic();
                 
             // May have already matched in UpdateOp, but do again to get details set correctly
-            if ( ! c->matcher().matches( c->currKey(), c->currLoc(), &details ) ){
+            if ( ! c->matcher()->matches( c->currKey(), c->currLoc(), &details ) ){
                 c->advance();
                     
                 if ( nscanned % 256 == 0 && ! atomic ){
diff --git a/jstests/or4.js b/jstests/or4.js
index 710d411024c..c1c3dc41670 100644
--- a/jstests/or4.js
+++ b/jstests/or4.js
@@ -52,9 +52,27 @@ assert.eq.automsg( "3", "t.count( {z:1} )" );
 
 assert.eq.automsg( "3", "t.find( {$or:[{a:2},{b:3}]} ).toArray().length" );
 checkArrs( "t.find().toArray()", "t.find( {$or:[{a:2},{b:3}]} ).toArray()" );
+assert.eq.automsg( "2", "t.find( {$or:[{a:2},{b:3}]} ).skip(1).toArray().length" );
 
 assert.eq.automsg( "3", "t.find( {$or:[{a:2},{b:3}]} ).batchSize( 2 ).toArray().length" );
 
+t.save( {a:1} );
+t.save( {b:4} );
 t.save( {a:2} );
 
-assert.eq.automsg( "4", "t.find( {$or:[{a:2},{b:3}]} ).batchSize( 2 ).toArray().length" );
\ No newline at end of file
+assert.eq.automsg( "4", "t.find( {$or:[{a:2},{b:3}]} ).batchSize( 2 ).toArray().length" );
+assert.eq.automsg( "4", "t.find( {$or:[{a:2},{b:3}]} ).snapshot().toArray().length" );
+
+t.save( {a:1,b:3} );
+assert.eq.automsg( "4", "t.find( {$or:[{a:2},{b:3}]} ).batchSize(-4).toArray().length" );
+
+assert.eq.automsg( "[1,2]", "t.distinct( 'a', {$or:[{a:2},{b:3}]} )" );
+
+assert.eq.automsg( "[{a:2},{a:null},{a:1}]", "t.group( {key:{a:1}, cond:{$or:[{a:2},{b:3}]}, reduce:function( x, y ) { }, initial:{} } )" );
+assert.eq.automsg( "5", "t.mapReduce( function() { emit( 'a', this.a ); }, function( key, vals ) { return vals.length; }, {query:{$or:[{a:2},{b:3}]}} ).counts.input" );
+
+t.remove( {} );
+
+t.save( {a:[1,2]} );
+assert.eq.automsg( "1", "t.find( {$or:[{a:1},{a:2}]} ).toArray().length" );
+assert.eq.automsg( "1", "t.count( {$or:[{a:1},{a:2}]} )" );