some tool cleaning, bsondump tool

7 files changed, 257 insertions, 82 deletions

diff --git a/.gitignore b/.gitignore
index 03424e335ab..a67985595e3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -75,6 +75,7 @@ mongoimport
 mongosniff
 mongobridge
 mongostat
+bsondump
 
 *.tgz
 *.zip
diff --git a/SConstruct b/SConstruct
index 8b14f5f49d9..669fc8cbad0 100644
--- a/SConstruct
+++ b/SConstruct
@@ -1135,6 +1135,8 @@ env.Alias( "tools" , [ add_exe( "mongo" + x ) for x in normalTools ] )
 for x in normalTools:
     env.Program( "mongo" + x , allToolFiles + [ "tools/" + x + ".cpp" ] )
 
+#some special tools
+env.Program( "bsondump" , allToolFiles + [ "tools/bsondump.cpp" ] )
 env.Program( "mongobridge" , allToolFiles + [ "tools/bridge.cpp" ] )
 
 # mongos
diff --git a/bson/bsoninlines.h b/bson/bsoninlines.h
index f79a71553f0..6ba914eb9ee 100644
--- a/bson/bsoninlines.h
+++ b/bson/bsoninlines.h
@@ -250,6 +250,12 @@ namespace mongo {
         case Symbol:
         case mongo::String: {
             int x = valuestrsize();
+            if ( x + fieldNameSize() + 5 != size() ){
+                StringBuilder buf;
+                buf << "Invalid string size.   element size: " << size() << " fieldNameSize: " << fieldNameSize() << " valuestrsize(): " << valuestrsize();
+                msgasserted( 13292 , buf.str() );
+            }
+
             if ( x > 0 && valuestr()[x-1] == 0 )
                 return;
             StringBuilder buf;
diff --git a/tools/bsondump.cpp b/tools/bsondump.cpp
new file mode 100644
index 00000000000..b631a11172a
--- /dev/null
+++ b/tools/bsondump.cpp
@@ -0,0 +1,124 @@
+// restore.cpp
+
+/**
+*    Copyright (C) 2008 10gen Inc.
+*
+*    This program is free software: you can redistribute it and/or  modify
+*    it under the terms of the GNU Affero General Public License, version 3,
+*    as published by the Free Software Foundation.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU Affero General Public License for more details.
+*
+*    You should have received a copy of the GNU Affero General Public License
+*    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "../pch.h"
+#include "../client/dbclient.h"
+#include "../util/mmap.h"
+#include "tool.h"
+
+#include <boost/program_options.hpp>
+
+#include <fcntl.h>
+
+using namespace mongo;
+
+namespace po = boost::program_options;
+
+class BSONDump : public BSONTool {
+
+    enum OutputType { JSON , DEBUG } _type;
+
+public:
+    
+    BSONDump() : BSONTool( "bsondump" ){
+        add_options()
+            ("type" , po::value<string>()->default_value("json") , "type of output: json,debug" )
+            ;
+        add_hidden_options()
+            ("file" , po::value<string>() , ".bson file" )
+            ;
+        addPositionArg( "file" , 1 );
+        _noconnection = true;
+    }
+    
+    virtual void printExtraHelp(ostream& out) {
+        out << "usage: " << _name << " [options] <bson filename>" << endl;
+    }
+    
+    virtual int doRun(){
+        {
+            string t = getParam( "type" );
+            if ( t == "json" )
+                _type = JSON;
+            else if ( t == "debug" )
+                _type = DEBUG;
+            else {
+                cerr << "bad type: " << t << endl;
+                return 1;
+            }
+        }
+        processFile( getParam( "file" ) );
+        return 0;
+    }
+    
+    bool debug( const BSONObj& o , int depth=0){
+        string prefix = "";
+        for ( int i=0; i<depth; i++ ){
+            prefix += "\t\t\t";
+        }
+        
+        int read = 4;
+
+        try {
+            cout << prefix << "--- new object ---\n";
+            cout << prefix << "\t size : " << o.objsize() << "\n";
+            BSONObjIterator i(o);
+            while ( i.more() ){
+                BSONElement e = i.next();
+                cout << prefix << "\t\t " << e.fieldName() << "\n" << prefix << "\t\t\t type:" << setw(3) << e.type() << " size: " << e.size() << endl;
+                if ( ( read + e.size() ) > o.objsize() ){
+                    cout << prefix << " SIZE DOES NOT WORK" << endl;
+                    return false;
+                }
+                read += e.size();
+                try {
+                    e.validate();
+                    if ( e.isABSONObj() ){
+                        if ( ! debug( e.Obj() , depth + 1 ) )
+                            return false;
+                    }
+                }
+                catch ( std::exception& e ){
+                    cout << prefix << "\t\t\t bad value: " << e.what() << endl;
+                }
+            }
+        }
+        catch ( std::exception& e ){
+            cout << prefix << "\t" << e.what() << endl;
+        }
+        return true;
+    }
+
+    virtual void gotObject( const BSONObj& o ){
+        switch ( _type ){
+        case JSON:
+            cout << o << endl;
+            break;
+        case DEBUG:
+            debug(o);
+            break;
+        default:
+            cerr << "bad type? : " << _type << endl;
+        }
+    }
+};
+
+int main( int argc , char ** argv ) {
+    BSONDump dump;
+    return dump.main( argc , argv );
+}
diff --git a/tools/restore.cpp b/tools/restore.cpp
index 14dd9ef5091..d73d716d98b 100644
--- a/tools/restore.cpp
+++ b/tools/restore.cpp
@@ -29,18 +29,15 @@ using namespace mongo;
 
 namespace po = boost::program_options;
 
-class Restore : public Tool {
+class Restore : public BSONTool {
 public:
     
     bool _drop;
-    bool _objcheck;
-    auto_ptr<Matcher> _matcher;
-    
-    Restore() : Tool( "restore" , true , "" , "" ) , _drop(false),_objcheck(false){
+    const char * _curns;
+
+    Restore() : BSONTool( "restore" ) , _drop(false){
         add_options()
             ("drop" , "drop each collection before import" )
-            ("objcheck" , "validate object before inserting" )
-            ("filter" , po::value<string>() , "filter to apply before inserting" )
             ;
         add_hidden_options()
             ("dir", po::value<string>()->default_value("dump"), "directory to restore from")
@@ -52,14 +49,10 @@ public:
         out << "usage: " << _name << " [options] [directory or filename to restore from]" << endl;
     }
 
-    int run(){
+    virtual int doRun(){
         auth();
         path root = getParam("dir");
         _drop = hasParam( "drop" );
-        _objcheck = hasParam( "objcheck" );
-        
-        if ( hasParam( "filter" ) )
-            _matcher.reset( new Matcher( fromjson( getParam( "filter" ) ) ) );
 
         /* If _db is not "" then the user specified a db name to restore as.
          *
@@ -135,84 +128,22 @@ public:
             ns += "." + l;
         }
 
-        long long fileLength = file_size( root );
-
-        if ( fileLength == 0 ) {
-            out() << "file " + root.native_file_string() + " empty, skipping" << endl;
-            return;
-        }
-
         out() << "\t going into namespace [" << ns << "]" << endl;
 
         if ( _drop ){
             out() << "\t dropping" << endl;
             conn().dropCollection( ns );
         }
+        
+        _curns = ns.c_str();
+        processFile( root );
+    }
 
-        string fileString = root.string();
-        ifstream file( fileString.c_str() , ios_base::in | ios_base::binary);
-        if ( ! file.is_open() ){
-            log() << "error opening file: " << fileString << endl;
-            return;
-        }
-
-        log(1) << "\t file size: " << fileLength << endl;
-
-        long long read = 0;
-        long long num = 0;
-        long long inserted = 0;
-
-        const int BUF_SIZE = 1024 * 1024 * 5;
-        boost::scoped_array<char> buf_holder(new char[BUF_SIZE]);
-        char * buf = buf_holder.get();
-
-        ProgressMeter m( fileLength );
-
-        while ( read < fileLength ) {
-            file.read( buf , 4 );
-            int size = ((int*)buf)[0];
-            if ( size >= BUF_SIZE ){
-                cerr << "got an object of size: " << size << "  terminating..." << endl;
-            }
-            uassert( 10264 ,  "invalid object size" , size < BUF_SIZE );
-
-            file.read( buf + 4 , size - 4 );
-
-            BSONObj o( buf );
-            if ( _objcheck && ! o.valid() ){
-                cerr << "INVALID OBJECT - going try and pring out " << endl;
-                cerr << "size: " << size << endl;
-                BSONObjIterator i(o);
-                while ( i.more() ){
-                    BSONElement e = i.next();
-                    try {
-                        e.validate();
-                    }
-                    catch ( ... ){
-                        cerr << "\t\t NEXT ONE IS INVALID" << endl;
-                    }
-                    cerr << "\t name : " << e.fieldName() << " " << e.type() << endl;
-                    cerr << "\t " << e << endl;
-                }
-            }
-            
-            if ( _matcher.get() == 0 || _matcher->matches( o ) ){
-                conn().insert( ns.c_str() , o );
-                inserted++;
-            }
-
-            read += o.objsize();
-            num++;
-
-            m.hit( o.objsize() );
-        }
-
-        uassert( 10265 ,  "counts don't match" , m.done() == fileLength );
-        out() << "\t "  << m.hits() << " objects found" << endl;
-        if ( _matcher.get() )
-            out() << "\t "  << inserted << " objects inserted" << endl;
-            
+    virtual void gotObject( const BSONObj& obj ){
+        conn().insert( _curns , obj );
     }
+
+    
 };
 
 int main( int argc , char ** argv ) {
diff --git a/tools/tool.cpp b/tools/tool.cpp
index 1af9c16bd26..bfc28a2f434 100644
--- a/tools/tool.cpp
+++ b/tools/tool.cpp
@@ -314,6 +314,101 @@ namespace mongo {
         throw UserException( 9997 , (string)"auth failed: " + errmsg );
     }
 
+    BSONTool::BSONTool( const char * name , bool objcheck ) 
+        : Tool( name , true , "" , "" ) , _objcheck( objcheck ){
+        
+        add_options()
+            ("objcheck" , "validate object before inserting" )
+            ("filter" , po::value<string>() , "filter to apply before inserting" )
+            ;
+    }
+
+
+    int BSONTool::run(){
+        _objcheck = hasParam( "objcheck" );
+        
+        if ( hasParam( "filter" ) )
+            _matcher.reset( new Matcher( fromjson( getParam( "filter" ) ) ) );
+        
+        return doRun();
+    }
+
+    long long BSONTool::processFile( const path& root ){
+        string fileString = root.string();
+        
+        long long fileLength = file_size( root );
+
+        if ( fileLength == 0 ) {
+            out() << "file " << fileString << " empty, skipping" << endl;
+            return 0;
+        }
+
+
+        ifstream file( fileString.c_str() , ios_base::in | ios_base::binary);
+        if ( ! file.is_open() ){
+            log() << "error opening file: " << fileString << endl;
+            return 0;
+        }
+
+        log(1) << "\t file size: " << fileLength << endl;
+
+        long long read = 0;
+        long long num = 0;
+        long long processed = 0;
+
+        const int BUF_SIZE = 1024 * 1024 * 5;
+        boost::scoped_array<char> buf_holder(new char[BUF_SIZE]);
+        char * buf = buf_holder.get();
+
+        ProgressMeter m( fileLength );
+
+        while ( read < fileLength ) {
+            file.read( buf , 4 );
+            int size = ((int*)buf)[0];
+            if ( size >= BUF_SIZE ){
+                cerr << "got an object of size: " << size << "  terminating..." << endl;
+            }
+            uassert( 10264 ,  "invalid object size" , size < BUF_SIZE );
+
+            file.read( buf + 4 , size - 4 );
+
+            BSONObj o( buf );
+            if ( _objcheck && ! o.valid() ){
+                cerr << "INVALID OBJECT - going try and pring out " << endl;
+                cerr << "size: " << size << endl;
+                BSONObjIterator i(o);
+                while ( i.more() ){
+                    BSONElement e = i.next();
+                    try {
+                        e.validate();
+                    }
+                    catch ( ... ){
+                        cerr << "\t\t NEXT ONE IS INVALID" << endl;
+                    }
+                    cerr << "\t name : " << e.fieldName() << " " << e.type() << endl;
+                    cerr << "\t " << e << endl;
+                }
+            }
+            
+            if ( _matcher.get() == 0 || _matcher->matches( o ) ){
+                gotObject( o );
+                processed++;
+            }
+
+            read += o.objsize();
+            num++;
+
+            m.hit( o.objsize() );
+        }
+
+        uassert( 10265 ,  "counts don't match" , m.done() == fileLength );
+        out() << "\t "  << m.hits() << " objects found" << endl;
+        if ( _matcher.get() )
+            out() << "\t "  << processed << " objects processed" << endl;
+        return processed;
+    }
+            
+
 
     void setupSignals(){}
 }
diff --git a/tools/tool.h b/tools/tool.h
index b22204485fd..70dd0e822f8 100644
--- a/tools/tool.h
+++ b/tools/tool.h
@@ -119,4 +119,20 @@ namespace mongo {
 
     };
 
+    class BSONTool : public Tool {
+        bool _objcheck;
+        auto_ptr<Matcher> _matcher;
+        
+    public:
+        BSONTool( const char * name , bool objcheck = false );
+        
+        virtual int doRun() = 0;
+        virtual void gotObject( const BSONObj& obj ) = 0;
+        
+        virtual int run();
+
+        long long processFile( const path& file );
+        
+    };
+
 }