1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
|
// importJSON.cpp
/**
* Copyright (C) 2008 10gen Inc.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License, version 3,
* as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "stdafx.h"
#include "client/dbclient.h"
#include "db/json.h"
#include "tool.h"
#include <fstream>
#include <iostream>
#include <boost/program_options.hpp>
using namespace mongo;
namespace po = boost::program_options;
class ImportJSON : public Tool {
public:
ImportJSON() : Tool( "importjson" ){
add_options()
("file",po::value<string>() , "file to import from; if not specified stdin is used" )
("drop", "drop collection first " )
;
addPositionArg( "file" , 1 );
}
int run(){
string filename = getParam( "file" );
long long fileSize = -1;
istream * in = &cin;
ifstream file( filename.c_str() , ios_base::in | ios_base::binary);
if ( filename.size() > 0 && filename != "-" ){
in = &file;
fileSize = file_size( filename );
}
string ns;
try {
ns = getNS();
} catch (...) {
printHelp(cerr);
return -1;
}
auth();
if ( hasParam( "drop" ) ){
cout << "dropping: " << ns << endl;
conn().dropCollection( ns.c_str() );
}
int num = 0;
time_t start = time(0);
ProgressMeter pm( fileSize );
const int BUF_SIZE = 1024 * 1024 * 4;
char line[ (1024 * 1024 * 4) + 128];
while ( *in ){
in->getline( line , BUF_SIZE );
char * buf = line;
while( isspace( buf[0] ) ) buf++;
int len = strlen( buf );
if ( ! len )
continue;
if ( in->rdstate() == ios_base::eofbit )
break;
assert( in->rdstate() == 0 );
try {
BSONObj o = fromjson( buf );
conn().insert( ns.c_str() , o );
}
catch ( MsgAssertionException& ma ){
cout << "exception:" << ma.toString() << endl;
cout << buf << endl;
}
num++;
if ( pm.hit( len + 1 ) ){
cout << "\t\t\t" << num << "\t" << ( num / ( time(0) - start ) ) << "/second" << endl;
}
}
return 0;
}
};
int main( int argc , char ** argv ) {
ImportJSON import;
return import.main( argc , argv );
}
|
