summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorBen Becker <benjamin.becker@10gen.com>2012-01-05 12:20:25 -0800
committerBen Becker <benjamin.becker@10gen.com>2012-01-05 12:20:25 -0800
commit186721566a69192a91d75f424757b5edb2a6c257 (patch)
treecfa55309e275a50b7e58441f73a5b993cbd5a6fe
parent37b331b75677016c9a8591c92befc369581a3b17 (diff)
downloadmongo-186721566a69192a91d75f424757b5edb2a6c257.tar.gz
SERVER-4593: Ensure mongoimport allows imports of up to 16MB in BSON size.
-rw-r--r--debian/mongoimport.12
-rw-r--r--src/mongo/bson/util/builder.h9
-rw-r--r--src/mongo/db/json.cpp16
-rw-r--r--src/mongo/tools/import.cpp18
4 files changed, 31 insertions, 14 deletions
diff --git a/debian/mongoimport.1 b/debian/mongoimport.1
index f75dbde326e..92688efd2b1 100644
--- a/debian/mongoimport.1
+++ b/debian/mongoimport.1
@@ -82,7 +82,7 @@ make sure this is indexed
stop importing at first error rather than continuing
.TP
.B \-\-jsonArray
-load a json array, not one item per line. Currently limited to 4MB.
+load a json array, not one item per line. Currently limited to 16MB.
.SH "COPYRIGHT"
.PP
Copyright 2007\-2011 10gen
diff --git a/src/mongo/bson/util/builder.h b/src/mongo/bson/util/builder.h
index a5c71a8db92..54b1ac1384c 100644
--- a/src/mongo/bson/util/builder.h
+++ b/src/mongo/bson/util/builder.h
@@ -19,6 +19,8 @@
#include <cfloat>
#include <string>
+#include <sstream>
+#include <iostream>
#include <string.h>
#include <stdio.h>
#include "../inline_decls.h"
@@ -203,8 +205,11 @@ namespace mongo {
a = 512;
if ( l > a )
a = l + 16 * 1024;
- if ( a > BufferMaxSize )
- msgasserted(13548, "BufBuilder grow() > 64MB");
+ if ( a > BufferMaxSize ) {
+ std::stringstream ss;
+ ss << "BufBuilder attempted to grow() to " << a << " bytes, past the 64MB limit.";
+ msgasserted(13548, ss.str().c_str());
+ }
data = (char *) al.Realloc(data, a);
size= a;
}
diff --git a/src/mongo/db/json.cpp b/src/mongo/db/json.cpp
index 73457a2bfbb..f27ccbf896e 100644
--- a/src/mongo/db/json.cpp
+++ b/src/mongo/db/json.cpp
@@ -44,15 +44,17 @@ namespace mongo {
struct ObjectBuilder : boost::noncopyable {
~ObjectBuilder() {
- unsigned i = builders.size();
- if ( i ) {
- i--;
- for ( ; i>=1; i-- ) {
- if ( builders[i] ) {
- builders[i]->done();
+ DESTRUCTOR_GUARD(
+ unsigned i = builders.size();
+ if ( i ) {
+ i--;
+ for ( ; i>=1; i-- ) {
+ if ( builders[i] ) {
+ builders[i]->done();
+ }
}
}
- }
+ );
}
BSONObjBuilder *back() {
return builders.back().get();
diff --git a/src/mongo/tools/import.cpp b/src/mongo/tools/import.cpp
index 24741ed46ad..4c3281f5a8c 100644
--- a/src/mongo/tools/import.cpp
+++ b/src/mongo/tools/import.cpp
@@ -30,6 +30,8 @@
#include <boost/algorithm/string.hpp>
using namespace mongo;
+using std::string;
+using std::stringstream;
namespace po = boost::program_options;
@@ -45,7 +47,7 @@ class Import : public Tool {
bool _doimport;
bool _jsonArray;
vector<string> _upsertFields;
- static const int BUF_SIZE = 1024 * 1024 * 4;
+ static const int BUF_SIZE = 1024 * 1024 * 16;
void csvTokenizeRow(const string& row, vector<string>& tokens) {
bool inQuotes = false;
@@ -146,7 +148,11 @@ class Import : public Tool {
return -1;
int jslen;
- o = fromjson(buf, &jslen);
+ try {
+ o = fromjson(buf, &jslen);
+ } catch ( MsgAssertionException& e ) {
+ uasserted(13293, string("BSON representation of supplied JSON array is too large: ") + e.what());
+ }
len += jslen;
return len;
@@ -176,7 +182,11 @@ class Import : public Tool {
*end = 0;
end--;
}
- o = fromjson( line );
+ try {
+ o = fromjson( line );
+ } catch ( MsgAssertionException& e ) {
+ uasserted(13504, string("BSON representation of supplied JSON is too large: ") + e.what());
+ }
return true;
}
@@ -259,7 +269,7 @@ public:
("upsert", "insert or update objects that already exist" )
("upsertFields", po::value<string>(), "comma-separated fields for the query part of the upsert. You should make sure this is indexed" )
("stopOnError", "stop importing at first error rather than continuing" )
- ("jsonArray", "load a json array, not one item per line. Currently limited to 4MB." )
+ ("jsonArray", "load a json array, not one item per line. Currently limited to 16MB." )
;
add_hidden_options()
("noimport", "don't actually import. useful for benchmarking parser" )