diff options
author | David Golden <xdg@xdg.me> | 2019-07-16 10:09:09 -0400 |
---|---|---|
committer | David Golden <xdg@xdg.me> | 2019-07-16 11:07:21 -0400 |
commit | c652c07f6c460a6bd2d1ed0c75d934017aa99fb9 (patch) | |
tree | 3bdd9c36c9be3fcc3dabd670639a6df7e7002c23 /src/mongo/gotools/src/github.com | |
parent | c8259b383f8e6f9ce9229774881a55c8dc41e222 (diff) | |
download | mongo-c652c07f6c460a6bd2d1ed0c75d934017aa99fb9.tar.gz |
Import tools: f6fdd97dbea59639732068720558c83b88947e15 from branch v4.1
ref: 4eba41d830..f6fdd97dbe
for: 4.3.1
TOOLS-2174 mongodump and mongoexport fail to dump autoIndexId false capped collection due to invalid hint
TOOLS-2276 dump/export/files query should use extJSON v2 and ordered BSON
TOOLS-2302 Update supported build platforms to match server 4.2
TOOLS-2332 oplog_replay_local_rs.js fails on server latest
Diffstat (limited to 'src/mongo/gotools/src/github.com')
10 files changed, 309 insertions, 69 deletions
diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common-pvt.yml b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common-pvt.yml index 1d625e37019..479b422f873 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common-pvt.yml +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common-pvt.yml @@ -196,7 +196,7 @@ buildvariants: - name: amazonlinux64-enterprise display_name: Amazon Linux 64 Enterprise run_on: - - linux-64-amzn-test + - amazon1-2018-test expansions: build_tags: "sasl gssapi ssl" tasks: *atlas_live_tasks @@ -248,7 +248,7 @@ buildvariants: - name: rhel62-enterprise display_name: RHEL 6.2 Enterprise run_on: - - rhel62-test + - rhel62-small expansions: build_tags: "sasl gssapi ssl" tasks: *atlas_live_tasks @@ -256,7 +256,7 @@ buildvariants: - name: rhel70-enterprise display_name: RHEL 7.0 Enterprise run_on: - - rhel70 + - rhel70-small expansions: build_tags: "sasl gssapi ssl" tasks: *atlas_live_tasks @@ -293,6 +293,14 @@ buildvariants: build_tags: "sasl gssapi ssl" tasks: *atlas_live_tasks +- name: ubuntu1804-enterprise + display_name: Ubuntu 18.04 Enterprise + run_on: + - ubuntu1804-test + expansions: + build_tags: "sasl gssapi ssl" + tasks: *atlas_live_tasks + ####################################### # Windows Buildvariants # ####################################### @@ -300,7 +308,7 @@ buildvariants: - name: windows-64-ssl display_name: Windows 64-bit SSL run_on: - - windows-64-vs2013-compile + - windows-64-vs2017-test expansions: build_tags: "ssl" tasks: *atlas_live_tasks @@ -319,6 +327,16 @@ buildvariants: build_tags: "ssl" tasks: *atlas_live_tasks +- name: ubuntu1804-arm64-ssl + display_name: ZAP ARM64 Ubuntu 18.04 SSL + run_on: + - ubuntu1804-arm64-small + stepback: false + batchtime: 10080 # weekly + expansions: + build_tags: "ssl" + tasks: *atlas_live_tasks + ####################################### # Power Buildvariants # ####################################### @@ -343,6 +361,16 @@ buildvariants: build_tags: 'ssl sasl gssapi' tasks: *atlas_live_tasks +- name: ubuntu1804-ppc64le-enterprise + display_name: ZAP PPC64LE Ubuntu 18.04 Enterprise + run_on: + - ubuntu1804-power8-test + stepback: false + batchtime: 10080 # weekly + expansions: + build_tags: 'ssl sasl gssapi' + tasks: *atlas_live_tasks + ####################################### # Z (s390x) Buildvariants # ####################################### @@ -357,6 +385,16 @@ buildvariants: build_tags: "sasl gssapi ssl" tasks: *atlas_live_tasks +- name: rhel72-s390x-enterprise + display_name: ZAP s390x RHEL 7.2 Enterprise + run_on: + - rhel72-zseries-test + stepback: false + batchtime: 10080 # weekly + expansions: + build_tags: "sasl gssapi ssl" + tasks: *atlas_live_tasks + - name: ubuntu1604-s390x-enterprise display_name: ZAP s390x Ubuntu 16.04 Enterprise run_on: diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common.yml b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common.yml index b756c2f19ce..db5fc62a185 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common.yml +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common.yml @@ -182,28 +182,28 @@ mongo_tools_variables: - name: integration-4.2 - name: integration-4.2-auth distros: - - windows-64-vs2013-test + - windows-64-vs2017-test - name: qa-tests-3.2 distros: - - windows-64-vs2013-test + - windows-64-vs2017-test - name: qa-tests-3.4 distros: - - windows-64-vs2013-test + - windows-64-vs2017-test - name: qa-tests-3.6 distros: - - windows-64-vs2013-test + - windows-64-vs2017-test - name: qa-tests-4.0 distros: - - windows-64-vs2013-test + - windows-64-vs2017-test - name: qa-tests-4.2 distros: - - windows-64-vs2013-test + - windows-64-vs2017-test - name: qa-dump-restore-with-archiving-current distros: - - windows-64-vs2013-test + - windows-64-vs2017-test - name: qa-dump-restore-with-gzip-current distros: - - windows-64-vs2013-test + - windows-64-vs2017-test - name: qa-tests-unstable - name: unit windows_64_ssl_task_list: &windows_64_ssl_tasks @@ -225,25 +225,25 @@ mongo_tools_variables: - name: integration-4.2-auth - name: kerberos distros: - - windows-64-vs2013-test + - windows-64-vs2017-test - name: qa-tests-3.2 distros: - - windows-64-vs2013-test + - windows-64-vs2017-test - name: qa-tests-3.4 distros: - - windows-64-vs2013-test + - windows-64-vs2017-test - name: qa-tests-3.6 distros: - - windows-64-vs2013-test + - windows-64-vs2017-test - name: qa-tests-4.0 distros: - - windows-64-vs2013-test + - windows-64-vs2017-test - name: qa-tests-4.2 distros: - - windows-64-vs2013-test + - windows-64-vs2017-test - name: qa-tests-unstable distros: - - windows-64-vs2013-test + - windows-64-vs2017-test - name: unit - name: native-cert-ssl-current rhel71_ppc64le_enterprise_task_list: &rhel71_ppc64le_enterprise_tasks @@ -1836,7 +1836,7 @@ buildvariants: - name: rhel62 display_name: RHEL 6.2 run_on: - - rhel62-test + - rhel62-small expansions: <<: *mongod_default_startup_args <<: *mongo_default_startup_args @@ -1851,7 +1851,7 @@ buildvariants: - name: rhel62-ssl display_name: RHEL 6.2 SSL run_on: - - rhel62-test + - rhel62-small expansions: <<: *mongod_ssl_startup_args <<: *mongo_ssl_startup_args @@ -1871,7 +1871,7 @@ buildvariants: - name: rhel62-enterprise display_name: RHEL 6.2 Enterprise run_on: - - rhel62-test + - rhel62-small expansions: <<: *mongod_default_startup_args <<: *mongo_default_startup_args @@ -1890,7 +1890,7 @@ buildvariants: - name: rhel70 display_name: RHEL 7.0 run_on: - - rhel70 + - rhel70-small expansions: build_tags: "" tasks: @@ -1900,7 +1900,7 @@ buildvariants: - name: rhel70-enterprise display_name: RHEL 7.0 Enterprise run_on: - - rhel70 + - rhel70-small expansions: build_tags: "sasl gssapi ssl" tasks: @@ -2035,7 +2035,7 @@ buildvariants: - name: windows-64 display_name: Windows 64-bit run_on: - - windows-64-vs2013-test + - windows-64-vs2017-test expansions: <<: *mongod_default_startup_args <<: *mongo_default_startup_args @@ -2052,7 +2052,7 @@ buildvariants: - name: windows-64-ssl display_name: Windows 64-bit SSL run_on: - - windows-64-vs2013-compile + - windows-64-vs2017-compile expansions: <<: *mongod_ssl_startup_args <<: *mongo_ssl_startup_args @@ -2074,7 +2074,7 @@ buildvariants: - name: windows-64-enterprise display_name: Windows 64-bit Enterprise run_on: - - windows-64-vs2013-compile + - windows-64-vs2017-compile expansions: <<: *mongod_default_startup_args <<: *mongo_default_startup_args @@ -2221,6 +2221,18 @@ buildvariants: integration_test_args: integration tasks: *rhel67_s390x_enterprise_tasks +- name: rhel72-s390x-enterprise + display_name: ZAP s390x RHEL 7.2 Enterprise + run_on: + - rhel72-zseries-test + stepback: false + batchtime: 10080 # weekly + expansions: + build_tags: "sasl gssapi ssl" + tasks: + - name: dist + - name: replay-dist + - name: ubuntu1604-s390x-enterprise display_name: ZAP s390x Ubuntu 16.04 Enterprise run_on: diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/import.data b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/import.data index 8126e685de1..c991b271e9c 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/import.data +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/import.data @@ -1,5 +1,5 @@ { - "commit": "4eba41d830a94075202cd6eed6e3669ced4f9579", + "commit": "f6fdd97dbea59639732068720558c83b88947e15", "github": "mongodb/mongo-tools.git", "vendor": "tools", "branch": "v4.1" diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongodump/mongodump.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongodump/mongodump.go index 3db90063c55..4c956895542 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongodump/mongodump.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongodump/mongodump.go @@ -12,11 +12,9 @@ import ( "github.com/mongodb/mongo-tools-common/archive" "github.com/mongodb/mongo-tools-common/auth" - "github.com/mongodb/mongo-tools-common/bsonutil" "github.com/mongodb/mongo-tools-common/db" "github.com/mongodb/mongo-tools-common/failpoint" "github.com/mongodb/mongo-tools-common/intents" - "github.com/mongodb/mongo-tools-common/json" "github.com/mongodb/mongo-tools-common/log" "github.com/mongodb/mongo-tools-common/options" "github.com/mongodb/mongo-tools-common/progress" @@ -54,7 +52,7 @@ type MongoDump struct { // useful internals that we don't directly expose as options SessionProvider *db.SessionProvider manager *intents.Manager - query bson.M + query bson.D oplogCollection string oplogStart primitive.Timestamp oplogEnd primitive.Timestamp @@ -197,26 +195,16 @@ func (dump *MongoDump) Dump() (err error) { dump.shutdownIntentsNotifier = newNotifier() if dump.InputOptions.HasQuery() { - // parse JSON then convert extended JSON values - var asJSON interface{} content, err := dump.InputOptions.GetQuery() if err != nil { return err } - err = json.Unmarshal(content, &asJSON) + var query bson.D + err = bson.UnmarshalExtJSON(content, false, &query) if err != nil { - return fmt.Errorf("error parsing query as json: %v", err) + return fmt.Errorf("error parsing query as Extended JSON: %v", err) } - convertedJSON, err := bsonutil.ConvertLegacyExtJSONValueToBSON(asJSON) - if err != nil { - return fmt.Errorf("error converting query to bson: %v", err) - } - asMap, ok := convertedJSON.(map[string]interface{}) - if !ok { - // unlikely to be reached - return fmt.Errorf("query is not in proper format") - } - dump.query = bson.M(asMap) + dump.query = query } if !dump.SkipUsersAndRoles && dump.OutputOptions.DumpDBUsersAndRoles { @@ -533,7 +521,11 @@ func (dump *MongoDump) DumpIntent(intent *intents.Intent, buffer resettableOutpu // Views have an implied aggregation which does not support snapshot. // These are all a no-op. default: - findQuery.Hint = bson.D{{"_id", 1}} + // Don't hint autoIndexId:false collections + autoIndexId, found := intent.Options["autoIndexId"] + if !found || autoIndexId == true { + findQuery.Hint = bson.D{{"_id", 1}} + } } var dumpCount int64 diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongodump/mongodump_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongodump/mongodump_test.go index d08a89ec00a..1de256efa52 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongodump/mongodump_test.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongodump/mongodump_test.go @@ -214,7 +214,7 @@ func readBSONIntoDatabase(dir, restoreDBName string) error { bsonSource := db.NewDecodedBSONSource(db.NewBSONSource(file)) defer bsonSource.Close() - var result bson.M + var result bson.D for bsonSource.Next(&result) { _, err = collection.InsertOne(nil, result) if err != nil { @@ -239,7 +239,7 @@ func setUpMongoDumpTestData() error { coll := session.Database(testDB).Collection(collectionName) for j := 0; j < 10*(i+1); j++ { - _, err = coll.InsertOne(nil, bson.M{"collectionName": collectionName, "age": j}) + _, err = coll.InsertOne(nil, bson.M{"collectionName": collectionName, "age": j, "coords": bson.D{{"x", i}, {"y", j}}}) if err != nil { return err } @@ -407,7 +407,7 @@ func testDumpOneCollection(md *MongoDump, dumpDir string) { iter, err := collOriginal.Find(nil, bson.D{}) So(err, ShouldBeNil) - var result bson.M + var result bson.D for iter.Next(nil) { iter.Decode(&result) restoredCount, err := collRestore.CountDocuments(nil, result) @@ -832,3 +832,97 @@ func TestMongoDumpOplog(t *testing.T) { }) } + +// Test dumping a collection with autoIndexId:false. As of MongoDB 4.0, +// this is only allowed on the 'local' database. +func TestMongoDumpTOOLS2174(t *testing.T) { + testtype.SkipUnlessTestType(t, testtype.IntegrationTestType) + log.SetWriter(ioutil.Discard) + + sessionProvider, _, err := testutil.GetBareSessionProvider() + if err != nil { + t.Fatalf("No cluster available: %v", err) + } + + collName := "tools-2174" + dbName := "local" + + var r1 bson.M + sessionProvider.Run(bson.D{{"drop", collName}}, &r1, dbName) + + createCmd := bson.D{ + {"create", collName}, + {"autoIndexId", false}, + } + var r2 bson.M + err = sessionProvider.Run(createCmd, &r2, dbName) + if err != nil { + t.Fatalf("Error creating capped, no-autoIndexId collection: %v", err) + } + + Convey("testing dumping a capped, autoIndexId:false collection", t, func() { + md := simpleMongoDumpInstance() + md.ToolOptions.Namespace.Collection = collName + md.ToolOptions.Namespace.DB = dbName + md.OutputOptions.Out = "dump" + err = md.Init() + So(err, ShouldBeNil) + err = md.Dump() + So(err, ShouldBeNil) + }) +} + +func TestMongoDumpOrderedQuery(t *testing.T) { + testtype.SkipUnlessTestType(t, testtype.IntegrationTestType) + log.SetWriter(ioutil.Discard) + + Convey("With a MongoDump instance", t, func() { + err := setUpMongoDumpTestData() + So(err, ShouldBeNil) + path, err := os.Getwd() + So(err, ShouldBeNil) + dumpDir := util.ToUniversalPath(filepath.Join(path, "dump")) + + Convey("testing that --query is order-preserving", func() { + // If order is not preserved, probabalistically, some of these + // loops will fail. + for i := 0; i < 100; i++ { + So(os.RemoveAll(dumpDir), ShouldBeNil) + + md := simpleMongoDumpInstance() + md.InputOptions.Query = `{"coords":{"x":0,"y":1}}` + md.ToolOptions.Namespace.Collection = testCollectionNames[0] + md.ToolOptions.Namespace.DB = testDB + md.OutputOptions.Out = "dump" + err = md.Init() + So(err, ShouldBeNil) + err = md.Dump() + So(err, ShouldBeNil) + + dumpBSON := util.ToUniversalPath(filepath.Join(dumpDir, testDB, testCollectionNames[0]+".bson")) + + file, err := os.Open(dumpBSON) + So(err, ShouldBeNil) + + bsonSource := db.NewDecodedBSONSource(db.NewBSONSource(file)) + + var count int + var result bson.M + for bsonSource.Next(&result) { + count++ + } + So(bsonSource.Err(), ShouldBeNil) + + So(count, ShouldEqual, 1) + + bsonSource.Close() + file.Close() + } + }) + + Reset(func() { + So(os.RemoveAll(dumpDir), ShouldBeNil) + So(tearDownMongoDumpTestData(), ShouldBeNil) + }) + }) +} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongodump/options.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongodump/options.go index 599fc21d910..0de15aad972 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongodump/options.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongodump/options.go @@ -21,8 +21,8 @@ See http://docs.mongodb.org/manual/reference/program/mongodump/ for more informa // InputOptions defines the set of options to use in retrieving data from the server. type InputOptions struct { - Query string `long:"query" short:"q" description:"query filter, as a JSON string, e.g., '{x:{$gt:1}}'"` - QueryFile string `long:"queryFile" description:"path to a file containing a query filter (JSON)"` + Query string `long:"query" short:"q" description:"query filter, as a v2 Extended JSON string, e.g., '{\"x\":{\"$gt\":1}}'"` + QueryFile string `long:"queryFile" description:"path to a file containing a query filter (v2 Extended JSON)"` ReadPreference string `long:"readPreference" value-name:"<string>|<json>" description:"specify either a preference mode (e.g. 'nearest') or a preference json object (e.g. '{mode: \"nearest\", tagSets: [{a: \"b\"}], maxStalenessSeconds: 123}')"` TableScan bool `long:"forceTableScan" description:"force a table scan"` } diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoexport/mongoexport.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoexport/mongoexport.go index bd2adee4a8d..5ba2ffaf36a 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoexport/mongoexport.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoexport/mongoexport.go @@ -305,16 +305,16 @@ func (exp *MongoExport) getCursor() (*mongo.Cursor, error) { findOpts.SetSort(sortD) } - query := map[string]interface{}{} + query := bson.D{} if exp.InputOpts != nil && exp.InputOpts.HasQuery() { var err error content, err := exp.InputOpts.GetQuery() if err != nil { return nil, err } - query, err = getObjectFromByteArg(content) + err = bson.UnmarshalExtJSON(content, false, &query) if err != nil { - return nil, err + return nil, fmt.Errorf("error parsing query as Extended JSON: %v", err) } } @@ -329,8 +329,11 @@ func (exp *MongoExport) getCursor() (*mongo.Cursor, error) { if !exp.InputOpts.ForceTableScan && len(query) == 0 && exp.InputOpts != nil && exp.InputOpts.Sort == "" && !exp.collInfo.IsView() && !exp.collInfo.IsSystemCollection() { - // Use a hint on the _id index instead of the deprecated snapshot option - findOpts.SetHint(bson.D{{"_id", 1}}) + // Don't hint autoIndexId:false collections + autoIndexId, found := exp.collInfo.Options["autoIndexId"] + if !found || autoIndexId == true { + findOpts.SetHint(bson.D{{"_id", 1}}) + } } if exp.InputOpts != nil { diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoexport/mongoexport_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoexport/mongoexport_test.go index 716724ef503..842d895cbc7 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoexport/mongoexport_test.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoexport/mongoexport_test.go @@ -7,17 +7,73 @@ package mongoexport import ( + "bytes" "encoding/json" + "io/ioutil" "os" "testing" "github.com/mongodb/mongo-tools-common/bsonutil" + "github.com/mongodb/mongo-tools-common/db" + "github.com/mongodb/mongo-tools-common/log" + "github.com/mongodb/mongo-tools-common/options" "github.com/mongodb/mongo-tools-common/testtype" + "github.com/mongodb/mongo-tools-common/testutil" . "github.com/smartystreets/goconvey/convey" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" ) +var ( + // database with test data + testDB = "mongoexport_test_db" + testCollectionName = "coll1" +) + +func simpleMongoExportOpts() Options { + var toolOptions *options.ToolOptions + + // get ToolOptions from URI or defaults + if uri := os.Getenv("MONGOD"); uri != "" { + fakeArgs := []string{"--uri=" + uri} + toolOptions = options.New("mongoexport", "", "", "", options.EnabledOptions{URI: true}) + toolOptions.URI.AddKnownURIParameters(options.KnownURIOptionsReadPreference) + _, err := toolOptions.ParseArgs(fakeArgs) + if err != nil { + panic("Could not parse MONGOD environment variable") + } + } else { + ssl := testutil.GetSSLOptions() + auth := testutil.GetAuthOptions() + connection := &options.Connection{ + Host: "localhost", + Port: db.DefaultTestPort, + } + toolOptions = &options.ToolOptions{ + SSL: &ssl, + Connection: connection, + Auth: &auth, + Verbosity: &options.Verbosity{}, + URI: &options.URI{}, + } + } + + // Limit ToolOptions to test database + toolOptions.Namespace = &options.Namespace{DB: testDB, Collection: testCollectionName} + + opts := Options{ + ToolOptions: toolOptions, + OutputFormatOptions: &OutputFormatOptions{ + Type: "json", + JSONFormat: "canonical", + }, + InputOptions: &InputOptions{}, + } + + log.SetVerbosity(toolOptions.Verbosity) + return opts +} + func TestExtendedJSON(t *testing.T) { testtype.SkipUnlessTestType(t, testtype.UnitTestType) @@ -50,3 +106,44 @@ func TestFieldSelect(t *testing.T) { So(makeFieldSelector("x,foo.baz"), ShouldResemble, bson.M{"_id": 1, "foo": 1, "x": 1}) }) } + +// Test exporting a collection with autoIndexId:false. As of MongoDB 4.0, +// this is only allowed on the 'local' database. +func TestMongoExportTOOLS2174(t *testing.T) { + testtype.SkipUnlessTestType(t, testtype.IntegrationTestType) + log.SetWriter(ioutil.Discard) + + sessionProvider, _, err := testutil.GetBareSessionProvider() + if err != nil { + t.Fatalf("No cluster available: %v", err) + } + + collName := "tools-2174" + dbName := "local" + + var r1 bson.M + sessionProvider.Run(bson.D{{"drop", collName}}, &r1, dbName) + + createCmd := bson.D{ + {"create", collName}, + {"autoIndexId", false}, + } + var r2 bson.M + err = sessionProvider.Run(createCmd, &r2, dbName) + if err != nil { + t.Fatalf("Error creating capped, no-autoIndexId collection: %v", err) + } + + Convey("testing dumping a capped, autoIndexId:false collection", t, func() { + opts := simpleMongoExportOpts() + opts.Collection = collName + opts.DB = dbName + + me, err := New(opts) + So(err, ShouldBeNil) + defer me.Close() + out := &bytes.Buffer{} + _, err = me.Export(out) + So(err, ShouldBeNil) + }) +} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongofiles/mongofiles.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongofiles/mongofiles.go index 5c15124cf74..e6786182ad1 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongofiles/mongofiles.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongofiles/mongofiles.go @@ -13,10 +13,9 @@ import ( "io" "os" "regexp" + "strings" - "github.com/mongodb/mongo-tools-common/bsonutil" "github.com/mongodb/mongo-tools-common/db" - "github.com/mongodb/mongo-tools-common/json" "github.com/mongodb/mongo-tools-common/log" "github.com/mongodb/mongo-tools-common/options" "github.com/mongodb/mongo-tools-common/util" @@ -287,22 +286,27 @@ func (mf *MongoFiles) handleDeleteID() error { // parse and convert input extended JSON _id. Generates a new ObjectID if no _id provided. func (mf *MongoFiles) parseOrCreateID() (interface{}, error) { - if mf.Id == "" { + trimmed := strings.Trim(mf.Id, " ") + + if trimmed == "" { return primitive.NewObjectID(), nil } - var asJSON interface{} - if err := json.Unmarshal([]byte(mf.Id), &asJSON); err != nil { - return nil, fmt.Errorf("error parsing provided extJSON: %v", err) + // Wrap JSON bytes into a document for unmarshaling, then pick out the value after. + var wrapped string + switch trimmed[0] { + case '{': + wrapped = fmt.Sprintf(`{"_id":%s}`, trimmed) + default: + wrapped = fmt.Sprintf(`{"_id":"%s"}`, trimmed) } - - // legacy extJSON parser - id, err := bsonutil.ConvertLegacyExtJSONValueToBSON(asJSON) + var idDoc bson.D + err := bson.UnmarshalExtJSON([]byte(wrapped), false, &idDoc) if err != nil { - return nil, fmt.Errorf("error converting extJSON vlaue to bson: %v", err) + return nil, fmt.Errorf("error parsing id as Extended JSON: %v", err) } - return id, nil + return idDoc[0].Value, nil } // writeGFSFileToLocal writes a file from gridFS to stdout or the filesystem. diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongofiles/mongofiles_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongofiles/mongofiles_test.go index bf14d399cee..ae5303c22c4 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongofiles/mongofiles_test.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongofiles/mongofiles_test.go @@ -198,7 +198,7 @@ func fileContentsCompare(file1, file2 *os.File, t *testing.T) (bool, error) { // get an id of an existing file, for _id access func idOfFile(filename string) string { - return fmt.Sprintf("ObjectId('%v')", testFiles[filename].Hex()) + return fmt.Sprintf(`{"$oid":"%s"}`, testFiles[filename].Hex()) } // test output needs some cleaning @@ -540,7 +540,7 @@ func TestMongoFilesCommands(t *testing.T) { }) Convey("Testing the 'put_id' command by putting some lorem ipsum file with 287613 bytes with different ids should succeed", func() { - for _, idToTest := range []string{"'test_id'", "'{a:\"b\"}'", "'{$numberlong:9999999999999999999999}'", "'{a:{b:{c:{}}}}'"} { + for _, idToTest := range []string{`test_id`, `{"a":"b"}`, `{"$numberLong":"999999999999999"}`, `{"a":{"b":{"c":{}}}}`} { runPutIDTestCase(idToTest, t) } }) |