diff options
author | Judah Schvimer <judah@mongodb.com> | 2015-12-16 15:41:49 -0500 |
---|---|---|
committer | Judah Schvimer <judah@mongodb.com> | 2015-12-16 15:41:49 -0500 |
commit | a3b7602a4b462b3b42d040d60a03071ade7d1eb5 (patch) | |
tree | 9d87a9837f58682d30f164117255c291def2d242 | |
parent | e3463ed7d542e2144c681585bdd7b08c591717b9 (diff) | |
download | mongo-a3b7602a4b462b3b42d040d60a03071ade7d1eb5.tar.gz |
TOOLS-954 Added bypassDocumentValidation option to mongorestore and mongoimport
-rw-r--r-- | common/db/db.go | 27 | ||||
-rw-r--r-- | mongoimport/main/mongoimport.go | 1 | ||||
-rw-r--r-- | mongoimport/mongoimport.go | 2 | ||||
-rw-r--r-- | mongoimport/options.go | 3 | ||||
-rw-r--r-- | mongorestore/main/mongorestore.go | 2 | ||||
-rw-r--r-- | mongorestore/options.go | 19 | ||||
-rw-r--r-- | test/qa-tests/jstests/import/import_document_validation.js | 113 | ||||
-rw-r--r-- | test/qa-tests/jstests/restore/restore_document_validation.js | 204 | ||||
-rw-r--r-- | test/qa-tests/jstests/unstable/restore_document_validation.js | 130 |
9 files changed, 358 insertions, 143 deletions
diff --git a/common/db/db.go b/common/db/db.go index afa3b994e49..cf91bc4d0f7 100644 --- a/common/db/db.go +++ b/common/db/db.go @@ -64,9 +64,10 @@ type SessionProvider struct { masterSession *mgo.Session // flags for generating the master session - flags sessionFlag - readPreference mgo.Mode - tags bson.D + bypassDocumentValidation bool + flags sessionFlag + readPreference mgo.Mode + tags bson.D } // ApplyOpsResponse represents the response from an 'applyOps' command. @@ -115,8 +116,12 @@ func (self *SessionProvider) GetSession() (*mgo.Session, error) { // session provider flags passed in with SetFlags. // This helper assumes a lock is already taken. func (self *SessionProvider) refresh() { + // handle bypassDocumentValidation + self.masterSession.SetBypassValidation(self.bypassDocumentValidation) + // handle readPreference self.masterSession.SetMode(self.readPreference, true) + // disable timeouts if (self.flags & DisableSocketTimeout) > 0 { self.masterSession.SetSocketTimeout(0) @@ -152,6 +157,19 @@ func (self *SessionProvider) SetReadPreference(pref mgo.Mode) { } } +// SetBypassDocumentValidation sets whether to bypass document validation in the SessionProvider +// and eventually in the masterSession +func (self *SessionProvider) SetBypassDocumentValidation(bypassDocumentValidation bool) { + self.masterSessionLock.Lock() + defer self.masterSessionLock.Unlock() + + self.bypassDocumentValidation = bypassDocumentValidation + + if self.masterSession != nil { + self.refresh() + } +} + // SetTags sets the server selection tags in the SessionProvider // and eventually in the masterSession func (self *SessionProvider) SetTags(tags bson.D) { @@ -170,7 +188,8 @@ func (self *SessionProvider) SetTags(tags bson.D) { func NewSessionProvider(opts options.ToolOptions) (*SessionProvider, error) { // create the provider provider := &SessionProvider{ - readPreference: mgo.Primary, + readPreference: mgo.Primary, + bypassDocumentValidation: false, } // finalize auth options, filling in missing passwords diff --git a/mongoimport/main/mongoimport.go b/mongoimport/main/mongoimport.go index 60d1bcfcf32..70d125ca5ad 100644 --- a/mongoimport/main/mongoimport.go +++ b/mongoimport/main/mongoimport.go @@ -53,6 +53,7 @@ func main() { log.Logf(log.Always, "error connecting to host: %v", err) os.Exit(util.ExitError) } + sessionProvider.SetBypassDocumentValidation(ingestOpts.BypassDocumentValidation) m := mongoimport.MongoImport{ ToolOptions: opts, diff --git a/mongoimport/mongoimport.go b/mongoimport/mongoimport.go index 6c38302c837..ef33efc6335 100644 --- a/mongoimport/mongoimport.go +++ b/mongoimport/mongoimport.go @@ -394,6 +394,7 @@ func (imp *MongoImport) ingestDocuments(readDocs chan bson.D) (retErr error) { // // 1. Sets the session to not timeout // 2. Sets the write concern on the session +// 3. Sets the session safety // // returns an error if it's unable to set the write concern func (imp *MongoImport) configureSession(session *mgo.Session) error { @@ -404,6 +405,7 @@ func (imp *MongoImport) configureSession(session *mgo.Session) error { return fmt.Errorf("write concern error: %v", err) } session.SetSafe(sessionSafety) + return nil } diff --git a/mongoimport/options.go b/mongoimport/options.go index 3762f00ad83..ebdba128504 100644 --- a/mongoimport/options.go +++ b/mongoimport/options.go @@ -57,6 +57,9 @@ type IngestOptions struct { // Sets write concern level for write operations. WriteConcern string `long:"writeConcern" default:"majority" value-name:"<write-concern-specifier>" default-mask:"-" description:"write concern options e.g. --writeConcern majority, --writeConcern '{w: 3, wtimeout: 500, fsync: true, j: true}' (defaults to 'majority')"` + + // Indicates that the server should bypass document validation on import. + BypassDocumentValidation bool `long:"bypassDocumentValidation" description:"bypass document validation"` } // Name returns a description of the IngestOptions struct. diff --git a/mongorestore/main/mongorestore.go b/mongorestore/main/mongorestore.go index 9673d5b36f0..6748907a16c 100644 --- a/mongorestore/main/mongorestore.go +++ b/mongorestore/main/mongorestore.go @@ -56,6 +56,8 @@ func main() { log.Logf(log.Always, "error connecting to host: %v", err) os.Exit(util.ExitError) } + provider.SetBypassDocumentValidation(outputOpts.BypassDocumentValidation) + // disable TCP timeouts for restore jobs provider.SetFlags(db.DisableSocketTimeout) restore := mongorestore.MongoRestore{ diff --git a/mongorestore/options.go b/mongorestore/options.go index 5f3b867c96a..5e3d5460f19 100644 --- a/mongorestore/options.go +++ b/mongorestore/options.go @@ -28,15 +28,16 @@ func (*InputOptions) Name() string { // OutputOptions defines the set of options for restoring dump data. type OutputOptions struct { - Drop bool `long:"drop" description:"drop each collection before import"` - WriteConcern string `long:"writeConcern" value-name:"<write-concern>" default:"majority" default-mask:"-" description:"write concern options e.g. --writeConcern majority, --writeConcern '{w: 3, wtimeout: 500, fsync: true, j: true}' (defaults to 'majority')"` - NoIndexRestore bool `long:"noIndexRestore" description:"don't restore indexes"` - NoOptionsRestore bool `long:"noOptionsRestore" description:"don't restore collection options"` - KeepIndexVersion bool `long:"keepIndexVersion" description:"don't update index version"` - MaintainInsertionOrder bool `long:"maintainInsertionOrder" description:"preserve order of documents during restoration"` - NumParallelCollections int `long:"numParallelCollections" short:"j" description:"number of collections to restore in parallel (4 by default)" default:"4" default-mask:"-"` - NumInsertionWorkers int `long:"numInsertionWorkersPerCollection" description:"number of insert operations to run concurrently per collection (1 by default)" default:"1" default-mask:"-"` - StopOnError bool `long:"stopOnError" description:"stop restoring if an error is encountered on insert (off by default)"` + Drop bool `long:"drop" description:"drop each collection before import"` + WriteConcern string `long:"writeConcern" value-name:"<write-concern>" default:"majority" default-mask:"-" description:"write concern options e.g. --writeConcern majority, --writeConcern '{w: 3, wtimeout: 500, fsync: true, j: true}' (defaults to 'majority')"` + NoIndexRestore bool `long:"noIndexRestore" description:"don't restore indexes"` + NoOptionsRestore bool `long:"noOptionsRestore" description:"don't restore collection options"` + KeepIndexVersion bool `long:"keepIndexVersion" description:"don't update index version"` + MaintainInsertionOrder bool `long:"maintainInsertionOrder" description:"preserve order of documents during restoration"` + NumParallelCollections int `long:"numParallelCollections" short:"j" description:"number of collections to restore in parallel (4 by default)" default:"4" default-mask:"-"` + NumInsertionWorkers int `long:"numInsertionWorkersPerCollection" description:"number of insert operations to run concurrently per collection (1 by default)" default:"1" default-mask:"-"` + StopOnError bool `long:"stopOnError" description:"stop restoring if an error is encountered on insert (off by default)"` + BypassDocumentValidation bool `long:"bypassDocumentValidation" description:"bypass document validation"` } // Name returns a human-readable group name for output options. diff --git a/test/qa-tests/jstests/import/import_document_validation.js b/test/qa-tests/jstests/import/import_document_validation.js new file mode 100644 index 00000000000..1f0d84e830c --- /dev/null +++ b/test/qa-tests/jstests/import/import_document_validation.js @@ -0,0 +1,113 @@ +/** + * import_document_validation.js + * + * This file test that mongoimport works with document validation. It both checks that when + * validation is turned on invalid documents are not imported and that when a user indicates + * they want to bypass validation, that all documents are imported. + */ + +(function() { + 'use strict'; + if (typeof getToolTest === 'undefined') { + load('jstests/configs/plain_28.config.js'); + } + + /** + * Part 1: Test that import follows document validation rules. + */ + jsTest.log('Testing that import reacts well to document validation'); + + var toolTest = getToolTest('import_document_validation'); + var commonToolArgs = getCommonToolArguments(); + + // the db we will use + var testDB = toolTest.db.getSiblingDB('test'); + + // create 1000 documents, half of which will pass the validation + for (var i = 0; i < 1000; i++) { + if (i%2 === 0) { + testDB.bar.insert({ _id: i, num: i+1, s: '' + i }); + } else { + testDB.bar.insert({ _id: i, num: i+1, s: '' + i, baz: i }); + } + } + // sanity check the insertion worked + assert.eq(1000, testDB.bar.count()); + + // export the data + var ret = toolTest.runTool.apply( + toolTest, + ['export', + '--out', toolTest.extFile, + '-d', 'test', + '-c', 'bar' + ].concat(commonToolArgs) + ); + assert.eq(0, ret, 'export should run successfully'); + + testDB.dropDatabase(); + assert.eq(0, testDB.bar.count(), 'after dropping the database, no documents should be seen'); + + // sanity check that we can import the data without validation + ret = toolTest.runTool.apply( + toolTest, + ['import', + '--file', toolTest.extFile, + '--db', 'test', + '-c', 'bar'].concat(commonToolArgs) + ); + assert.eq(0, ret); + + assert.eq(1000, testDB.bar.count(), 'after import, the documents should be seen again'); + + testDB.dropDatabase(); + assert.eq(0, testDB.bar.count(), 'after dropping the database, no documents should be seen'); + + // turn on validation + var r = testDB.createCollection('bar', { validator: { baz: { $exists: true } } }); + assert.eq(r,{ ok: 1 }, 'create collection with validation works'); + + // test that it's working + r = testDB.bar.insert({ num: 10000 }); + assert.eq(r.nInserted, 0, "invalid documents shouldn't be inserted"); + + // import the 1000 records of which only 500 are valid + ret = toolTest.runTool.apply( + toolTest, + ['import', + '--file', toolTest.extFile, + '--db', 'test', + '-c', 'bar'].concat(commonToolArgs) + ); + assert.eq(0, ret, 'import against a collection with validation on still succeeds'); + + assert.eq(500, testDB.bar.count(), 'only the valid documents are imported'); + + /** + * Part 2: Test that import can bypass document validation rules. + */ + jsTest.log('Testing that bypass document validation works'); + + testDB.dropDatabase(); + + // turn on validation + r = testDB.createCollection('bar',{ validator:{ baz: { $exists: true } } }); + assert.eq(r, { ok: 1 }, 'create collection with validation should work'); + + // test that we cannot insert an 'invalid' document + r = testDB.bar.insert({ num: 10000 }); + assert.eq(r.nInserted, 0, 'invalid documents should not be inserted'); + + // import the 1000 records again with bypassDocumentValidation turned on + ret = toolTest.runTool.apply( + toolTest, + ['import', + '--file', toolTest.extFile, + '--db', 'test', + '-c', 'bar', + '--bypassDocumentValidation'].concat(commonToolArgs) + ); + assert.eq(0, ret, 'importing documents should work with bypass document validation set'); + assert.eq(1000, testDB.bar.count(), + 'all documents should be imported with bypass document validation set'); +}()); diff --git a/test/qa-tests/jstests/restore/restore_document_validation.js b/test/qa-tests/jstests/restore/restore_document_validation.js new file mode 100644 index 00000000000..5f649a73937 --- /dev/null +++ b/test/qa-tests/jstests/restore/restore_document_validation.js @@ -0,0 +1,204 @@ +/** + * restore_document_validation.js + * + * This file test that mongorestore works with document validation. It both checks that when + * validation is turned on invalid documents are not restored and that when a user indicates + * they want to bypass validation, that all documents are restored. + */ + +(function() { + 'use strict'; + if (typeof getToolTest === 'undefined') { + load('jstests/configs/plain_28.config.js'); + } + + /** + * Part 1: Test that restore follows document validation rules. + */ + jsTest.log('Testing that restore reacts well to document validation'); + + var toolTest = getToolTest('document_validation'); + var commonToolArgs = getCommonToolArguments(); + + // where we'll put the dump + var dumpTarget = 'doc_validation'; + resetDbpath(dumpTarget); + + // the db we will use + var testDB = toolTest.db.getSiblingDB('test'); + + // create 1000 documents, half of which will pass the validation + for (var i = 0; i < 1000; i++) { + if (i%2 === 0) { + testDB.bar.insert({ _id: i, num: i+1, s: ''+i }); + } else { + testDB.bar.insert({ _id: i, num: i+1, s: ''+i, baz: i }); + } + } + // sanity check the insertion worked + assert.eq(1000, testDB.bar.count(), 'all documents should be inserted'); + + var ret = toolTest.runTool.apply( + toolTest, + ['dump','-v']. + concat(getDumpTarget(dumpTarget)). + concat(commonToolArgs) + ); + assert.eq(0, ret, 'dumping should run successfully'); + + testDB.dropDatabase(); + assert.eq(0, testDB.bar.count(), 'after the drop, no documents should be seen'); + + // sanity check that we can restore the data without validation + ret = toolTest.runTool.apply( + toolTest, + ['restore']. + concat(getRestoreTarget(dumpTarget)). + concat(commonToolArgs) + ); + assert.eq(0, ret); + + assert.eq(1000, testDB.bar.count(), 'after the restore, all documents should be seen'); + + testDB.dropDatabase(); + assert.eq(0, testDB.bar.count(), 'after the drop, no documents should be seen'); + + // turn on validation + var r = testDB.createCollection('bar', { validator:{ baz: { $exists: true } } }); + assert.eq(r, { ok: 1 }, 'create collection with validation should work'); + + // test that it's working + r = testDB.bar.insert({ num: 10000 }); + assert.eq(r.nInserted, 0, "invalid documents shouldn't be inserted"); + + // restore the 1000 records of which only 500 are valid + ret = toolTest.runTool.apply( + toolTest, + ['restore','-v']. + concat(getRestoreTarget(dumpTarget)). + concat(commonToolArgs) + ); + assert.eq(0, ret, 'restoring against a collection with validation on should still succeed'); + + assert.eq(500, testDB.bar.count(), 'only the valid documents should be restored'); + + /** + * Part 2: Test that restore can bypass document validation rules. + */ + jsTest.log('Testing that bypass document validation works'); + + testDB.dropDatabase(); + + // turn on validation + r = testDB.createCollection('bar',{ validator: { baz: { $exists: true } } }); + assert.eq(r, { ok: 1 }, 'create collection with validation should work'); + + // test that we cannot insert an 'invalid' document + r = testDB.bar.insert({ num: 10000 }); + assert.eq(r.nInserted, 0, 'invalid documents should not be inserted'); + + // restore the 1000 records again with bypassDocumentValidation turned on + ret = toolTest.runTool.apply( + toolTest, + ['restore', + '--bypassDocumentValidation']. + concat(getRestoreTarget(dumpTarget)). + concat(commonToolArgs) + ); + assert.eq(0, ret, 'restoring documents should work with bypass document validation set'); + assert.eq(1000, testDB.bar.count(), + 'all documents should be restored with bypass document validation set'); + + /** + * Part 3: Test that restore can restore the document validation rules, + * if they're dumped with the collection. + */ + jsTest.log('Testing that dump and restore restores the validation rules themselves'); + + // clear out the database, including the validation rules + testDB.dropDatabase(); + assert.eq(0, testDB.bar.count(), 'after the drop, no documents should be seen'); + + // test that we can insert an 'invalid' document + r = testDB.bar.insert({ num: 10000 }); + assert.eq(r.nInserted, 1, + 'invalid documents should be inserted after validation rules are dropped'); + + testDB.dropDatabase(); + assert.eq(0, testDB.bar.count(), 'after the drop, no documents should be seen'); + + // restore the 1000 records again + ret = toolTest.runTool.apply( + toolTest, + ['restore']. + concat(getRestoreTarget(dumpTarget)). + concat(commonToolArgs) + ); + assert.eq(0, ret); + assert.eq(1000, testDB.bar.count()); + + // turn on validation on a existing collection + testDB.runCommand({ 'collMod': 'bar', 'validator' : { baz: { $exists: true } } }); + + // re-dump everything, this time dumping the validation rules themselves + ret = toolTest.runTool.apply( + toolTest, + ['dump','-v']. + concat(getDumpTarget(dumpTarget)). + concat(commonToolArgs) + ); + assert.eq(0, ret, 'the dump should run successfully'); + + // clear out the database, including the validation rules + testDB.dropDatabase(); + assert.eq(0, testDB.bar.count(), 'after the drop, no documents should be seen'); + + // test that we can insert an 'invalid' document + r = testDB.bar.insert({ num: 10000 }); + assert.eq(r.nInserted, 1, + 'invalid documents should be inserted after we drop validation rules'); + + testDB.dropDatabase(); + assert.eq(0, testDB.bar.count(), 'after the drop, no documents should be seen'); + + // restore the 1000 records again + ret = toolTest.runTool.apply( + toolTest, + ['restore']. + concat(getRestoreTarget(dumpTarget)). + concat(commonToolArgs) + ); + assert.eq(0, ret, 'restoring rules and some invalid documents should run successfully'); + assert.eq(500, testDB.bar.count(), + 'restoring the validation rules and documents should only restore valid documents'); + + /** + * Part 4: Test that restore can bypass the document validation rules, + * even if they're dumped with the collection and restored with the collection. + */ + jsTest.log('Testing that bypass document validation works when restoring the rules as well'); + + // clear out the database, including the validation rules + testDB.dropDatabase(); + assert.eq(0, testDB.bar.count(), 'after the drop, no documents should be seen'); + + // test that we can insert an 'invalid' document + r = testDB.bar.insert({ num: 10000 }); + assert.eq(r.nInserted, 1, + 'invalid documents should be inserted after validation rules are dropped'); + + testDB.dropDatabase(); + assert.eq(0, testDB.bar.count(), 'after the drop, no documents should be seen'); + + // restore the 1000 records again with bypassDocumentValidation turned on + ret = toolTest.runTool.apply( + toolTest, + ['restore', + '--bypassDocumentValidation']. + concat(getRestoreTarget(dumpTarget)). + concat(commonToolArgs) + ); + assert.eq(0, ret, 'restoring documents should work with bypass document validation set'); + assert.eq(1000, testDB.bar.count(), + 'all documents should be restored with bypass document validation set'); +}()); diff --git a/test/qa-tests/jstests/unstable/restore_document_validation.js b/test/qa-tests/jstests/unstable/restore_document_validation.js deleted file mode 100644 index d151901f800..00000000000 --- a/test/qa-tests/jstests/unstable/restore_document_validation.js +++ /dev/null @@ -1,130 +0,0 @@ -(function() { - - if (typeof getToolTest === 'undefined') { - load('jstests/configs/plain_28.config.js'); - } - - jsTest.log('Testing that restore reacts well to document validation'); - - var toolTest = getToolTest('document_validation'); - var commonToolArgs = getCommonToolArguments(); - - // where we'll put the dump - var dumpTarget = 'doc_validation'; - resetDbpath(dumpTarget); - - // the db we will use - var testDB = toolTest.db.getSiblingDB('test'); - - // crate 1000 documents, half of which will pass the validation - for (var i = 0; i < 1000; i++) { - if (i%2==0) { - testDB.bar.insert({ _id: i, num: i+1, s: ''+i }); - } else { - testDB.bar.insert({ _id: i, num: i+1, s: ''+i, baz: i }); - } - } - // sanity check the insertion worked - assert.eq(1000, testDB.bar.count()); - - ret = toolTest.runTool.apply( - toolTest, - ['dump','-v']. - concat(getDumpTarget(dumpTarget)). - concat(commonToolArgs) - ); - assert.eq(0, ret,"the dump runs successfully"); - - testDB.dropDatabase(); - assert.eq(0, testDB.bar.count(),"after the drop, the documents not seen"); - - // sanity check that we can restore the data without validation - ret = toolTest.runTool.apply( - toolTest, - ['restore']. - concat(getRestoreTarget(dumpTarget)). - concat(commonToolArgs) - ); - assert.eq(0, ret); - - assert.eq(1000, testDB.bar.count(),"after the restore, the documents are seen again"); - - testDB.dropDatabase(); - assert.eq(0, testDB.bar.count(),"after the drop, the documents not seen"); - - // turn on validation - r = testDB.createCollection("bar",{validator:{baz:{$exists:true}}}); - assert.eq(r,{ok:1},"create collection with validation works"); - - // test that it's working - r = testDB.bar.insert({num:10000}); - assert.eq(r.nInserted,0,"invalid documents can't be inserted") - - // restore the 1000 records of which only 500 are valid - ret = toolTest.runTool.apply( - toolTest, - ['restore','-v']. - concat(getRestoreTarget(dumpTarget)). - concat(commonToolArgs) - ); - assert.eq(0, ret,"restore against a collection with validation on still succeeds"); - - assert.eq(500, testDB.bar.count(),"only the valid documents are restored"); - - jsTest.log('Testing that dump and restore the validation rules themselves'); - - // clear out the database, including the validation rules - testDB.dropDatabase(); - assert.eq(0, testDB.bar.count(),"after the drop, the documents not seen"); - - // test that we can insert an "invalid" document - r = testDB.bar.insert({num:10000}); - assert.eq(r.nInserted,1,"invalid documents can be inserted") - - testDB.dropDatabase(); - assert.eq(0, testDB.bar.count(),"after the drop, the documents not seen"); - - // restore the 1000 records again - ret = toolTest.runTool.apply( - toolTest, - ['restore']. - concat(getRestoreTarget(dumpTarget)). - concat(commonToolArgs) - ); - assert.eq(0, ret); - assert.eq(1000, testDB.bar.count()); - - // turn on validation on a existing collection - testDB.runCommand({"collMod": "bar", "validator" : {baz: {$exists: true}}}); - - // re-dump everything, this time dumping the validation rules themselves - ret = toolTest.runTool.apply( - toolTest, - ['dump','-v']. - concat(getDumpTarget(dumpTarget)). - concat(commonToolArgs) - ); - assert.eq(0, ret,"the dump runs successfully"); - - // clear out the database, including the validation rules - testDB.dropDatabase(); - assert.eq(0, testDB.bar.count(),"after the drop, the documents not seen"); - - // test that we can insert an "invalid" document - r = testDB.bar.insert({num:10000}); - assert.eq(r.nInserted,1,"invalid documents can be inserted") - - testDB.dropDatabase(); - assert.eq(0, testDB.bar.count(),"after the drop, the documents not seen"); - - // restore the 1000 records again - ret = toolTest.runTool.apply( - toolTest, - ['restore']. - concat(getRestoreTarget(dumpTarget)). - concat(commonToolArgs) - ); - assert.eq(0, ret,"restoring rules and some invalid documents runs"); - assert.eq(500, testDB.bar.count(),"restore the validation rules and the valid documents"); - -}()); |