diff options
author | Ramon Fernandez <ramon@mongodb.com> | 2016-12-08 12:06:39 -0500 |
---|---|---|
committer | Ramon Fernandez <ramon@mongodb.com> | 2016-12-08 15:05:03 -0500 |
commit | a094b2d8d72d29ebb8197dbc6f549fde2f309508 (patch) | |
tree | edd8048160b439bc454914f249fc109ba210b23b | |
parent | 16a31a6b8f49ac0d6b4c489f9968c52a8b2a3c63 (diff) | |
download | mongo-a094b2d8d72d29ebb8197dbc6f549fde2f309508.tar.gz |
Import tools: 4a0fbf5245669b55915adf7547ac592223681fe1 from branch v3.4
ref: 3cc9a07766..4a0fbf5245
for: 3.4.1
TOOLS-1498 Stats collection has large playback performance impact
TOOLS-1501 Add option to set capture buffer size to avoid packet loss
TOOLS-1502 Playback file contains full reply payload
TOOLS-1516 mongoreplay: out of bounds error in "shortenreply" during record
TOOLS-1534 Running mongodump then mongorestore should restore the _id index with its exact original spec
TOOLS-1535 Add test for restoring a collection with a default collation
TOOLS-1541 support exporting views
TOOLS-1558 use mongodb 3.4 "current" tests in master
TOOLS-1561 nil pointer dereference in mongoreplay when error on new playback file creation
19 files changed, 497 insertions, 94 deletions
diff --git a/src/mongo/gotools/common.yml b/src/mongo/gotools/common.yml index f5d14936b6b..7453ec02145 100644 --- a/src/mongo/gotools/common.yml +++ b/src/mongo/gotools/common.yml @@ -841,7 +841,7 @@ tasks: value: "${args} -test.types=db" - func: "download mongod" vars: - mongo_version: "3.2" + mongo_version: "3.4" - func: "start mongod" - func: "wait for mongod to be ready" - func: "setup integration test" @@ -925,7 +925,7 @@ tasks: value: "${args} -test.types=${integration_test_args}" - func: "download mongod" vars: - mongo_version: "3.2" + mongo_version: "3.4" - func: "start mongod" - func: "wait for mongod to be ready" - func: "run tool integration tests" @@ -949,7 +949,7 @@ tasks: value: "db.createUser({ user: '${auth_username}', pwd: '${auth_password}', roles: [{ role: '__system', db: 'admin' }] });" - func: "download mongod" vars: - mongo_version: "3.2" + mongo_version: "3.4" - func: "start mongod" - func: "wait for mongod to be ready" - func: "run tool integration tests" @@ -1193,7 +1193,7 @@ tasks: - func: "setup credentials" - func: "download mongod" vars: - mongo_version: "3.2" + mongo_version: "3.4" - func: "fetch tool" vars: tool: mongoimport @@ -1288,7 +1288,7 @@ tasks: - func: "setup credentials" - func: "download mongod" vars: - mongo_version: "3.2" + mongo_version: "3.4" - func: "fetch tool" vars: tool: mongodump @@ -1309,7 +1309,7 @@ tasks: - func: "setup credentials" - func: "download mongod" vars: - mongo_version: "3.2" + mongo_version: "3.4" - func: "fetch tool" vars: tool: mongodump @@ -1330,7 +1330,7 @@ tasks: - func: "setup credentials" - func: "download mongod" vars: - mongo_version: "3.2" + mongo_version: "3.4" - func: "fetch tool" vars: tool: mongoimport @@ -1452,7 +1452,7 @@ tasks: tool: mongoreplay - func: "download mongod" vars: - mongo_version: "3.2" + mongo_version: "3.4" - func: "start mongod" - func: "wait for mongod to be ready" - command: shell.exec @@ -1482,7 +1482,7 @@ tasks: pcapFname: getmore_single_channel.pcap - func: "download mongod" vars: - mongo_version: "3.2" + mongo_version: "3.4" - func: "start mongod" - func: "wait for mongod to be ready" - func: "run go_test" @@ -1507,7 +1507,7 @@ tasks: pcapFname: getmore_single_channel.pcap - func: "download mongod" vars: - mongo_version: "3.2" + mongo_version: "3.4" - func: "create sharded_cluster" - func: "run go_test" vars: @@ -1531,7 +1531,7 @@ tasks: pcapFname: getmore_single_channel.pcap - func: "download mongod" vars: - mongo_version: "3.2" + mongo_version: "3.4" - func: "start mongod" vars: additional_args: --auth @@ -1561,7 +1561,7 @@ tasks: pcapFname: getmore_single_channel.pcap - func: "download mongod" vars: - mongo_version: "3.2" + mongo_version: "3.4" - func: "create repl_set" vars: mongod_port: ${mongod_port} @@ -1581,7 +1581,7 @@ tasks: tool: mongoreplay - func: "download mongod" vars: - mongo_version: "3.2" + mongo_version: "3.4" - func: "fetch ftdc" - command: shell.exec params: diff --git a/src/mongo/gotools/common/db/namespaces.go b/src/mongo/gotools/common/db/namespaces.go index 149400543ef..908687b1c56 100644 --- a/src/mongo/gotools/common/db/namespaces.go +++ b/src/mongo/gotools/common/db/namespaces.go @@ -129,6 +129,7 @@ func GetCollectionOptions(coll *mgo.Collection) (*bson.D, error) { if err != nil { return nil, err } + defer iter.Close() comparisonName := coll.Name if useFullName { comparisonName = coll.FullName @@ -143,17 +144,23 @@ func GetCollectionOptions(coll *mgo.Collection) (*bson.D, error) { if nameStr, ok := name.(string); ok { if nameStr == comparisonName { // we've found the collection we're looking for - return collInfo, nil + break } } else { collInfo = nil continue } } - err = iter.Err() - if err != nil { - return nil, err + + if collInfo != nil { + optsInterface, _ := bsonutil.FindValueByKey("options", collInfo) + if optsInterface != nil { + optsD, ok := optsInterface.(bson.D) + if !ok { + return nil, fmt.Errorf("Cannot unmarshal collection options for collection %v.%v", coll.Database, coll.Name) + } + return &optsD, nil + } } - // The given collection was not found, but no error encountered. - return nil, nil + return nil, iter.Err() } diff --git a/src/mongo/gotools/import.data b/src/mongo/gotools/import.data new file mode 100644 index 00000000000..f62f351837f --- /dev/null +++ b/src/mongo/gotools/import.data @@ -0,0 +1,5 @@ +{ + "commit": "4a0fbf5245669b55915adf7547ac592223681fe1", + "github": "mongodb/mongo-tools.git", + "branch": "v3.4" +} diff --git a/src/mongo/gotools/mongodump/prepare.go b/src/mongo/gotools/mongodump/prepare.go index ff36bdfb02a..6c22748a810 100644 --- a/src/mongo/gotools/mongodump/prepare.go +++ b/src/mongo/gotools/mongodump/prepare.go @@ -4,7 +4,6 @@ import ( "bytes" "fmt" "github.com/mongodb/mongo-tools/common/archive" - "github.com/mongodb/mongo-tools/common/bsonutil" "github.com/mongodb/mongo-tools/common/db" "github.com/mongodb/mongo-tools/common/intents" "github.com/mongodb/mongo-tools/common/log" @@ -301,23 +300,11 @@ func (dump *MongoDump) CreateCollectionIntent(dbName, colName string) error { } defer session.Close() - opts, err := db.GetCollectionOptions(session.DB(dbName).C(colName)) + intent.Options, err = db.GetCollectionOptions(session.DB(dbName).C(colName)) if err != nil { return fmt.Errorf("error getting collection options: %v", err) } - intent.Options = nil - if opts != nil { - optsInterface, _ := bsonutil.FindValueByKey("options", opts) - if optsInterface != nil { - if optsD, ok := optsInterface.(bson.D); ok { - intent.Options = &optsD - } else { - return fmt.Errorf("Failed to parse collection options as bson.D") - } - } - } - dump.manager.Put(intent) log.Logvf(log.DebugLow, "enqueued collection '%v'", intent.Namespace()) diff --git a/src/mongo/gotools/mongoexport/mongoexport.go b/src/mongo/gotools/mongoexport/mongoexport.go index 6b4e96c68f1..4feab4894d2 100644 --- a/src/mongo/gotools/mongoexport/mongoexport.go +++ b/src/mongo/gotools/mongoexport/mongoexport.go @@ -233,16 +233,31 @@ func (exp *MongoExport) getCursor() (*mgo.Iter, *mgo.Session, error) { } } - flags := 0 - if len(query) == 0 && exp.InputOpts != nil && - exp.InputOpts.ForceTableScan != true && exp.InputOpts.Sort == "" { - flags = flags | db.Snapshot + session, err := exp.SessionProvider.GetSession() + if err != nil { + return nil, nil, err } + collection := session.DB(exp.ToolOptions.Namespace.DB).C(exp.ToolOptions.Namespace.Collection) - session, err := exp.SessionProvider.GetSession() + // figure out if we're exporting a view + isView := false + opts, err := db.GetCollectionOptions(collection) if err != nil { return nil, nil, err } + if opts != nil { + viewOn, _ := bsonutil.FindValueByKey("viewOn", opts) + if viewOn != nil { + isView = true + } + } + + flags := 0 + // don't snapshot if we've been asked not to, + // or if we cannot because we are querying, sorting, or if the collection is a view + if !exp.InputOpts.ForceTableScan && len(query) == 0 && exp.InputOpts != nil && exp.InputOpts.Sort == "" && !isView { + flags = flags | db.Snapshot + } skip := 0 if exp.InputOpts != nil { @@ -265,9 +280,7 @@ func (exp *MongoExport) getCursor() (*mgo.Iter, *mgo.Session, error) { } // build the query - q := session.DB(exp.ToolOptions.Namespace.DB). - C(exp.ToolOptions.Namespace.Collection).Find(query).Sort(sortFields...). - Skip(skip).Limit(limit) + q := collection.Find(query).Sort(sortFields...).Skip(skip).Limit(limit) if len(exp.OutputOpts.Fields) > 0 { q.Select(makeFieldSelector(exp.OutputOpts.Fields)) diff --git a/src/mongo/gotools/mongoreplay/auth_test.go b/src/mongo/gotools/mongoreplay/auth_test.go index 8b7a0038b81..58ba964a753 100644 --- a/src/mongo/gotools/mongoreplay/auth_test.go +++ b/src/mongo/gotools/mongoreplay/auth_test.go @@ -29,7 +29,7 @@ func TestCommandsAgainstAuthedDBWhenAuthed(t *testing.T) { t.Error(err) } }() - statCollector, _ := newStatCollector(testCollectorOpts, true, true) + statCollector, _ := newStatCollector(testCollectorOpts, "format", true, true) context := NewExecutionContext(statCollector) t.Logf("Beginning mongoreplay playback of generated traffic against host: %v\n", urlAuth) err := Play(context, generator.opChan, testSpeed, urlAuth, 1, 10) @@ -94,7 +94,7 @@ func TestCommandsAgainstAuthedDBWhenNotAuthed(t *testing.T) { t.Error(err) } }() - statCollector, _ := newStatCollector(testCollectorOpts, true, true) + statCollector, _ := newStatCollector(testCollectorOpts, "format", true, true) context := NewExecutionContext(statCollector) err := Play(context, generator.opChan, testSpeed, urlNonAuth, 1, 10) if err != nil { diff --git a/src/mongo/gotools/mongoreplay/mongoreplay_test.go b/src/mongo/gotools/mongoreplay/mongoreplay_test.go index 5b54ebda216..8df441765e2 100644 --- a/src/mongo/gotools/mongoreplay/mongoreplay_test.go +++ b/src/mongo/gotools/mongoreplay/mongoreplay_test.go @@ -559,6 +559,16 @@ func TestShortenLegacyReply(t *testing.T) { } } +type cursorDoc struct { + Batch []interface{} `bson:"firstBatch"` + Id int64 `bson:"id"` + Ns string `bson:"ns"` +} +type findReply struct { + Cursor cursorDoc `bson:"cursor"` + Ok int `bson:"ok"` +} + func TestShortenCommandReply(t *testing.T) { generator := newRecordedOpGenerator() @@ -568,11 +578,6 @@ func TestShortenCommandReply(t *testing.T) { DocumentNumber: 100000, Success: true, } - op.CommandReply = &testDoc{ - Name: "Command Reply", - DocumentNumber: 200000, - Success: true, - } doc1 := testDoc{ Name: "Op Raw Short Reply Test 1", @@ -584,14 +589,21 @@ func TestShortenCommandReply(t *testing.T) { DocumentNumber: 2, Success: true, } - op.OutputDocs = []interface{}{doc1, doc2} + + batch := []interface{}{doc1, doc2} + + cursorDocIn := cursorDoc{ + batch, 12345678, "test"} + + op.CommandReply = findReply{cursorDocIn, 1} + op.OutputDocs = []interface{}{} result, err := generator.fetchRecordedOpsFromConn(&op.CommandReplyOp) // reply should be functional and parseable parsed, err := result.RawOp.Parse() if err != nil { - t.Errorf("error parsing op: %v", err) + t.Errorf("error parsing op: %#v", err) } t.Logf("parsed Op: %v", parsed) @@ -600,8 +612,23 @@ func TestShortenCommandReply(t *testing.T) { if !ok { t.Errorf("parsed op was wrong type") } - if !(len(fullReply.OutputDocs) == 2) { - t.Errorf("parsed reply has wrong number of docs: %d", len(fullReply.OutputDocs)) + + commandReplyCheckRaw, ok := fullReply.CommandReply.(*bson.Raw) + if !ok { + t.Errorf("comamndReply not bson.Raw") + } + + commandReplyCheck := &findReply{ + Cursor: cursorDoc{}, + } + err = bson.Unmarshal(commandReplyCheckRaw.Data, commandReplyCheck) + if err != nil { + t.Errorf("error unmarshaling commandReply %v", err) + } + + // ensure that the reply now has 2 document + if !(len(commandReplyCheck.Cursor.Batch) == 2) { + t.Errorf("parsed reply has wrong number of docs: %d", len(commandReplyCheck.Cursor.Batch)) } // shorten the reply @@ -617,9 +644,22 @@ func TestShortenCommandReply(t *testing.T) { t.Errorf("parsed op was wrong type") } - // ensure that the reply now has only 1 document - if !(len(fullReply.OutputDocs) == 1) { - t.Errorf("parsed reply has wrong number of docs: %d", len(fullReply.OutputDocs)) + commandReplyRaw, ok := fullReply.CommandReply.(*bson.Raw) + if !ok { + t.Errorf("comamndReply not bson.Raw") + } + + commandReplyOut := &findReply{ + Cursor: cursorDoc{}, + } + err = bson.Unmarshal(commandReplyRaw.Data, commandReplyOut) + if err != nil { + t.Errorf("error unmarshaling commandReply %v", err) + } + + // ensure that the reply now has 0 documents + if !(len(commandReplyOut.Cursor.Batch) == 0) { + t.Errorf("parsed reply has wrong number of docs: %d", len(commandReplyOut.Cursor.Batch)) } } diff --git a/src/mongo/gotools/mongoreplay/monitor.go b/src/mongo/gotools/mongoreplay/monitor.go index 502dc27ca7e..f79578e70c0 100644 --- a/src/mongo/gotools/mongoreplay/monitor.go +++ b/src/mongo/gotools/mongoreplay/monitor.go @@ -14,6 +14,7 @@ type MonitorCommand struct { GlobalOpts *Options `no-flag:"true"` StatOptions OpStreamSettings + Collect string `long:"collect" description:"Stat collection format; 'format' option uses the --format string" choice:"json" choice:"format" choice:"none" default:"format"` PairedMode bool `long:"paired" description:"Output only one line for a request/reply pair"` Gzip bool `long:"gzip" description:"decompress gzipped input"` PlaybackFile string `short:"p" description:"path to playback file to read from" long:"playback-file"` @@ -125,7 +126,7 @@ func (monitor *MonitorCommand) Execute(args []string) error { ctx.packetHandler.Close() }() } - statColl, err := newStatCollector(monitor.StatOptions, monitor.PairedMode, false) + statColl, err := newStatCollector(monitor.StatOptions, monitor.Collect, monitor.PairedMode, false) if err != nil { return err } diff --git a/src/mongo/gotools/mongoreplay/pcap_test.go b/src/mongo/gotools/mongoreplay/pcap_test.go index 97334a43989..185e9673f1e 100644 --- a/src/mongo/gotools/mongoreplay/pcap_test.go +++ b/src/mongo/gotools/mongoreplay/pcap_test.go @@ -165,7 +165,7 @@ func pcapTestHelper(t *testing.T, pcapFname string, preprocess bool, verifier ve t.Errorf("error opening playback file to write: %v\n", err) } - statCollector, _ := newStatCollector(testCollectorOpts, true, true) + statCollector, _ := newStatCollector(testCollectorOpts, "format", true, true) statRec := statCollector.StatRecorder.(*BufferedStatRecorder) context := NewExecutionContext(statCollector) diff --git a/src/mongo/gotools/mongoreplay/play.go b/src/mongo/gotools/mongoreplay/play.go index 220ee2b6664..baee4db33dd 100644 --- a/src/mongo/gotools/mongoreplay/play.go +++ b/src/mongo/gotools/mongoreplay/play.go @@ -21,6 +21,7 @@ type PlayCommand struct { QueueTime int `long:"queueTime" description:"don't queue ops much further in the future than this number of seconds" default:"15"` NoPreprocess bool `long:"no-preprocess" description:"don't preprocess the input file to premap data such as mongo cursorIDs"` Gzip bool `long:"gzip" description:"decompress gzipped input"` + Collect string `long:"collect" description:"Stat collection format; 'format' option uses the --format string" choice:"json" choice:"format" choice:"none" default:"none"` } const queueGranularity = 1000 @@ -179,7 +180,7 @@ func (play *PlayCommand) Execute(args []string) error { } play.GlobalOpts.SetLogging() - statColl, err := newStatCollector(play.StatOptions, true, true) + statColl, err := newStatCollector(play.StatOptions, play.Collect, true, true) if err != nil { return err } diff --git a/src/mongo/gotools/mongoreplay/play_livedb_test.go b/src/mongo/gotools/mongoreplay/play_livedb_test.go index 9f5e5c7e9e7..54838c60c53 100644 --- a/src/mongo/gotools/mongoreplay/play_livedb_test.go +++ b/src/mongo/gotools/mongoreplay/play_livedb_test.go @@ -29,7 +29,6 @@ var ( authTestServerMode bool isMongosTestServer bool testCollectorOpts = StatOptions{ - Collect: "format", Buffered: true, } ) @@ -181,7 +180,7 @@ func TestOpInsertLiveDB(t *testing.T) { } }() - statCollector, _ := newStatCollector(testCollectorOpts, true, true) + statCollector, _ := newStatCollector(testCollectorOpts, "format", true, true) statRec := statCollector.StatRecorder.(*BufferedStatRecorder) context := NewExecutionContext(statCollector) @@ -297,7 +296,7 @@ func TestUpdateOpLiveDB(t *testing.T) { } }() - statCollector, _ := newStatCollector(testCollectorOpts, true, true) + statCollector, _ := newStatCollector(testCollectorOpts, "format", true, true) statRec := statCollector.StatRecorder.(*BufferedStatRecorder) context := NewExecutionContext(statCollector) @@ -400,7 +399,7 @@ func TestQueryOpLiveDB(t *testing.T) { } }() - statCollector, _ := newStatCollector(testCollectorOpts, true, true) + statCollector, _ := newStatCollector(testCollectorOpts, "format", true, true) statRec := statCollector.StatRecorder.(*BufferedStatRecorder) context := NewExecutionContext(statCollector) @@ -488,7 +487,7 @@ func TestOpGetMoreLiveDB(t *testing.T) { } } }() - statCollector, _ := newStatCollector(testCollectorOpts, true, true) + statCollector, _ := newStatCollector(testCollectorOpts, "format", true, true) statRec := statCollector.StatRecorder.(*BufferedStatRecorder) context := NewExecutionContext(statCollector) @@ -589,7 +588,7 @@ func TestOpGetMoreMultiCursorLiveDB(t *testing.T) { } } }() - statCollector, _ := newStatCollector(testCollectorOpts, true, true) + statCollector, _ := newStatCollector(testCollectorOpts, "format", true, true) statRec := statCollector.StatRecorder.(*BufferedStatRecorder) context := NewExecutionContext(statCollector) @@ -710,7 +709,7 @@ func TestOpKillCursorsLiveDB(t *testing.T) { t.Error(err) } }() - statCollector, _ := newStatCollector(testCollectorOpts, true, true) + statCollector, _ := newStatCollector(testCollectorOpts, "format", true, true) statRec := statCollector.StatRecorder.(*BufferedStatRecorder) context := NewExecutionContext(statCollector) @@ -776,7 +775,7 @@ func TestCommandOpInsertLiveDB(t *testing.T) { } }() - statCollector, _ := newStatCollector(testCollectorOpts, true, true) + statCollector, _ := newStatCollector(testCollectorOpts, "format", true, true) statRec := statCollector.StatRecorder.(*BufferedStatRecorder) context := NewExecutionContext(statCollector) @@ -876,7 +875,7 @@ func TestCommandOpFindLiveDB(t *testing.T) { } }() - statCollector, _ := newStatCollector(testCollectorOpts, true, true) + statCollector, _ := newStatCollector(testCollectorOpts, "format", true, true) statRec := statCollector.StatRecorder.(*BufferedStatRecorder) context := NewExecutionContext(statCollector) @@ -968,7 +967,7 @@ func TestCommandOpGetMoreLiveDB(t *testing.T) { } } }() - statCollector, _ := newStatCollector(testCollectorOpts, true, true) + statCollector, _ := newStatCollector(testCollectorOpts, "format", true, true) statRec := statCollector.StatRecorder.(*BufferedStatRecorder) context := NewExecutionContext(statCollector) diff --git a/src/mongo/gotools/mongoreplay/raw_op.go b/src/mongo/gotools/mongoreplay/raw_op.go index bbe6871aa9e..2ab0c890cd0 100644 --- a/src/mongo/gotools/mongoreplay/raw_op.go +++ b/src/mongo/gotools/mongoreplay/raw_op.go @@ -6,8 +6,11 @@ import ( "io" mgo "github.com/10gen/llmgo" + "github.com/10gen/llmgo/bson" ) +const maxBSONSize = 16 * 1024 * 1024 // 16MB - maximum BSON document size + // RawOp may be exactly the same as OpUnknown. type RawOp struct { Header MsgHeader @@ -49,6 +52,16 @@ func (op *RawOp) FromReader(r io.Reader) error { return err } +type CommandReplyStruct struct { + Cursor struct { + Id int64 `bson:"id"` + Ns string `bson:"ns"` + FirstBatch bson.Raw `bson:"firstBatch,omitempty"` + NextBatch bson.Raw `bson:"nextBatch,omitempty"` + } `bson:"cursor"` + Ok int `bson:"ok"` +} + // ShortReplyFromReader reads an op from the given reader. It only holds on // to header-related information and the first document. func (op *RawOp) ShortenReply() error { @@ -66,18 +79,51 @@ func (op *RawOp) ShortenReply() error { return nil } firstDocSize := getInt32(op.Body, 20+MsgHeaderLen) + if 20+MsgHeaderLen+int(firstDocSize) > len(op.Body) || firstDocSize > maxBSONSize { + return fmt.Errorf("the size of the first document is greater then the size of the message") + } op.Body = op.Body[0:(20 + MsgHeaderLen + firstDocSize)] case OpCodeCommandReply: - commandReplyDocSize := getInt32(op.Body, MsgHeaderLen) - metadataDocSize := getInt32(op.Body, int(commandReplyDocSize)+MsgHeaderLen) - if op.Header.MessageLength <= commandReplyDocSize+metadataDocSize+MsgHeaderLen { - //there are no reply docs + // unmarshal the needed fields for replacing into the buffer + commandReply := &CommandReplyStruct{} + + err := bson.Unmarshal(op.Body[MsgHeaderLen:], commandReply) + if err != nil { + return fmt.Errorf("unmarshaling op to shorten: %v", err) + } + switch { + case commandReply.Cursor.FirstBatch.Data != nil: + commandReply.Cursor.FirstBatch.Data, _ = bson.Marshal([0]byte{}) + + case commandReply.Cursor.NextBatch.Data != nil: + commandReply.Cursor.NextBatch.Data, _ = bson.Marshal([0]byte{}) + + default: + // it's not a findReply so we don't care about it return nil } - firstOutputDocSize := getInt32(op.Body, int(commandReplyDocSize+metadataDocSize)+MsgHeaderLen) - shortReplySize := commandReplyDocSize + metadataDocSize + firstOutputDocSize + MsgHeaderLen - op.Body = op.Body[0:shortReplySize] + + out, err := bson.Marshal(commandReply) + if err != nil { + return err + } + + // calculate the new sizes for offsets into the new buffer + commandReplySize := getInt32(op.Body, MsgHeaderLen) + newCommandReplySize := getInt32(out, 0) + sizeDiff := commandReplySize - newCommandReplySize + newSize := op.Header.MessageLength - sizeDiff + newBody := make([]byte, newSize) + + // copy the new data into a buffer that will replace the old buffer + copy(newBody, op.Body[:MsgHeaderLen]) + copy(newBody[MsgHeaderLen:], out) + copy(newBody[MsgHeaderLen+newCommandReplySize:], op.Body[MsgHeaderLen+commandReplySize:]) + // update the size of this message in the headers + SetInt32(newBody, 0, newSize) + op.Header.MessageLength = newSize + op.Body = newBody default: return fmt.Errorf("unexpected op type : %v", op.Header.OpCode) diff --git a/src/mongo/gotools/mongoreplay/record.go b/src/mongo/gotools/mongoreplay/record.go index 670239ab3fe..3137488b923 100644 --- a/src/mongo/gotools/mongoreplay/record.go +++ b/src/mongo/gotools/mongoreplay/record.go @@ -51,13 +51,13 @@ func getOpstream(cfg OpStreamSettings) (*packetHandlerContext, error) { } } else if len(cfg.NetworkInterface) > 0 { inactive, err := pcap.NewInactiveHandle(cfg.NetworkInterface) - // This is safe; calling `Activate()` steals the underlying ptr. - defer inactive.CleanUp() if err != nil { return nil, fmt.Errorf("error creating a pcap handle: %v", err) } + // This is safe; calling `Activate()` steals the underlying ptr. + defer inactive.CleanUp() - err = inactive.SetSnapLen(32*1024*1024) + err = inactive.SetSnapLen(64 * 1024) if err != nil { return nil, fmt.Errorf("error setting snaplen on pcap handle: %v", err) } @@ -140,7 +140,7 @@ func (record *RecordCommand) ValidateParams(args []string) error { } if record.OpStreamSettings.CaptureBufSize == 0 { // default capture buffer size to 2 MiB (same as libpcap) - record.OpStreamSettings.CaptureBufSize = 2*1024 + record.OpStreamSettings.CaptureBufSize = 2 * 1024 } return nil } @@ -169,10 +169,10 @@ func (record *RecordCommand) Execute(args []string) error { ctx.packetHandler.Close() }() playbackWriter, err := NewPlaybackWriter(record.PlaybackFile, record.Gzip) - defer playbackWriter.Close() if err != nil { return err } + defer playbackWriter.Close() return Record(ctx, playbackWriter, record.FullReplies) @@ -186,23 +186,35 @@ func Record(ctx *packetHandlerContext, ch := make(chan error) go func() { defer close(ch) + var fail error for op := range ctx.mongoOpStream.Ops { + // since we don't currently have a way to shutdown packetHandler.Handle() + // continue to read from ctx.mongoOpStream.Ops even after a faltal error + if fail != nil { + toolDebugLogger.Logvf(DebugHigh, "not recording op because of record error %v", fail) + continue + } if (op.Header.OpCode == OpCodeReply || op.Header.OpCode == OpCodeCommandReply) && !noShortenReply { - op.ShortenReply() + err := op.ShortenReply() + if err != nil { + userInfoLogger.Logvf(DebugLow, "stream %v problem shortening reply: %v", op.SeenConnectionNum, err) + continue + } } bsonBytes, err := bson.Marshal(op) if err != nil { - ch <- fmt.Errorf("error marshaling message: %v", err) - return + userInfoLogger.Logvf(DebugLow, "stream %v error marshaling message: %v", op.SeenConnectionNum, err) + continue } _, err = playbackWriter.Write(bsonBytes) if err != nil { - ch <- fmt.Errorf("error writing message: %v", err) - return + fail = fmt.Errorf("error writing message: %v", err) + userInfoLogger.Logvf(Always, "%v", err) + continue } } - ch <- nil + ch <- fail }() if err := ctx.packetHandler.Handle(ctx.mongoOpStream, -1); err != nil { diff --git a/src/mongo/gotools/mongoreplay/stat_collector.go b/src/mongo/gotools/mongoreplay/stat_collector.go index 94f133c5d20..91009fe060e 100644 --- a/src/mongo/gotools/mongoreplay/stat_collector.go +++ b/src/mongo/gotools/mongoreplay/stat_collector.go @@ -18,8 +18,8 @@ const TruncateLength = 350 // StatOptions stores settings for the mongoreplay subcommands which have stat // output type StatOptions struct { - Collect string `long:"collect" description:"Stat collection format; 'format' option uses the --format string" choice:"json" choice:"format" choice:"none" default:"format"` Buffered bool `hidden:"yes"` + BufferSize int `long:"stats-buffer-size" description:"the size (in events) of the stat collector buffer" default:"1024"` Report string `long:"report" description:"Write report on execution to given output path"` NoTruncate bool `long:"no-truncate" description:"Disable truncation of large payload data in log output"` Format string `long:"format" description:"Format for terminal output, %-escaped. Arguments are provided immediately after the escape, surrounded in curly braces. Supported escapes are:\n %n namespace\n%l latency\n%t time (optional arg -- specify date layout, e.g. '%t{3:04PM}')\n%T op type\n%c command\n%o number of connections\n%i request ID\n%q request (optional arg -- dot-delimited field within the JSON structure, e.g. '%q{command_args.documents}')\n%r response (optional arg -- same as %q)\n%Q{<arg>} conditionally show <arg> on presence of request data\n%R{<arg>} conditionally show <arg> on presence of response data\nANSI escape sequences, start/end:\n%B/%b bold\n%U/%u underline\n%S/%s standout\n%F/%f text color (required arg -- word or number, 8-color)\n%K/%k background color (required arg -- same as %F/%f)\n" default:"%F{blue}%t%f %F{cyan}(Connection: %o:%i)%f %F{yellow}%l%f %F{red}%T %c%f %F{white}%n%f %F{green}%Q{Request:}%f%q %F{green}%R{Response:}%f%r"` @@ -34,6 +34,7 @@ type StatCollector struct { sync.Once done chan struct{} statStream chan *OpStat + statStreamSize int StatGenerator StatRecorder noop bool @@ -50,11 +51,11 @@ func (statColl *StatCollector) Close() error { return statColl.StatRecorder.Close() } -func newStatCollector(opts StatOptions, isPairedMode bool, isComparative bool) (*StatCollector, error) { +func newStatCollector(opts StatOptions, collectFormat string, isPairedMode bool, isComparative bool) (*StatCollector, error) { if opts.Buffered { - opts.Collect = "buffered" + collectFormat = "buffered" } - if opts.Collect == "none" { + if collectFormat == "none" { return &StatCollector{noop: true}, nil } @@ -84,7 +85,7 @@ func newStatCollector(opts StatOptions, isPairedMode bool, isComparative bool) ( } var statRec StatRecorder - switch opts.Collect { + switch collectFormat { case "json": statRec = &JSONStatRecorder{ out: o, @@ -101,9 +102,14 @@ func newStatCollector(opts StatOptions, isPairedMode bool, isComparative bool) ( } } + if opts.BufferSize < 1 { + opts.BufferSize = 1 + } + return &StatCollector{ - StatGenerator: statGen, - StatRecorder: statRec, + StatGenerator: statGen, + StatRecorder: statRec, + statStreamSize: opts.BufferSize, }, nil } @@ -145,7 +151,7 @@ func (statColl *StatCollector) Collect(op *RecordedOp, replayedOp Op, reply Repl return } statColl.Do(func() { - statColl.statStream = make(chan *OpStat, 1024) + statColl.statStream = make(chan *OpStat, statColl.statStreamSize) statColl.done = make(chan struct{}) go func() { for stat := range statColl.statStream { diff --git a/src/mongo/gotools/mongorestore/restore.go b/src/mongo/gotools/mongorestore/restore.go index 81adf69e63a..c97f1844161 100644 --- a/src/mongo/gotools/mongorestore/restore.go +++ b/src/mongo/gotools/mongorestore/restore.go @@ -141,6 +141,30 @@ func (restore *MongoRestore) RestoreIntent(intent *intents.Intent) error { return fmt.Errorf("error parsing metadata from %v: %v", intent.MetadataLocation, err) } + // The only way to specify options on the idIndex is at collection creation time. + // This loop pulls out the idIndex from `indexes` and sets it in `options`. + for i, index := range indexes { + // The index with the name "_id_" will always be the idIndex. + if index.Options["name"].(string) == "_id_" { + // Remove the index version (to use the default) unless otherwise specified. + if !restore.OutputOptions.KeepIndexVersion { + delete(index.Options, "v") + } + index.Options["ns"] = intent.Namespace() + + // If the collection has an idIndex, then we are about to create it, so + // ignore the value of autoIndexId. + for j, opt := range options { + if opt.Name == "autoIndexId" { + options = append(options[:j], options[j+1:]...) + } + } + options = append(options, bson.DocElem{"idIndex", index}) + indexes = append(indexes[:i], indexes[i+1:]...) + break + } + } + if restore.OutputOptions.NoOptionsRestore { log.Logv(log.Info, "not restoring collection options") logMessageSuffix = "with no collection options" diff --git a/src/mongo/gotools/test/qa-tests/jstests/export/export_views.js b/src/mongo/gotools/test/qa-tests/jstests/export/export_views.js new file mode 100644 index 00000000000..77e94c6af38 --- /dev/null +++ b/src/mongo/gotools/test/qa-tests/jstests/export/export_views.js @@ -0,0 +1,79 @@ +(function() { + if (typeof getToolTest === 'undefined') { + load('jstests/configs/plain_28.config.js'); + } + var dbName = 'test'; + var toolTest = getToolTest('views'); + var db = toolTest.db.getSiblingDB(dbName); + var commonToolArgs = getCommonToolArguments(); + + var exportTarget = 'views_export'; + removeFile(exportTarget); + + function addCitiesData() { + db.cities.insertMany([{ + city: 'Boise', + state: 'ID', + }, { + city: 'Pocatello', + state: 'ID', + }, { + city: 'Nampa', + state: 'ID', + }, { + city: 'Albany', + state: 'NY', + }, { + city: 'New York', + state: 'NY', + }, { + city: 'Los Angeles', + state: 'CA', + }, { + city: 'San Jose', + state: 'CA', + }, { + city: 'Cupertino', + state: 'CA', + }, { + city: 'San Francisco', + state: 'CA', + }]); + } + + function addStateView(state) { + db.createCollection('cities'+state, { + viewOn: 'cities', + pipeline: [{ + $match: {state: state}, + }], + }); + } + + addCitiesData(); + addStateView('ID'); + addStateView('NY'); + addStateView('CA'); + + assert.eq(9, db.cities.count(), 'should have 9 cities'); + assert.eq(3, db.citiesID.count(), 'should have 3 cities in Idaho view'); + assert.eq(2, db.citiesNY.count(), 'should have 2 cities in New York view'); + assert.eq(4, db.citiesCA.count(), 'should have 4 cities in California view'); + + var ret; + + ret = toolTest.runTool.apply(toolTest, ['export', '-o', exportTarget, '-d', dbName, '-c' , 'citiesCA'] + .concat(commonToolArgs)); + assert.eq(0, ret, 'export should succeed'); + + db.dropDatabase(); + + ret = toolTest.runTool.apply(toolTest, ['import', exportTarget, '-d', dbName, '-c' , 'CACities'] + .concat(commonToolArgs)); + assert.eq(0, ret, 'export should succeed'); + + assert.eq(4, db.CACities.count(), 'restored view should have correct number of rows'); + + removeFile(exportTarget); + toolTest.stop(); +}()); diff --git a/src/mongo/gotools/test/qa-tests/jstests/restore/collation.js b/src/mongo/gotools/test/qa-tests/jstests/restore/collation.js new file mode 100644 index 00000000000..ff7ebdbc997 --- /dev/null +++ b/src/mongo/gotools/test/qa-tests/jstests/restore/collation.js @@ -0,0 +1,73 @@ +(function() { + + if (typeof getToolTest === 'undefined') { + load('jstests/configs/plain_28.config.js'); + } + + // Tests that mongorestore correctly restores collections with a default collation. + + jsTest.log('Testing restoration of a collection with a default collation'); + + var toolTest = getToolTest('collation'); + var commonToolArgs = getCommonToolArguments(); + + var dumpTarget = 'collation_dump'; + resetDbpath(dumpTarget); + + var testDB = toolTest.db.getSiblingDB('test'); + var testColl = testDB.coll; + + // Create a collection with a default collation. + assert.commandWorked(testDB.createCollection('coll', {collation: {locale: 'fr_CA'}})); + var collectionInfos = testDB.getCollectionInfos({name: 'coll'}); + assert.eq(collectionInfos.length, 1); + assert(collectionInfos[0].options.hasOwnProperty('collation'), tojson(collectionInfos[0])); + var collationBefore = collectionInfos[0].options.collation; + + // Dump the data. + var ret = toolTest.runTool.apply(toolTest, ['dump'] + .concat(getDumpTarget(dumpTarget)) + .concat(commonToolArgs)); + assert.eq(0, ret); + + // Drop the collection. + testColl.drop(); + + // Restore the data. + ret = toolTest.runTool.apply(toolTest, ['restore'] + .concat(getRestoreTarget(dumpTarget)) + .concat(commonToolArgs)); + assert.eq(0, ret); + + collectionInfos = testDB.getCollectionInfos({name: 'coll'}); + assert.eq(collectionInfos.length, 1); + assert(collectionInfos[0].options.hasOwnProperty('collation'), tojson(collectionInfos[0])); + var collationAfter = collectionInfos[0].options.collation; + + // Check that the collection was restored with the same collation. + assert.docEq(collationBefore, collationAfter, tojson(collationBefore) + tojson(collationAfter)); + + if (dump_targets === 'archive') { + jsTest.log('skipping bson file restore test while running with archiving'); + } else { + // Drop the collection. + testColl.drop(); + + // Restore the data, but this time mentioning the bson file specifically. + ret = toolTest.runTool.apply(toolTest, ['restore'] + .concat(getRestoreTarget(dumpTarget+'/test/coll.bson')) + .concat(commonToolArgs)); + assert.eq(0, ret); + + collectionInfos = testDB.getCollectionInfos({name: 'coll'}); + assert.eq(collectionInfos.length, 1); + assert(collectionInfos[0].options.hasOwnProperty('collation'), tojson(collectionInfos[0])); + collationAfter = collectionInfos[0].options.collation; + + // Check that the collection was restored with the same collation. + assert.docEq(collationBefore, collationAfter, tojson(collationBefore) + tojson(collationAfter)); + } + + // success + toolTest.stop(); +}()); diff --git a/src/mongo/gotools/test/qa-tests/jstests/restore/index_version_roundtrip.js b/src/mongo/gotools/test/qa-tests/jstests/restore/index_version_roundtrip.js new file mode 100644 index 00000000000..48ffd42a7e4 --- /dev/null +++ b/src/mongo/gotools/test/qa-tests/jstests/restore/index_version_roundtrip.js @@ -0,0 +1,106 @@ +(function() { + + if (typeof getToolTest === 'undefined') { + load('jstests/configs/plain_28.config.js'); + } + + // Tests that mongorestore correctly round-trips _id index versions. + + jsTest.log('Testing restoration of different types of indexes'); + + var toolTest = getToolTest('index_version_roundtrip'); + var commonToolArgs = getCommonToolArguments(); + + // where we'll put the dump + var name = 'idx_version_rt_dump'; + resetDbpath(name); + + var testDB = toolTest.db.getSiblingDB(name); + + // drop the db + testDB.dropDatabase(); + + assert.commandWorked(testDB.runCommand({ + create: "coll1", + idIndex: { + v: 1, + key: { + _id: 1 + }, + name: "_id_", + ns: name + ".coll1", + } + })); + assert.commandWorked(testDB.runCommand({ + create: "coll2", + idIndex: { + v: 2, + key: { + _id: 1 + }, + name: "_id_", + ns: name + ".coll2", + } + })); + + // create an aditional index to verify non _id indexes work + assert.commandWorked(testDB.coll1.ensureIndex({a: 1}, {v: 1})); + assert.commandWorked(testDB.coll2.ensureIndex({a: 1}, {v: 2})); + + // insert arbitrary data so the collections aren't empty + testDB.coll1.insert({a: 123}); + testDB.coll2.insert({a: 123}); + + // store the index specs, for comparison after dump / restore + var idxSorter = function(a, b) { + return a.name.localeCompare(b.name); + }; + + var idxPre1 = testDB.coll1.getIndexSpecs(); + idxPre1.sort(idxSorter); + var idxPre2 = testDB.coll2.getIndexSpecs(); + idxPre2.sort(idxSorter); + + // dump the data + var ret = toolTest.runTool.apply(toolTest, ['dump'] + .concat(getDumpTarget(name)) + .concat(commonToolArgs)); + assert.eq(0, ret); + + // drop the db + testDB.dropDatabase(); + // sanity check that the drop worked + assert.eq(0, db.runCommand({ + listCollections: 1 + }).cursor.firstBatch.length); + + // restore the data + ret = toolTest.runTool.apply(toolTest, ['restore', '--keepIndexVersion'] + .concat(getRestoreTarget(name)) + .concat(commonToolArgs)); + assert.eq(0, ret); + + // make sure the data was restored + assert.eq(1, testDB.coll1.find().itcount()); + assert.eq(1, testDB.coll2.find().itcount()); + + // make sure the indexes were restored correctly + var idxPost1 = testDB.coll1.getIndexSpecs(); + idxPost1.sort(idxSorter); + assert.eq(idxPre1.length, idxPost1.length, + "indexes before: " + tojson(idxPre1) + "\nindexes after: " + tojson(idxPost1)); + for (var i = 0; i < idxPre1.length; i++) { + assert.eq(idxPre1[i], idxPost1[i]); + } + + var idxPost2 = testDB.coll2.getIndexSpecs(); + idxPost2.sort(idxSorter); + assert.eq(idxPre2.length, idxPost2.length, + "indexes before: " + tojson(idxPre2) + "\nindexes after: " + tojson(idxPost2)); + for (i = 0; i < idxPre2.length; i++) { + assert.eq(idxPre2[i], idxPost2[i]); + } + + // success + toolTest.stop(); +}()); diff --git a/src/mongo/gotools/test/qa-tests/jstests/restore/no_options_restore.js b/src/mongo/gotools/test/qa-tests/jstests/restore/no_options_restore.js index 9f57dc64bd9..89dd29551dd 100644 --- a/src/mongo/gotools/test/qa-tests/jstests/restore/no_options_restore.js +++ b/src/mongo/gotools/test/qa-tests/jstests/restore/no_options_restore.js @@ -89,6 +89,10 @@ // make sure the options were restored correctly var cappedOptionsFromDB = extractCollectionOptions(testDB, 'capped'); + // Restore no longer honors autoIndexId. + if (!cappedOptionsFromDB.hasOwnProperty('autoIndexId')) { + cappedOptionsFromDB.autoIndexId = true; + } assert.eq(baseCappedOptionsFromDB, cappedOptionsFromDB); var withOptionsFromDB = extractCollectionOptions(testDB, 'withOptions'); assert.eq(baseWithOptionsFromDB, withOptionsFromDB); |