Import tools: 4a0fbf5245669b55915adf7547ac592223681fe1 from branch v3.4

ref: 3cc9a07766..4a0fbf5245
for: 3.4.1

TOOLS-1498    Stats collection has large playback performance impact
TOOLS-1501    Add option to set capture buffer size to avoid packet loss
TOOLS-1502    Playback file contains full reply payload
TOOLS-1516    mongoreplay: out of bounds error in "shortenreply" during record
TOOLS-1534    Running mongodump then mongorestore should restore the _id index with its exact original spec
TOOLS-1535    Add test for restoring a collection with a default collation
TOOLS-1541    support exporting views
TOOLS-1558    use mongodb 3.4 "current" tests in master
TOOLS-1561    nil pointer dereference in mongoreplay when error on new playback file creation

19 files changed, 497 insertions, 94 deletions

diff --git a/src/mongo/gotools/common.yml b/src/mongo/gotools/common.yml
index f5d14936b6b..7453ec02145 100644
--- a/src/mongo/gotools/common.yml
+++ b/src/mongo/gotools/common.yml
@@ -841,7 +841,7 @@ tasks:
             value: "${args} -test.types=db"
     - func: "download mongod"
       vars:
-        mongo_version: "3.2"
+        mongo_version: "3.4"
     - func: "start mongod"
     - func: "wait for mongod to be ready"
     - func: "setup integration test"
@@ -925,7 +925,7 @@ tasks:
             value: "${args} -test.types=${integration_test_args}"
     - func: "download mongod"
       vars:
-        mongo_version: "3.2"
+        mongo_version: "3.4"
     - func: "start mongod"
     - func: "wait for mongod to be ready"
     - func: "run tool integration tests"
@@ -949,7 +949,7 @@ tasks:
             value: "db.createUser({ user: '${auth_username}', pwd: '${auth_password}', roles: [{ role: '__system', db: 'admin' }] });"
     - func: "download mongod"
       vars:
-        mongo_version: "3.2"
+        mongo_version: "3.4"
     - func: "start mongod"
     - func: "wait for mongod to be ready"
     - func: "run tool integration tests"
@@ -1193,7 +1193,7 @@ tasks:
     - func: "setup credentials"
     - func: "download mongod"
       vars:
-        mongo_version: "3.2"
+        mongo_version: "3.4"
     - func: "fetch tool"
       vars:
         tool: mongoimport
@@ -1288,7 +1288,7 @@ tasks:
     - func: "setup credentials"
     - func: "download mongod"
       vars:
-        mongo_version: "3.2"
+        mongo_version: "3.4"
     - func: "fetch tool"
       vars:
         tool: mongodump
@@ -1309,7 +1309,7 @@ tasks:
     - func: "setup credentials"
     - func: "download mongod"
       vars:
-        mongo_version: "3.2"
+        mongo_version: "3.4"
     - func: "fetch tool"
       vars:
         tool: mongodump
@@ -1330,7 +1330,7 @@ tasks:
     - func: "setup credentials"
     - func: "download mongod"
       vars:
-        mongo_version: "3.2"
+        mongo_version: "3.4"
     - func: "fetch tool"
       vars:
         tool: mongoimport
@@ -1452,7 +1452,7 @@ tasks:
       tool: mongoreplay
   - func: "download mongod"
     vars:
-      mongo_version: "3.2"
+      mongo_version: "3.4"
   - func: "start mongod"
   - func: "wait for mongod to be ready"
   - command: shell.exec
@@ -1482,7 +1482,7 @@ tasks:
       pcapFname: getmore_single_channel.pcap
   - func: "download mongod"
     vars:
-      mongo_version: "3.2"
+      mongo_version: "3.4"
   - func: "start mongod"
   - func: "wait for mongod to be ready"
   - func: "run go_test"
@@ -1507,7 +1507,7 @@ tasks:
       pcapFname: getmore_single_channel.pcap
   - func: "download mongod"
     vars:
-      mongo_version: "3.2"
+      mongo_version: "3.4"
   - func: "create sharded_cluster"
   - func: "run go_test"
     vars:
@@ -1531,7 +1531,7 @@ tasks:
       pcapFname: getmore_single_channel.pcap
   - func: "download mongod"
     vars:
-      mongo_version: "3.2"
+      mongo_version: "3.4"
   - func: "start mongod"
     vars:
         additional_args: --auth
@@ -1561,7 +1561,7 @@ tasks:
       pcapFname: getmore_single_channel.pcap
   - func: "download mongod"
     vars:
-      mongo_version: "3.2"
+      mongo_version: "3.4"
   - func: "create repl_set"
     vars:
         mongod_port: ${mongod_port}
@@ -1581,7 +1581,7 @@ tasks:
       tool: mongoreplay
   - func: "download mongod"
     vars:
-      mongo_version: "3.2"
+      mongo_version: "3.4"
   - func: "fetch ftdc"
   - command: shell.exec
     params:
diff --git a/src/mongo/gotools/common/db/namespaces.go b/src/mongo/gotools/common/db/namespaces.go
index 149400543ef..908687b1c56 100644
--- a/src/mongo/gotools/common/db/namespaces.go
+++ b/src/mongo/gotools/common/db/namespaces.go
@@ -129,6 +129,7 @@ func GetCollectionOptions(coll *mgo.Collection) (*bson.D, error) {
 	if err != nil {
 		return nil, err
 	}
+	defer iter.Close()
 	comparisonName := coll.Name
 	if useFullName {
 		comparisonName = coll.FullName
@@ -143,17 +144,23 @@ func GetCollectionOptions(coll *mgo.Collection) (*bson.D, error) {
 		if nameStr, ok := name.(string); ok {
 			if nameStr == comparisonName {
 				// we've found the collection we're looking for
-				return collInfo, nil
+				break
 			}
 		} else {
 			collInfo = nil
 			continue
 		}
 	}
-	err = iter.Err()
-	if err != nil {
-		return nil, err
+
+	if collInfo != nil {
+		optsInterface, _ := bsonutil.FindValueByKey("options", collInfo)
+		if optsInterface != nil {
+			optsD, ok := optsInterface.(bson.D)
+			if !ok {
+				return nil, fmt.Errorf("Cannot unmarshal collection options for collection %v.%v", coll.Database, coll.Name)
+			}
+			return &optsD, nil
+		}
 	}
-	// The given collection was not found, but no error encountered.
-	return nil, nil
+	return nil, iter.Err()
 }
diff --git a/src/mongo/gotools/import.data b/src/mongo/gotools/import.data
new file mode 100644
index 00000000000..f62f351837f
--- /dev/null
+++ b/src/mongo/gotools/import.data
@@ -0,0 +1,5 @@
+{
+    "commit": "4a0fbf5245669b55915adf7547ac592223681fe1", 
+    "github": "mongodb/mongo-tools.git", 
+    "branch": "v3.4"
+}
diff --git a/src/mongo/gotools/mongodump/prepare.go b/src/mongo/gotools/mongodump/prepare.go
index ff36bdfb02a..6c22748a810 100644
--- a/src/mongo/gotools/mongodump/prepare.go
+++ b/src/mongo/gotools/mongodump/prepare.go
@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"fmt"
 	"github.com/mongodb/mongo-tools/common/archive"
-	"github.com/mongodb/mongo-tools/common/bsonutil"
 	"github.com/mongodb/mongo-tools/common/db"
 	"github.com/mongodb/mongo-tools/common/intents"
 	"github.com/mongodb/mongo-tools/common/log"
@@ -301,23 +300,11 @@ func (dump *MongoDump) CreateCollectionIntent(dbName, colName string) error {
 	}
 	defer session.Close()
 
-	opts, err := db.GetCollectionOptions(session.DB(dbName).C(colName))
+	intent.Options, err = db.GetCollectionOptions(session.DB(dbName).C(colName))
 	if err != nil {
 		return fmt.Errorf("error getting collection options: %v", err)
 	}
 
-	intent.Options = nil
-	if opts != nil {
-		optsInterface, _ := bsonutil.FindValueByKey("options", opts)
-		if optsInterface != nil {
-			if optsD, ok := optsInterface.(bson.D); ok {
-				intent.Options = &optsD
-			} else {
-				return fmt.Errorf("Failed to parse collection options as bson.D")
-			}
-		}
-	}
-
 	dump.manager.Put(intent)
 
 	log.Logvf(log.DebugLow, "enqueued collection '%v'", intent.Namespace())
diff --git a/src/mongo/gotools/mongoexport/mongoexport.go b/src/mongo/gotools/mongoexport/mongoexport.go
index 6b4e96c68f1..4feab4894d2 100644
--- a/src/mongo/gotools/mongoexport/mongoexport.go
+++ b/src/mongo/gotools/mongoexport/mongoexport.go
@@ -233,16 +233,31 @@ func (exp *MongoExport) getCursor() (*mgo.Iter, *mgo.Session, error) {
 		}
 	}
 
-	flags := 0
-	if len(query) == 0 && exp.InputOpts != nil &&
-		exp.InputOpts.ForceTableScan != true && exp.InputOpts.Sort == "" {
-		flags = flags | db.Snapshot
+	session, err := exp.SessionProvider.GetSession()
+	if err != nil {
+		return nil, nil, err
 	}
+	collection := session.DB(exp.ToolOptions.Namespace.DB).C(exp.ToolOptions.Namespace.Collection)
 
-	session, err := exp.SessionProvider.GetSession()
+	// figure out if we're exporting a view
+	isView := false
+	opts, err := db.GetCollectionOptions(collection)
 	if err != nil {
 		return nil, nil, err
 	}
+	if opts != nil {
+		viewOn, _ := bsonutil.FindValueByKey("viewOn", opts)
+		if viewOn != nil {
+			isView = true
+		}
+	}
+
+	flags := 0
+	// don't snapshot if we've been asked not to,
+	// or if we cannot because  we are querying, sorting, or if the collection is a view
+	if !exp.InputOpts.ForceTableScan && len(query) == 0 && exp.InputOpts != nil && exp.InputOpts.Sort == "" && !isView {
+		flags = flags | db.Snapshot
+	}
 
 	skip := 0
 	if exp.InputOpts != nil {
@@ -265,9 +280,7 @@ func (exp *MongoExport) getCursor() (*mgo.Iter, *mgo.Session, error) {
 	}
 
 	// build the query
-	q := session.DB(exp.ToolOptions.Namespace.DB).
-		C(exp.ToolOptions.Namespace.Collection).Find(query).Sort(sortFields...).
-		Skip(skip).Limit(limit)
+	q := collection.Find(query).Sort(sortFields...).Skip(skip).Limit(limit)
 
 	if len(exp.OutputOpts.Fields) > 0 {
 		q.Select(makeFieldSelector(exp.OutputOpts.Fields))
diff --git a/src/mongo/gotools/mongoreplay/auth_test.go b/src/mongo/gotools/mongoreplay/auth_test.go
index 8b7a0038b81..58ba964a753 100644
--- a/src/mongo/gotools/mongoreplay/auth_test.go
+++ b/src/mongo/gotools/mongoreplay/auth_test.go
@@ -29,7 +29,7 @@ func TestCommandsAgainstAuthedDBWhenAuthed(t *testing.T) {
 			t.Error(err)
 		}
 	}()
-	statCollector, _ := newStatCollector(testCollectorOpts, true, true)
+	statCollector, _ := newStatCollector(testCollectorOpts, "format", true, true)
 	context := NewExecutionContext(statCollector)
 	t.Logf("Beginning mongoreplay playback of generated traffic against host: %v\n", urlAuth)
 	err := Play(context, generator.opChan, testSpeed, urlAuth, 1, 10)
@@ -94,7 +94,7 @@ func TestCommandsAgainstAuthedDBWhenNotAuthed(t *testing.T) {
 			t.Error(err)
 		}
 	}()
-	statCollector, _ := newStatCollector(testCollectorOpts, true, true)
+	statCollector, _ := newStatCollector(testCollectorOpts, "format", true, true)
 	context := NewExecutionContext(statCollector)
 	err := Play(context, generator.opChan, testSpeed, urlNonAuth, 1, 10)
 	if err != nil {
diff --git a/src/mongo/gotools/mongoreplay/mongoreplay_test.go b/src/mongo/gotools/mongoreplay/mongoreplay_test.go
index 5b54ebda216..8df441765e2 100644
--- a/src/mongo/gotools/mongoreplay/mongoreplay_test.go
+++ b/src/mongo/gotools/mongoreplay/mongoreplay_test.go
@@ -559,6 +559,16 @@ func TestShortenLegacyReply(t *testing.T) {
 	}
 }
 
+type cursorDoc struct {
+	Batch []interface{} `bson:"firstBatch"`
+	Id    int64         `bson:"id"`
+	Ns    string        `bson:"ns"`
+}
+type findReply struct {
+	Cursor cursorDoc `bson:"cursor"`
+	Ok     int       `bson:"ok"`
+}
+
 func TestShortenCommandReply(t *testing.T) {
 	generator := newRecordedOpGenerator()
 
@@ -568,11 +578,6 @@ func TestShortenCommandReply(t *testing.T) {
 		DocumentNumber: 100000,
 		Success:        true,
 	}
-	op.CommandReply = &testDoc{
-		Name:           "Command Reply",
-		DocumentNumber: 200000,
-		Success:        true,
-	}
 
 	doc1 := testDoc{
 		Name:           "Op Raw Short Reply Test 1",
@@ -584,14 +589,21 @@ func TestShortenCommandReply(t *testing.T) {
 		DocumentNumber: 2,
 		Success:        true,
 	}
-	op.OutputDocs = []interface{}{doc1, doc2}
+
+	batch := []interface{}{doc1, doc2}
+
+	cursorDocIn := cursorDoc{
+		batch, 12345678, "test"}
+
+	op.CommandReply = findReply{cursorDocIn, 1}
+	op.OutputDocs = []interface{}{}
 
 	result, err := generator.fetchRecordedOpsFromConn(&op.CommandReplyOp)
 
 	// reply should be functional and parseable
 	parsed, err := result.RawOp.Parse()
 	if err != nil {
-		t.Errorf("error parsing op: %v", err)
+		t.Errorf("error parsing op: %#v", err)
 	}
 
 	t.Logf("parsed Op: %v", parsed)
@@ -600,8 +612,23 @@ func TestShortenCommandReply(t *testing.T) {
 	if !ok {
 		t.Errorf("parsed op was wrong type")
 	}
-	if !(len(fullReply.OutputDocs) == 2) {
-		t.Errorf("parsed reply has wrong number of docs: %d", len(fullReply.OutputDocs))
+
+	commandReplyCheckRaw, ok := fullReply.CommandReply.(*bson.Raw)
+	if !ok {
+		t.Errorf("comamndReply not bson.Raw")
+	}
+
+	commandReplyCheck := &findReply{
+		Cursor: cursorDoc{},
+	}
+	err = bson.Unmarshal(commandReplyCheckRaw.Data, commandReplyCheck)
+	if err != nil {
+		t.Errorf("error unmarshaling commandReply %v", err)
+	}
+
+	// ensure that the reply now has 2 document
+	if !(len(commandReplyCheck.Cursor.Batch) == 2) {
+		t.Errorf("parsed reply has wrong number of docs: %d", len(commandReplyCheck.Cursor.Batch))
 	}
 
 	// shorten the reply
@@ -617,9 +644,22 @@ func TestShortenCommandReply(t *testing.T) {
 		t.Errorf("parsed op was wrong type")
 	}
 
-	// ensure that the reply now has only 1 document
-	if !(len(fullReply.OutputDocs) == 1) {
-		t.Errorf("parsed reply has wrong number of docs: %d", len(fullReply.OutputDocs))
+	commandReplyRaw, ok := fullReply.CommandReply.(*bson.Raw)
+	if !ok {
+		t.Errorf("comamndReply not bson.Raw")
+	}
+
+	commandReplyOut := &findReply{
+		Cursor: cursorDoc{},
+	}
+	err = bson.Unmarshal(commandReplyRaw.Data, commandReplyOut)
+	if err != nil {
+		t.Errorf("error unmarshaling commandReply %v", err)
+	}
+
+	// ensure that the reply now has 0 documents
+	if !(len(commandReplyOut.Cursor.Batch) == 0) {
+		t.Errorf("parsed reply has wrong number of docs: %d", len(commandReplyOut.Cursor.Batch))
 	}
 }
 
diff --git a/src/mongo/gotools/mongoreplay/monitor.go b/src/mongo/gotools/mongoreplay/monitor.go
index 502dc27ca7e..f79578e70c0 100644
--- a/src/mongo/gotools/mongoreplay/monitor.go
+++ b/src/mongo/gotools/mongoreplay/monitor.go
@@ -14,6 +14,7 @@ type MonitorCommand struct {
 	GlobalOpts *Options `no-flag:"true"`
 	StatOptions
 	OpStreamSettings
+	Collect      string `long:"collect" description:"Stat collection format; 'format' option uses the --format string" choice:"json" choice:"format" choice:"none" default:"format"`
 	PairedMode   bool   `long:"paired" description:"Output only one line for a request/reply pair"`
 	Gzip         bool   `long:"gzip" description:"decompress gzipped input"`
 	PlaybackFile string `short:"p" description:"path to playback file to read from" long:"playback-file"`
@@ -125,7 +126,7 @@ func (monitor *MonitorCommand) Execute(args []string) error {
 			ctx.packetHandler.Close()
 		}()
 	}
-	statColl, err := newStatCollector(monitor.StatOptions, monitor.PairedMode, false)
+	statColl, err := newStatCollector(monitor.StatOptions, monitor.Collect, monitor.PairedMode, false)
 	if err != nil {
 		return err
 	}
diff --git a/src/mongo/gotools/mongoreplay/pcap_test.go b/src/mongo/gotools/mongoreplay/pcap_test.go
index 97334a43989..185e9673f1e 100644
--- a/src/mongo/gotools/mongoreplay/pcap_test.go
+++ b/src/mongo/gotools/mongoreplay/pcap_test.go
@@ -165,7 +165,7 @@ func pcapTestHelper(t *testing.T, pcapFname string, preprocess bool, verifier ve
 		t.Errorf("error opening playback file to write: %v\n", err)
 	}
 
-	statCollector, _ := newStatCollector(testCollectorOpts, true, true)
+	statCollector, _ := newStatCollector(testCollectorOpts, "format", true, true)
 	statRec := statCollector.StatRecorder.(*BufferedStatRecorder)
 	context := NewExecutionContext(statCollector)
 
diff --git a/src/mongo/gotools/mongoreplay/play.go b/src/mongo/gotools/mongoreplay/play.go
index 220ee2b6664..baee4db33dd 100644
--- a/src/mongo/gotools/mongoreplay/play.go
+++ b/src/mongo/gotools/mongoreplay/play.go
@@ -21,6 +21,7 @@ type PlayCommand struct {
 	QueueTime    int     `long:"queueTime" description:"don't queue ops much further in the future than this number of seconds" default:"15"`
 	NoPreprocess bool    `long:"no-preprocess" description:"don't preprocess the input file to premap data such as mongo cursorIDs"`
 	Gzip         bool    `long:"gzip" description:"decompress gzipped input"`
+	Collect      string  `long:"collect" description:"Stat collection format; 'format' option uses the --format string" choice:"json" choice:"format" choice:"none" default:"none"`
 }
 
 const queueGranularity = 1000
@@ -179,7 +180,7 @@ func (play *PlayCommand) Execute(args []string) error {
 	}
 	play.GlobalOpts.SetLogging()
 
-	statColl, err := newStatCollector(play.StatOptions, true, true)
+	statColl, err := newStatCollector(play.StatOptions, play.Collect, true, true)
 	if err != nil {
 		return err
 	}
diff --git a/src/mongo/gotools/mongoreplay/play_livedb_test.go b/src/mongo/gotools/mongoreplay/play_livedb_test.go
index 9f5e5c7e9e7..54838c60c53 100644
--- a/src/mongo/gotools/mongoreplay/play_livedb_test.go
+++ b/src/mongo/gotools/mongoreplay/play_livedb_test.go
@@ -29,7 +29,6 @@ var (
 	authTestServerMode  bool
 	isMongosTestServer  bool
 	testCollectorOpts   = StatOptions{
-		Collect:  "format",
 		Buffered: true,
 	}
 )
@@ -181,7 +180,7 @@ func TestOpInsertLiveDB(t *testing.T) {
 		}
 	}()
 
-	statCollector, _ := newStatCollector(testCollectorOpts, true, true)
+	statCollector, _ := newStatCollector(testCollectorOpts, "format", true, true)
 	statRec := statCollector.StatRecorder.(*BufferedStatRecorder)
 	context := NewExecutionContext(statCollector)
 
@@ -297,7 +296,7 @@ func TestUpdateOpLiveDB(t *testing.T) {
 		}
 	}()
 
-	statCollector, _ := newStatCollector(testCollectorOpts, true, true)
+	statCollector, _ := newStatCollector(testCollectorOpts, "format", true, true)
 	statRec := statCollector.StatRecorder.(*BufferedStatRecorder)
 	context := NewExecutionContext(statCollector)
 
@@ -400,7 +399,7 @@ func TestQueryOpLiveDB(t *testing.T) {
 		}
 	}()
 
-	statCollector, _ := newStatCollector(testCollectorOpts, true, true)
+	statCollector, _ := newStatCollector(testCollectorOpts, "format", true, true)
 	statRec := statCollector.StatRecorder.(*BufferedStatRecorder)
 	context := NewExecutionContext(statCollector)
 
@@ -488,7 +487,7 @@ func TestOpGetMoreLiveDB(t *testing.T) {
 			}
 		}
 	}()
-	statCollector, _ := newStatCollector(testCollectorOpts, true, true)
+	statCollector, _ := newStatCollector(testCollectorOpts, "format", true, true)
 	statRec := statCollector.StatRecorder.(*BufferedStatRecorder)
 	context := NewExecutionContext(statCollector)
 
@@ -589,7 +588,7 @@ func TestOpGetMoreMultiCursorLiveDB(t *testing.T) {
 			}
 		}
 	}()
-	statCollector, _ := newStatCollector(testCollectorOpts, true, true)
+	statCollector, _ := newStatCollector(testCollectorOpts, "format", true, true)
 	statRec := statCollector.StatRecorder.(*BufferedStatRecorder)
 	context := NewExecutionContext(statCollector)
 
@@ -710,7 +709,7 @@ func TestOpKillCursorsLiveDB(t *testing.T) {
 			t.Error(err)
 		}
 	}()
-	statCollector, _ := newStatCollector(testCollectorOpts, true, true)
+	statCollector, _ := newStatCollector(testCollectorOpts, "format", true, true)
 	statRec := statCollector.StatRecorder.(*BufferedStatRecorder)
 	context := NewExecutionContext(statCollector)
 
@@ -776,7 +775,7 @@ func TestCommandOpInsertLiveDB(t *testing.T) {
 		}
 	}()
 
-	statCollector, _ := newStatCollector(testCollectorOpts, true, true)
+	statCollector, _ := newStatCollector(testCollectorOpts, "format", true, true)
 	statRec := statCollector.StatRecorder.(*BufferedStatRecorder)
 	context := NewExecutionContext(statCollector)
 
@@ -876,7 +875,7 @@ func TestCommandOpFindLiveDB(t *testing.T) {
 		}
 	}()
 
-	statCollector, _ := newStatCollector(testCollectorOpts, true, true)
+	statCollector, _ := newStatCollector(testCollectorOpts, "format", true, true)
 	statRec := statCollector.StatRecorder.(*BufferedStatRecorder)
 	context := NewExecutionContext(statCollector)
 
@@ -968,7 +967,7 @@ func TestCommandOpGetMoreLiveDB(t *testing.T) {
 			}
 		}
 	}()
-	statCollector, _ := newStatCollector(testCollectorOpts, true, true)
+	statCollector, _ := newStatCollector(testCollectorOpts, "format", true, true)
 	statRec := statCollector.StatRecorder.(*BufferedStatRecorder)
 	context := NewExecutionContext(statCollector)
 
diff --git a/src/mongo/gotools/mongoreplay/raw_op.go b/src/mongo/gotools/mongoreplay/raw_op.go
index bbe6871aa9e..2ab0c890cd0 100644
--- a/src/mongo/gotools/mongoreplay/raw_op.go
+++ b/src/mongo/gotools/mongoreplay/raw_op.go
@@ -6,8 +6,11 @@ import (
 	"io"
 
 	mgo "github.com/10gen/llmgo"
+	"github.com/10gen/llmgo/bson"
 )
 
+const maxBSONSize = 16 * 1024 * 1024 // 16MB - maximum BSON document size
+
 // RawOp may be exactly the same as OpUnknown.
 type RawOp struct {
 	Header MsgHeader
@@ -49,6 +52,16 @@ func (op *RawOp) FromReader(r io.Reader) error {
 	return err
 }
 
+type CommandReplyStruct struct {
+	Cursor struct {
+		Id         int64    `bson:"id"`
+		Ns         string   `bson:"ns"`
+		FirstBatch bson.Raw `bson:"firstBatch,omitempty"`
+		NextBatch  bson.Raw `bson:"nextBatch,omitempty"`
+	} `bson:"cursor"`
+	Ok int `bson:"ok"`
+}
+
 // ShortReplyFromReader reads an op from the given reader. It only holds on
 // to header-related information and the first document.
 func (op *RawOp) ShortenReply() error {
@@ -66,18 +79,51 @@ func (op *RawOp) ShortenReply() error {
 			return nil
 		}
 		firstDocSize := getInt32(op.Body, 20+MsgHeaderLen)
+		if 20+MsgHeaderLen+int(firstDocSize) > len(op.Body) || firstDocSize > maxBSONSize {
+			return fmt.Errorf("the size of the first document is greater then the size of the message")
+		}
 		op.Body = op.Body[0:(20 + MsgHeaderLen + firstDocSize)]
 
 	case OpCodeCommandReply:
-		commandReplyDocSize := getInt32(op.Body, MsgHeaderLen)
-		metadataDocSize := getInt32(op.Body, int(commandReplyDocSize)+MsgHeaderLen)
-		if op.Header.MessageLength <= commandReplyDocSize+metadataDocSize+MsgHeaderLen {
-			//there are no reply docs
+		// unmarshal the needed fields for replacing into the buffer
+		commandReply := &CommandReplyStruct{}
+
+		err := bson.Unmarshal(op.Body[MsgHeaderLen:], commandReply)
+		if err != nil {
+			return fmt.Errorf("unmarshaling op to shorten: %v", err)
+		}
+		switch {
+		case commandReply.Cursor.FirstBatch.Data != nil:
+			commandReply.Cursor.FirstBatch.Data, _ = bson.Marshal([0]byte{})
+
+		case commandReply.Cursor.NextBatch.Data != nil:
+			commandReply.Cursor.NextBatch.Data, _ = bson.Marshal([0]byte{})
+
+		default:
+			// it's not a findReply so we don't care about it
 			return nil
 		}
-		firstOutputDocSize := getInt32(op.Body, int(commandReplyDocSize+metadataDocSize)+MsgHeaderLen)
-		shortReplySize := commandReplyDocSize + metadataDocSize + firstOutputDocSize + MsgHeaderLen
-		op.Body = op.Body[0:shortReplySize]
+
+		out, err := bson.Marshal(commandReply)
+		if err != nil {
+			return err
+		}
+
+		// calculate the new sizes for offsets into the new buffer
+		commandReplySize := getInt32(op.Body, MsgHeaderLen)
+		newCommandReplySize := getInt32(out, 0)
+		sizeDiff := commandReplySize - newCommandReplySize
+		newSize := op.Header.MessageLength - sizeDiff
+		newBody := make([]byte, newSize)
+
+		// copy the new data into a buffer that will replace the old buffer
+		copy(newBody, op.Body[:MsgHeaderLen])
+		copy(newBody[MsgHeaderLen:], out)
+		copy(newBody[MsgHeaderLen+newCommandReplySize:], op.Body[MsgHeaderLen+commandReplySize:])
+		// update the size of this message in the headers
+		SetInt32(newBody, 0, newSize)
+		op.Header.MessageLength = newSize
+		op.Body = newBody
 
 	default:
 		return fmt.Errorf("unexpected op type : %v", op.Header.OpCode)
diff --git a/src/mongo/gotools/mongoreplay/record.go b/src/mongo/gotools/mongoreplay/record.go
index 670239ab3fe..3137488b923 100644
--- a/src/mongo/gotools/mongoreplay/record.go
+++ b/src/mongo/gotools/mongoreplay/record.go
@@ -51,13 +51,13 @@ func getOpstream(cfg OpStreamSettings) (*packetHandlerContext, error) {
 		}
 	} else if len(cfg.NetworkInterface) > 0 {
 		inactive, err := pcap.NewInactiveHandle(cfg.NetworkInterface)
-		// This is safe; calling `Activate()` steals the underlying ptr.
-		defer inactive.CleanUp()
 		if err != nil {
 			return nil, fmt.Errorf("error creating a pcap handle: %v", err)
 		}
+		// This is safe; calling `Activate()` steals the underlying ptr.
+		defer inactive.CleanUp()
 
-		err = inactive.SetSnapLen(32*1024*1024)
+		err = inactive.SetSnapLen(64 * 1024)
 		if err != nil {
 			return nil, fmt.Errorf("error setting snaplen on pcap handle: %v", err)
 		}
@@ -140,7 +140,7 @@ func (record *RecordCommand) ValidateParams(args []string) error {
 	}
 	if record.OpStreamSettings.CaptureBufSize == 0 {
 		// default capture buffer size to 2 MiB (same as libpcap)
-		record.OpStreamSettings.CaptureBufSize = 2*1024
+		record.OpStreamSettings.CaptureBufSize = 2 * 1024
 	}
 	return nil
 }
@@ -169,10 +169,10 @@ func (record *RecordCommand) Execute(args []string) error {
 		ctx.packetHandler.Close()
 	}()
 	playbackWriter, err := NewPlaybackWriter(record.PlaybackFile, record.Gzip)
-	defer playbackWriter.Close()
 	if err != nil {
 		return err
 	}
+	defer playbackWriter.Close()
 
 	return Record(ctx, playbackWriter, record.FullReplies)
 
@@ -186,23 +186,35 @@ func Record(ctx *packetHandlerContext,
 	ch := make(chan error)
 	go func() {
 		defer close(ch)
+		var fail error
 		for op := range ctx.mongoOpStream.Ops {
+			// since we don't currently have a way to shutdown packetHandler.Handle()
+			// continue to read from ctx.mongoOpStream.Ops even after a faltal error
+			if fail != nil {
+				toolDebugLogger.Logvf(DebugHigh, "not recording op because of record error %v", fail)
+				continue
+			}
 			if (op.Header.OpCode == OpCodeReply || op.Header.OpCode == OpCodeCommandReply) &&
 				!noShortenReply {
-				op.ShortenReply()
+				err := op.ShortenReply()
+				if err != nil {
+					userInfoLogger.Logvf(DebugLow, "stream %v problem shortening reply: %v", op.SeenConnectionNum, err)
+					continue
+				}
 			}
 			bsonBytes, err := bson.Marshal(op)
 			if err != nil {
-				ch <- fmt.Errorf("error marshaling message: %v", err)
-				return
+				userInfoLogger.Logvf(DebugLow, "stream %v error marshaling message: %v", op.SeenConnectionNum, err)
+				continue
 			}
 			_, err = playbackWriter.Write(bsonBytes)
 			if err != nil {
-				ch <- fmt.Errorf("error writing message: %v", err)
-				return
+				fail = fmt.Errorf("error writing message: %v", err)
+				userInfoLogger.Logvf(Always, "%v", err)
+				continue
 			}
 		}
-		ch <- nil
+		ch <- fail
 	}()
 
 	if err := ctx.packetHandler.Handle(ctx.mongoOpStream, -1); err != nil {
diff --git a/src/mongo/gotools/mongoreplay/stat_collector.go b/src/mongo/gotools/mongoreplay/stat_collector.go
index 94f133c5d20..91009fe060e 100644
--- a/src/mongo/gotools/mongoreplay/stat_collector.go
+++ b/src/mongo/gotools/mongoreplay/stat_collector.go
@@ -18,8 +18,8 @@ const TruncateLength = 350
 // StatOptions stores settings for the mongoreplay subcommands which have stat
 // output
 type StatOptions struct {
-	Collect    string `long:"collect" description:"Stat collection format; 'format' option uses the --format string" choice:"json" choice:"format" choice:"none" default:"format"`
 	Buffered   bool   `hidden:"yes"`
+	BufferSize int    `long:"stats-buffer-size" description:"the size (in events) of the stat collector buffer" default:"1024"`
 	Report     string `long:"report" description:"Write report on execution to given output path"`
 	NoTruncate bool   `long:"no-truncate" description:"Disable truncation of large payload data in log output"`
 	Format     string `long:"format" description:"Format for terminal output, %-escaped. Arguments are provided immediately after the escape, surrounded in curly braces. Supported escapes are:\n	%n namespace\n%l latency\n%t time (optional arg -- specify date layout, e.g. '%t{3:04PM}')\n%T op type\n%c command\n%o number of connections\n%i request ID\n%q request (optional arg -- dot-delimited field within the JSON structure, e.g. '%q{command_args.documents}')\n%r response (optional arg -- same as %q)\n%Q{<arg>} conditionally show <arg> on presence of request data\n%R{<arg>} conditionally show <arg> on presence of response data\nANSI escape sequences, start/end:\n%B/%b bold\n%U/%u underline\n%S/%s standout\n%F/%f text color (required arg -- word or number, 8-color)\n%K/%k background color (required arg -- same as %F/%f)\n" default:"%F{blue}%t%f %F{cyan}(Connection: %o:%i)%f %F{yellow}%l%f %F{red}%T %c%f %F{white}%n%f %F{green}%Q{Request:}%f%q %F{green}%R{Response:}%f%r"`
@@ -34,6 +34,7 @@ type StatCollector struct {
 	sync.Once
 	done       chan struct{}
 	statStream chan *OpStat
+	statStreamSize int
 	StatGenerator
 	StatRecorder
 	noop bool
@@ -50,11 +51,11 @@ func (statColl *StatCollector) Close() error {
 	return statColl.StatRecorder.Close()
 }
 
-func newStatCollector(opts StatOptions, isPairedMode bool, isComparative bool) (*StatCollector, error) {
+func newStatCollector(opts StatOptions, collectFormat string, isPairedMode bool, isComparative bool) (*StatCollector, error) {
 	if opts.Buffered {
-		opts.Collect = "buffered"
+		collectFormat = "buffered"
 	}
-	if opts.Collect == "none" {
+	if collectFormat == "none" {
 		return &StatCollector{noop: true}, nil
 	}
 
@@ -84,7 +85,7 @@ func newStatCollector(opts StatOptions, isPairedMode bool, isComparative bool) (
 	}
 
 	var statRec StatRecorder
-	switch opts.Collect {
+	switch collectFormat {
 	case "json":
 		statRec = &JSONStatRecorder{
 			out: o,
@@ -101,9 +102,14 @@ func newStatCollector(opts StatOptions, isPairedMode bool, isComparative bool) (
 		}
 	}
 
+	if opts.BufferSize < 1 {
+		opts.BufferSize = 1
+	}
+
 	return &StatCollector{
-		StatGenerator: statGen,
-		StatRecorder:  statRec,
+		StatGenerator:  statGen,
+		StatRecorder:   statRec,
+		statStreamSize: opts.BufferSize,
 	}, nil
 }
 
@@ -145,7 +151,7 @@ func (statColl *StatCollector) Collect(op *RecordedOp, replayedOp Op, reply Repl
 		return
 	}
 	statColl.Do(func() {
-		statColl.statStream = make(chan *OpStat, 1024)
+		statColl.statStream = make(chan *OpStat, statColl.statStreamSize)
 		statColl.done = make(chan struct{})
 		go func() {
 			for stat := range statColl.statStream {
diff --git a/src/mongo/gotools/mongorestore/restore.go b/src/mongo/gotools/mongorestore/restore.go
index 81adf69e63a..c97f1844161 100644
--- a/src/mongo/gotools/mongorestore/restore.go
+++ b/src/mongo/gotools/mongorestore/restore.go
@@ -141,6 +141,30 @@ func (restore *MongoRestore) RestoreIntent(intent *intents.Intent) error {
 			return fmt.Errorf("error parsing metadata from %v: %v", intent.MetadataLocation, err)
 		}
 
+		// The only way to specify options on the idIndex is at collection creation time.
+		// This loop pulls out the idIndex from `indexes` and sets it in `options`.
+		for i, index := range indexes {
+			// The index with the name "_id_" will always be the idIndex.
+			if index.Options["name"].(string) == "_id_" {
+				// Remove the index version (to use the default) unless otherwise specified.
+				if !restore.OutputOptions.KeepIndexVersion {
+					delete(index.Options, "v")
+				}
+				index.Options["ns"] = intent.Namespace()
+
+				// If the collection has an idIndex, then we are about to create it, so
+				// ignore the value of autoIndexId.
+				for j, opt := range options {
+					if opt.Name == "autoIndexId" {
+						options = append(options[:j], options[j+1:]...)
+					}
+				}
+				options = append(options, bson.DocElem{"idIndex", index})
+				indexes = append(indexes[:i], indexes[i+1:]...)
+				break
+			}
+		}
+
 		if restore.OutputOptions.NoOptionsRestore {
 			log.Logv(log.Info, "not restoring collection options")
 			logMessageSuffix = "with no collection options"
diff --git a/src/mongo/gotools/test/qa-tests/jstests/export/export_views.js b/src/mongo/gotools/test/qa-tests/jstests/export/export_views.js
new file mode 100644
index 00000000000..77e94c6af38
--- /dev/null
+++ b/src/mongo/gotools/test/qa-tests/jstests/export/export_views.js
@@ -0,0 +1,79 @@
+(function() {
+  if (typeof getToolTest === 'undefined') {
+    load('jstests/configs/plain_28.config.js');
+  }
+  var dbName = 'test';
+  var toolTest = getToolTest('views');
+  var db = toolTest.db.getSiblingDB(dbName);
+  var commonToolArgs = getCommonToolArguments();
+
+  var exportTarget = 'views_export';
+  removeFile(exportTarget);
+
+  function addCitiesData() {
+    db.cities.insertMany([{
+      city: 'Boise',
+      state: 'ID',
+    }, {
+      city: 'Pocatello',
+      state: 'ID',
+    }, {
+      city: 'Nampa',
+      state: 'ID',
+    }, {
+      city: 'Albany',
+      state: 'NY',
+    }, {
+      city: 'New York',
+      state: 'NY',
+    }, {
+      city: 'Los Angeles',
+      state: 'CA',
+    }, {
+      city: 'San Jose',
+      state: 'CA',
+    }, {
+      city: 'Cupertino',
+      state: 'CA',
+    }, {
+      city: 'San Francisco',
+      state: 'CA',
+    }]);
+  }
+
+  function addStateView(state) {
+    db.createCollection('cities'+state, {
+      viewOn: 'cities',
+      pipeline: [{
+        $match: {state: state},
+      }],
+    });
+  }
+
+  addCitiesData();
+  addStateView('ID');
+  addStateView('NY');
+  addStateView('CA');
+
+  assert.eq(9, db.cities.count(), 'should have 9 cities');
+  assert.eq(3, db.citiesID.count(), 'should have 3 cities in Idaho view');
+  assert.eq(2, db.citiesNY.count(), 'should have 2 cities in New York view');
+  assert.eq(4, db.citiesCA.count(), 'should have 4 cities in California view');
+
+  var ret;
+
+  ret = toolTest.runTool.apply(toolTest, ['export', '-o', exportTarget, '-d', dbName, '-c' , 'citiesCA']
+      .concat(commonToolArgs));
+  assert.eq(0, ret, 'export should succeed');
+
+  db.dropDatabase();
+
+  ret = toolTest.runTool.apply(toolTest, ['import', exportTarget, '-d', dbName, '-c' , 'CACities']
+      .concat(commonToolArgs));
+  assert.eq(0, ret, 'export should succeed');
+
+  assert.eq(4, db.CACities.count(), 'restored view should have correct number of rows');
+
+  removeFile(exportTarget);
+  toolTest.stop();
+}());
diff --git a/src/mongo/gotools/test/qa-tests/jstests/restore/collation.js b/src/mongo/gotools/test/qa-tests/jstests/restore/collation.js
new file mode 100644
index 00000000000..ff7ebdbc997
--- /dev/null
+++ b/src/mongo/gotools/test/qa-tests/jstests/restore/collation.js
@@ -0,0 +1,73 @@
+(function() {
+
+  if (typeof getToolTest === 'undefined') {
+    load('jstests/configs/plain_28.config.js');
+  }
+
+  // Tests that mongorestore correctly restores collections with a default collation.
+
+  jsTest.log('Testing restoration of a collection with a default collation');
+
+  var toolTest = getToolTest('collation');
+  var commonToolArgs = getCommonToolArguments();
+
+  var dumpTarget = 'collation_dump';
+  resetDbpath(dumpTarget);
+
+  var testDB = toolTest.db.getSiblingDB('test');
+  var testColl = testDB.coll;
+
+  // Create a collection with a default collation.
+  assert.commandWorked(testDB.createCollection('coll', {collation: {locale: 'fr_CA'}}));
+  var collectionInfos = testDB.getCollectionInfos({name: 'coll'});
+  assert.eq(collectionInfos.length, 1);
+  assert(collectionInfos[0].options.hasOwnProperty('collation'), tojson(collectionInfos[0]));
+  var collationBefore = collectionInfos[0].options.collation;
+
+  // Dump the data.
+  var ret = toolTest.runTool.apply(toolTest, ['dump']
+    .concat(getDumpTarget(dumpTarget))
+    .concat(commonToolArgs));
+  assert.eq(0, ret);
+
+  // Drop the collection.
+  testColl.drop();
+
+  // Restore the data.
+  ret = toolTest.runTool.apply(toolTest, ['restore']
+    .concat(getRestoreTarget(dumpTarget))
+    .concat(commonToolArgs));
+  assert.eq(0, ret);
+
+  collectionInfos = testDB.getCollectionInfos({name: 'coll'});
+  assert.eq(collectionInfos.length, 1);
+  assert(collectionInfos[0].options.hasOwnProperty('collation'), tojson(collectionInfos[0]));
+  var collationAfter = collectionInfos[0].options.collation;
+
+  // Check that the collection was restored with the same collation.
+  assert.docEq(collationBefore, collationAfter, tojson(collationBefore) + tojson(collationAfter));
+
+  if (dump_targets === 'archive') {
+    jsTest.log('skipping bson file restore test while running with archiving');
+  } else {
+    // Drop the collection.
+    testColl.drop();
+
+    // Restore the data, but this time mentioning the bson file specifically.
+    ret = toolTest.runTool.apply(toolTest, ['restore']
+      .concat(getRestoreTarget(dumpTarget+'/test/coll.bson'))
+      .concat(commonToolArgs));
+    assert.eq(0, ret);
+
+    collectionInfos = testDB.getCollectionInfos({name: 'coll'});
+    assert.eq(collectionInfos.length, 1);
+    assert(collectionInfos[0].options.hasOwnProperty('collation'), tojson(collectionInfos[0]));
+    collationAfter = collectionInfos[0].options.collation;
+
+    // Check that the collection was restored with the same collation.
+    assert.docEq(collationBefore, collationAfter, tojson(collationBefore) + tojson(collationAfter));
+  }
+
+  // success
+  toolTest.stop();
+}());
diff --git a/src/mongo/gotools/test/qa-tests/jstests/restore/index_version_roundtrip.js b/src/mongo/gotools/test/qa-tests/jstests/restore/index_version_roundtrip.js
new file mode 100644
index 00000000000..48ffd42a7e4
--- /dev/null
+++ b/src/mongo/gotools/test/qa-tests/jstests/restore/index_version_roundtrip.js
@@ -0,0 +1,106 @@
+(function() {
+
+  if (typeof getToolTest === 'undefined') {
+    load('jstests/configs/plain_28.config.js');
+  }
+
+  // Tests that mongorestore correctly round-trips _id index versions.
+
+  jsTest.log('Testing restoration of different types of indexes');
+
+  var toolTest = getToolTest('index_version_roundtrip');
+  var commonToolArgs = getCommonToolArguments();
+
+  // where we'll put the dump
+  var name = 'idx_version_rt_dump';
+  resetDbpath(name);
+
+  var testDB = toolTest.db.getSiblingDB(name);
+
+  // drop the db
+  testDB.dropDatabase();
+
+  assert.commandWorked(testDB.runCommand({
+    create: "coll1",
+    idIndex: {
+      v: 1,
+      key: {
+        _id: 1
+      },
+      name: "_id_",
+      ns: name + ".coll1",
+    }
+  }));
+  assert.commandWorked(testDB.runCommand({
+    create: "coll2",
+    idIndex: {
+      v: 2,
+      key: {
+        _id: 1
+      },
+      name: "_id_",
+      ns: name + ".coll2",
+    }
+  }));
+
+  // create an aditional index to verify non _id indexes work
+  assert.commandWorked(testDB.coll1.ensureIndex({a: 1}, {v: 1}));
+  assert.commandWorked(testDB.coll2.ensureIndex({a: 1}, {v: 2}));
+
+  // insert arbitrary data so the collections aren't empty
+  testDB.coll1.insert({a: 123});
+  testDB.coll2.insert({a: 123});
+
+  // store the index specs, for comparison after dump / restore
+  var idxSorter = function(a, b) {
+    return a.name.localeCompare(b.name);
+  };
+
+  var idxPre1 = testDB.coll1.getIndexSpecs();
+  idxPre1.sort(idxSorter);
+  var idxPre2 = testDB.coll2.getIndexSpecs();
+  idxPre2.sort(idxSorter);
+
+  // dump the data
+  var ret = toolTest.runTool.apply(toolTest, ['dump']
+    .concat(getDumpTarget(name))
+    .concat(commonToolArgs));
+  assert.eq(0, ret);
+
+  // drop the db
+  testDB.dropDatabase();
+  // sanity check that the drop worked
+  assert.eq(0, db.runCommand({
+    listCollections: 1
+  }).cursor.firstBatch.length);
+
+  // restore the data
+  ret = toolTest.runTool.apply(toolTest, ['restore', '--keepIndexVersion']
+    .concat(getRestoreTarget(name))
+    .concat(commonToolArgs));
+  assert.eq(0, ret);
+
+  // make sure the data was restored
+  assert.eq(1, testDB.coll1.find().itcount());
+  assert.eq(1, testDB.coll2.find().itcount());
+
+  // make sure the indexes were restored correctly
+  var idxPost1 = testDB.coll1.getIndexSpecs();
+  idxPost1.sort(idxSorter);
+  assert.eq(idxPre1.length, idxPost1.length,
+    "indexes before: " + tojson(idxPre1) + "\nindexes after: " + tojson(idxPost1));
+  for (var i = 0; i < idxPre1.length; i++) {
+    assert.eq(idxPre1[i], idxPost1[i]);
+  }
+
+  var idxPost2 = testDB.coll2.getIndexSpecs();
+  idxPost2.sort(idxSorter);
+  assert.eq(idxPre2.length, idxPost2.length,
+    "indexes before: " + tojson(idxPre2) + "\nindexes after: " + tojson(idxPost2));
+  for (i = 0; i < idxPre2.length; i++) {
+    assert.eq(idxPre2[i], idxPost2[i]);
+  }
+
+  // success
+  toolTest.stop();
+}());
diff --git a/src/mongo/gotools/test/qa-tests/jstests/restore/no_options_restore.js b/src/mongo/gotools/test/qa-tests/jstests/restore/no_options_restore.js
index 9f57dc64bd9..89dd29551dd 100644
--- a/src/mongo/gotools/test/qa-tests/jstests/restore/no_options_restore.js
+++ b/src/mongo/gotools/test/qa-tests/jstests/restore/no_options_restore.js
@@ -89,6 +89,10 @@
 
   // make sure the options were restored correctly
   var cappedOptionsFromDB = extractCollectionOptions(testDB, 'capped');
+  // Restore no longer honors autoIndexId.
+  if (!cappedOptionsFromDB.hasOwnProperty('autoIndexId')) {
+    cappedOptionsFromDB.autoIndexId = true;
+  }
   assert.eq(baseCappedOptionsFromDB, cappedOptionsFromDB);
   var withOptionsFromDB = extractCollectionOptions(testDB, 'withOptions');
   assert.eq(baseWithOptionsFromDB, withOptionsFromDB);