Import tools: af4004cbd1f136d24b6484d66371d37a591b0cd8 from branch v3.3

ref: 959ed7c66b..af4004cbd1
for: 3.3.13

TOOLS-1397    mongoplay / truncate reply messages after first doc in record
TOOLS-1442    make spacemonkeygo/openssl compile on suse11

8 files changed, 174 insertions, 26 deletions

diff --git a/src/mongo/gotools/mongoreplay/command_reply.go b/src/mongo/gotools/mongoreplay/command_reply.go
index f95053f3ff2..55d0c5dd6ea 100644
--- a/src/mongo/gotools/mongoreplay/command_reply.go
+++ b/src/mongo/gotools/mongoreplay/command_reply.go
@@ -149,17 +149,24 @@ func (op *CommandReplyOp) FromReader(r io.Reader) error {
 		return err
 	}
 
-	lengthRead := len(commandReplyAsSlice) + len(metadataAsSlice)
 	op.OutputDocs = make([]interface{}, 0)
-	docLen := 0
-	for lengthRead+docLen < int(op.Header.MessageLength)-MsgHeaderLen {
+	for {
 		docAsSlice, err := ReadDocument(r)
+		if err != nil {
+			if err != io.EOF {
+				// Broken BSON in reply data. TODO log something here?
+				return err
+			}
+			break
+		}
+		if len(docAsSlice) == 0 {
+			break
+		}
 		doc := &bson.Raw{}
 		err = bson.Unmarshal(docAsSlice, doc)
 		if err != nil {
 			return err
 		}
-		docLen += len(docAsSlice)
 		op.OutputDocs = append(op.OutputDocs, doc)
 	}
 	return nil
diff --git a/src/mongo/gotools/mongoreplay/cursors_test.go b/src/mongo/gotools/mongoreplay/cursors_test.go
index eb619e275e2..7a8e727e6ff 100644
--- a/src/mongo/gotools/mongoreplay/cursors_test.go
+++ b/src/mongo/gotools/mongoreplay/cursors_test.go
@@ -174,12 +174,12 @@ func TestBlockOnUnresolvedCursor(t *testing.T) {
 			t.Error("Cursor map returned result before live cursor was mapped")
 		}
 		// Retrieve cursorInfo from map
-		*lock.RLock()
+		lock.RLock()
 		cursorInfo, ok := cursorManager.cursorInfos[fileCursor]
 		if !ok {
 			t.Errorf("Cursor %v was supposed to be mapped, but wasn't", testCursorID)
 		}
-		*lock.RUnlock()
+		lock.RUnlock()
 
 		t.Log("Verifying that successChan not closed before cursor was set")
 		// Verify that its successChan is not closed, which indicates that
@@ -323,12 +323,12 @@ func TestSkipOnMarkFailed(t *testing.T) {
 		if retrievedCursor != -1 {
 			t.Error("Cursor map returned result before cursor was marked as failed")
 		}
-		*lock.RLock()
+		lock.RLock()
 		cursorInfo, ok := preprocessManager.cursorInfos[testCursorID]
 		if !ok {
 			t.Errorf("Cursor %v was supposed to be mapped, but wasn't", testCursorID)
 		}
-		*lock.RUnlock()
+		lock.RUnlock()
 
 		t.Log("Checking that successChan and failChan are still open before marking op as failed")
 		select {
diff --git a/src/mongo/gotools/mongoreplay/execute_test.go b/src/mongo/gotools/mongoreplay/execute_test.go
index 6cac6c0cc04..1b4d23559d4 100644
--- a/src/mongo/gotools/mongoreplay/execute_test.go
+++ b/src/mongo/gotools/mongoreplay/execute_test.go
@@ -4,15 +4,11 @@ import (
 	"testing"
 
 	mgo "github.com/10gen/llmgo"
-	"github.com/mongodb/mongo-tools/common/log"
-	"github.com/mongodb/mongo-tools/common/options"
 )
 
 func TestCompleteReply(t *testing.T) {
 	context := NewExecutionContext(&StatCollector{})
 
-	log.SetVerbosity(&options.Verbosity{[]bool{true, true, true, true, true}, false})
-
 	// AddFromWire takes a recorded request and a live reply to the re-execution
 	// of that reply
 	reply1 := &ReplyOp{}
diff --git a/src/mongo/gotools/mongoreplay/mongotape_test.go b/src/mongo/gotools/mongoreplay/mongotape_test.go
index 9771138300a..dc9be128c37 100644
--- a/src/mongo/gotools/mongoreplay/mongotape_test.go
+++ b/src/mongo/gotools/mongoreplay/mongotape_test.go
@@ -492,6 +492,136 @@ func TestOpCommandReplyGetCursorID(t *testing.T) {
 	}
 }
 
+func TestShortenLegacyReply(t *testing.T) {
+	generator := newRecordedOpGenerator()
+
+	op := ReplyOp{}
+	op.ReplyDocs = 2
+
+	result, err := generator.fetchRecordedOpsFromConn(&op.ReplyOp)
+
+	doc1 := &testDoc{
+		Name:           "Op Raw Short Reply Test 1",
+		DocumentNumber: 1,
+		Success:        true,
+	}
+	doc2 := &testDoc{
+		Name:           "Op Raw Short Reply Test 2",
+		DocumentNumber: 2,
+		Success:        true,
+	}
+
+	asByte1, err := bson.Marshal(doc1)
+	if err != nil {
+		t.Errorf("could not marshal bson: %v", err)
+	}
+
+	asByte2, err := bson.Marshal(doc2)
+	if err != nil {
+		t.Errorf("could not marshal bson: %v", err)
+	}
+
+	// add the two docs as the docs from the reply
+	result.RawOp.Body = append(result.RawOp.Body, asByte1...)
+	result.RawOp.Body = append(result.RawOp.Body, asByte2...)
+
+	// reply should be functional and parseable
+	parsed, err := result.RawOp.Parse()
+	if err != nil {
+		t.Errorf("error parsing op: %v", err)
+	}
+
+	fullReply, ok := parsed.(*ReplyOp)
+	if !ok {
+		t.Errorf("parsed op was wrong type")
+	}
+	if !(len(fullReply.Docs) == 2) {
+		t.Errorf("parsed reply has wrong number of docs: %d", len(fullReply.Docs))
+	}
+
+	// shorten the reply
+	result.ShortenReply()
+
+	parsed, err = result.RawOp.Parse()
+	if err != nil {
+		t.Errorf("error parsing op: %v", err)
+	}
+
+	fullReply, ok = parsed.(*ReplyOp)
+	if !ok {
+		t.Errorf("parsed op was wrong type")
+	}
+
+	// ensure that the reply now has only 1 document
+	if !(len(fullReply.Docs) == 1) {
+		t.Errorf("parsed reply has wrong number of docs: %d", len(fullReply.Docs))
+	}
+}
+
+func TestShortenCommandReply(t *testing.T) {
+	generator := newRecordedOpGenerator()
+
+	op := CommandReplyOp{}
+	op.Metadata = &testDoc{
+		Name:           "Metadata",
+		DocumentNumber: 100000,
+		Success:        true,
+	}
+	op.CommandReply = &testDoc{
+		Name:           "Command Reply",
+		DocumentNumber: 200000,
+		Success:        true,
+	}
+
+	doc1 := testDoc{
+		Name:           "Op Raw Short Reply Test 1",
+		DocumentNumber: 1,
+		Success:        true,
+	}
+	doc2 := testDoc{
+		Name:           "Op Raw Short Reply Test 2",
+		DocumentNumber: 2,
+		Success:        true,
+	}
+	op.OutputDocs = []interface{}{doc1, doc2}
+
+	result, err := generator.fetchRecordedOpsFromConn(&op.CommandReplyOp)
+
+	// reply should be functional and parseable
+	parsed, err := result.RawOp.Parse()
+	if err != nil {
+		t.Errorf("error parsing op: %v", err)
+	}
+
+	t.Logf("parsed Op: %v", parsed)
+
+	fullReply, ok := parsed.(*CommandReplyOp)
+	if !ok {
+		t.Errorf("parsed op was wrong type")
+	}
+	if !(len(fullReply.OutputDocs) == 2) {
+		t.Errorf("parsed reply has wrong number of docs: %d", len(fullReply.OutputDocs))
+	}
+
+	// shorten the reply
+	result.ShortenReply()
+
+	parsed, err = result.RawOp.Parse()
+	if err != nil {
+		t.Errorf("error parsing op: %v", err)
+	}
+
+	fullReply, ok = parsed.(*CommandReplyOp)
+	if !ok {
+		t.Errorf("parsed op was wrong type")
+	}
+
+	// ensure that the reply now has only 1 document
+	if !(len(fullReply.OutputDocs) == 1) {
+		t.Errorf("parsed reply has wrong number of docs: %d", len(fullReply.OutputDocs))
+	}
+}
+
 func TestLegacyOpReplyGetCursorID(t *testing.T) {
 	testCursorID := int64(123)
 	doc := &struct {
diff --git a/src/mongo/gotools/mongoreplay/pcap_test.go b/src/mongo/gotools/mongoreplay/pcap_test.go
index eb777323f15..df0c9325660 100644
--- a/src/mongo/gotools/mongoreplay/pcap_test.go
+++ b/src/mongo/gotools/mongoreplay/pcap_test.go
@@ -155,7 +155,7 @@ func pcapTestHelper(t *testing.T, pcapFname string, preprocess bool, verifier ve
 	}
 
 	t.Log("Recording playbackfile from pcap file")
-	err = Record(ctx, playbackWriter)
+	err = Record(ctx, playbackWriter, false)
 	if err != nil {
 		t.Errorf("error makign tape file: %v\n", err)
 	}
diff --git a/src/mongo/gotools/mongoreplay/raw_op.go b/src/mongo/gotools/mongoreplay/raw_op.go
index 3429ce6338f..e430bea339d 100644
--- a/src/mongo/gotools/mongoreplay/raw_op.go
+++ b/src/mongo/gotools/mongoreplay/raw_op.go
@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"fmt"
 	"io"
-	"io/ioutil"
 
 	mgo "github.com/10gen/llmgo"
 )
@@ -51,21 +50,31 @@ func (op *RawOp) FromReader(r io.Reader) error {
 }
 
 // ShortReplyFromReader reads an op from the given reader. It only holds on
-// to header-related information
-func (op *RawOp) ShortReplyFromReader(r io.Reader) error {
+// to header-related information and the first document.
+func (op *RawOp) ShortenReply() error {
 	if op.Header.MessageLength < MsgHeaderLen {
-		return nil
+		return fmt.Errorf("expected message header to have length: %d bytes but was %d bytes", MsgHeaderLen, op.Header.MessageLength)
 	}
 	if op.Header.MessageLength > MaxMessageSize {
 		return fmt.Errorf("wire message size, %v, was greater then the maximum, %v bytes", op.Header.MessageLength, MaxMessageSize)
 	}
-	op.Body = make([]byte, 20) // op_replies have an additional 20 bytes of header that we capture
-	_, err := io.ReadFull(r, op.Body)
-	if err != nil {
-		return err
+
+	switch op.Header.OpCode {
+	case OpCodeReply:
+		firstDocSize := getInt32(op.Body, 20+MsgHeaderLen)
+		op.Body = op.Body[0:(20 + MsgHeaderLen + firstDocSize)]
+
+	case OpCodeCommandReply:
+		commandReplyDocSize := getInt32(op.Body, MsgHeaderLen)
+		metadataDocSize := getInt32(op.Body, int(commandReplyDocSize)+MsgHeaderLen)
+		firstOutputDocSize := getInt32(op.Body, int(commandReplyDocSize+metadataDocSize)+MsgHeaderLen)
+		shortReplySize := commandReplyDocSize + metadataDocSize + firstOutputDocSize + MsgHeaderLen
+		op.Body = op.Body[0:shortReplySize]
+
+	default:
+		return fmt.Errorf("unexpected op type : %v", op.Header.OpCode)
 	}
-	_, err = io.CopyN(ioutil.Discard, r, int64(op.Header.MessageLength-MsgHeaderLen-20))
-	return err
+	return nil
 }
 
 // Parse returns the underlying op from its given RawOp form.
diff --git a/src/mongo/gotools/mongoreplay/record.go b/src/mongo/gotools/mongoreplay/record.go
index ae87055d71b..aaecb157348 100644
--- a/src/mongo/gotools/mongoreplay/record.go
+++ b/src/mongo/gotools/mongoreplay/record.go
@@ -17,6 +17,7 @@ type RecordCommand struct {
 	GlobalOpts *Options `no-flag:"true"`
 	OpStreamSettings
 	Gzip         bool   `long:"gzip" description:"compress output file with Gzip"`
+	FullReplies  bool   `long:"full-replies" description:"save full reply payload in playback file"`
 	PlaybackFile string `short:"p" description:"path to playback file to record to" long:"playback-file" required:"yes"`
 }
 
@@ -139,18 +140,23 @@ func (record *RecordCommand) Execute(args []string) error {
 		return err
 	}
 
-	return Record(ctx, playbackWriter)
+	return Record(ctx, playbackWriter, record.FullReplies)
 
 }
 
 // Record writes pcap data into a playback file
 func Record(ctx *packetHandlerContext,
-	playbackWriter *PlaybackWriter) error {
+	playbackWriter *PlaybackWriter,
+	noShortenReply bool) error {
 
 	ch := make(chan error)
 	go func() {
 		defer close(ch)
 		for op := range ctx.mongoOpStream.Ops {
+			if (op.Header.OpCode == OpCodeReply || op.Header.OpCode == OpCodeCommandReply) &&
+				!noShortenReply {
+				op.ShortenReply()
+			}
 			bsonBytes, err := bson.Marshal(op)
 			if err != nil {
 				ch <- fmt.Errorf("error marshaling message: %v", err)
diff --git a/src/mongo/gotools/mongoreplay/sanity_check.sh b/src/mongo/gotools/mongoreplay/sanity_check.sh
index 7c7e4b804fe..7b840913732 100755
--- a/src/mongo/gotools/mongoreplay/sanity_check.sh
+++ b/src/mongo/gotools/mongoreplay/sanity_check.sh
@@ -28,7 +28,7 @@ done
 
 command -v mongoreplay >/dev/null
 if [ $? != 0 ]; then
-  log "mongoreplay must be in PATH"
+  echo "mongoreplay must be in PATH"
   exit 1
 fi