Import tools: 3cc9a07766fb55de63e81a13e72f3c5a7c07f477 from branch v3.3

ref: a604ced46f..3cc9a07766 for: 3.4.0 TOOLS-1392 Move mongoreplay tests to the mongotools project TOOLS-1489 fix per collection overhead when doing gzip dumps TOOLS-1491 mongoreplay date is wrong TOOLS-1497 A single invocation of mongoreplay replays the ops twice TOOLS-1501 Add option to set capture buffer size to avoid packet loss TOOLS-1503 Truncated playback file when using --gzip option TOOLS-1505 Connections never closed during replay TOOLS-1511 mongodump does not report number of dumped documents when query is specified TOOLS-1517 Use dynamically allocated ports in tests TOOLS-1518 wait a few seconds after the kill -ABRT to get time to dump stacks in timeout handler TOOLS-1519 fix dumping of view in archives TOOLS-1524 tests are not logging to buildlogger
author: Ramon Fernandez <ramon@mongodb.com> 2016-11-08 12:23:36 -0500
committer: Ramon Fernandez <ramon@mongodb.com> 2016-11-08 12:23:36 -0500
commit: bf31ef57f79a58e79c8fd0dd01d05d843252208a (patch)
tree: 93c6b9fe5f5333baa426eb6786b00f58c26a3e24
parent: bb78a26fcdca87e808cab52b6daa91be581bcbbc (diff)
download: mongo-bf31ef57f79a58e79c8fd0dd01d05d843252208a.tar.gz
23 files changed, 772 insertions, 442 deletions
diff --git a/src/mongo/gotools/.evergreen.yml b/src/mongo/gotools/.evergreen.yml
index dba4b8f3870..43535b8124e 100644
--- a/src/mongo/gotools/.evergreen.yml
+++ b/src/mongo/gotools/.evergreen.yml
@@ -190,7 +190,7 @@ post:
   - command: shell.cleanup
 
 tasks:
-- name: build
+- name: replay-dist
   commands:
   - func: "fetch source"
   - func: "build tool"
@@ -201,9 +201,9 @@ tasks:
       tool: mongotape
   - func: "build tool"
 
-- name: sanity_check
+- name: replay-sanity_check
   depends_on:
-  - name: build
+  - name: replay-dist
   commands:
   - func: "fetch source"
   - func: "fetch tool"
@@ -221,9 +221,9 @@ tasks:
         echo "Running sanity check"
         ./sanity_check.sh -p ${mongod_port}
 
-- name: go_test
+- name: replay-go_test
   depends_on:
-  - name: build
+  - name: replay-dist
   commands:
   - func: "fetch source"
   - func: "fetch tool"
@@ -243,9 +243,9 @@ tasks:
     vars:
         filename: playtest
 
-- name: sharded_test
+- name: replay-sharded_test
   depends_on:
-  - name: build
+  - name: replay-dist
   commands:
   - func: "fetch source"
   - func: "fetch tool"
@@ -267,9 +267,9 @@ tasks:
         environment_vars: DB_PORT=20010
         additional_args: --run "LiveDB"
 
-- name: auth_test
+- name: replay-auth_test
   depends_on:
-  - name: build
+  - name: replay-dist
   commands:
   - func: "fetch source"
   - func: "fetch tool"
@@ -295,9 +295,9 @@ tasks:
         filename: authtest
         additional_args: --run "(LiveDB)|(Authed)"
 
-- name: repl_test
+- name: replay-repl_test
   depends_on:
-  - name: build
+  - name: replay-dist
   commands:
   - func: "fetch source"
   - func: "fetch tool"
@@ -318,9 +318,9 @@ tasks:
         filename: repltest
         additional_args: --run "LiveDB"
 
-- name: replay_test
+- name: replay-replay_test
   depends_on:
-  - name: build
+  - name: replay-dist
   commands:
   - func: "fetch source"
   - func: "fetch tool"
@@ -358,10 +358,10 @@ buildvariants:
   run_on:
   - ubuntu1404-test
   tasks:
-  - name: build
-  - name: sanity_check
-  - name: go_test
-  - name: auth_test
-  - name: sharded_test
-  - name: repl_test
-  - name: replay_test
+  - name: replay-dist
+  - name: replay-sanity_check
+  - name: replay-go_test
+  - name: replay-auth_test
+  - name: replay-sharded_test
+  - name: replay-repl_test
+  - name: replay-replay_test
diff --git a/src/mongo/gotools/common.yml b/src/mongo/gotools/common.yml
index e63d3b4beb2..f5d14936b6b 100644
--- a/src/mongo/gotools/common.yml
+++ b/src/mongo/gotools/common.yml
@@ -24,6 +24,14 @@ mongo_tools_variables:
       - name: qa-dump-restore-archiving
       - name: qa-dump-restore-gzip
       - name: unit
+# disabled until BUILD-2273 is done
+#      - name: replay-dist
+#      - name: replay-sanity_check
+#      - name: replay-go_test
+#      - name: replay-auth_test
+#      - name: replay-sharded_test
+#      - name: replay-repl_test
+#      - name: replay-replay_test
     osx_1010_ssl_task_list: &osx_1010_ssl_tasks
       - name: dist
       - name: qa-tests
@@ -52,6 +60,13 @@ mongo_tools_variables:
       - name: qa-tests-wt
       - name: unit
       - name: vet
+      - name: replay-dist
+      - name: replay-sanity_check
+      - name: replay-go_test
+      - name: replay-auth_test
+      - name: replay-sharded_test
+      - name: replay-repl_test
+      - name: replay-replay_test
     ubuntu1204_ssl_task_list: &ubuntu1204_ssl_tasks
       - name: dist
       - name: integration
@@ -71,6 +86,13 @@ mongo_tools_variables:
       - name: legacy24
       - name: qa-tests
       - name: native-cert-ssl
+      - name: replay-dist
+      - name: replay-sanity_check
+      - name: replay-go_test
+      - name: replay-auth_test
+      - name: replay-sharded_test
+      - name: replay-repl_test
+      - name: replay-replay_test
     ubuntu1204_race_task_list: &ubuntu1204_race_tasks
       - name: db
       - name: dist
@@ -83,6 +105,13 @@ mongo_tools_variables:
       - name: qa-tests
       - name: qa-tests-wt
       - name: unit
+      - name: replay-dist
+      - name: replay-sanity_check
+      - name: replay-go_test
+      - name: replay-auth_test
+      - name: replay-sharded_test
+      - name: replay-repl_test
+      - name: replay-replay_test
     windows_32_task_list: &windows_32_tasks
       - name: db
         distros:
@@ -157,6 +186,14 @@ mongo_tools_variables:
       - name: qa-tests-unstable
       - name: qa-tests-wt
       - name: native-cert-ssl
+# disabled until BUILD-2273 is done
+#      - name: replay-dist
+#      - name: replay-sanity_check
+#      - name: replay-go_test
+#      - name: replay-auth_test
+#      - name: replay-sharded_test
+#      - name: replay-repl_test
+#      - name: replay-replay_test
     rhel72_enterprise_task_list: &rhel72_enterprise_tasks
       - name: db
       - name: dist
@@ -168,6 +205,14 @@ mongo_tools_variables:
       - name: qa-tests-unstable
       - name: qa-tests-wt
       - name: native-cert-ssl
+# disabled until BUILD-2273 is done
+#      - name: replay-dist
+#      - name: replay-sanity_check
+#      - name: replay-go_test
+#      - name: replay-auth_test
+#      - name: replay-sharded_test
+#      - name: replay-repl_test
+#      - name: replay-replay_test
     ubuntu1604_ssl_task_list: &ubuntu1604_ssl_tasks
       - name: db
       - name: dist
@@ -183,18 +228,23 @@ mongo_tools_variables:
 ## Common mongodb arguments
   mongod_arguments:
     default: &mongod_default_startup_args
-      mongod_args: &mongod_default_startup_args_string "--port 33333"
+      mongod_args: ""
+      mongod_port: 33333
     ssl: &mongod_ssl_startup_args
-      mongod_args: "--port 33333 --sslMode requireSSL --sslCAFile common/db/openssl/testdata/ca.pem --sslPEMKeyFile common/db/openssl/testdata/server.pem"
+      mongod_args: "--sslMode requireSSL --sslCAFile common/db/openssl/testdata/ca.pem --sslPEMKeyFile common/db/openssl/testdata/server.pem"
+      mongod_port: 33333
     # Set storage engine as mmapv1 for 32 bit variants because WiredTiger requires 64 bit support.
     win32: &mongod_win32_startup_args
-      mongod_args: "--port 33333 --storageEngine=mmapv1"
+      mongod_args: "--storageEngine=mmapv1"
+      mongod_port: 33333
 
   mongo_arguments:
     default: &mongo_default_startup_args
       mongo_args: &mongo_default_startup_args_string "--port 33333"
+      mongod_port: 33333
     ssl: &mongo_ssl_startup_args
       mongo_args: "--port 33333 --ssl --sslCAFile ./common/db/openssl/testdata/ca.pem --sslPEMKeyFile ./common/db/openssl/testdata/server.pem --sslAllowInvalidCertificates"
+      mongod_port: 33333
 
 functions:
   "run legacy tests":
@@ -203,11 +253,13 @@ functions:
     params:
       working_dir: src
       script: |
-        mv ./mongodb/mongod${extension} .
-        mv ./mongodb/mongo${extension} .
-        mv ${test_path}/* .
-        chmod +x mongo*
-        python buildscripts/smoke.py ${smoke_use_ssl} --nopreallocj --with-cleanbb --mongod ./mongod --mongo ./mongo --report-file report.json --continue-on-failure --buildlogger-builder MCI_${build_variant} --buildlogger-buildnum ${builder_num|} --buildlogger-credentials ./mci.buildlogger --buildlogger-phase ${task_name}_${execution} ${smoke_args} tool
+        set -x
+        set -v
+        set -e
+        chmod +x bin/*
+        mv bin/* ${test_path}/
+        cd ${test_path}
+        python buildscripts/smoke.py ${smoke_use_ssl} --nopreallocj --with-cleanbb --mongod ./mongod --mongo ./mongo --report-file ../../report.json --continue-on-failure --buildlogger-builder MCI_${build_variant} --buildlogger-buildnum ${builder_num|} --buildlogger-credentials ./mci.buildlogger --buildlogger-phase ${task_name}_${execution} ${smoke_args} tool
 
   "run qa-tests":
     command: shell.exec
@@ -215,16 +267,12 @@ functions:
     params:
       working_dir: src
       script: |
-        set -o verbose
+        set -x
+        set -v
         set -e
-        mv ./mongodb/mongod${extension} .
-        mv ./mongodb/mongo${extension} .
-        mv ./mongodb/mongos${extension} .
-        mv test/qa-tests/* .
-        chmod +x mongo*
-        if [ -f bsondump ]; then
-          chmod +x bsondump
-        fi
+        chmod +x bin/*
+        mv bin/* test/qa-tests/
+        cd test/qa-tests
         rm -rf /data/mci/install /data/mci/multiversion
         mkdir -p /data/mci/install /data/mci/multiversion
         if [ "${multiversion_override}" != "skip" ]; then
@@ -232,7 +280,7 @@ functions:
         fi
         chmod 400 jstests/libs/key*
 
-        PATH=$PATH:/data/mci/multiversion python buildscripts/resmoke.py --suite=${resmoke_suite} --continueOnFailure --log=buildlogger --reportFile=report.json ${resmoke_args} --excludeWithAnyTags="${excludes}"
+        PATH=$PATH:/data/mci/multiversion python buildscripts/resmoke.py --suite=${resmoke_suite} --continueOnFailure --log=buildlogger --reportFile=../../report.json ${resmoke_args} --excludeWithAnyTags="${excludes}"
 
   "build tool":
     command: shell.exec
@@ -240,14 +288,13 @@ functions:
     params:
       working_dir: src
       script: |
+        set -x
+        set -v
+        set -e
         echo "Building ${tool}..."
         if [ "Windows_NT" = "$OS" ]; then
             set -o igncr
         fi;
-        if [ '${library_path}' != '' ]; then
-            export ${library_path}
-        fi
-
         # In RHEL 5.5, /usr/bin/ld can't handle --build-id parameters, so
         # use a wrapper if it's present on the system
         #
@@ -269,21 +316,24 @@ functions:
       working_dir: src
       script: |
         set -x
+        set -v
+        set -e
         mongotarget=$(if [ "${mongo_target}" ]; then echo "${mongo_target}"; else echo "${mongo_os}"; fi)
         mongoversion=$(if [ "${mongo_version_always_use_latest}" ]; then echo "latest"; else echo "${mongo_version}"; fi)
         dlurl=$(python binaryurl.py --edition=${mongo_edition} --target=$mongotarget --version=$mongoversion --arch=${mongo_arch|x86_64})
         filename=$(echo $dlurl | sed -e "s_.*/__")
-        mkdir mongodb
-        cd mongodb
+        mkdir -p bin
         curl -s $dlurl --output $filename
         ${decompress} $filename
         rm $filename
-        chmod +x ./mongodb-*/bin/*
         if [ "${only_shell}" ]; then
-          mv -f ./mongodb-*/bin/mongo${extension} .
+          mv -f ./mongodb-*/bin/mongo${extension} ./bin/
         else
-          mv -f ./mongodb-*/bin/* .
+          mv -f ./mongodb-*/bin/mongo${extension} ./bin/
+          mv -f ./mongodb-*/bin/mongos${extension} ./bin/
+          mv -f ./mongodb-*/bin/mongod${extension} ./bin/
         fi
+        chmod +x ./bin/*
         rm -rf ./mongodb-*
 
   "fetch tool" :
@@ -292,7 +342,7 @@ functions:
       bucket: mciuploads
       aws_key: ${aws_key}
       aws_secret: ${aws_secret}
-      local_file: src/${tool}${extension}
+      local_file: src/bin/${tool}${extension}
       remote_file: mongo-tools/binaries/${build_id}/${edition|community}/${tool}${extension}
 
   "generate coverage html + text":
@@ -301,14 +351,16 @@ functions:
     params:
       working_dir: src
       script: |
-        set -o verbose
+        set -x
+        set -v
+        set -e
         if [ "${coverage}" = "true" ]; then
           if [ "Windows_NT" = "$OS" ]; then
             set -o igncr
           fi;
           . ./set_gopath.sh
-          go tool cover -html=coverage.out -o coverage.html
-          go tool cover -func=coverage.out -o coverage.txt
+          ${gorootvars} go tool cover -html=coverage.out -o coverage.html
+          ${gorootvars} go tool cover -func=coverage.out -o coverage.txt
         fi;
 
   "get buildnumber":
@@ -322,7 +374,9 @@ functions:
     params:
       working_dir: src
       script: |
-        set -o verbose
+        set -x
+        set -v
+        set -e
         if [ "${coverage}" = "true" ]; then
           mv ${package}/coverage.out .
         fi
@@ -333,17 +387,16 @@ functions:
     params:
       working_dir: src
       script: |
+        set -x
+        set -v
         if [ "Windows_NT" = "$OS" ]; then
             set -o igncr
         fi;
         . ./set_gopath.sh
-        if [ '${library_path}' != '' ]; then
-            export ${library_path}
-        fi
         export MONGODB_KERBEROS_PASSWORD=${kerberos_password}
-        export basedir=`pwd`
+        export basedir=$PWD
         cd ${package}
-        go test ${coverage_args} ${args} -tags "${build_tags}" -test.v > unit.suite
+        ${gorootvars} go test ${coverage_args} ${args} -tags "${build_tags}" -test.v > unit.suite
         export exitcode=$?
         cat unit.suite
         cp unit.suite $basedir/.
@@ -356,6 +409,9 @@ functions:
       # Set up Kerberos stuff: run kinit if necessary, and add KDC to registry
       # on Windows (see https://wiki.mongodb.com/display/DH/Testing+Kerberos)
       script: |
+        set -x
+        set -v
+        set -e
         if [ '${run_kinit}' = 'true' ]
         then
           echo "${kerberos_password}" | kinit -p drivers@LDAPTEST.10GEN.CC;
@@ -369,6 +425,9 @@ functions:
     params:
       working_dir: src
       script: |
+        set -x
+        set -v
+        set -e
         chmod 700 set_gopath.sh
 
   "run tool unit tests":
@@ -377,6 +436,8 @@ functions:
     params:
       working_dir: src
       script: |
+        set -x
+        set -v
         if [ "Windows_NT" = "$OS" ]; then
             set -o igncr
         fi;
@@ -390,7 +451,7 @@ functions:
             if [ "${run_coverage}" ]; then
                 COVERAGE_ARGS="-coverprofile=coverage_$COMMON_SUBPKG.out"
             fi;
-            ${library_path} go test $COVERAGE_ARGS ${args} -tags "failpoints ${build_tags}" -test.v > $COMMON_SUBPKG.suite
+            ${gorootvars} go test $COVERAGE_ARGS ${args} -tags "failpoints ${build_tags}" -test.v > $COMMON_SUBPKG.suite
             if [ $? -ne 0 ]; then
                 echo "suite failed, will exit with error on finish"
                 export exitcode=1
@@ -407,7 +468,7 @@ functions:
             if [ "${run_coverage}" ]; then
                 COVERAGE_ARGS="-coverprofile=coverage_$i.out"
             fi;
-            ${library_path} go test $COVERAGE_ARGS ${args} -tags "${build_tags}" -test.v > $i.suite
+            ${gorootvars} go test $COVERAGE_ARGS ${args} -tags "${build_tags}" -test.v > $i.suite
             if [ $? -ne 0 ]; then
                 echo "suite failed, will exit with error on finish"
                 export exitcode=1
@@ -424,19 +485,18 @@ functions:
     params:
       working_dir: src
       script: |
+        set -x
+        set -v
         if [ "Windows_NT" = "$OS" ]; then
             set -o igncr
             export MONGODB_KERBEROS_PASSWORD=${kerberos_password}
         fi;
         . ./set_gopath.sh
-        if [ "${library_path}" != "" ]; then
-            export ${library_path}
-        fi;
 
         if [ "${create_mongod_users_command}" != "" ]; then
           export AUTH_USERNAME=${auth_username}
           export AUTH_PASSWORD=${auth_password}
-          echo "${create_mongod_users_command}" | ./mongodb/mongo${extension} ${mongo_args} admin
+          echo "${create_mongod_users_command}" | ./bin/mongo${extension} ${mongo_args} admin
         fi;
 
         for i in mongoimport mongoexport mongostat mongooplog mongorestore mongodump mongofiles; do
@@ -445,7 +505,7 @@ functions:
             if [ "${run_coverage}" ]; then
                 COVERAGE_ARGS="-coverprofile=coverage_$i.out"
             fi;
-            ${library_path} go test $COVERAGE_ARGS -test.v ${args} -tags "${build_tags}" > $i.suite
+            ${gorootvars} go test $COVERAGE_ARGS -test.v ${args} -tags "${build_tags}" > $i.suite
             if [ $? -ne 0 ]; then
                 echo "suite failed, will exit with error on finish"
                 export exitcode=1
@@ -462,6 +522,10 @@ functions:
     params:
       working_dir: src
       script: |
+        set -x
+        set -v
+        set -e
+        set -o verbose
         if [ "Windows_NT" = "$OS" ]; then
             set -o igncr
         fi;
@@ -471,8 +535,8 @@ functions:
         for i in mongoimport mongoexport mongostat mongooplog mongorestore mongodump mongofiles; do
             cd $i
             perl -pe 's/.*src/github.com\/mongodb\/mongo-tools/' coverage_$i.out > coverage_$i_rewrite.out
-            ${library_path} go tool cover -html=coverage_$i_rewrite.out -o coverage_$i.html
-            ${library_path} go tool cover -func=coverage_$i_rewrite.out -o coverage_$i.txt
+            ${gorootvars} go tool cover -html=coverage_$i_rewrite.out -o coverage_$i.html
+            ${gorootvars} go tool cover -func=coverage_$i_rewrite.out -o coverage_$i.txt
             cd ..
         done
 
@@ -508,6 +572,9 @@ functions:
       working_dir: src
       silent: true
       script: |
+        set -x
+        set -v
+        set -e
         cat > mci.buildlogger <<END_OF_CREDS
         slavename='${slave}'
         passwd='${passwd}'
@@ -515,6 +582,9 @@ functions:
         build_num=${builder_num}
         build_phase='${task_name}_${execution}'
         END_OF_CREDS
+        # Resmoke hardcodes the location of this file so we need to copy it to the working directory
+        # we run resmoke from.
+        cp mci.buildlogger test/qa-tests
 
   "start mongod":
     command: shell.exec
@@ -522,21 +592,25 @@ functions:
       working_dir: src
       background: true
       script: |
-        set -o verbose
-        rm -rf mongodb/${db_files_dir|db_files} mongodb/${logfile|run.log};
-        mkdir mongodb/${db_files_dir|db_files};
-        echo "Starting mongod...";
-        ./mongodb/mongod${extension} ${mongod_args} --dbpath mongodb/db_files --setParameter=enableTestCommands=1
+        set -x
+        set -v
+        set -e
+        rm -rf mongodb/db_files mongodb/${logfile|run.log}
+        mkdir -p mongodb/db_files
+        echo "Starting mongod..."
+        PATH=$PWD/bin:$PATH ./bin/mongod${extension}  --port ${mongod_port} ${mongod_args} ${additional_args} --dbpath mongodb/db_files --setParameter=enableTestCommands=1
 
   "wait for mongod to be ready":
     command: shell.exec
     params:
       working_dir: src
       script: |
-        set -o verbose
+        set -x
+        set -v
         SECONDS=0
         while true ; do
-            ./mongodb/mongo${extension} ${mongo_args} </dev/null 2>/dev/null >/dev/null
+            set -o verbose
+            ./bin/mongo${extension} ${mongo_args} </dev/null 2>/dev/null >/dev/null
             if [ "$?" = "0" ]; then
                 echo "mongod ready";
                 exit 0
@@ -546,7 +620,7 @@ functions:
                     echo "mongod not ready after 20 seconds"
                     exit 1
                 fi
-                echo "waiting for mongod to be ready..." ;
+                echo "waiting for mongod ( ${mongo_args} ) to be ready..."  ;
                 sleep 1 ;
             fi
         done
@@ -563,11 +637,150 @@ functions:
       content_type: application/octet-stream
       display_name: ${tool}
 
+  "fetch source" :
+    - command: git.get_project
+      params:
+        directory: src
+    - command: git.apply_patch
+      params:
+        directory: src
+    - command: shell.exec
+      params:
+        working_dir: src
+        script: |
+          set -x
+          set -v
+          set -e
+          mkdir -p bin
+
+  "upload archive" :
+    command: s3.put
+    params:
+      aws_key: ${aws_key}
+      aws_secret: ${aws_secret}
+      local_file: src/${filename}
+      remote_file: mongo-tools/mongoreplay/archive/${build_id}/${filename}
+      bucket: mciuploads
+      permissions: public-read
+      content_type: application/gzip
+      display_name: ${filename}
+
+  "create timeseries" :
+    command: shell.exec
+    params:
+      working_dir: src
+      script: |
+        set -x
+        set -v
+        set -e
+        git clone git@github.com:10gen/support-tools
+        sudo pip install argparse python-dateutil pytz
+        python support-tools/timeseries/timeseries.py /data/mongoreplay/diagnostic.data --html timeseries.html
+
+  "upload timeseries" :
+    command: s3.put
+    params:
+      aws_key: ${aws_key}
+      aws_secret: ${aws_secret}
+      local_file: src/timeseries.html
+      remote_file: mongo-tools/mongoreplay/timeseries/${build_id}/timeseries.html
+      bucket: mciuploads
+      permissions: public-read
+      content_type: text/html
+      display_name: timeseries.html
+
+  "fetch pcap" :
+      command: s3.get
+      params:
+        bucket: boxes.10gen.com
+        aws_key: ${aws_key}
+        aws_secret: ${aws_secret}
+        local_file: src/mongoreplay/testPcap/${pcapFname}
+        remote_file: build/mongotape/${pcapFname}
+
+  "fetch ftdc" :
+  - command: shell.exec
+    type: test
+    params:
+      working_dir: src
+      script: |
+        set -x
+        set -v
+        set -e
+        ${gorootvars} GOPATH=$PWD go get github.com/10gen/ftdc-utils/cmd/ftdc
+
+  "create auth_user" :
+  - command: shell.exec
+    params:
+      working_dir: src
+      script: |
+        set -x
+        set -v
+        set -e
+        ./bin/mongo --port ${mongod_port} admin --eval "db.createUser({user:\"authorizedUser\", pwd: \"authorizedPwd\", roles:[\"readWriteAnyDatabase\", \"clusterManager\"]});"
+
+  "create sharded_cluster" :
+    - command: shell.exec
+      params:
+        working_dir: src
+        background: true
+        script: |
+          set -x
+          set -v
+          set -e
+          echo "starting sharded cluster"
+          mkdir -p /data/db/
+          PATH=./bin:$PATH ./bin/mongo --nodb --eval 'var d = new ShardingTest({shards:3, mongos:[{port:${mongod_port}}]}); while(true){sleep(1000)}' 
+    - command: shell.exec
+      params:
+        working_dir: src
+        script: |
+          set -x
+          set -v
+          set -e
+          ./bin/mongo --nodb --eval 'var d; assert.soon(function(x){try{d = new Mongo("localhost:${mongod_port}"); return true} catch(e){return false}}, "timed out connection");d.setLogLevel(5, "write");'
+
+  "create repl_set" :
+    - command: shell.exec
+      params:
+        working_dir: src
+        background: true
+        script: |
+          set -x
+          set -v
+          set -e
+          echo "starting repl set"
+          mkdir -p /data/db/
+          PATH=./bin:$PATH ./bin/mongo --nodb --eval 'TestData = new Object(); TestData.minPort="${mongod_port}"; var repl = new ReplSetTest({nodes:3});repl.startSet();repl.initiate();repl.awaitSecondaryNodes();while(true){sleep(1000);}'
+    - command: shell.exec
+      params:
+        working_dir: src
+        script: |
+          set -x
+          set -v
+          set -e
+          ./bin/mongo --nodb --eval 'assert.soon(function(x){try{var d = new Mongo("localhost:${mongod_port}"); return true} catch(e){return false}}, "timed out connection")'
+
+  "run go_test" :
+  - command: shell.exec
+    type: test
+    params:
+      working_dir: src
+      script: |
+        set -x
+        set -v
+        set -e
+        . ./set_gopath.sh
+        ${gorootvars} ${environment_vars} go test ${additional_args} -v > ${filename}.suite
+
+
 pre:
   - command: shell.exec
     params:
       silent: true
       script: |
+        set -x
+          set -v
         ${killall_mci|pkill -9 mongo; pkill -9 mongodump; pkill -9 mongoexport; pkill -9 mongoimport; pkill -9 mongofiles; pkill -9 mongooplog; pkill -9 mongorestore; pkill -9 mongostat; pkill -9 mongotop; pkill -9 mongod; pkill -9 mongos; pkill -f buildlogger.py; pkill -f smoke.py} >/dev/null 2>&1
         rm -rf src /data/db/*
         exit 0
@@ -580,6 +793,8 @@ post:
     params:
       silent: true
       script: |
+        set -x
+          set -v
         ${killall_mci|pkill -9 mongo; pkill -9 mongodump; pkill -9 mongoexport; pkill -9 mongoimport; pkill -9 mongofiles; pkill -9 mongooplog; pkill -9 mongorestore; pkill -9 mongostat; pkill -9 mongotop; pkill -9 mongod; pkill -9 mongos; pkill -f buildlogger.py; pkill -f smoke.py} >/dev/null 2>&1
         exit 0
   - command: gotest.parse_files
@@ -588,6 +803,8 @@ post:
   - command: shell.exec
     params:
       script: |
+        set -x
+          set -v
         rm -rf /data/db/*
         rm -rf /data/mci/multiversion
         rm -rf /data/mci/install
@@ -598,24 +815,23 @@ timeout:
     params:
       silent: true
       script: |
+        set -x
+          set -v
         # don't attempt to abort on any distro which has a special way of
         # killing everything (i.e. using taskkill on Windows)
         if [ "${killall_mci}" = "" ]; then
           all_tools="bsondump mongodump mongoexport mongofiles mongoimport mongooplog mongorestore mongostat mongotop"
           # send SIGABRT to print a stacktrace for any hung tool
           pkill -ABRT "^($(echo -n $all_tools | tr ' ' '|'))\$"
+          # git the processes a second or two to dump their stacks
+          sleep 10
         fi
 
 
 tasks:
 - name: db
   commands:
-    - command: git.get_project
-      params:
-        directory: src
-    - command: git.apply_patch
-      params:
-        directory: src
+    - func: "fetch source"
     - command: expansions.update
       params:
         updates:
@@ -634,18 +850,7 @@ tasks:
 - name: dist
   depends_on:
   commands:
-    - command: git.get_project
-      params:
-        directory: src
-    - command: git.apply_patch
-      params:
-        directory: src
-    - command: shell.exec
-      params:
-        working_dir: src
-        script: |
-          rm -rf bin/
-          mkdir bin
+    - func: "fetch source"
     # bsondump
     - func: "build tool"
       vars:
@@ -712,12 +917,7 @@ tasks:
 
 - name: integration
   commands:
-    - command: git.get_project
-      params:
-        directory: src
-    - command: git.apply_patch
-      params:
-        directory: src
+    - func: "fetch source"
     - command: expansions.update
       params:
         updates:
@@ -732,12 +932,7 @@ tasks:
 
 - name: integration-auth
   commands:
-    - command: git.get_project
-      params:
-        directory: src
-    - command: git.apply_patch
-      params:
-        directory: src
+    - func: "fetch source"
     # Concat auth args
     - command: expansions.update
       params:
@@ -761,12 +956,7 @@ tasks:
 
 - name: kerberos
   commands:
-    - command: git.get_project
-      params:
-        directory: src
-    - command: git.apply_patch
-      params:
-        directory: src
+    - func: "fetch source"
     # Explicitly run ONLY Kerberos tests
     - command: expansions.update
       params:
@@ -780,12 +970,7 @@ tasks:
   depends_on:
   - name: dist
   commands:
-    - command: git.get_project
-      params:
-        directory: src
-    - command: git.apply_patch
-      params:
-        directory: src
+    - func: "fetch source"
     - func: "get buildnumber"
     - func: "setup credentials"
     - func: "download mongod"
@@ -821,12 +1006,7 @@ tasks:
   depends_on:
   - name: dist
   commands:
-    - command: git.get_project
-      params:
-        directory: src
-    - command: git.apply_patch
-      params:
-        directory: src
+    - func: "fetch source"
     - func: "get buildnumber"
     - func: "setup credentials"
     - func: "download mongod"
@@ -862,12 +1042,7 @@ tasks:
   depends_on:
   - name: dist
   commands:
-    - command: git.get_project
-      params:
-        directory: src
-    - command: git.apply_patch
-      params:
-        directory: src
+    - func: "fetch source"
     - func: "get buildnumber"
     - func: "setup credentials"
     - func: "download mongod"
@@ -903,12 +1078,7 @@ tasks:
   depends_on:
   - name: dist
   commands:
-    - command: git.get_project
-      params:
-        directory: src
-    - command: git.apply_patch
-      params:
-        directory: src
+    - func: "fetch source"
     - func: "get buildnumber"
     - func: "setup credentials"
     - func: "download mongod"
@@ -946,36 +1116,29 @@ tasks:
 
 - name: lint-go
   commands:
-    - command: git.get_project
-      params:
-        directory: src
-    - command: git.apply_patch
-      params:
-        directory: src
+    - func: "fetch source"
     - command: shell.exec
       type: test
       params:
         working_dir: src
         script: |
-          set -o errexit
-          set -o verbose
-          retVal=$(. ./set_gopath.sh && go run vendor/src/github.com/3rf/mongo-lint/golint/golint.go mongo* bson* common/*);
+          set -x
+          set -v
+          set -e
+          ${gorootvars} retVal=$(. ./set_gopath.sh && go run vendor/src/github.com/3rf/mongo-lint/golint/golint.go mongo* bson* common/*);
           if [ "$retVal" = "" ]; then exit 0; else echo $retVal; exit 1; fi;
 
 - name: lint-js
   commands:
-    - command: git.get_project
-      params:
-        directory: src
-    - command: git.apply_patch
-      params:
-        directory: src
+    - func: "fetch source"
     - command: shell.exec
       type: test
       params:
         working_dir: src
         script: |
-          set -o verbose
+          set -x
+          set -v
+          set -e
           PATH="/opt/node/bin:$PATH"
           /opt/node/bin/npm install eslint@3.2
           /opt/node/bin/node node_modules/eslint/bin/eslint.js test/qa-tests/jstests/**/*.js
@@ -984,12 +1147,7 @@ tasks:
   depends_on:
   - name: dist
   commands:
-    - command: git.get_project
-      params:
-        directory: src
-    - command: git.apply_patch
-      params:
-        directory: src
+    - func: "fetch source"
     - func: "get buildnumber"
     - func: "setup credentials"
     - func: "download mongod"
@@ -1030,12 +1188,7 @@ tasks:
   depends_on:
   - name: dist
   commands:
-    - command: git.get_project
-      params:
-        directory: src
-    - command: git.apply_patch
-      params:
-        directory: src
+    - func: "fetch source"
     - func: "get buildnumber"
     - func: "setup credentials"
     - func: "download mongod"
@@ -1077,12 +1230,7 @@ tasks:
   depends_on:
   - name: dist
   commands:
-    - command: git.get_project
-      params:
-        directory: src
-    - command: git.apply_patch
-      params:
-        directory: src
+    - func: "fetch source"
     - func: "get buildnumber"
     - func: "setup credentials"
     - func: "download mongod"
@@ -1118,33 +1266,24 @@ tasks:
     - command: shell.exec
       type: test
       params:
-        working_dir: src
+        working_dir: src/test/qa-tests
         script: |
-          set -o verbose
+          set -x
+          set -v
           set -e
-          mv ./mongodb/mongod${extension} .
-          mv ./mongodb/mongo${extension} .
-          mv ./mongodb/mongos${extension} .
-          mv test/qa-tests/* .
-          chmod +x mongo*
-          chmod +x bsondump*
+          chmod +x ../../bin/*
+          mv ../../bin/* .
           rm -rf /data/mci/install /data/mci/multiversion
           mkdir -p /data/mci/install /data/mci/multiversion
           python buildscripts/setup_multiversion_mongodb.py /data/mci/install /data/mci/multiversion ${arch} "2.6" "2.4" --latest ${smoke_use_ssl} --os="${mongo_os}" || true
           chmod 400 jstests/libs/key*
-
-          PATH=$PATH:/data/mci/multiversion python buildscripts/resmoke.py --suite=native_cert_ssl  --continueOnFailure --log=buildlogger --reportFile=report.json ${resmoke_args} --excludeWithAnyTags="${excludes}"
+          PATH=$PATH:/data/mci/multiversion python buildscripts/resmoke.py --suite=native_cert_ssl  --continueOnFailure --log=buildlogger --reportFile=../../report.json ${resmoke_args} --excludeWithAnyTags="${excludes}"
 
 - name: qa-dump-restore-archiving
   depends_on:
   - name: dist
   commands:
-    - command: git.get_project
-      params:
-        directory: src
-    - command: git.apply_patch
-      params:
-        directory: src
+    - func: "fetch source"
     - func: "get buildnumber"
     - func: "setup credentials"
     - func: "download mongod"
@@ -1165,12 +1304,7 @@ tasks:
   depends_on:
   - name: dist
   commands:
-    - command: git.get_project
-      params:
-        directory: src
-    - command: git.apply_patch
-      params:
-        directory: src
+    - func: "fetch source"
     - func: "get buildnumber"
     - func: "setup credentials"
     - func: "download mongod"
@@ -1191,12 +1325,7 @@ tasks:
   depends_on:
   - name: dist
   commands:
-    - command: git.get_project
-      params:
-        directory: src
-    - command: git.apply_patch
-      params:
-        directory: src
+    - func: "fetch source"
     - func: "get buildnumber"
     - func: "setup credentials"
     - func: "download mongod"
@@ -1242,12 +1371,7 @@ tasks:
         updates:
           - key: "run_coverage"
             value: "true"
-    - command: git.get_project
-      params:
-        directory: src
-    - command: git.apply_patch
-      params:
-        directory: src
+    - func: "fetch source"
     - func: "run tool unit tests"
     - func: "create coverage reports"
     - command: expansions.update
@@ -1295,20 +1419,194 @@ tasks:
 
 - name: vet
   commands:
-    - command: git.get_project
-      params:
-        directory: src
-    - command: git.apply_patch
-      params:
-        directory: src
+    - func: "fetch source"
     - command: shell.exec
       type: test
       params:
         working_dir: src
         script: |
-          set -o verbose
+          set -x
+          set -v
+          set -e
           . ./set_gopath.sh
-          go tool vet bsondump common mongo*
+          ${gorootvars} go tool vet bsondump common mongo*
+
+- name: replay-dist
+  commands:
+  - func: "fetch source"
+  - func: "build tool"
+    vars:
+      tool: mongoreplay
+  - func: "upload tool"
+    vars:
+      tool: mongoreplay
+  - func: "build tool"
+
+- name: replay-sanity_check
+  depends_on:
+  - name: replay-dist
+  commands:
+  - func: "fetch source"
+  - func: "fetch tool"
+    vars:
+      tool: mongoreplay
+  - func: "download mongod"
+    vars:
+      mongo_version: "3.2"
+  - func: "start mongod"
+  - func: "wait for mongod to be ready"
+  - command: shell.exec
+    params:
+      working_dir: src
+      script: |
+        set -x
+        set -v
+        set -e
+        chmod +x bin/mongoreplay
+        echo "Running sanity check"
+        PATH=$PATH:$PWD/bin ./mongoreplay/sanity_check.sh -p ${mongod_port}
+
+- name: replay-go_test
+  depends_on:
+  - name: replay-dist
+  commands:
+  - func: "fetch source"
+  - func: "fetch tool"
+    vars:
+      tool: mongoreplay
+  - func: "fetch pcap"
+    vars:
+      pcapFname: getmore_multi_channel.pcap
+  - func: "fetch pcap"
+    vars:
+      pcapFname: getmore_single_channel.pcap
+  - func: "download mongod"
+    vars:
+      mongo_version: "3.2"
+  - func: "start mongod"
+  - func: "wait for mongod to be ready"
+  - func: "run go_test"
+    vars:
+        filename: playtest
+        environment_vars: DB_PORT=${mongod_port}
+        additional_args: github.com/mongodb/mongo-tools/mongoreplay
+
+- name: replay-sharded_test
+  depends_on:
+  - name: replay-dist
+  commands:
+  - func: "fetch source"
+  - func: "fetch tool"
+    vars:
+      tool: mongoreplay
+  - func: "fetch pcap"
+    vars:
+      pcapFname: getmore_multi_channel.pcap
+  - func: "fetch pcap"
+    vars:
+      pcapFname: getmore_single_channel.pcap
+  - func: "download mongod"
+    vars:
+      mongo_version: "3.2"
+  - func: "create sharded_cluster"
+  - func: "run go_test"
+    vars:
+        filename: sharded
+        environment_vars: DB_PORT=${mongod_port}
+        additional_args: github.com/mongodb/mongo-tools/mongoreplay --run "LiveDB"
+
+- name: replay-auth_test
+  depends_on:
+  - name: replay-dist
+  commands:
+  - func: "fetch source"
+  - func: "fetch tool"
+    vars:
+      tool: mongoreplay
+  - func: "fetch pcap"
+    vars:
+      pcapFname: getmore_multi_channel.pcap
+  - func: "fetch pcap"
+    vars:
+      pcapFname: getmore_single_channel.pcap
+  - func: "download mongod"
+    vars:
+      mongo_version: "3.2"
+  - func: "start mongod"
+    vars:
+        additional_args: --auth
+  - func: "wait for mongod to be ready"
+  - func: "create auth_user"
+    vars:
+        mongod_port: ${mongod_port}
+  - func: "run go_test"
+    vars:
+        filename: authtest
+        environment_vars: AUTH=1 DB_PORT=${mongod_port}
+        additional_args: github.com/mongodb/mongo-tools/mongoreplay --run "(LiveDB)|(Authed)"
+
+- name: replay-repl_test
+  depends_on:
+  - name: replay-dist
+  commands:
+  - func: "fetch source"
+  - func: "fetch tool"
+    vars:
+      tool: mongoreplay
+  - func: "fetch pcap"
+    vars:
+      pcapFname: getmore_multi_channel.pcap
+  - func: "fetch pcap"
+    vars:
+      pcapFname: getmore_single_channel.pcap
+  - func: "download mongod"
+    vars:
+      mongo_version: "3.2"
+  - func: "create repl_set"
+    vars:
+        mongod_port: ${mongod_port}
+  - func: "run go_test"
+    vars:
+        filename: repltest
+        environment_vars: DB_PORT=${mongod_port}
+        additional_args: github.com/mongodb/mongo-tools/mongoreplay --run "LiveDB"
+
+- name: replay-replay_test
+  depends_on:
+  - name: replay-dist
+  commands:
+  - func: "fetch source"
+  - func: "fetch tool"
+    vars:
+      tool: mongoreplay
+  - func: "download mongod"
+    vars:
+      mongo_version: "3.2"
+  - func: "fetch ftdc"
+  - command: shell.exec
+    params:
+      working_dir: src
+      script: |
+        set -x
+        set -v
+        set -e
+        . ./set_gopath.sh
+        chmod +x bin/mongoreplay
+        echo "Running replay test"
+        PATH=$PATH:$PWD/bin:$PWD/vendor/bin ./mongoreplay/replay_test.sh --verbose --explicit --keep
+  - command: shell.exec
+    params:
+      working_dir: src
+      script: |
+        set -x
+        set -v
+        set -e
+        tar czf replay.tar.gz tmp.*
+  - func: "upload archive"
+    vars:
+      filename: replay.tar.gz
+  - func: "create timeseries"
+  - func: "upload timeseries"
 
 buildvariants:
 #######################################
@@ -1413,7 +1711,7 @@ buildvariants:
     args: -gccgoflags "$(pkg-config --libs --cflags libssl)"
     build_tags: 'ssl'
     resmoke_use_ssl: _ssl
-    library_path: PATH="/opt/mongodbtoolchain/v2/bin/:$PATH"
+    gorootvars: PATH="/opt/mongodbtoolchain/v2/bin/:$PATH"
     resmoke_args: -j 4
     excludes: requires_mmap_available,requires_large_ram,requires_mongo_24,requires_mongo_26,requires_mongo_30
     multiversion_override: "skip"
@@ -1436,7 +1734,7 @@ buildvariants:
     args: -gccgoflags "$(pkg-config --libs --cflags libssl libsasl2)"
     build_tags: "sasl ssl"
     resmoke_use_ssl: _ssl
-    library_path: PATH="/opt/mongodbtoolchain/v2/bin/:$PATH"
+    gorootvars: PATH="/opt/mongodbtoolchain/v2/bin/:$PATH"
     excludes: requires_mmap_available,requires_mongo_24,requires_mongo_26,requires_mongo_30
     resmoke_args: -j 2
     multiversion_override: "skip"
@@ -1460,7 +1758,7 @@ buildvariants:
     args: -gccgoflags "$(pkg-config --libs --cflags libcrypto libssl)"
     build_tags: "ssl"
     resmoke_use_ssl: _ssl
-    library_path: PATH="/opt/mongodbtoolchain/v2/bin/:$PATH"
+    gorootvars: PATH="/opt/mongodbtoolchain/v2/bin/:$PATH"
     excludes: requires_mmap_available,requires_large_ram,requires_mongo_24,requires_mongo_26,requires_mongo_30
     resmoke_args: -j 2
     multiversion_override: "skip"
@@ -1481,7 +1779,7 @@ buildvariants:
     <<: *mongod_default_startup_args
     <<: *mongo_default_startup_args
     mongo_os: "sunos5"
-    library_path: PATH="/opt/mongodbtoolchain/v2/bin/:$PATH"
+    gorootvars: PATH="/opt/mongodbtoolchain/v2/bin/:$PATH"
     args: -gccgoflags "-lsocket -lnsl"
     excludes: requires_large_ram
     resmoke_args: -j$(kstat cpu | sort -u | grep -c "^module")
@@ -1538,7 +1836,7 @@ buildvariants:
     multiversion_override: "2.6"
     extension: .exe
     arch: "win32/x86_64"
-    library_path: PATH="/cygdrive/c/mingw-w64/x86_64-4.9.1-posix-seh-rt_v3-rev1/mingw64/bin:/cygdrive/c/sasl/:$PATH"
+    gorootvars: PATH="/cygdrive/c/mingw-w64/x86_64-4.9.1-posix-seh-rt_v3-rev1/mingw64/bin:/cygdrive/c/sasl/:$PATH"
     preproc_gpm: "perl -pi -e 's/\\r\\n/\\n/g' "
     integration_test_args: "integration,ssl"
   tasks: *windows_64_ssl_tasks
@@ -1561,7 +1859,7 @@ buildvariants:
     edition: enterprise
     extension: .exe
     arch: "win32/x86_64"
-    library_path: PATH="/cygdrive/c/mingw-w64/x86_64-4.9.1-posix-seh-rt_v3-rev1/mingw64/bin:/cygdrive/c/sasl/:$PATH"
+    gorootvars: PATH="/cygdrive/c/mingw-w64/x86_64-4.9.1-posix-seh-rt_v3-rev1/mingw64/bin:/cygdrive/c/sasl/:$PATH"
     preproc_gpm: "perl -pi -e 's/\\r\\n/\\n/g' "
     integration_test_args: "integration"
   tasks: *windows_64_enterprise_tasks
@@ -1615,7 +1913,7 @@ buildvariants:
   run_on:
   - rhel62-test
   expansions:
-    gorootvars: GOROOT=/opt/go PATH="/opt/go/bin:$PATH"
+    gorootvars: PATH="/opt/go/bin:$PATH"
     build_tags: "sasl ssl"
   tasks:
   - name: dist
@@ -1625,7 +1923,7 @@ buildvariants:
   run_on:
   - rhel70
   expansions:
-    gorootvars: GOROOT=/opt/go PATH="/opt/go/bin:$PATH"
+    gorootvars: PATH="/opt/go/bin:$PATH"
     build_tags: "sasl ssl"
   tasks:
   - name: dist
@@ -1644,7 +1942,7 @@ buildvariants:
   run_on:
   - debian71-test
   expansions:
-    gorootvars: GOROOT=/opt/go PATH="/opt/go/bin:$PATH"
+    gorootvars: PATH="/opt/go/bin:$PATH"
     build_tags: "sasl ssl"
   tasks:
   - name: dist
diff --git a/src/mongo/gotools/common/util/file.go b/src/mongo/gotools/common/util/file.go
index ce5ad160fd6..f5d0ca722af 100644
--- a/src/mongo/gotools/common/util/file.go
+++ b/src/mongo/gotools/common/util/file.go
@@ -2,6 +2,7 @@ package util
 
 import (
 	"bufio"
+	"io"
 	"os"
 	"path/filepath"
 )
@@ -32,3 +33,31 @@ func GetFieldsFromFile(path string) ([]string, error) {
 func ToUniversalPath(path string) string {
 	return filepath.FromSlash(path)
 }
+
+type WrappedReadCloser struct {
+	io.ReadCloser
+	Inner io.ReadCloser
+}
+
+func (wrc *WrappedReadCloser) Close() error {
+	outerErr := wrc.ReadCloser.Close()
+	innerErr := wrc.Inner.Close()
+	if outerErr != nil {
+		return outerErr
+	}
+	return innerErr
+}
+
+type WrappedWriteCloser struct {
+	io.WriteCloser
+	Inner io.WriteCloser
+}
+
+func (wwc *WrappedWriteCloser) Close() error {
+	outerErr := wwc.WriteCloser.Close()
+	innerErr := wwc.Inner.Close()
+	if outerErr != nil {
+		return outerErr
+	}
+	return innerErr
+}
diff --git a/src/mongo/gotools/mongodump/metadata_dump.go b/src/mongo/gotools/mongodump/metadata_dump.go
index ce30c005efd..e1c6d594364 100644
--- a/src/mongo/gotools/mongodump/metadata_dump.go
+++ b/src/mongo/gotools/mongodump/metadata_dump.go
@@ -1,8 +1,9 @@
 package mongodump
 
 import (
-	"bufio"
 	"fmt"
+	"io"
+
 	"github.com/mongodb/mongo-tools/common/bsonutil"
 	"github.com/mongodb/mongo-tools/common/db"
 	"github.com/mongodb/mongo-tools/common/intents"
@@ -25,10 +26,8 @@ type IndexDocumentFromDB struct {
 
 // dumpMetadata gets the metadata for a collection and writes it
 // in readable JSON format.
-func (dump *MongoDump) dumpMetadata(intent *intents.Intent) error {
-	var err error
+func (dump *MongoDump) dumpMetadata(intent *intents.Intent, buffer resettableOutputBuffer) (err error) {
 
-	nsID := fmt.Sprintf("%v.%v", intent.DB, intent.C)
 	meta := Metadata{
 		// We have to initialize Indexes to an empty slice, not nil, so that an empty
 		// array is marshalled into json instead of null. That is, {indexes:[]} is okay
@@ -51,7 +50,7 @@ func (dump *MongoDump) dumpMetadata(intent *intents.Intent) error {
 	// We keep a running list of all the indexes
 	// for the current collection as we iterate over the cursor, and include
 	// that list as the "indexes" field of the metadata document.
-	log.Logvf(log.DebugHigh, "\treading indexes for `%v`", nsID)
+	log.Logvf(log.DebugHigh, "\treading indexes for `%v`", intent.Namespace())
 
 	session, err := dump.sessionProvider.GetSession()
 	if err != nil {
@@ -82,30 +81,42 @@ func (dump *MongoDump) dumpMetadata(intent *intents.Intent) error {
 		}
 
 		if err := indexesIter.Err(); err != nil {
-			return fmt.Errorf("error getting indexes for collection `%v`: %v", nsID, err)
+			return fmt.Errorf("error getting indexes for collection `%v`: %v", intent.Namespace(), err)
 		}
 	}
 
 	// Finally, we send the results to the writer as JSON bytes
 	jsonBytes, err := json.Marshal(meta)
 	if err != nil {
-		return fmt.Errorf("error marshalling metadata json for collection `%v`: %v", nsID, err)
+		return fmt.Errorf("error marshalling metadata json for collection `%v`: %v", intent.Namespace(), err)
 	}
 
 	err = intent.MetadataFile.Open()
 	if err != nil {
 		return err
 	}
-	defer intent.MetadataFile.Close()
-	// make a buffered writer for nicer disk i/o
-	w := bufio.NewWriter(intent.MetadataFile)
-	_, err = w.Write(jsonBytes)
-	if err != nil {
-		return fmt.Errorf("error writing metadata for collection `%v` to disk: %v", nsID, err)
+	defer func() {
+		closeErr := intent.MetadataFile.Close()
+		if err == nil && closeErr != nil {
+			err = fmt.Errorf("error writing metadata for collection `%v` to disk: %v", intent.Namespace(), closeErr)
+		}
+	}()
+
+	var f io.Writer
+	f = intent.MetadataFile
+	if buffer != nil {
+		buffer.Reset(f)
+		f = buffer
+		defer func() {
+			closeErr := buffer.Close()
+			if err == nil && closeErr != nil {
+				err = fmt.Errorf("error writing metadata for collection `%v` to disk: %v", intent.Namespace(), closeErr)
+			}
+		}()
 	}
-	err = w.Flush()
+	_, err = f.Write(jsonBytes)
 	if err != nil {
-		return fmt.Errorf("error writing metadata for collection `%v` to disk: %v", nsID, err)
+		err = fmt.Errorf("error writing metadata for collection `%v` to disk: %v", intent.Namespace(), err)
 	}
-	return nil
+	return
 }
diff --git a/src/mongo/gotools/mongodump/mongodump.go b/src/mongo/gotools/mongodump/mongodump.go
index d87eea246df..079ec3c737a 100644
--- a/src/mongo/gotools/mongodump/mongodump.go
+++ b/src/mongo/gotools/mongodump/mongodump.go
@@ -16,6 +16,7 @@ import (
 	"gopkg.in/mgo.v2"
 	"gopkg.in/mgo.v2/bson"
 
+	"bufio"
 	"compress/gzip"
 	"fmt"
 	"io"
@@ -426,6 +427,29 @@ func (dump *MongoDump) Dump() (err error) {
 	return err
 }
 
+type resettableOutputBuffer interface {
+	io.Writer
+	Close() error
+	Reset(io.Writer)
+}
+
+type closableBufioWriter struct {
+	*bufio.Writer
+}
+
+func (w closableBufioWriter) Close() error {
+	return w.Flush()
+}
+
+func (dump *MongoDump) getResettableOutputBuffer() resettableOutputBuffer {
+	if dump.OutputOptions.Archive != "" {
+		return nil
+	} else if dump.OutputOptions.Gzip {
+		return gzip.NewWriter(nil)
+	}
+	return &closableBufioWriter{bufio.NewWriter(nil)}
+}
+
 // DumpIntents iterates through the previously-created intents and
 // dumps all of the found collections.
 func (dump *MongoDump) DumpIntents() error {
@@ -447,6 +471,7 @@ func (dump *MongoDump) DumpIntents() error {
 	// start a goroutine for each job thread
 	for i := 0; i < jobs; i++ {
 		go func(id int) {
+			buffer := dump.getResettableOutputBuffer()
 			log.Logvf(log.DebugHigh, "starting dump routine with id=%v", id)
 			for {
 				intent := dump.manager.Pop()
@@ -456,7 +481,7 @@ func (dump *MongoDump) DumpIntents() error {
 					return
 				}
 				if intent.BSONFile != nil {
-					err := dump.DumpIntent(intent)
+					err := dump.DumpIntent(intent, buffer)
 					if err != nil {
 						resultChan <- err
 						return
@@ -478,7 +503,7 @@ func (dump *MongoDump) DumpIntents() error {
 }
 
 // DumpIntent dumps the specified database's collection.
-func (dump *MongoDump) DumpIntent(intent *intents.Intent) error {
+func (dump *MongoDump) DumpIntent(intent *intents.Intent, buffer resettableOutputBuffer) error {
 	session, err := dump.sessionProvider.GetSession()
 	if err != nil {
 		return err
@@ -489,12 +514,6 @@ func (dump *MongoDump) DumpIntent(intent *intents.Intent) error {
 	// duplicates the behavior of an exhaust cursor.
 	session.SetPrefetch(1.0)
 
-	err = intent.BSONFile.Open()
-	if err != nil {
-		return err
-	}
-	defer intent.BSONFile.Close()
-
 	var findQuery *mgo.Query
 	switch {
 	case len(dump.query) > 0:
@@ -513,7 +532,7 @@ func (dump *MongoDump) DumpIntent(intent *intents.Intent) error {
 
 	if dump.OutputOptions.Out == "-" {
 		log.Logvf(log.Always, "writing %v to stdout", intent.Namespace())
-		dumpCount, err = dump.dumpQueryToWriter(findQuery, intent)
+		dumpCount, err = dump.dumpQueryToIntent(findQuery, intent, buffer)
 		if err == nil {
 			// on success, print the document count
 			log.Logvf(log.Always, "dumped %v %v", dumpCount, docPlural(dumpCount))
@@ -532,7 +551,7 @@ func (dump *MongoDump) DumpIntent(intent *intents.Intent) error {
 
 	if !dump.OutputOptions.Repair {
 		log.Logvf(log.Always, "writing %v to %v", intent.Namespace(), intent.Location)
-		if dumpCount, err = dump.dumpQueryToWriter(findQuery, intent); err != nil {
+		if dumpCount, err = dump.dumpQueryToIntent(findQuery, intent, buffer); err != nil {
 			return err
 		}
 	} else {
@@ -540,7 +559,7 @@ func (dump *MongoDump) DumpIntent(intent *intents.Intent) error {
 		log.Logvf(log.Always, "writing repair of %v to %v", intent.Namespace(), intent.Location)
 		repairIter := session.DB(intent.DB).C(intent.C).Repair()
 		repairCounter := progress.NewCounter(1) // this counter is ignored
-		if err := dump.dumpIterToWriter(repairIter, intent.BSONFile, repairCounter); err != nil {
+		if err := dump.dumpIterToWriter(repairIter, buffer, repairCounter); err != nil {
 			return fmt.Errorf("repair error: %v", err)
 		}
 		_, repairCount := repairCounter.Progress()
@@ -552,17 +571,29 @@ func (dump *MongoDump) DumpIntent(intent *intents.Intent) error {
 	return nil
 }
 
-// dumpQueryToWriter takes an mgo Query, its intent, and a writer, performs the query,
+// dumpQueryToIntent takes an mgo Query, its intent, and a writer, performs the query,
 // and writes the raw bson results to the writer. Returns a final count of documents
 // dumped, and any errors that occured.
-func (dump *MongoDump) dumpQueryToWriter(
-	query *mgo.Query, intent *intents.Intent) (int64, error) {
+func (dump *MongoDump) dumpQueryToIntent(
+	query *mgo.Query, intent *intents.Intent, buffer resettableOutputBuffer) (dumpCount int64, err error) {
+
+	// restore of views from archives require an empty collection as the trigger to create the view
+	// so, we open here before the early return if IsView so that we write an empty collection to the archive
+	err = intent.BSONFile.Open()
+	if err != nil {
+		return 0, err
+	}
+	defer func() {
+		closeErr := intent.BSONFile.Close()
+		if err == nil && closeErr != nil {
+			err = fmt.Errorf("error writing data for collection `%v` to disk: %v", intent.Namespace(), closeErr)
+		}
+	}()
 	// don't dump any data for views being dumped as views
 	if intent.IsView() && !dump.OutputOptions.ViewsAsCollections {
 		return 0, nil
 	}
 	var total int
-	var err error
 	if len(dump.query) == 0 {
 		total, err = query.Count()
 		if err != nil {
@@ -579,10 +610,25 @@ func (dump *MongoDump) dumpQueryToWriter(
 		defer dump.ProgressManager.Detach(intent.Namespace())
 	}
 
-	err = dump.dumpIterToWriter(query.Iter(), intent.BSONFile, dumpProgressor)
-	_, dumpCount := dumpProgressor.Progress()
+	var f io.Writer
+	f = intent.BSONFile
+	if buffer != nil {
+		buffer.Reset(f)
+		f = buffer
+		defer func() {
+			closeErr := buffer.Close()
+			if err == nil && closeErr != nil {
+				err = fmt.Errorf("error writing data for collection `%v` to disk: %v", intent.Namespace(), closeErr)
+			}
+		}()
+	}
 
-	return dumpCount, err
+	err = dump.dumpIterToWriter(query.Iter(), f, dumpProgressor)
+	dumpCount, _ = dumpProgressor.Progress()
+	if err != nil {
+		err = fmt.Errorf("error writing data for collection `%v` to disk: %v", intent.Namespace(), err)
+	}
+	return
 }
 
 // dumpIterToWriter takes an mgo iterator, a writer, and a pointer to
@@ -641,6 +687,7 @@ func (dump *MongoDump) dumpIterToWriter(
 // database. Only works with an authentication schema version >= 3.
 func (dump *MongoDump) DumpUsersAndRolesForDB(db string) error {
 	session, err := dump.sessionProvider.GetSession()
+	buffer := dump.getResettableOutputBuffer()
 	if err != nil {
 		return err
 	}
@@ -648,37 +695,19 @@ func (dump *MongoDump) DumpUsersAndRolesForDB(db string) error {
 
 	dbQuery := bson.M{"db": db}
 	usersQuery := session.DB("admin").C("system.users").Find(dbQuery)
-	intent := dump.manager.Users()
-	err = intent.BSONFile.Open()
-	if err != nil {
-		return fmt.Errorf("error opening output stream for dumping Users: %v", err)
-	}
-	defer intent.BSONFile.Close()
-	_, err = dump.dumpQueryToWriter(usersQuery, intent)
+	_, err = dump.dumpQueryToIntent(usersQuery, dump.manager.Users(), buffer)
 	if err != nil {
 		return fmt.Errorf("error dumping db users: %v", err)
 	}
 
 	rolesQuery := session.DB("admin").C("system.roles").Find(dbQuery)
-	intent = dump.manager.Roles()
-	err = intent.BSONFile.Open()
-	if err != nil {
-		return fmt.Errorf("error opening output stream for dumping Roles: %v", err)
-	}
-	defer intent.BSONFile.Close()
-	_, err = dump.dumpQueryToWriter(rolesQuery, intent)
+	_, err = dump.dumpQueryToIntent(rolesQuery, dump.manager.Roles(), buffer)
 	if err != nil {
 		return fmt.Errorf("error dumping db roles: %v", err)
 	}
 
 	versionQuery := session.DB("admin").C("system.version").Find(nil)
-	intent = dump.manager.AuthVersion()
-	err = intent.BSONFile.Open()
-	if err != nil {
-		return fmt.Errorf("error opening output stream for dumping AuthVersion: %v", err)
-	}
-	defer intent.BSONFile.Close()
-	_, err = dump.dumpQueryToWriter(versionQuery, intent)
+	_, err = dump.dumpQueryToIntent(versionQuery, dump.manager.AuthVersion(), buffer)
 	if err != nil {
 		return fmt.Errorf("error dumping db auth version: %v", err)
 	}
@@ -690,20 +719,21 @@ func (dump *MongoDump) DumpUsersAndRolesForDB(db string) error {
 // TODO: This and DumpUsersAndRolesForDB should be merged, correctly
 func (dump *MongoDump) DumpUsersAndRoles() error {
 	var err error
+	buffer := dump.getResettableOutputBuffer()
 	if dump.manager.Users() != nil {
-		err = dump.DumpIntent(dump.manager.Users())
+		err = dump.DumpIntent(dump.manager.Users(), buffer)
 		if err != nil {
 			return err
 		}
 	}
 	if dump.manager.Roles() != nil {
-		err = dump.DumpIntent(dump.manager.Roles())
+		err = dump.DumpIntent(dump.manager.Roles(), buffer)
 		if err != nil {
 			return err
 		}
 	}
 	if dump.manager.AuthVersion() != nil {
-		err = dump.DumpIntent(dump.manager.AuthVersion())
+		err = dump.DumpIntent(dump.manager.AuthVersion(), buffer)
 		if err != nil {
 			return err
 		}
@@ -714,8 +744,9 @@ func (dump *MongoDump) DumpUsersAndRoles() error {
 
 // DumpSystemIndexes dumps all of the system.indexes
 func (dump *MongoDump) DumpSystemIndexes() error {
+	buffer := dump.getResettableOutputBuffer()
 	for _, dbName := range dump.manager.SystemIndexDBs() {
-		err := dump.DumpIntent(dump.manager.SystemIndexes(dbName))
+		err := dump.DumpIntent(dump.manager.SystemIndexes(dbName), buffer)
 		if err != nil {
 			return err
 		}
@@ -727,9 +758,10 @@ func (dump *MongoDump) DumpSystemIndexes() error {
 // that has metadata
 func (dump *MongoDump) DumpMetadata() error {
 	allIntents := dump.manager.Intents()
+	buffer := dump.getResettableOutputBuffer()
 	for _, intent := range allIntents {
 		if intent.MetadataFile != nil {
-			err := dump.dumpMetadata(intent)
+			err := dump.dumpMetadata(intent, buffer)
 			if err != nil {
 				return err
 			}
@@ -748,23 +780,6 @@ func (*nopCloseWriter) Close() error {
 	return nil
 }
 
-// wrappedWriteCloser implements io.WriteCloser. It wraps up two WriteClosers. The Write method
-// of the io.WriteCloser is implemented by the embedded io.WriteCloser
-type wrappedWriteCloser struct {
-	io.WriteCloser
-	inner io.WriteCloser
-}
-
-// Close is part of the io.WriteCloser interface. Close closes both the embedded io.WriteCloser as
-// well as the inner io.WriteCloser
-func (wwc *wrappedWriteCloser) Close() error {
-	err := wwc.WriteCloser.Close()
-	if err != nil {
-		return err
-	}
-	return wwc.inner.Close()
-}
-
 func (dump *MongoDump) getArchiveOut() (out io.WriteCloser, err error) {
 	if dump.OutputOptions.Archive == "-" {
 		out = &nopCloseWriter{dump.stdout}
@@ -788,10 +803,7 @@ func (dump *MongoDump) getArchiveOut() (out io.WriteCloser, err error) {
 		}
 	}
 	if dump.OutputOptions.Gzip {
-		return &wrappedWriteCloser{
-			WriteCloser: gzip.NewWriter(out),
-			inner:       out,
-		}, nil
+		return &util.WrappedWriteCloser{gzip.NewWriter(out), out}, nil
 	}
 	return out, nil
 }
diff --git a/src/mongo/gotools/mongodump/oplog_dump.go b/src/mongo/gotools/mongodump/oplog_dump.go
index dba7835edfa..b0800ff4318 100644
--- a/src/mongo/gotools/mongodump/oplog_dump.go
+++ b/src/mongo/gotools/mongodump/oplog_dump.go
@@ -73,16 +73,10 @@ func (dump *MongoDump) DumpOplogAfterTimestamp(ts bson.MongoTimestamp) error {
 		return err
 	}
 	defer session.Close()
-	intent := dump.manager.Oplog()
-	err = intent.BSONFile.Open()
-	if err != nil {
-		return fmt.Errorf("error opening output stream for dumping oplog: %v", err)
-	}
-	defer intent.BSONFile.Close()
 	session.SetPrefetch(1.0) // mimic exhaust cursor
 	queryObj := bson.M{"ts": bson.M{"$gt": ts}}
 	oplogQuery := session.DB("local").C(dump.oplogCollection).Find(queryObj).LogReplay()
-	oplogCount, err := dump.dumpQueryToWriter(oplogQuery, dump.manager.Oplog())
+	oplogCount, err := dump.dumpQueryToIntent(oplogQuery, dump.manager.Oplog(), dump.getResettableOutputBuffer())
 	if err == nil {
 		log.Logvf(log.Always, "\tdumped %v oplog %v",
 			oplogCount, util.Pluralize(int(oplogCount), "entry", "entries"))
diff --git a/src/mongo/gotools/mongodump/prepare.go b/src/mongo/gotools/mongodump/prepare.go
index 45a29a2e881..ff36bdfb02a 100644
--- a/src/mongo/gotools/mongodump/prepare.go
+++ b/src/mongo/gotools/mongodump/prepare.go
@@ -1,9 +1,7 @@
 package mongodump
 
 import (
-	"bufio"
 	"bytes"
-	"compress/gzip"
 	"fmt"
 	"github.com/mongodb/mongo-tools/common/archive"
 	"github.com/mongodb/mongo-tools/common/bsonutil"
@@ -45,18 +43,6 @@ type writeFlushCloser struct {
 	writeFlusher
 }
 
-// availableWriteFlusher wraps a writeFlusher and adds an Available function.
-type availableWriteFlusher interface {
-	Available() int
-	writeFlusher
-}
-
-// atomicFlusher is a availableWriteFlusher implementation
-// which guarantees atomic writes.
-type atomicFlusher struct {
-	availableWriteFlusher
-}
-
 // errorReader implements io.Reader.
 type errorReader struct{}
 
@@ -79,7 +65,6 @@ type realBSONFile struct {
 	// intent.file ( a ReadWriteOpenCloser )
 	errorReader
 	intent *intents.Intent
-	gzip   bool
 	NilPos
 }
 
@@ -97,45 +82,14 @@ func (f *realBSONFile) Open() (err error) {
 			filepath.Dir(f.path), err)
 	}
 
-	fileName := f.path
-	file, err := os.Create(fileName)
+	f.WriteCloser, err = os.Create(f.path)
 	if err != nil {
-		return fmt.Errorf("error creating BSON file %v: %v", fileName, err)
-	}
-	var writeCloser io.WriteCloser
-	if f.gzip {
-		writeCloser = gzip.NewWriter(file)
-	} else {
-		// wrap writer in buffer to reduce load on disk
-		writeCloser = writeFlushCloser{
-			atomicFlusher{
-				bufio.NewWriterSize(file, 32*1024),
-			},
-		}
-	}
-	f.WriteCloser = &wrappedWriteCloser{
-		WriteCloser: writeCloser,
-		inner:       file,
+		return fmt.Errorf("error creating BSON file %v: %v", f.path, err)
 	}
 
 	return nil
 }
 
-// Write guarantees that when it returns, either the entire
-// contents of buf or none of it, has been flushed by the writer.
-// This is useful in the unlikely case that mongodump crashes.
-func (f atomicFlusher) Write(buf []byte) (int, error) {
-	if len(buf) > f.availableWriteFlusher.Available() {
-		f.availableWriteFlusher.Flush()
-	}
-	if len(buf) > f.availableWriteFlusher.Available() {
-		l, e := f.availableWriteFlusher.Write(buf)
-		f.availableWriteFlusher.Flush()
-		return l, e
-	}
-	return f.availableWriteFlusher.Write(buf)
-}
-
 // realMetadataFile implements intent.file, and corresponds to a Metadata file on disk
 type realMetadataFile struct {
 	io.WriteCloser
@@ -144,12 +98,10 @@ type realMetadataFile struct {
 	// errorWrite adds a Read() method to this object allowing it to be an
 	// intent.file ( a ReadWriteOpenCloser )
 	intent *intents.Intent
-	gzip   bool
 	NilPos
 }
 
 // Open opens the file on disk that the intent indicates. Any directories needed are created.
-// If compression is needed, the File gets wrapped in a gzip.Writer
 func (f *realMetadataFile) Open() (err error) {
 	if f.path == "" {
 		return fmt.Errorf("No metadata path for %v.%v", f.intent.DB, f.intent.C)
@@ -160,16 +112,9 @@ func (f *realMetadataFile) Open() (err error) {
 			filepath.Dir(f.path), err)
 	}
 
-	fileName := f.path
-	f.WriteCloser, err = os.Create(fileName)
+	f.WriteCloser, err = os.Create(f.path)
 	if err != nil {
-		return fmt.Errorf("error creating metadata file %v: %v", fileName, err)
-	}
-	if f.gzip {
-		f.WriteCloser = &wrappedWriteCloser{
-			WriteCloser: gzip.NewWriter(f.WriteCloser),
-			inner:       f.WriteCloser,
-		}
+		return fmt.Errorf("error creating metadata file %v: %v", f.path, err)
 	}
 	return nil
 }
@@ -252,7 +197,7 @@ func (dump *MongoDump) NewIntent(dbName, colName string) (*intents.Intent, error
 					`and can't be dumped to the filesystem`, dbName, colName, c)
 			}
 			path := nameGz(dump.OutputOptions.Gzip, dump.outputPath(dbName, colName)+".bson")
-			intent.BSONFile = &realBSONFile{path: path, intent: intent, gzip: dump.OutputOptions.Gzip}
+			intent.BSONFile = &realBSONFile{path: path, intent: intent}
 		}
 		if !intent.IsSystemIndexes() {
 			if dump.OutputOptions.Archive != "" {
@@ -262,7 +207,7 @@ func (dump *MongoDump) NewIntent(dbName, colName string) (*intents.Intent, error
 				}
 			} else {
 				path := nameGz(dump.OutputOptions.Gzip, dump.outputPath(dbName, colName+".metadata.json"))
-				intent.MetadataFile = &realMetadataFile{path: path, intent: intent, gzip: dump.OutputOptions.Gzip}
+				intent.MetadataFile = &realMetadataFile{path: path, intent: intent}
 			}
 		}
 	}
@@ -296,7 +241,7 @@ func (dump *MongoDump) CreateOplogIntents() error {
 	if dump.OutputOptions.Archive != "" {
 		oplogIntent.BSONFile = &archive.MuxIn{Mux: dump.archive.Mux, Intent: oplogIntent}
 	} else {
-		oplogIntent.BSONFile = &realBSONFile{path: dump.outputPath("oplog.bson", ""), intent: oplogIntent, gzip: dump.OutputOptions.Gzip}
+		oplogIntent.BSONFile = &realBSONFile{path: dump.outputPath("oplog.bson", ""), intent: oplogIntent}
 	}
 	dump.manager.Put(oplogIntent)
 	return nil
@@ -326,9 +271,9 @@ func (dump *MongoDump) CreateUsersRolesVersionIntentsForDB(db string) error {
 		rolesIntent.BSONFile = &archive.MuxIn{Intent: rolesIntent, Mux: dump.archive.Mux}
 		versionIntent.BSONFile = &archive.MuxIn{Intent: versionIntent, Mux: dump.archive.Mux}
 	} else {
-		usersIntent.BSONFile = &realBSONFile{path: filepath.Join(outDir, nameGz(dump.OutputOptions.Gzip, "$admin.system.users.bson")), intent: usersIntent, gzip: dump.OutputOptions.Gzip}
-		rolesIntent.BSONFile = &realBSONFile{path: filepath.Join(outDir, nameGz(dump.OutputOptions.Gzip, "$admin.system.roles.bson")), intent: rolesIntent, gzip: dump.OutputOptions.Gzip}
-		versionIntent.BSONFile = &realBSONFile{path: filepath.Join(outDir, nameGz(dump.OutputOptions.Gzip, "$admin.system.version.bson")), intent: versionIntent, gzip: dump.OutputOptions.Gzip}
+		usersIntent.BSONFile = &realBSONFile{path: filepath.Join(outDir, nameGz(dump.OutputOptions.Gzip, "$admin.system.users.bson")), intent: usersIntent}
+		rolesIntent.BSONFile = &realBSONFile{path: filepath.Join(outDir, nameGz(dump.OutputOptions.Gzip, "$admin.system.roles.bson")), intent: rolesIntent}
+		versionIntent.BSONFile = &realBSONFile{path: filepath.Join(outDir, nameGz(dump.OutputOptions.Gzip, "$admin.system.version.bson")), intent: versionIntent}
 	}
 	dump.manager.Put(usersIntent)
 	dump.manager.Put(rolesIntent)
diff --git a/src/mongo/gotools/mongoreplay/aggregation/mongotape_aggregations.js b/src/mongo/gotools/mongoreplay/aggregation/mongoreplay_aggregations.js
index 3bd258f80e2..3bd258f80e2 100644
--- a/src/mongo/gotools/mongoreplay/aggregation/mongotape_aggregations.js
+++ b/src/mongo/gotools/mongoreplay/aggregation/mongoreplay_aggregations.js
diff --git a/src/mongo/gotools/mongoreplay/execute.go b/src/mongo/gotools/mongoreplay/execute.go
index 5d039c33b50..dce06fbb2f6 100644
--- a/src/mongo/gotools/mongoreplay/execute.go
+++ b/src/mongo/gotools/mongoreplay/execute.go
@@ -150,6 +150,7 @@ func (context *ExecutionContext) newExecutionSession(url string, start time.Time
 		var connected bool
 		time.Sleep(start.Add(-5 * time.Second).Sub(now)) // Sleep until five seconds before the start time
 		session, err := mgo.Dial(url)
+		defer session.Close()
 		if err == nil {
 			userInfoLogger.Logvf(Info, "(Connection %v) New connection CREATED.", connectionNum)
 			connected = true
diff --git a/src/mongo/gotools/mongoreplay/main/mongoreplay.go b/src/mongo/gotools/mongoreplay/main/mongoreplay.go
index 5e2fc6dccfd..5a7fd02c809 100644
--- a/src/mongo/gotools/mongoreplay/main/mongoreplay.go
+++ b/src/mongo/gotools/mongoreplay/main/mongoreplay.go
@@ -15,11 +15,23 @@ const (
 )
 
 func main() {
+	versionOpts := mongoreplay.VersionOptions{}
+	versionFlagParser := flags.NewParser(&versionOpts, flags.Default)
+	versionFlagParser.Options = flags.IgnoreUnknown
+	_, err := versionFlagParser.Parse()
+	if err != nil {
+		os.Exit(ExitError)
+	}
+
+	if versionOpts.PrintVersion() {
+		os.Exit(ExitOk)
+	}
+
 	opts := mongoreplay.Options{}
 
 	var parser = flags.NewParser(&opts, flags.Default)
 
-	_, err := parser.AddCommand("play", "Play captured traffic against a mongodb instance", "",
+	_, err = parser.AddCommand("play", "Play captured traffic against a mongodb instance", "",
 		&mongoreplay.PlayCommand{GlobalOpts: &opts})
 	if err != nil {
 		panic(err)
@@ -37,13 +49,6 @@ func main() {
 		panic(err)
 	}
 
-	parser.Options = flags.IgnoreUnknown
-	parser.Parse()
-	if opts.PrintVersion() {
-		os.Exit(ExitOk)
-	}
-
-	parser.Options = flags.Default
 	_, err = parser.Parse()
 
 	if err != nil {
diff --git a/src/mongo/gotools/mongoreplay/mongo_op_handler.go b/src/mongo/gotools/mongoreplay/mongo_op_handler.go
index 311d72248f2..a48a2d9d31a 100644
--- a/src/mongo/gotools/mongoreplay/mongo_op_handler.go
+++ b/src/mongo/gotools/mongoreplay/mongo_op_handler.go
@@ -15,6 +15,7 @@ import (
 type OpStreamSettings struct {
 	PcapFile         string `short:"f" description:"path to the pcap file to be read"`
 	PacketBufSize    int    `short:"b" description:"Size of heap used to merge separate streams together"`
+	CaptureBufSize   int    `long:"capSize" description:"Size in KiB of the PCAP capture buffer"`
 	Expression       string `short:"e" long:"expr" description:"BPF filter expression to apply to packets"`
 	NetworkInterface string `short:"i" description:"network interface to listen on"`
 }
diff --git a/src/mongo/gotools/mongoreplay/mongotape.go b/src/mongo/gotools/mongoreplay/mongoreplay.go
index 5bf2b6d428a..fca27f0701d 100644
--- a/src/mongo/gotools/mongoreplay/mongotape.go
+++ b/src/mongo/gotools/mongoreplay/mongoreplay.go
@@ -5,7 +5,6 @@ type Options struct {
 	Verbosity []bool `short:"v" long:"verbosity" description:"increase the detail regarding the tools performance on the input file that is output to logs (include multiple times for increased logging verbosity, e.g. -vvv)"`
 	Debug     []bool `short:"d" long:"debug" description:"increase the detail regarding the operations and errors of the tool that is output to the logs(include multiple times for increased debugging information, e.g. -ddd)"`
 	Silent    bool   `short:"s" long:"silent" description:"silence all log output"`
-	Version   bool   `long:"version" description:"display the version and exit"`
 }
 
 // SetLogging sets the verbosity/debug level for log output.
@@ -19,3 +18,7 @@ func (opts *Options) SetLogging() {
 	userInfoLogger.setVerbosity(v)
 	toolDebugLogger.setVerbosity(d)
 }
+
+type VersionOptions struct {
+	Version bool `long:"version" description:"display the version and exit"`
+}
diff --git a/src/mongo/gotools/mongoreplay/mongotape_test.go b/src/mongo/gotools/mongoreplay/mongoreplay_test.go
index dc9be128c37..5b54ebda216 100644
--- a/src/mongo/gotools/mongoreplay/mongotape_test.go
+++ b/src/mongo/gotools/mongoreplay/mongoreplay_test.go
@@ -524,6 +524,7 @@ func TestShortenLegacyReply(t *testing.T) {
 	// add the two docs as the docs from the reply
 	result.RawOp.Body = append(result.RawOp.Body, asByte1...)
 	result.RawOp.Body = append(result.RawOp.Body, asByte2...)
+	result.Header.MessageLength = int32(len(result.RawOp.Body))
 
 	// reply should be functional and parseable
 	parsed, err := result.RawOp.Parse()
diff --git a/src/mongo/gotools/mongoreplay/pcap_test.go b/src/mongo/gotools/mongoreplay/pcap_test.go
index df0c9325660..97334a43989 100644
--- a/src/mongo/gotools/mongoreplay/pcap_test.go
+++ b/src/mongo/gotools/mongoreplay/pcap_test.go
@@ -128,7 +128,7 @@ func TestMultiChannelGetMoreLiveDB(t *testing.T) {
 
 func pcapTestHelper(t *testing.T, pcapFname string, preprocess bool, verifier verifyFunc) {
 
-	pcapFile := "testPcap/" + pcapFname
+	pcapFile := "mongoreplay/testPcap/" + pcapFname
 	if _, err := os.Stat(pcapFile); err != nil {
 		t.Skipf("pcap file %v not present, skipping test", pcapFile)
 	}
diff --git a/src/mongo/gotools/mongoreplay/play_livedb_test.go b/src/mongo/gotools/mongoreplay/play_livedb_test.go
index b4607dc42e8..9f5e5c7e9e7 100644
--- a/src/mongo/gotools/mongoreplay/play_livedb_test.go
+++ b/src/mongo/gotools/mongoreplay/play_livedb_test.go
@@ -12,7 +12,7 @@ import (
 )
 
 const (
-	defaultTestPort      = 20000
+	defaultTestPort      = "20000"
 	nonAuthTestServerURL = "mongodb://localhost"
 	authTestServerURL    = "mongodb://authorizedUser:authorizedPwd@localhost"
 	testDB               = "mongoreplay"
@@ -43,24 +43,31 @@ type recordedOpGenerator struct {
 }
 
 func setConnectionURL() error {
+	testPort := os.Getenv("DB_PORT")
+	if testPort == "" {
+		testPort = defaultTestPort
+	}
 	var url string
 	if os.Getenv("AUTH") == "1" {
 		url = authTestServerURL
 	} else {
 		url = nonAuthTestServerURL
 	}
-	dialURL := fmt.Sprintf("%s:%v", url, defaultTestPort)
+	dialURL := fmt.Sprintf("%s:%v", url, testPort)
 	if os.Getenv("AUTH") == "1" {
 		dialURL += "/admin"
 	}
-	session, err := mgo.Dial(dialURL)
+	session, err := mgo.DialWithTimeout(dialURL, 30*time.Second)
 	if err != nil {
-		return err
+		return fmt.Errorf("%v:%v", dialURL, err)
 	}
 
-	port, err := getTestDBPort(session)
+	port, err := getPrimaryPort(session)
 	if err != nil {
-		return err
+		return fmt.Errorf("%v:%v", dialURL, err)
+	}
+	if port == "" {
+		port = testPort
 	}
 	urlNonAuth = fmt.Sprintf("%s:%s", nonAuthTestServerURL, port)
 	urlAuth = fmt.Sprintf("%s:%v/admin", authTestServerURL, port)
@@ -71,10 +78,7 @@ func setConnectionURL() error {
 	return nil
 }
 
-func getTestDBPort(session *mgo.Session) (string, error) {
-	if port := os.Getenv("DB_PORT"); port != "" {
-		return port, nil
-	}
+func getPrimaryPort(session *mgo.Session) (string, error) {
 
 	result := struct {
 		Members []struct {
@@ -83,13 +87,22 @@ func getTestDBPort(session *mgo.Session) (string, error) {
 		} `bson:"members"`
 	}{}
 
-	err := session.DB("admin").Run("replSetGetStatus", &result)
-	if err != nil && err.Error() != "not running with --replSet" {
-		return "", err
+	res := &struct {
+		Msg string
+	}{}
+	session.Run("ismaster", res)
+	isMongosTestServer = (res.Msg == "isdbgrid")
+	if isMongosTestServer {
+		return "", nil
 	}
 
-	if err != nil && err.Error() == "not running with --replSet" {
-		return fmt.Sprintf("%d", defaultTestPort), nil
+	err := session.DB("admin").Run("replSetGetStatus", &result)
+
+	if err != nil {
+		if err.Error() == "not running with --replSet" {
+			return "", nil
+		}
+		return "", err
 	}
 
 	for _, member := range result.Members {
@@ -98,7 +111,7 @@ func getTestDBPort(session *mgo.Session) (string, error) {
 		}
 	}
 
-	return "", fmt.Errorf("unable to determine database port")
+	return "", fmt.Errorf("replset status has no primary")
 }
 
 func TestMain(m *testing.M) {
@@ -112,17 +125,6 @@ func TestMain(m *testing.M) {
 		authTestServerMode = false
 	}
 
-	session, err := mgo.Dial(currentTestURL)
-	if err != nil {
-		panic(err)
-	}
-	res := &struct {
-		Msg string
-	}{}
-	session.Run("ismaster", res)
-	isMongosTestServer = (res.Msg == "isdbgrid")
-	session.Close()
-
 	os.Exit(m.Run())
 }
 
@@ -192,7 +194,7 @@ func TestOpInsertLiveDB(t *testing.T) {
 	t.Log("Completed mongoreplay playback of generated traffic")
 
 	// prepare a query for the database
-	session, err := mgo.Dial(currentTestURL)
+	session, err := mgo.DialWithTimeout(currentTestURL, 30*time.Second)
 	if err != nil {
 		t.Errorf("Error connecting to test server: %v", err)
 	}
@@ -308,7 +310,7 @@ func TestUpdateOpLiveDB(t *testing.T) {
 	t.Log("Completed mongoreplay playback of generated traffic")
 
 	// prepare a query for the database
-	session, err := mgo.Dial(currentTestURL)
+	session, err := mgo.DialWithTimeout(currentTestURL, 30*time.Second)
 	if err != nil {
 		t.Errorf("Error connecting to test server: %v", err)
 	}
@@ -787,7 +789,7 @@ func TestCommandOpInsertLiveDB(t *testing.T) {
 	t.Log("Completed mongoreplay playback of generated traffic")
 
 	// prepare a query for the database
-	session, err := mgo.Dial(currentTestURL)
+	session, err := mgo.DialWithTimeout(currentTestURL, 30*time.Second)
 	if err != nil {
 		t.Errorf("Error connecting to test server: %v", err)
 	}
@@ -1005,7 +1007,7 @@ func TestCommandOpGetMoreLiveDB(t *testing.T) {
 }
 
 func teardownDB() error {
-	session, err := mgo.Dial(currentTestURL)
+	session, err := mgo.DialWithTimeout(currentTestURL, 30*time.Second)
 	if err != nil {
 		return err
 	}
diff --git a/src/mongo/gotools/mongoreplay/record.go b/src/mongo/gotools/mongoreplay/record.go
index aaecb157348..670239ab3fe 100644
--- a/src/mongo/gotools/mongoreplay/record.go
+++ b/src/mongo/gotools/mongoreplay/record.go
@@ -10,6 +10,7 @@ import (
 
 	"github.com/10gen/llmgo/bson"
 	"github.com/google/gopacket/pcap"
+	"github.com/mongodb/mongo-tools/common/util"
 )
 
 // RecordCommand stores settings for the mongoreplay 'record' subcommand
@@ -49,7 +50,35 @@ func getOpstream(cfg OpStreamSettings) (*packetHandlerContext, error) {
 			return nil, fmt.Errorf("error opening pcap file: %v", err)
 		}
 	} else if len(cfg.NetworkInterface) > 0 {
-		pcapHandle, err = pcap.OpenLive(cfg.NetworkInterface, 32*1024*1024, false, pcap.BlockForever)
+		inactive, err := pcap.NewInactiveHandle(cfg.NetworkInterface)
+		// This is safe; calling `Activate()` steals the underlying ptr.
+		defer inactive.CleanUp()
+		if err != nil {
+			return nil, fmt.Errorf("error creating a pcap handle: %v", err)
+		}
+
+		err = inactive.SetSnapLen(32*1024*1024)
+		if err != nil {
+			return nil, fmt.Errorf("error setting snaplen on pcap handle: %v", err)
+		}
+
+		err = inactive.SetPromisc(false)
+		if err != nil {
+			return nil, fmt.Errorf("error setting promisc on pcap handle: %v", err)
+		}
+
+		err = inactive.SetTimeout(pcap.BlockForever)
+		if err != nil {
+			return nil, fmt.Errorf("error setting timeout on pcap handle: %v", err)
+		}
+
+		// CaptureBufSize is in KiB to match units on `tcpdump -B`.
+		err = inactive.SetBufferSize(cfg.CaptureBufSize * 1024)
+		if err != nil {
+			return nil, fmt.Errorf("error setting buffer size on pcap handle: %v", err)
+		}
+
+		pcapHandle, err = inactive.Activate()
 		if err != nil {
 			return nil, fmt.Errorf("error listening to network interface: %v", err)
 		}
@@ -90,7 +119,7 @@ func NewPlaybackWriter(playbackFileName string, isGzipWriter bool) (*PlaybackWri
 		return nil, fmt.Errorf("error opening playback file to write to: %v", err)
 	}
 	if isGzipWriter {
-		pbWriter.WriteCloser = gzip.NewWriter(file)
+		pbWriter.WriteCloser = &util.WrappedWriteCloser{gzip.NewWriter(file), file}
 	} else {
 		pbWriter.WriteCloser = file
 	}
@@ -109,6 +138,10 @@ func (record *RecordCommand) ValidateParams(args []string) error {
 		// default heap size
 		record.OpStreamSettings.PacketBufSize = 1000
 	}
+	if record.OpStreamSettings.CaptureBufSize == 0 {
+		// default capture buffer size to 2 MiB (same as libpcap)
+		record.OpStreamSettings.CaptureBufSize = 2*1024
+	}
 	return nil
 }
 
@@ -136,6 +169,7 @@ func (record *RecordCommand) Execute(args []string) error {
 		ctx.packetHandler.Close()
 	}()
 	playbackWriter, err := NewPlaybackWriter(record.PlaybackFile, record.Gzip)
+	defer playbackWriter.Close()
 	if err != nil {
 		return err
 	}
diff --git a/src/mongo/gotools/mongoreplay/sanity_check.sh b/src/mongo/gotools/mongoreplay/sanity_check.sh
index 7b840913732..74a4cf49c6b 100755
--- a/src/mongo/gotools/mongoreplay/sanity_check.sh
+++ b/src/mongo/gotools/mongoreplay/sanity_check.sh
@@ -1,8 +1,9 @@
 #!/bin/bash
 
 PORT=27017
-PCAPFILE="mongoreplay_test.out"
 STARTMONGO=false
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+PCAPFILE="$SCRIPT_DIR/mongoreplay_test.out"
 
 while test $# -gt 0; do
 	case "$1" in
@@ -33,27 +34,33 @@ if [ $? != 0 ]; then
 fi
 
 set -e
+set -o verbose
 
 OUTFILE="$(echo $PCAPFILE | cut -f 1 -d '.').playback"
-mongoreplay record --silent -f $PCAPFILE -p $OUTFILE
+mongoreplay record -f $PCAPFILE -p $OUTFILE
 
 if [ "$STARTMONGO" = true ]; then
 	rm -rf /data/mongoreplay/
 	mkdir /data/mongoreplay/
 	echo "starting MONGOD"
-	mongod --port=$PORT --dbpath=/data/mongoreplay > /dev/null 2>&1 &
+	mongod --port=$PORT --dbpath=/data/mongoreplay &
 	MONGOPID=$!
 fi
 
-mongo --port=$PORT mongoplay_test --eval "db.setProfilingLevel(2);" >/dev/null
-mongo --port=$PORT mongoplay_test --eval "db.createCollection('sanity_check', {});" >/dev/null
+mongo --port=$PORT mongoplay_test --eval "db.setProfilingLevel(2);" 
+mongo --port=$PORT mongoplay_test --eval "db.createCollection('sanity_check', {});" 
 
-mongoreplay play --silent --collect=none -p $OUTFILE
+mongoreplay play --host mongodb://localhost:$PORT -p $OUTFILE
 mongo --port=$PORT mongoplay_test --eval "var profile_results = db.system.profile.find({'ns':'mongoplay_test.sanity_check'});
-assert.gt(profile_results.size(), 0);" >/dev/null
+assert.gt(profile_results.size(), 0);" 
 
 mongo --port=$PORT mongoplay_test --eval "var query_results = db.sanity_check.find({'test_success':1});
-assert.gt(query_results.size(), 0);" >/dev/null
+assert.gt(query_results.size(), 0);" 
+
+# test that files are correctly gziped ( TOOLS-1503 )
+mongoreplay record -f $PCAPFILE -p ${OUTFILE} --gzip
+gunzip -t ${OUTFILE}
+
 echo "Success!"
 
 if [ "$STARTMONGO" = true ]; then
diff --git a/src/mongo/gotools/mongoreplay/stat_format.go b/src/mongo/gotools/mongoreplay/stat_format.go
index 720c4168096..ad64ef7bd95 100644
--- a/src/mongo/gotools/mongoreplay/stat_format.go
+++ b/src/mongo/gotools/mongoreplay/stat_format.go
@@ -158,7 +158,7 @@ func (stat *OpStat) getRequestID() string {
 }
 func (stat *OpStat) getTime(layout string) string {
 	if layout == "" {
-		layout = "2/15 15:04:05.000"
+		layout = time.RFC822Z
 	}
 	t := stat.Seen
 	if stat.PlayedAt != nil {
diff --git a/src/mongo/gotools/mongoreplay/version.go b/src/mongo/gotools/mongoreplay/version.go
index b957364eb0e..8d147b8e1ff 100644
--- a/src/mongo/gotools/mongoreplay/version.go
+++ b/src/mongo/gotools/mongoreplay/version.go
@@ -8,7 +8,7 @@ import (
 
 // Print the tool version to stdout.  Returns whether or not the version flag
 // is specified.
-func (o *Options) PrintVersion() bool {
+func (o *VersionOptions) PrintVersion() bool {
 	if o.Version {
 		fmt.Printf("%v version: %v\n", "mongoreplay", options.VersionStr)
 		fmt.Printf("git version: %v\n", options.Gitspec)
diff --git a/src/mongo/gotools/mongorestore/filepath.go b/src/mongo/gotools/mongorestore/filepath.go
index 88d730aaf26..c3d434f68a0 100644
--- a/src/mongo/gotools/mongorestore/filepath.go
+++ b/src/mongo/gotools/mongorestore/filepath.go
@@ -152,7 +152,7 @@ func (f *realMetadataFile) Open() (err error) {
 		if err != nil {
 			return fmt.Errorf("error reading compressed metadata %v: %v", f.path, err)
 		}
-		f.ReadCloser = &wrappedReadCloser{gzFile, file}
+		f.ReadCloser = &util.WrappedReadCloser{gzFile, file}
 	} else {
 		f.ReadCloser = file
 	}
diff --git a/src/mongo/gotools/mongorestore/mongorestore.go b/src/mongo/gotools/mongorestore/mongorestore.go
index 2607bb45501..3e96bba0b0e 100644
--- a/src/mongo/gotools/mongorestore/mongorestore.go
+++ b/src/mongo/gotools/mongorestore/mongorestore.go
@@ -486,19 +486,6 @@ func (restore *MongoRestore) Restore() error {
 	return nil
 }
 
-type wrappedReadCloser struct {
-	io.ReadCloser
-	inner io.ReadCloser
-}
-
-func (wrc *wrappedReadCloser) Close() error {
-	err := wrc.ReadCloser.Close()
-	if err != nil {
-		return err
-	}
-	return wrc.inner.Close()
-}
-
 func (restore *MongoRestore) getArchiveReader() (rc io.ReadCloser, err error) {
 	if restore.InputOptions.Archive == "-" {
 		rc = ioutil.NopCloser(restore.stdin)
@@ -528,7 +515,7 @@ func (restore *MongoRestore) getArchiveReader() (rc io.ReadCloser, err error) {
 		if err != nil {
 			return nil, err
 		}
-		return &wrappedReadCloser{gzrc, rc}, nil
+		return &util.WrappedReadCloser{gzrc, rc}, nil
 	}
 	return rc, nil
 }
diff --git a/src/mongo/gotools/test/qa-tests/jstests/oplog/asymmetric_ssl_test.js b/src/mongo/gotools/test/qa-tests/jstests/oplog/asymmetric_ssl_test.js
index 720b6c53bf4..0e897349c9d 100644
--- a/src/mongo/gotools/test/qa-tests/jstests/oplog/asymmetric_ssl_test.js
+++ b/src/mongo/gotools/test/qa-tests/jstests/oplog/asymmetric_ssl_test.js
@@ -18,7 +18,7 @@
   ];
 
   if (toolTest.useSSL) {
-    var port = 26999;
+    var port = allocatePort();
 
     // this mongod is actually started with SSL flags because of `useSSL`
     startMongod('--auth', '--port', port,
diff --git a/src/mongo/gotools/test/qa-tests/jstests/oplog/oplog_server_ko_test.js b/src/mongo/gotools/test/qa-tests/jstests/oplog/oplog_server_ko_test.js
index 9bd5f0996bb..007897db30e 100644
--- a/src/mongo/gotools/test/qa-tests/jstests/oplog/oplog_server_ko_test.js
+++ b/src/mongo/gotools/test/qa-tests/jstests/oplog/oplog_server_ko_test.js
@@ -12,7 +12,7 @@
   db = toolTest.db.getSiblingDB('foo'); // eslint-disable-line no-native-reassign
   db.dropDatabase();
 
-  var port = 26999;
+  var port = allocatePort();
   startMongod('--auth', '--port', port,
       '--dbpath', MongoRunner.dataPath + 'oplogServerKOTest2');
author	Ramon Fernandez <ramon@mongodb.com>	2016-11-08 12:23:36 -0500
committer	Ramon Fernandez <ramon@mongodb.com>	2016-11-08 12:23:36 -0500
commit	bf31ef57f79a58e79c8fd0dd01d05d843252208a (patch)
tree	93c6b9fe5f5333baa426eb6786b00f58c26a3e24
parent	bb78a26fcdca87e808cab52b6daa91be581bcbbc (diff)
download	mongo-bf31ef57f79a58e79c8fd0dd01d05d843252208a.tar.gz