Import tools: a604ced46f11689e5fc5f83c5752436568582889 from branch v3.3

ref: cd0ae5ddb4..a604ced46f
for: 3.4.0

TOOLS-1445    correctly merge resmoke excludes
TOOLS-1457    different_db.js should not create collections that start with "system"
TOOLS-1466    fix node path in lint-js
TOOLS-1467    move multiversion dirs into /data/mci
TOOLS-1485     mongotop_reports.js, mongotop_stress.js, and dump_server_ko_test.js need to check startParallelShell exit code
TOOLS-1488    provide IO buffers to oplog archive intents

7 files changed, 86 insertions, 57 deletions

diff --git a/src/mongo/gotools/common.yml b/src/mongo/gotools/common.yml
index c8e13e872b1..e63d3b4beb2 100644
--- a/src/mongo/gotools/common.yml
+++ b/src/mongo/gotools/common.yml
@@ -225,13 +225,14 @@ functions:
         if [ -f bsondump ]; then
           chmod +x bsondump
         fi
-        rm -rf /data/install /data/multiversion
+        rm -rf /data/mci/install /data/mci/multiversion
+        mkdir -p /data/mci/install /data/mci/multiversion
         if [ "${multiversion_override}" != "skip" ]; then
-          python buildscripts/setup_multiversion_mongodb.py /data/install /data/multiversion ${arch} ${multiversion_override|2.6 2.4} --latest ${smoke_use_ssl} --os="${mongo_os}"
+          python buildscripts/setup_multiversion_mongodb.py /data/mci/install /data/mci/multiversion ${arch} ${multiversion_override|2.6 2.4} --latest ${smoke_use_ssl} --os="${mongo_os}"
         fi
         chmod 400 jstests/libs/key*
 
-        PATH=$PATH:/data/multiversion python buildscripts/resmoke.py --suite=${resmoke_suite} --continueOnFailure --log=buildlogger --reportFile=report.json ${resmoke_args}
+        PATH=$PATH:/data/mci/multiversion python buildscripts/resmoke.py --suite=${resmoke_suite} --continueOnFailure --log=buildlogger --reportFile=report.json ${resmoke_args} --excludeWithAnyTags="${excludes}"
 
   "build tool":
     command: shell.exec
@@ -588,6 +589,8 @@ post:
     params:
       script: |
         rm -rf /data/db/*
+        rm -rf /data/mci/multiversion
+        rm -rf /data/mci/install
         exit 0
 
 timeout:
@@ -973,6 +976,7 @@ tasks:
         working_dir: src
         script: |
           set -o verbose
+          PATH="/opt/node/bin:$PATH"
           /opt/node/bin/npm install eslint@3.2
           /opt/node/bin/node node_modules/eslint/bin/eslint.js test/qa-tests/jstests/**/*.js
 
@@ -1067,7 +1071,7 @@ tasks:
     - func: "run qa-tests"
       vars:
         resmoke_suite: "core${resmoke_use_ssl}"
-        resmoke_args: "--excludeWithAnyTags=requires_unstable ${resmoke_args}"
+        excludes: "requires_unstable,${excludes}"
 
 - name: native-cert-ssl
   depends_on:
@@ -1124,11 +1128,12 @@ tasks:
           mv test/qa-tests/* .
           chmod +x mongo*
           chmod +x bsondump*
-          rm -rf /data/install /data/multiversion
-          python buildscripts/setup_multiversion_mongodb.py /data/install /data/multiversion ${arch} "2.6" "2.4" --latest ${smoke_use_ssl} --os="${mongo_os}" || true
+          rm -rf /data/mci/install /data/mci/multiversion
+          mkdir -p /data/mci/install /data/mci/multiversion
+          python buildscripts/setup_multiversion_mongodb.py /data/mci/install /data/mci/multiversion ${arch} "2.6" "2.4" --latest ${smoke_use_ssl} --os="${mongo_os}" || true
           chmod 400 jstests/libs/key*
 
-          PATH=$PATH:/data/multiversion python buildscripts/resmoke.py --suite=native_cert_ssl  --continueOnFailure --log=buildlogger --reportFile=report.json ${resmoke_args}
+          PATH=$PATH:/data/mci/multiversion python buildscripts/resmoke.py --suite=native_cert_ssl  --continueOnFailure --log=buildlogger --reportFile=report.json ${resmoke_args} --excludeWithAnyTags="${excludes}"
 
 - name: qa-dump-restore-archiving
   depends_on:
@@ -1154,7 +1159,7 @@ tasks:
     - func: "run qa-tests"
       vars:
         resmoke_suite: "restore_archive"
-        resmoke_args: "--excludeWithAnyTags=requires_unstable ${resmoke_args}"
+        excludes: "requires_unstable,${excludes}"
 
 - name: qa-dump-restore-gzip
   depends_on:
@@ -1180,7 +1185,7 @@ tasks:
     - func: "run qa-tests"
       vars:
         resmoke_suite: "restore_gzip"
-        resmoke_args: "--excludeWithAnyTags=requires_unstable ${resmoke_args}"
+        excludes: "requires_unstable,${excludes}"
 
 - name: qa-tests-wt
   depends_on:
@@ -1227,7 +1232,8 @@ tasks:
     - func: "run qa-tests"
       vars:
         resmoke_suite: "core"
-        resmoke_args: "--excludeWithAnyTags=requires_unstable --storageEngine=wiredTiger ${resmoke_args}"
+        resmoke_args: "--storageEngine=wiredTiger ${resmoke_args}"
+        excludes: "requires_unstable,${excludes}"
 
 - name: unit
   commands:
@@ -1318,7 +1324,7 @@ buildvariants:
     mongo_os: "osx"
     arch: "osx/x86_64"
     build_tags: "ssl"
-    resmoke_args: --excludeWithAnyTags=requires_many_files
+    excludes: requires_many_files
   tasks: *osx_1010_tasks
 
 - name: osx-1010-ssl
@@ -1333,7 +1339,7 @@ buildvariants:
     arch: "osx/x86_64"
     build_tags: "ssl"
     edition: ssl
-    resmoke_args: --excludeWithAnyTags=requires_many_files
+    excludes: requires_many_files
   tasks: *osx_1010_ssl_tasks
 
 #######################################
@@ -1408,7 +1414,8 @@ buildvariants:
     build_tags: 'ssl'
     resmoke_use_ssl: _ssl
     library_path: PATH="/opt/mongodbtoolchain/v2/bin/:$PATH"
-    resmoke_args: --excludeWithAnyTags=requires_mmap_available,requires_large_ram,requires_mongo_24,requires_mongo_26,requires_mongo_30 -j 4
+    resmoke_args: -j 4
+    excludes: requires_mmap_available,requires_large_ram,requires_mongo_24,requires_mongo_26,requires_mongo_30
     multiversion_override: "skip"
     arch: "linux/ppc64le"
     edition: enterprise
@@ -1430,7 +1437,8 @@ buildvariants:
     build_tags: "sasl ssl"
     resmoke_use_ssl: _ssl
     library_path: PATH="/opt/mongodbtoolchain/v2/bin/:$PATH"
-    resmoke_args: --excludeWithAnyTags=requires_mmap_available,requires_mongo_24,requires_mongo_26,requires_mongo_30 -j 2
+    excludes: requires_mmap_available,requires_mongo_24,requires_mongo_26,requires_mongo_30
+    resmoke_args: -j 2
     multiversion_override: "skip"
     mongo_version_always_use_latest: true
     arch: "linux/s390x"
@@ -1453,7 +1461,8 @@ buildvariants:
     build_tags: "ssl"
     resmoke_use_ssl: _ssl
     library_path: PATH="/opt/mongodbtoolchain/v2/bin/:$PATH"
-    resmoke_args: --excludeWithAnyTags=requires_mmap_available,requires_large_ram,requires_mongo_24,requires_mongo_26,requires_mongo_30 -j 2
+    excludes: requires_mmap_available,requires_large_ram,requires_mongo_24,requires_mongo_26,requires_mongo_30
+    resmoke_args: -j 2
     multiversion_override: "skip"
     mongo_version_always_use_latest: true
     arch: "linux/arm64"
@@ -1474,7 +1483,8 @@ buildvariants:
     mongo_os: "sunos5"
     library_path: PATH="/opt/mongodbtoolchain/v2/bin/:$PATH"
     args: -gccgoflags "-lsocket -lnsl"
-    resmoke_args: --excludeWithAnyTags=requires_large_ram -j$(kstat cpu | sort -u | grep -c "^module")
+    excludes: requires_large_ram
+    resmoke_args: -j$(kstat cpu | sort -u | grep -c "^module")
   tasks: *solaris_tasks
 
 #######################################
@@ -1491,7 +1501,7 @@ buildvariants:
     mongo_arch: "i386"
     extension: .exe
     preproc_gpm: "perl -pi -e 's/\\r\\n/\\n/g' "
-    resmoke_args: --excludeWithAnyTags=requires_large_ram
+    excludes: requires_large_ram
     integration_test_args: "integration"
   tasks: *windows_32_tasks
 
@@ -1504,7 +1514,7 @@ buildvariants:
     <<: *mongo_default_startup_args
     mongo_os: "windows-64"
     mongo_target: "windows_x86_64-2008plus"
-    resmoke_args: --excludeWithAnyTags=requires_large_ram
+    excludes: requires_large_ram
     extension: .exe
     arch: "win32/x86_64"
     preproc_gpm: "perl -pi -e 's/\\r\\n/\\n/g' "
@@ -1524,7 +1534,7 @@ buildvariants:
     edition: ssl
     smoke_use_ssl: --use-ssl
     resmoke_use_ssl: _ssl
-    resmoke_args: --excludeWithAnyTags=requires_large_ram,requires_mongo_24
+    excludes: requires_large_ram,requires_mongo_24
     multiversion_override: "2.6"
     extension: .exe
     arch: "win32/x86_64"
@@ -1546,7 +1556,7 @@ buildvariants:
     build_tags: "sasl ssl"
     smoke_use_ssl: --use-ssl
     resmoke_use_ssl: _ssl
-    resmoke_args: --excludeWithAnyTags=requires_large_ram,requires_mongo_24
+    excludes: requires_large_ram,requires_mongo_24
     multiversion_override: "2.6"
     edition: enterprise
     extension: .exe
@@ -1574,7 +1584,7 @@ buildvariants:
     build_tags: "ssl"
     arch: "linux/x86_64"
     args: "-race"
-    resmoke_args: --excludeWithAnyTags=requires_large_ram
+    excludes: requires_large_ram
     integration_test_args: integration
   tasks: *ubuntu1204_race_tasks
 
diff --git a/src/mongo/gotools/mongorestore/oplog.go b/src/mongo/gotools/mongorestore/oplog.go
index 2aec289806a..68e24d20b8f 100644
--- a/src/mongo/gotools/mongorestore/oplog.go
+++ b/src/mongo/gotools/mongorestore/oplog.go
@@ -6,6 +6,7 @@ import (
 	"strings"
 
 	"github.com/mongodb/mongo-tools/common/db"
+	"github.com/mongodb/mongo-tools/common/intents"
 	"github.com/mongodb/mongo-tools/common/log"
 	"github.com/mongodb/mongo-tools/common/progress"
 	"github.com/mongodb/mongo-tools/common/util"
@@ -31,6 +32,9 @@ func (restore *MongoRestore) RestoreOplog() error {
 	if err := intent.BSONFile.Open(); err != nil {
 		return err
 	}
+	if fileNeedsIOBuffer, ok := intent.BSONFile.(intents.FileNeedsIOBuffer); ok {
+		fileNeedsIOBuffer.TakeIOBuffer(make([]byte, db.MaxBSONSize))
+	}
 	defer intent.BSONFile.Close()
 	// NewBufferlessBSONSource reads each bson document into its own buffer
 	// because bson.Unmarshal currently can't unmarshal binary types without
@@ -84,6 +88,9 @@ func (restore *MongoRestore) RestoreOplog() error {
 			return fmt.Errorf("error applying oplog: %v", err)
 		}
 	}
+	if fileNeedsIOBuffer, ok := intent.BSONFile.(intents.FileNeedsIOBuffer); ok {
+		fileNeedsIOBuffer.ReleaseIOBuffer()
+	}
 
 	log.Logvf(log.Info, "applied %v ops", totalOps)
 	return nil
diff --git a/src/mongo/gotools/test/qa-tests/jstests/libs/servers.js b/src/mongo/gotools/test/qa-tests/jstests/libs/servers.js
index f72007c0202..2715b369e46 100644
--- a/src/mongo/gotools/test/qa-tests/jstests/libs/servers.js
+++ b/src/mongo/gotools/test/qa-tests/jstests/libs/servers.js
@@ -387,13 +387,14 @@
       };
 
       var addOptionsToFullArgs = function(k, v) {
-        if (v === undefined || v === null)
-            return;
+        if (v === undefined || v === null) {
+          return;
+        }
 
         fullArgs.push("--" + k);
 
-        if (v != "") {
-            fullArgs.push("" + v);
+        if (v !== "") {
+          fullArgs.push("" + v);
         }
       };
 
@@ -420,11 +421,12 @@
         } else if (k === "setParameter" && isObject(o[k])) {
             // If the value associated with the setParameter option is an object, we want
             // to add all key-value pairs in that object as separate --setParameters.
-            Object.keys(o[k]).forEach(function(paramKey) {
-                addOptionsToFullArgs(k, "" + paramKey + "=" + o[k][paramKey]);
-            });
+          for (var l = 0; l < Object.keys(o[k]).length; l++) {
+            var paramKey = Object.keys(o[k])[l];
+            addOptionsToFullArgs(k, "" + paramKey + "=" + o[k][paramKey]);
+          }
         } else {
-            addOptionsToFullArgs(k, o[k]);
+          addOptionsToFullArgs(k, o[k]);
         }
       }
     } else {
diff --git a/src/mongo/gotools/test/qa-tests/jstests/restore/different_db.js b/src/mongo/gotools/test/qa-tests/jstests/restore/different_db.js
index c4723069098..b92d9cda9c3 100644
--- a/src/mongo/gotools/test/qa-tests/jstests/restore/different_db.js
+++ b/src/mongo/gotools/test/qa-tests/jstests/restore/different_db.js
@@ -68,15 +68,15 @@
   // restore the data to another different db
   ret = toolTest.runTool.apply(toolTest, ['restore',
       '--nsFrom', '$db$.$collection$',
-      '--nsTo', 'otherdest.$collection$_$db$']
+      '--nsTo', 'otherdest.$db$_$collection$']
     .concat(getRestoreTarget(dumpTarget))
     .concat(commonToolArgs));
   assert.eq(0, ret);
   destDB = toolTest.db.getSiblingDB('otherdest');
   collNames.forEach(function(collName) {
-    assert.eq(500, destDB[collName+'_source'].count());
+    assert.eq(500, destDB['source_'+collName].count());
     for (var i = 0; i < 500; i++) {
-      assert.eq(1, destDB[collName+'_source'].count({_id: i+'_'+collName}));
+      assert.eq(1, destDB['source_'+collName].count({_id: i+'_'+collName}));
     }
   });
 
diff --git a/src/mongo/gotools/test/qa-tests/jstests/ssl/ssl_with_system_ca.js b/src/mongo/gotools/test/qa-tests/jstests/ssl/ssl_with_system_ca.js
index 16fc55e88a6..5abfe227f9d 100644
--- a/src/mongo/gotools/test/qa-tests/jstests/ssl/ssl_with_system_ca.js
+++ b/src/mongo/gotools/test/qa-tests/jstests/ssl/ssl_with_system_ca.js
@@ -3,41 +3,41 @@
 // https://github.com/10gen/buildslave-cookbooks/commit/af7cabe5b6e0885902ebd4902f7f974b64cc8961
 // for details.
 ((function() {
-    'use strict';
-    const HOST_TYPE = getBuildInfo().buildEnvironment.target_os;
+  'use strict';
+  const HOST_TYPE = getBuildInfo().buildEnvironment.target_os;
 
-    if (HOST_TYPE == "windows") {
-        runProgram(
+  if (HOST_TYPE === "windows") {
+    runProgram(
             "certutil.exe", "-addstore", "-user", "-f", "CA", "jstests\\libs\\trusted-ca.pem");
-    }
+  }
 
-    var testWithCerts = function(serverPem,clientPem) {
-        jsTest.log(`Testing with SSL certs $ {
+  var testWithCerts = function(serverPem, clientPem) {
+    jsTest.log(`Testing with SSL certs $ {
             serverPem
         }`);
         // allowSSL instead of requireSSL so that the non-SSL connection succeeds.
-        var conn = MongoRunner.runMongod(
+    var conn = MongoRunner.runMongod(
             {sslMode: 'requireSSL', sslPEMKeyFile: "jstests/libs/" + serverPem});
 
         // Should not be able to authenticate with x509.
         // Authenticate call will return 1 on success, 0 on error.
-        var argv =
-            ['./mongodump', '-v', '--ssl', '--port', conn.port, "--sslPEMKeyFile", "jstests/libs/" + clientPem ];
-        if (HOST_TYPE == "linux") {
+    var argv =
+            ['./mongodump', '-v', '--ssl', '--port', conn.port, "--sslPEMKeyFile", "jstests/libs/" + clientPem];
+    if (HOST_TYPE === "linux") {
             // On Linux we override the default path to the system CA store to point to our
             // "trusted" CA. On Windows, this CA will have been added to the user's trusted CA list
-            argv.unshift("env", "SSL_CERT_FILE=jstests/libs/trusted-ca.pem");
-        }
-        var exitStatus = runMongoProgram.apply(null, argv);
-        assert.eq(exitStatus, 0, "successfully connected with SSL");
+      argv.unshift("env", "SSL_CERT_FILE=jstests/libs/trusted-ca.pem");
+    }
+    var exitStatus = runMongoProgram.apply(null, argv);
+    assert.eq(exitStatus, 0, "successfully connected with SSL");
 
-        MongoRunner.stopMongod(conn.port);
-    };
+    MongoRunner.stopMongod(conn.port);
+  };
 
-    assert.throws(function() {
-        testWithCerts("server.pem", "client.pem");
-    });
-    assert.doesNotThrow(function() {
-        testWithCerts("trusted-server.pem", "trusted-client.pem");
-    });
+  assert.throws(function() {
+    testWithCerts("server.pem", "client.pem");
+  });
+  assert.doesNotThrow(function() {
+    testWithCerts("trusted-server.pem", "trusted-client.pem");
+  });
 })());
diff --git a/src/mongo/gotools/test/qa-tests/jstests/top/mongotop_reports.js b/src/mongo/gotools/test/qa-tests/jstests/top/mongotop_reports.js
index 24f2f13be84..853669dc853 100644
--- a/src/mongo/gotools/test/qa-tests/jstests/top/mongotop_reports.js
+++ b/src/mongo/gotools/test/qa-tests/jstests/top/mongotop_reports.js
@@ -22,7 +22,7 @@ load('jstests/top/util/mongotop_common.js');
       var authCommand = '\n db.getSiblingDB(\'admin\').auth(\'' + authUser + '\',\'' + authPassword + '\'); \n';
       test.shellCommand = authCommand + test.shellCommand;
     }
-    startParallelShell(test.shellCommand);
+    var shellWorkload = startParallelShell(test.shellCommand);
 
     // allow for command to actually start
     sleep(5000);
@@ -91,6 +91,9 @@ load('jstests/top/util/mongotop_common.js');
       });
     }
     t.stop();
+
+    // Swallow the exit code for the shell per SERVER-25777.
+    shellWorkload();
   };
 
   var runTests = function(topology, passthrough) {
diff --git a/src/mongo/gotools/test/qa-tests/jstests/top/mongotop_stress.js b/src/mongo/gotools/test/qa-tests/jstests/top/mongotop_stress.js
index 0283fa00b3b..5693a1b8254 100644
--- a/src/mongo/gotools/test/qa-tests/jstests/top/mongotop_stress.js
+++ b/src/mongo/gotools/test/qa-tests/jstests/top/mongotop_stress.js
@@ -25,8 +25,9 @@ load('jstests/top/util/mongotop_common.js');
     '       sleep(1);\n' +
     '   }\n';
 
+    var shells = [];
     for (var i = 0; i < 10; ++i) {
-      startParallelShell(stressShell);
+      shells.push(startParallelShell(stressShell));
     }
 
     // wait a bit for the stress to kick in
@@ -39,6 +40,12 @@ load('jstests/top/util/mongotop_common.js');
     assert.eq(runMongoProgram.apply(this, ['mongotop', '--port', conn.port, '--json', '--rowcount', 1].concat(passthrough.args)), 0, 'failed 1');
 
     t.stop();
+
+    // Wait for all the shells to finish per SERVER-25777.
+    shells.forEach(function(join) {
+      join();
+    });
+
   };
 
   // run with plain and auth passthroughs