Import tools: cc74524a447ba7e9049cc1de1ae83995618bfbfa from branch v3.6

ref: c1d0d6967c..cc74524a44
for: 3.6.11

TOOLS-1906    Ignore mongorestore error "x509 certificate routines:X509_STORE_add_cert:cert already in hash table"
TOOLS-2030    mongodump does not dump system.js collections
TOOLS-2109    Build Tools with Go 1.11
TOOLS-2158    mongodump failing on Windows with  "error opening system CA store: Access is denied."
TOOLS-2167    Add CGO flags for building on Windows
TOOLS-2168    Add CGO flags for building on MacOS
TOOLS-2210    Build with tools with Address Space Layout Randomised (ASLR) flags enabled

13 files changed, 87 insertions, 30 deletions

diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index 09a4a12c55c..ebdcdd1036e 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -539,8 +539,9 @@ functions:
         fi
 
         for i in $build_tools; do
-            go build -ldflags "$(print_ldflags)" ${args} -tags "$(print_tags ${tooltags})" -o "../../../../../../mongo-tools/$i${exe|}" $i/main/$i.go
+            go build $(buildflags) -ldflags "$(print_ldflags)" ${args} -tags "$(print_tags ${tooltags})" -o "../../../../../../mongo-tools/$i${exe|}" $i/main/$i.go
             "../../../../../../mongo-tools/$i${exe|}" --version
+            file "../../../../../../mongo-tools/$i${exe|}"
         done
 
   "send benchmark results" :
diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/Godeps b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/Godeps
index 257ef245d00..29218783294 100644
--- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/Godeps
+++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/Godeps
@@ -6,7 +6,7 @@ github.com/smartystreets/assertions     287b4346dc4e71a038c346375a9d572453bc469b
 github.com/smartystreets/goconvey       bf58a9a1291224109919756b4dcc469c670cc7e4
 github.com/jessevdk/go-flags            97448c91aac742cbca3d020b3e769013a420a06f
 github.com/3rf/mongo-lint               3550fdcf1f43b89aaeabaa4559eaae6dc4407e42
-github.com/10gen/openssl                bed982ee200d16c1cfc6c0845d0be025864c1c1b
+github.com/10gen/openssl                fc9a1d560ec3549c695198fe39b9de7f89a7503d
 github.com/spacemonkeygo/spacelog       f936fb050dc6b5fe4a96b485a6f069e8bdc59aeb
 github.com/howeyc/gopass                44476384cd4721b68705e72f19e95d1a3a504370
 github.com/nsf/termbox-go               0723e7c3d0a317dea811f0fbe4d6edd81908c971
diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/build.sh b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/build.sh
index 17337594540..f234cf372ce 100755
--- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/build.sh
+++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/build.sh
@@ -21,7 +21,7 @@ mkdir -p bin
 ec=0
 for i in bsondump mongostat mongofiles mongoexport mongoimport mongorestore mongodump mongotop mongoreplay; do
         echo "Building ${i}..."
-        go build -o "bin/$i" -ldflags "$(print_ldflags)" -tags "$(print_tags $tags)" "$i/main/$i.go" || { echo "Error building $i"; ec=1; break; }
+        go build -o "bin/$i" $(buildflags) -ldflags "$(print_ldflags)" -tags "$(print_tags $tags)" "$i/main/$i.go" || { echo "Error building $i"; ec=1; break; }
         ./bin/$i --version | head -1
 done
 
diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common.yml b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common.yml
index 92e3d8875aa..3f06088718f 100644
--- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common.yml
+++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common.yml
@@ -283,7 +283,7 @@ functions:
         GOROOT="" set_goenv || exit
         go version
         env | grep ^GO
-        go build -ldflags "$(print_ldflags)" ${args} -tags "$(print_tags failpoints ${build_tags})" -o bin/${tool} ${tool}/main/${tool}.go
+        go build $(buildflags) -ldflags "$(print_ldflags)" ${args} -tags "$(print_tags failpoints ${build_tags})" -o bin/${tool} ${tool}/main/${tool}.go
         ./bin/${tool} --version
 
   "download mongod":
@@ -372,7 +372,7 @@ functions:
         . ./set_goenv.sh
         GOROOT="" set_goenv || exit
         cd ${package}
-        go test -ldflags "$(print_ldflags)" ${coverage_args} ${args} -tags "$(print_tags ${build_tags})" -test.v > unit.suite
+        go test $(buildflags) -ldflags "$(print_ldflags)" ${coverage_args} ${args} -tags "$(print_tags ${build_tags})" -test.v > unit.suite
         export exitcode=$?
         cat unit.suite
         cp unit.suite $basedir/.
@@ -750,7 +750,7 @@ functions:
         set -e
         . ./set_goenv.sh
         GOROOT="" set_goenv || exit
-        ${environment_vars} go test -ldflags "$(print_ldflags)" ${additional_args} -v > ${filename}.suite
+        ${environment_vars} go test $(buildflags) -ldflags "$(print_ldflags)" ${additional_args} -v > ${filename}.suite
 
 
 pre:
@@ -2015,7 +2015,7 @@ buildvariants:
     mongo_edition: "enterprise"
     build_tags: "sasl ssl"
     arch: "linux/x86_64"
-    args: "-race"
+    args: "-buildmode=default -race"
     excludes: requires_large_ram
     integration_test_args: integration
   tasks: *ubuntu1404_race_tasks
diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/import.data b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/import.data
index 9e86781b8ae..bf1f06eb01c 100644
--- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/import.data
+++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/import.data
@@ -1,5 +1,5 @@
 {
-    "commit": "c1d0d6967c84c62b5b578f7b18a82226b5c76526", 
+    "commit": "cc74524a447ba7e9049cc1de1ae83995618bfbfa", 
     "github": "mongodb/mongo-tools.git", 
     "vendor": "tools", 
     "branch": "v3.6"
diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongodump/prepare.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongodump/prepare.go
index c874d4a9f8c..be2edf1de63 100644
--- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongodump/prepare.go
+++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongodump/prepare.go
@@ -135,10 +135,10 @@ func (f *stdoutFile) Close() error {
 	return nil
 }
 
-// isReservedSystemNamespace returns true when a namespace (database +
+// shouldSkipSystemNamespace returns true when a namespace (database +
 // collection name) match certain reserved system namespaces that must
 // not be dumped.
-func (dump *MongoDump) isReservedSystemNamespace(dbName, collName string) bool {
+func shouldSkipSystemNamespace(dbName, collName string) bool {
 	// ignore <db>.system.* except for admin; ignore other specific
 	// collections in config and admin databases used for 3.6 features.
 	switch dbName {
@@ -151,6 +151,9 @@ func (dump *MongoDump) isReservedSystemNamespace(dbName, collName string) bool {
 			return true
 		}
 	default:
+		if collName == "system.js" {
+			return false
+		}
 		if strings.HasPrefix(collName, "system.") {
 			return true
 		}
@@ -385,7 +388,7 @@ func (dump *MongoDump) CreateIntentsForDatabase(dbName string) error {
 			}
 			collInfo.Name = collName
 		}
-		if dump.isReservedSystemNamespace(dbName, collInfo.Name) {
+		if shouldSkipSystemNamespace(dbName, collInfo.Name) {
 			log.Logvf(log.DebugHigh, "will not dump system collection '%s.%s'", dbName, collInfo.Name)
 			continue
 		}
diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongodump/prepare_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongodump/prepare_test.go
index 1ee11101128..0d44adb3d3a 100644
--- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongodump/prepare_test.go
+++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongodump/prepare_test.go
@@ -56,3 +56,40 @@ func TestSkipCollection(t *testing.T) {
 	})
 
 }
+
+type testTable struct {
+	db     string
+	coll   string
+	output bool
+}
+
+func TestShouldSkipSystemNamespace(t *testing.T) {
+	tests := []testTable{
+		testTable{
+			db:     "test",
+			coll:   "system",
+			output: false,
+		},
+		testTable{
+			db:     "test",
+			coll:   "system.nonsense",
+			output: true,
+		},
+		testTable{
+			db:     "test",
+			coll:   "system.js",
+			output: false,
+		},
+		testTable{
+			db:     "test",
+			coll:   "test",
+			output: false,
+		},
+	}
+
+	for _, testVals := range tests {
+		if shouldSkipSystemNamespace(testVals.db, testVals.coll) != testVals.output {
+			t.Errorf("%s.%s should have been %v but failed\n", testVals.db, testVals.coll, testVals.output)
+		}
+	}
+}
diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoreplay/packet_handler.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoreplay/packet_handler.go
index 86fa4e0adde..64156113e6c 100644
--- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoreplay/packet_handler.go
+++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoreplay/packet_handler.go
@@ -63,7 +63,7 @@ func (p *PacketHandler) Handle(streamHandler StreamHandler, numToHandle int) err
 	count := int64(0)
 	start := time.Now()
 	if p.Verbose && numToHandle > 0 {
-		userInfoLogger.Logvf(Always, "Processing", numToHandle, "packets")
+		userInfoLogger.Logvf(Always, "Processing %v %v", numToHandle, "packets")
 	}
 	source := gopacket.NewPacketSource(p.pcap, p.pcap.LinkType())
 	streamPool := NewStreamPool(streamHandler)
diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongorestore/filepath.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongorestore/filepath.go
index 1ff7c8e4e0a..1db7186b3d8 100644
--- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongorestore/filepath.go
+++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongorestore/filepath.go
@@ -360,7 +360,7 @@ func (restore *MongoRestore) CreateIntentsForDB(db string, dir archive.DirLike)
 				// Server versions >= 3.0.3 disallow user inserts to system.profile so
 				// it would likely fail anyway.
 				if collection == "system.profile" {
-					log.Logvf(log.DebugLow, "skipping restore of system.profile collection", db)
+					log.Logvf(log.DebugLow, "skipping restore of system.profile collection in %v", db)
 					skip = true
 				}
 				// skip restoring the indexes collection if we are using metadata
diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/set_goenv.sh b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/set_goenv.sh
index 348a131d86a..4e3b5b6641f 100755
--- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/set_goenv.sh
+++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/set_goenv.sh
@@ -11,16 +11,27 @@ set_goenv() {
     UNAME_S=$(PATH="/usr/bin:/bin" uname -s)
     case $UNAME_S in
         CYGWIN*)
-            PREF_GOROOT="c:/golang/go1.10"
-            PREF_PATH="/cygdrive/c/golang/go1.10/bin:/cygdrive/c/mingw-w64/x86_64-4.9.1-posix-seh-rt_v3-rev1/mingw64/bin:$PATH"
+            PREF_GOROOT="c:/golang/go1.11"
+            PREF_PATH="/cygdrive/c/golang/go1.11/bin:/cygdrive/c/mingw-w64/x86_64-4.9.1-posix-seh-rt_v3-rev1/mingw64/bin:$PATH"
         ;;
         *)
-            PREF_GOROOT="/opt/golang/go1.10"
+            PREF_GOROOT="/opt/golang/go1.11"
             # XXX might not need mongodbtoolchain anymore
             PREF_PATH="$PREF_GOROOT/bin:/opt/mongodbtoolchain/v2/bin/:$PATH"
         ;;
     esac
 
+    # Set OS-level compilation flags
+    case $UNAME_S in
+        'CYGWIN*')
+            export CGO_CFLAGS="-D_WIN32_WINNT=0x0601 -DNTDDI_VERSION=0x06010000"
+            ;;
+        'Darwin')
+            export CGO_CFLAGS="-mmacosx-version-min=10.11"
+            export CGO_LDFLAGS="-mmacosx-version-min=10.11"
+            ;;
+    esac
+
     # XXX Setting the compiler might not be necessary anymore now that we're
     # using standard Go toolchain and if we don't put mongodbtoolchain into the
     # path.  But if we need to keep mongodbtoolchain for other tools (eg. python),
@@ -93,3 +104,15 @@ print_tags() {
     esac
     echo "$tags"
 }
+
+# On linux, we want to set buildmode=pie for ASLR support
+buildflags() {
+    flags=""
+    UNAME_S=$(PATH="/usr/bin:/bin" uname -s)
+    case $UNAME_S in
+        Linux)
+            flags="-buildmode=pie"
+        ;;
+    esac
+    echo "$flags"
+}
diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/vendor/github.com/10gen/openssl/.evergreen/config.yml b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/vendor/github.com/10gen/openssl/.evergreen/config.yml
index b8bbabba9a9..a51e615a122 100644
--- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/vendor/github.com/10gen/openssl/.evergreen/config.yml
+++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/vendor/github.com/10gen/openssl/.evergreen/config.yml
@@ -210,17 +210,6 @@ buildvariants:
 #     SUSE Buildvariants              #
 #######################################
 
-- name: suse11
-  display_name: SUSE 11 (Go 1.8)
-  run_on:
-  - suse11-test
-  expansions:
-    gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go'
-    build_tags: "openssl_pre_1.0"
-  tasks:
-  - name: build
-  - name: test
-
 - name: suse12
   display_name: SUSE 12 (Go 1.8)
   run_on:
diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/vendor/github.com/10gen/openssl/system_certs.c b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/vendor/github.com/10gen/openssl/system_certs.c
index 056f524aa1e..e744254a5da 100644
--- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/vendor/github.com/10gen/openssl/system_certs.c
+++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/vendor/github.com/10gen/openssl/system_certs.c
@@ -27,6 +27,7 @@ static int checkX509_STORE_error(char* err, size_t err_len) {
                  ERR_reason_error_string(errCode));
         return 0;
     }
+    ERR_clear_error();
     return 1;
 }
 
@@ -54,7 +55,7 @@ static int importCertStoreToX509_STORE(
     int status = 1;
     X509* x509Cert = NULL;
     HCERTSTORE systemStore =
-        CertOpenStore(CERT_STORE_PROV_SYSTEM_W, 0, (HCRYPTPROV)NULL, storeLocation, storeName);
+        CertOpenStore(CERT_STORE_PROV_SYSTEM_W, 0, (HCRYPTPROV)NULL, storeLocation | CERT_STORE_READONLY_FLAG, storeName);
     if (systemStore == NULL) {
 	formatError(GetLastError(),"error opening system CA store",err,err_len);
         status = 0;
diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/vendor/github.com/10gen/openssl/system_certs.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/vendor/github.com/10gen/openssl/system_certs.go
index 9751622f837..40807f01a61 100644
--- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/vendor/github.com/10gen/openssl/system_certs.go
+++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/vendor/github.com/10gen/openssl/system_certs.go
@@ -1,7 +1,8 @@
 package openssl
 
 import (
-	"errors"
+	"fmt"
+	"runtime"
 	"unsafe"
 )
 
@@ -19,11 +20,13 @@ extern int _setupSystemCA(SSL_CTX* context, char * err, size_t err_len);
 import "C"
 
 func (c *Ctx) SetupSystemCA() error {
+	runtime.LockOSThread()
+	defer runtime.UnlockOSThread()
 	err_buf := make([]byte, 1024, 1024)
 	cstr := (*C.char)(unsafe.Pointer(&err_buf[0]))
 	r := C._setupSystemCA(c.ctx, cstr, 1024)
 	if r == 1 {
 		return nil
 	}
-	return errors.New(string(err_buf))
+	return fmt.Errorf("%s: %v", string(err_buf), errorFromErrorQueue())
 }