Bug 1355041 - Add Windows gyp builds to TC r=franziskus

12 files changed, 132 insertions, 33 deletions

diff --git a/automation/taskcluster/graph/src/extend.js b/automation/taskcluster/graph/src/extend.js
index 2c5aff897..d9f6938c0 100644
--- a/automation/taskcluster/graph/src/extend.js
+++ b/automation/taskcluster/graph/src/extend.js
@@ -42,7 +42,8 @@ queue.filter(task => {
 
   if (task.tests == "bogo" || task.tests == "interop") {
     // No windows
-    if (task.platform == "windows2012-64") {
+    if (task.platform == "windows2012-64" ||
+        task.platform == "windows2012-32") {
       return false;
     }
 
@@ -53,8 +54,7 @@ queue.filter(task => {
   }
 
   // Only old make builds have -Ddisable_libpkix=0 and can run chain tests.
-  if (task.tests == "chains" && task.collection != "make" &&
-      task.platform != "windows2012-64") {
+  if (task.tests == "chains" && task.collection != "make") {
     return false;
   }
 
@@ -65,7 +65,6 @@ queue.filter(task => {
     }
   }
 
-
   // Don't run additional hardware tests on ARM (we don't have anything there).
   if (task.group == "Cipher" && task.platform == "aarch64" && task.env &&
       (task.env.NSS_DISABLE_PCLMUL == "1" || task.env.NSS_DISABLE_HW_AES == "1"
@@ -154,13 +153,34 @@ export default async function main() {
     features: ["allowPtrace"],
   }, "--ubsan --asan");
 
+  await scheduleWindows("Windows 2012 64 (debug, make)", {
+    platform: "windows2012-64",
+    collection: "make",
+    env: {USE_64: "1"}
+  }, "build.sh");
+
+  await scheduleWindows("Windows 2012 32 (debug, make)", {
+    platform: "windows2012-32",
+    collection: "make"
+  }, "build.sh");
+
   await scheduleWindows("Windows 2012 64 (opt)", {
-    env: {BUILD_OPT: "1"}
-  });
+    platform: "windows2012-64",
+  }, "build_gyp.sh --opt");
 
   await scheduleWindows("Windows 2012 64 (debug)", {
+    platform: "windows2012-64",
     collection: "debug"
-  });
+  }, "build_gyp.sh");
+
+  await scheduleWindows("Windows 2012 32 (opt)", {
+    platform: "windows2012-32",
+  }, "build_gyp.sh --opt -m32");
+
+  await scheduleWindows("Windows 2012 32 (debug)", {
+    platform: "windows2012-32",
+    collection: "debug"
+  }, "build_gyp.sh -m32");
 
   await scheduleFuzzing();
   await scheduleFuzzing32();
@@ -575,10 +595,9 @@ async function scheduleTestBuilds(base, args = "") {
 
 /*****************************************************************************/
 
-async function scheduleWindows(name, base) {
+async function scheduleWindows(name, base, build_script) {
   base = merge(base, {
     workerType: "nss-win2012r2",
-    platform: "windows2012-64",
     env: {
       PATH: "c:\\mozilla-build\\python;c:\\mozilla-build\\msys\\local\\bin;" +
             "c:\\mozilla-build\\7zip;c:\\mozilla-build\\info-zip;" +
@@ -588,7 +607,6 @@ async function scheduleWindows(name, base) {
             "c:\\mozilla-build\\wget",
       DOMSUF: "localdomain",
       HOST: "localhost",
-      USE_64: "1"
     }
   });
 
@@ -596,7 +614,7 @@ async function scheduleWindows(name, base) {
   let build_base = merge(base, {
     command: [
       WINDOWS_CHECKOUT_CMD,
-      "bash -c nss/automation/taskcluster/windows/build.sh"
+      `bash -c 'nss/automation/taskcluster/windows/${build_script}'`
     ],
     artifacts: [{
       expires: 24 * 7,
diff --git a/automation/taskcluster/graph/src/try_syntax.js b/automation/taskcluster/graph/src/try_syntax.js
index eddb7c21a..7748e068a 100644
--- a/automation/taskcluster/graph/src/try_syntax.js
+++ b/automation/taskcluster/graph/src/try_syntax.js
@@ -22,8 +22,10 @@ function parseOptions(opts) {
   }
 
   // Parse platforms.
-  let allPlatforms = ["linux", "linux64", "linux64-asan", "win64",
-                      "linux64-make", "linux-make", "linux-fuzz", "linux64-fuzz", "aarch64"];
+  let allPlatforms = ["linux", "linux64", "linux64-asan",
+                      "win", "win64", "win-make", "win64-make",
+                      "linux64-make", "linux-make", "linux-fuzz",
+                      "linux64-fuzz", "aarch64"];
   let platforms = intersect(opts.platform.split(/\s*,\s*/), allPlatforms);
 
   // If the given value is nonsense or "none" default to all platforms.
@@ -85,6 +87,9 @@ function filter(opts) {
         if (task.group && task.group.toLowerCase() == "ssl" && test == "ssl") {
           return true;
         }
+        if (task.group && task.group.toLowerCase() == "cipher" && test == "cipher") {
+          return true;
+        }
         return task.symbol.toLowerCase().startsWith(test);
       });
 
@@ -109,7 +114,10 @@ function filter(opts) {
         "linux64-fuzz": "linux64",
         "linux64-make": "linux64",
         "linux-make": "linux32",
-        "win64": "windows2012-64"
+        "win64-make": "windows2012-64",
+        "win-make": "windows2012-32",
+        "win64": "windows2012-64",
+        "win": "windows2012-32"
       };
 
       // Check the platform name.
@@ -118,7 +126,8 @@ function filter(opts) {
       // Additional checks.
       if (platform == "linux64-asan") {
         keep &= coll("asan");
-      } else if (platform == "linux64-make" || platform == "linux-make") {
+      } else if (platform == "linux64-make" || platform == "linux-make" ||
+                 platform == "win64-make" || platform == "win-make") {
         keep &= coll("make");
       } else if (platform == "linux64-fuzz" || platform == "linux-fuzz") {
         keep &= coll("fuzz");
diff --git a/automation/taskcluster/windows/build.sh b/automation/taskcluster/windows/build.sh
index 6c8a47470..46136153d 100644
--- a/automation/taskcluster/windows/build.sh
+++ b/automation/taskcluster/windows/build.sh
@@ -3,7 +3,11 @@
 set -v -e -x
 
 # Set up the toolchain.
-source $(dirname $0)/setup.sh
+if [ "$USE_64" = 1 ]; then
+  source $(dirname $0)/setup64.sh
+else
+  source $(dirname $0)/setup32.sh
+fi
 
 # Clone NSPR.
 hg_clone https://hg.mozilla.org/projects/nspr nspr default
diff --git a/automation/taskcluster/windows/build_gyp.sh b/automation/taskcluster/windows/build_gyp.sh
new file mode 100644
index 000000000..cc829ca99
--- /dev/null
+++ b/automation/taskcluster/windows/build_gyp.sh
@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+set -v -e -x
+
+# Set up the toolchain.
+if [[ "$@" == *"-m32"* ]]; then
+  source $(dirname $0)/setup32.sh
+else
+  source $(dirname $0)/setup64.sh
+fi
+
+# Install GYP.
+cd gyp
+python -m virtualenv test-env
+test-env/Scripts/python setup.py install
+test-env/Scripts/python -m pip install --upgrade pip
+test-env/Scripts/pip install --upgrade setuptools
+cd ..
+
+export GYP_MSVS_OVERRIDE_PATH="${VSPATH}"
+export GYP_MSVS_VERSION="2015"
+export GYP="${PWD}/gyp/test-env/Scripts/gyp"
+
+# Fool GYP.
+touch "${VSPATH}/VC/vcvarsall.bat"
+
+# Clone NSPR.
+hg_clone https://hg.mozilla.org/projects/nspr nspr default
+
+# Build with gyp.
+GYP=${GYP} ./nss/build.sh -g -v "$@"
+
+# Package.
+7z a public/build/dist.7z dist
diff --git a/automation/taskcluster/windows/releng.manifest b/automation/taskcluster/windows/releng.manifest
index 403be2b04..68d2c1d9e 100644
--- a/automation/taskcluster/windows/releng.manifest
+++ b/automation/taskcluster/windows/releng.manifest
@@ -6,5 +6,21 @@
     "algorithm": "sha512",
     "filename": "vs2015u3.zip",
     "unpack": true
+  },
+  {
+    "version": "Ninja 1.7.1",
+    "size": 184821,
+    "digest": "e4f9a1ae624a2630e75264ba37d396d9c7407d6e6aea3763056210ba6e1387908bd31cf4037a6a3661a418e86c4d2761e0c333e6a3bd0d66549d2b0d72d3f43b",
+    "algorithm": "sha512",
+    "filename": "ninja171.zip",
+    "unpack": true
+  },
+  {
+    "size": 13063963,
+    "visibility": "public",
+    "digest": "47a19f8f863eab3414abab2b9e9bd901ab896c799b3d9254b456b2f59374b085b99de805e21069a0819f01eecb3f43f7e2395a8c644c04bcbfa5711261cca29d",
+    "algorithm": "sha512",
+    "filename": "gyp-2017-05-23.zip",
+    "unpack": true
   }
 ]
diff --git a/automation/taskcluster/windows/setup.sh b/automation/taskcluster/windows/setup.sh
index 32732774a..7def50db4 100644
--- a/automation/taskcluster/windows/setup.sh
+++ b/automation/taskcluster/windows/setup.sh
@@ -2,6 +2,13 @@
 
 set -v -e -x
 
+export VSPATH="$(pwd)/vs2015u3"
+export NINJA_PATH="$(pwd)/ninja/bin"
+
+export WINDOWSSDKDIR="${VSPATH}/SDK"
+export VS90COMNTOOLS="${VSPATH}/VC"
+export INCLUDE="${VSPATH}/VC/include:${VSPATH}/SDK/Include/10.0.14393.0/ucrt:${VSPATH}/SDK/Include/10.0.14393.0/shared:${VSPATH}/SDK/Include/10.0.14393.0/um"
+
 # Usage: hg_clone repo dir [revision=@]
 hg_clone() {
     repo=$1
@@ -16,15 +23,4 @@ hg_clone() {
 }
 
 hg_clone https://hg.mozilla.org/build/tools tools default
-
 tools/scripts/tooltool/tooltool_wrapper.sh $(dirname $0)/releng.manifest https://api.pub.build.mozilla.org/tooltool/ non-existant-file.sh /c/mozilla-build/python/python.exe /c/builds/tooltool.py --authentication-file /c/builds/relengapi.tok -c /c/builds/tooltool_cache
-VSPATH="$(pwd)/vs2015u3"
-
-export WINDOWSSDKDIR="${VSPATH}/SDK"
-export WIN32_REDIST_DIR="${VSPATH}/VC/redist/x64/Microsoft.VC140.CRT"
-export WIN_UCRT_REDIST_DIR="${VSPATH}/SDK/Redist/ucrt/DLLs/x64"
-
-export PATH="${VSPATH}/VC/bin/amd64:${VSPATH}/VC/bin:${VSPATH}/SDK/bin/x64:${VSPATH}/VC/redist/x64/Microsoft.VC140.CRT:${VSPATH}/SDK/Redist/ucrt/DLLs/x64:${PATH}"
-
-export INCLUDE="${VSPATH}/VC/include:${VSPATH}/SDK/Include/10.0.14393.0/ucrt:${VSPATH}/SDK/Include/10.0.14393.0/shared:${VSPATH}/SDK/Include/10.0.14393.0/um"
-export LIB="${VSPATH}/VC/lib/amd64:${VSPATH}/SDK/lib/10.0.14393.0/ucrt/x64:${VSPATH}/SDK/lib/10.0.14393.0/um/x64"
diff --git a/automation/taskcluster/windows/setup32.sh b/automation/taskcluster/windows/setup32.sh
new file mode 100644
index 000000000..bcddabfa3
--- /dev/null
+++ b/automation/taskcluster/windows/setup32.sh
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+set -v -e -x
+
+source $(dirname $0)/setup.sh
+
+export WIN32_REDIST_DIR="${VSPATH}/VC/redist/x86/Microsoft.VC140.CRT"
+export WIN_UCRT_REDIST_DIR="${VSPATH}/SDK/Redist/ucrt/DLLs/x86"
+export PATH="${NINJA_PATH}:${VSPATH}/VC/bin/amd64_x86:${VSPATH}/VC/bin/amd64:${VSPATH}/VC/bin:${VSPATH}/SDK/bin/x86:${VSPATH}/SDK/bin/x64:${VSPATH}/VC/redist/x86/Microsoft.VC140.CRT:${VSPATH}/VC/redist/x64/Microsoft.VC140.CRT:${VSPATH}/SDK/Redist/ucrt/DLLs/x86:${VSPATH}/SDK/Redist/ucrt/DLLs/x64:${PATH}"
+export LIB="${VSPATH}/VC/lib:${VSPATH}/SDK/lib/10.0.14393.0/ucrt/x86:${VSPATH}/SDK/lib/10.0.14393.0/um/x86"
diff --git a/automation/taskcluster/windows/setup64.sh b/automation/taskcluster/windows/setup64.sh
new file mode 100644
index 000000000..f308298c1
--- /dev/null
+++ b/automation/taskcluster/windows/setup64.sh
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+set -v -e -x
+
+source $(dirname $0)/setup.sh
+
+export WIN32_REDIST_DIR="${VSPATH}/VC/redist/x64/Microsoft.VC140.CRT"
+export WIN_UCRT_REDIST_DIR="${VSPATH}/SDK/Redist/ucrt/DLLs/x64"
+export PATH="${NINJA_PATH}:${VSPATH}/VC/bin/amd64:${VSPATH}/VC/bin:${VSPATH}/SDK/bin/x64:${VSPATH}/VC/redist/x64/Microsoft.VC140.CRT:${VSPATH}/SDK/Redist/ucrt/DLLs/x64:${PATH}"
+export LIB="${VSPATH}/VC/lib/amd64:${VSPATH}/SDK/lib/10.0.14393.0/ucrt/x64:${VSPATH}/SDK/lib/10.0.14393.0/um/x64"
diff --git a/build.sh b/build.sh
index ae191b3c8..0a9ce762e 100755
--- a/build.sh
+++ b/build.sh
@@ -15,6 +15,7 @@ set -e
 cwd=$(cd $(dirname $0); pwd -P)
 source "$cwd"/coreconf/nspr.sh
 source "$cwd"/coreconf/sanitizers.sh
+GYP=${GYP:-gyp}
 
 # Usage info
 show_help()
@@ -183,7 +184,7 @@ if [[ "$rebuild_nspr" = 1 && "$no_local_nspr" = 0 ]]; then
     mv -f "$nspr_config".new "$nspr_config"
 fi
 if [ "$rebuild_gyp" = 1 ]; then
-    if ! hash gyp 2> /dev/null; then
+    if ! hash ${GYP} 2> /dev/null; then
         echo "Please install gyp" 1>&2
         exit 1
     fi
@@ -194,7 +195,7 @@ if [ "$rebuild_gyp" = 1 ]; then
         set_nspr_path "$obj_dir/include/nspr:$obj_dir/lib"
     fi
 
-    run_verbose run_scanbuild gyp -f ninja "${gyp_params[@]}" "$cwd"/nss.gyp
+    run_verbose run_scanbuild ${GYP} -f ninja "${gyp_params[@]}" "$cwd"/nss.gyp
 
     mv -f "$gyp_config".new "$gyp_config"
 fi
diff --git a/gtests/freebl_gtest/prng_kat_unittest.cc b/gtests/freebl_gtest/prng_kat_unittest.cc
index 6e3c9320f..7bc9beea3 100644
--- a/gtests/freebl_gtest/prng_kat_unittest.cc
+++ b/gtests/freebl_gtest/prng_kat_unittest.cc
@@ -34,7 +34,7 @@ bool contains(std::string& s, const char* to_find) {
 }
 
 std::string trim(std::string str) {
-  std::string whitespace = " \t";
+  std::string whitespace = " \t\r\n";
   const auto strBegin = str.find_first_not_of(whitespace);
   if (strBegin == std::string::npos) {
     return "";
@@ -46,7 +46,8 @@ std::string trim(std::string str) {
 
 std::vector<uint8_t> hex_string_to_bytes(std::string s) {
   std::vector<uint8_t> bytes;
-  for (size_t i = 0; i < s.length() - 1; i += 2) {
+  assert(s.length() % 2 == 0);
+  for (size_t i = 0; i < s.length(); i += 2) {
     bytes.push_back(std::stoul(s.substr(i, 2), nullptr, 16));
   }
   return bytes;
diff --git a/lib/softoken/legacydb/legacydb.gyp b/lib/softoken/legacydb/legacydb.gyp
index 6431fb5c1..34c0235bd 100644
--- a/lib/softoken/legacydb/legacydb.gyp
+++ b/lib/softoken/legacydb/legacydb.gyp
@@ -57,7 +57,7 @@
     'defines': [
       'SHLIB_SUFFIX=\"<(dll_suffix)\"',
       'SHLIB_PREFIX=\"<(dll_prefix)\"',
-      'LG_LIB_NAME=\"libnssdbm3.so\"'
+      'LG_LIB_NAME=\"<(dll_prefix)nssdbm3.<(dll_suffix)\"'
     ]
   },
   'variables': {
diff --git a/lib/softoken/softoken.gyp b/lib/softoken/softoken.gyp
index f32bacf4b..ba917cfc8 100644
--- a/lib/softoken/softoken.gyp
+++ b/lib/softoken/softoken.gyp
@@ -91,7 +91,7 @@
     'defines': [
       'SHLIB_SUFFIX=\"<(dll_suffix)\"',
       'SHLIB_PREFIX=\"<(dll_prefix)\"',
-      'SOFTOKEN_LIB_NAME=\"libsoftokn3.so\"',
+      'SOFTOKEN_LIB_NAME=\"<(dll_prefix)softokn3.<(dll_suffix)\"',
       'SHLIB_VERSION=\"3\"'
     ]
   },