Bug 1409516, NSS Tests detect FIPS buildconfiguration using certutil --build-flags. gyp builds with --enable-fips enable init tests. Enable cert_rsa_exponent test. Add Linux64 FIPS gyp build to taskcluster/CI. r=franziskus

9 files changed, 107 insertions, 21 deletions

diff --git a/automation/taskcluster/graph/src/extend.js b/automation/taskcluster/graph/src/extend.js
index afe3e82bc..90e23ae60 100644
--- a/automation/taskcluster/graph/src/extend.js
+++ b/automation/taskcluster/graph/src/extend.js
@@ -82,8 +82,8 @@ queue.filter(task => {
   }
 
   if (task.group == "Test") {
-    // Don't run test builds on old make platforms
-    if (task.collection == "make") {
+    // Don't run test builds on old make platforms, and not for fips gyp.
+    if (task.collection == "make" || task.collection == "fips") {
       return false;
     }
   }
@@ -196,6 +196,12 @@ export default async function main() {
     features: ["allowPtrace"],
   }, "--ubsan --asan");
 
+  await scheduleLinux("Linux 64 (FIPS opt)", {
+    platform: "linux64",
+    collection: "fips",
+    image: LINUX_IMAGE,
+  }, "--enable-fips --opt");
+
   await scheduleWindows("Windows 2012 64 (debug, make)", {
     platform: "windows2012-64",
     collection: "make",
@@ -368,7 +374,6 @@ async function scheduleLinux(name, base, args = "") {
       parent: extra_build,
       symbol: "Certs-F",
       group: "FIPS",
-      env: { NSS_TEST_ENABLE_FIPS: "1" }
     }));
 
     // Schedule FIPS tests.
@@ -811,7 +816,6 @@ async function scheduleWindows(name, base, build_script) {
       parent: extra_build,
       symbol: "Certs-F",
       group: "FIPS",
-      env: { NSS_TEST_ENABLE_FIPS: "1" }
     }));
 
     // Schedule FIPS tests.
diff --git a/automation/taskcluster/graph/src/try_syntax.js b/automation/taskcluster/graph/src/try_syntax.js
index 2c4075364..1f4e12eee 100644
--- a/automation/taskcluster/graph/src/try_syntax.js
+++ b/automation/taskcluster/graph/src/try_syntax.js
@@ -22,7 +22,7 @@ function parseOptions(opts) {
   }
 
   // Parse platforms.
-  let allPlatforms = ["linux", "linux64", "linux64-asan",
+  let allPlatforms = ["linux", "linux64", "linux64-asan", "linux64-fips",
                       "win", "win64", "win-make", "win64-make",
                       "linux64-make", "linux-make", "linux-fuzz",
                       "linux64-fuzz", "aarch64", "mac"];
@@ -111,6 +111,7 @@ function filter(opts) {
         "linux": "linux32",
         "linux-fuzz": "linux32",
         "linux64-asan": "linux64",
+        "linux64-fips": "linux64",
         "linux64-fuzz": "linux64",
         "linux64-make": "linux64",
         "linux-make": "linux32",
@@ -126,6 +127,8 @@ function filter(opts) {
       // Additional checks.
       if (platform == "linux64-asan") {
         keep &= coll("asan");
+      } else if (platform == "linux64-fips") {
+        keep &= coll("fips");
       } else if (platform == "linux64-make" || platform == "linux-make" ||
                  platform == "win64-make" || platform == "win-make") {
         keep &= coll("make");
diff --git a/cmd/certutil/certutil.c b/cmd/certutil/certutil.c
index 254182763..945ab7184 100644
--- a/cmd/certutil/certutil.c
+++ b/cmd/certutil/certutil.c
@@ -1054,6 +1054,18 @@ ListModules(void)
 }
 
 static void
+PrintBuildFlags()
+{
+#ifdef NSS_FIPS_DISABLED
+    PR_fprintf(PR_STDOUT, "NSS_FIPS_DISABLED\n");
+#endif
+#ifdef NSS_NO_INIT_SUPPORT
+    PR_fprintf(PR_STDOUT, "NSS_NO_INIT_SUPPORT\n");
+#endif
+    exit(0);
+}
+
+static void
 PrintSyntax(char *progName)
 {
 #define FPS fprintf(stderr,
@@ -1100,6 +1112,7 @@ PrintSyntax(char *progName)
     FPS "\t%s -L [-n cert-name] [-h token-name] [--email email-address]\n",
         progName);
     FPS "\t\t [-X] [-r] [-a] [--dump-ext-val OID] [-d certdir] [-P dbprefix]\n");
+    FPS "\t%s --build-flags\n", progName);
     FPS "\t%s -M -n cert-name -t trustargs [-d certdir] [-P dbprefix]\n",
         progName);
     FPS "\t%s -O -n cert-name [-X] [-d certdir] [-a] [-P dbprefix]\n", progName);
@@ -1813,6 +1826,18 @@ luS(enum usage_level ul, const char *command)
 }
 
 static void
+luBuildFlags(enum usage_level ul, const char *command)
+{
+    int is_my_command = (command && 0 == strcmp(command, "build-flags"));
+    if (ul == usage_all || !command || is_my_command)
+    FPS "%-15s Print enabled build flags relevant for NSS test execution\n",
+        "--build-flags");
+    if (ul == usage_selected && !is_my_command)
+        return;
+    FPS "\n");
+}
+
+static void
 LongUsage(char *progName, enum usage_level ul, const char *command)
 {
     luA(ul, command);
@@ -1826,6 +1851,7 @@ LongUsage(char *progName, enum usage_level ul, const char *command)
     luU(ul, command);
     luK(ul, command);
     luL(ul, command);
+    luBuildFlags(ul, command);
     luM(ul, command);
     luN(ul, command);
     luT(ul, command);
@@ -2401,6 +2427,7 @@ enum {
     cmd_Merge,
     cmd_UpgradeMerge, /* test only */
     cmd_Rename,
+    cmd_BuildFlags,
     max_cmd
 };
 
@@ -2503,7 +2530,9 @@ static const secuCommandFlag commands_init[] =
       { /* cmd_UpgradeMerge        */ 0, PR_FALSE, 0, PR_FALSE,
         "upgrade-merge" },
       { /* cmd_Rename              */ 0, PR_FALSE, 0, PR_FALSE,
-        "rename" }
+        "rename" },
+      { /* cmd_BuildFlags          */ 0, PR_FALSE, 0, PR_FALSE,
+        "build-flags" }
     };
 #define NUM_COMMANDS ((sizeof commands_init) / (sizeof commands_init[0]))
 
@@ -2690,6 +2719,10 @@ certutil_main(int argc, char **argv, PRBool initialize)
         exit(1);
     }
 
+    if (certutil.commands[cmd_BuildFlags].activated) {
+        PrintBuildFlags();
+    }
+
     if (certutil.options[opt_PasswordFile].arg) {
         pwdata.source = PW_FROMFILE;
         pwdata.data = certutil.options[opt_PasswordFile].arg;
diff --git a/coreconf/config.gypi b/coreconf/config.gypi
index 61582c491..f4c3fbd0f 100644
--- a/coreconf/config.gypi
+++ b/coreconf/config.gypi
@@ -128,6 +128,7 @@
       [ 'disable_fips==1', {
         'defines': [
           'NSS_FIPS_DISABLED',
+          'NSS_NO_INIT_SUPPORT',
         ],
       }],
       [ 'OS!="android" and OS!="mac" and OS!="win"', {
@@ -299,7 +300,6 @@
       'Common': {
         'abstract': 1,
         'defines': [
-          'NSS_NO_INIT_SUPPORT',
           'USE_UTIL_DIRECTLY',
           'NO_NSPR_10_SUPPORT',
           'SSL_DISABLE_DEPRECATED_CIPHER_SUITE_NAMES',
diff --git a/gtests/freebl_gtest/rsa_unittest.cc b/gtests/freebl_gtest/rsa_unittest.cc
index c2c435330..5c667a1d1 100644
--- a/gtests/freebl_gtest/rsa_unittest.cc
+++ b/gtests/freebl_gtest/rsa_unittest.cc
@@ -53,5 +53,9 @@ TEST_F(RSANewKeyTest, WrongKeysizeTest) {
 
 TEST_F(RSANewKeyTest, expThreeTest) {
   ScopedRSAPrivateKey key(CreateKeyWithExponent(2048, 0x03));
+#ifdef NSS_FIPS_DISABLED
   ASSERT_TRUE(key != nullptr);
+#else
+  ASSERT_TRUE(key == nullptr);
+#endif
 }
diff --git a/readme.md b/readme.md
index 41e8b4b16..17b99e805 100644
--- a/readme.md
+++ b/readme.md
@@ -137,3 +137,50 @@ The nss directory contains the following important subdirectories:
 A more comprehensible overview of the NSS folder structure and API guidelines
 can be found
 [here](https://developer.mozilla.org/en-US/docs/Mozilla/Projects/NSS/NSS_API_Guidelines).
+
+## Build mechanisms related to FIPS compliance
+
+NSS supports build configurations for FIPS-140 compliance, and alternative build
+configurations that disable functionality specific to FIPS-140 compliance.
+
+This section documents the environment variables and build parameters that
+control these configurations.
+
+### Build FIPS startup tests
+
+The C macro NSS_NO_INIT_SUPPORT controls the FIPS startup self tests.
+If NSS_NO_INIT_SUPPORT is defined, the startup tests are disabled.
+
+The legacy build system (make) by default disables these tests.
+To enable these tests, set environment variable NSS_FORCE_FIPS=1 at build time.
+
+The gyp build system by default disables these tests.
+To enable these tests, pass parameter --enable-fips to build.sh.
+
+### Building either FIPS compliant or alternative compliant code
+
+The C macro NSS_FIPS_DISABLED can be used to disable some FIPS compliant code
+and enable alternative implementations.
+
+The legacy build system (make) never defines NSS_FIPS_DISABLED and always uses
+the FIPS compliant code.
+
+The gyp build system by default defines NSS_FIPS_DISABLED.
+To use the FIPS compliant code, pass parameter --enable-fips to build.sh.
+
+### Test execution
+
+The NSS test suite may contain tests that are included, excluded, or are
+different based on the FIPS build configuration. To execute the correct tests,
+it's necessary to determine which build configuration was used.
+
+The legacy build system (make) uses environment variables to control all
+aspects of the build configuration, including FIPS build configuration.
+
+Because the gyp build system doesn't use environment variables to control the
+build configuration, the NSS tests cannot rely on environment variables to
+determine the build configuration.
+
+A helper binary named nss-build-flags is produced as part of the NSS build,
+which prints the C macro symbols that were defined at build time, and which are
+relevant to test execution.
diff --git a/tests/all.sh b/tests/all.sh
index 31af100eb..8d5bd2dbb 100755
--- a/tests/all.sh
+++ b/tests/all.sh
@@ -295,9 +295,9 @@ fi
 cycles="standard pkix upgradedb sharedb"
 CYCLES=${NSS_CYCLES:-$cycles}
 
-if [ -n "$NSS_FORCE_FIPS" ]; then
+NO_INIT_SUPPORT=`certutil --build-flags |grep -cw NSS_NO_INIT_SUPPORT`
+if [ $NO_INIT_SUPPORT -eq 0 ]; then
     RUN_FIPS="fips"
-    export NSS_TEST_ENABLE_FIPS=1
 fi
 
 tests="cipher lowhash libpkix cert dbtests tools $RUN_FIPS sdr crmf smime ssl ocsp merge pkits ec gtests ssl_gtests"
@@ -310,7 +310,7 @@ TESTS=${NSS_TESTS:-$tests}
 ALL_TESTS=${TESTS}
 
 nss_ssl_tests="crl iopr policy"
-if [ -n "$NSS_FORCE_FIPS" ]; then
+if [ $NO_INIT_SUPPORT -eq 0 ]; then
     nss_ssl_tests="$nss_ssl_tests fips_normal normal_fips"
 fi
 NSS_SSL_TESTS="${NSS_SSL_TESTS:-$nss_ssl_tests}"
diff --git a/tests/cert/cert.sh b/tests/cert/cert.sh
index 1bf7bc652..12594405c 100755
--- a/tests/cert/cert.sh
+++ b/tests/cert/cert.sh
@@ -1359,7 +1359,7 @@ MODSCRIPT
 # local shell function to verify small rsa exponent can be used (only
 # run if FIPS has not been turned on in the build).
 ##############################################################################
-cert_rsa_exponent()
+cert_rsa_exponent_nonfips()
 {
   echo "$SCRIPTNAME: Verify that small RSA exponents still work  =============="
   CU_ACTION="Attempt to generate a key with exponent of 3"
@@ -2431,16 +2431,12 @@ cert_test_implicit_db_init
 cert_extended_ssl
 cert_ssl
 cert_smime_client
-if [[ -n "$NSS_TEST_ENABLE_FIPS" ]]; then
-    cert_fips
+IS_FIPS_DISABLED=`certutil --build-flags |grep -cw NSS_FIPS_DISABLED`
+if [ $IS_FIPS_DISABLED -ne 0 ]; then
+  cert_rsa_exponent_nonfips
+else
+  cert_fips
 fi
-# We currently have difficulties to know if the build is a non-FIPS build,
-# because of differences between the "make" and "gyp" build systems.
-# As soon as we have a reliable way to detect that based on a variable,
-# we should enable the following test call. See bug 1409516.
-# if SYMBOL_THAT_TELLS_US_FIPS_IS_DISABLED
-#   cert_rsa_exponent
-# fi
 cert_eccurves
 cert_extensions
 cert_san_and_generic_extensions
diff --git a/tests/fips/fips.sh b/tests/fips/fips.sh
index 11bd70b63..4153e61aa 100755
--- a/tests/fips/fips.sh
+++ b/tests/fips/fips.sh
@@ -23,7 +23,6 @@
 ########################################################################
 fips_init()
 {
-  export NSS_TEST_ENABLE_FIPS=1
   SCRIPTNAME=fips.sh      # sourced - $0 would point to all.sh
 
   if [ -z "${CLEANUP}" ] ; then     # if nobody else is responsible for