Bug 1370667 - don't do startup tests when not in fips mode, r=ttaubert,rrelyea

Differential Revision: https://nss-review.dev.mozaws.net/D348

16 files changed, 165 insertions, 19 deletions

diff --git a/automation/taskcluster/graph/src/extend.js b/automation/taskcluster/graph/src/extend.js
index 3603afaa4..f18b24955 100644
--- a/automation/taskcluster/graph/src/extend.js
+++ b/automation/taskcluster/graph/src/extend.js
@@ -91,11 +91,19 @@ queue.filter(task => {
 queue.map(task => {
   if (task.collection == "asan") {
     // CRMF and FIPS tests still leak, unfortunately.
-    if (task.tests == "crmf" || task.tests == "fips") {
+    if (task.tests == "crmf") {
       task.env.ASAN_OPTIONS = "detect_leaks=0";
     }
   }
 
+  // We don't run FIPS SSL tests
+  if (task.tests == "ssl") {
+    if (!task.env) {
+      task.env = {};
+    }
+    task.env.NSS_SSL_TESTS = "1";
+  }
+
   // Windows is slow.
   if (task.platform == "windows2012-64" && task.tests == "chains") {
     task.maxRunTime = 7200;
@@ -320,6 +328,46 @@ async function scheduleLinux(name, base, args = "") {
   // The task that builds NSPR+NSS.
   let task_build = queue.scheduleTask(merge(build_base, {name}));
 
+  // Make builds run FIPS tests, which need an extra FIPS build.
+  if (base.collection == "make") {
+    let extra_build = queue.scheduleTask(merge(build_base, {
+      env: { NSS_FORCE_FIPS: "1" },
+      group: "FIPS",
+      name: `${name} w/ NSS_FORCE_FIPS`
+    }));
+
+    // The task that generates certificates.
+    let task_cert = queue.scheduleTask(merge(build_base, {
+      name: "Certificates",
+      command: [
+        "/bin/bash",
+        "-c",
+        "bin/checkout.sh && nss/automation/taskcluster/scripts/gen_certs.sh"
+      ],
+      parent: extra_build,
+      symbol: "Certs-F",
+      group: "FIPS",
+      env: { NSS_TEST_ENABLE_FIPS: "1" }
+    }));
+
+    // Schedule FIPS tests.
+    queue.scheduleTask(merge(base, {
+      parent: task_cert,
+      name: "FIPS",
+      command: [
+        "/bin/bash",
+        "-c",
+        "bin/checkout.sh && nss/automation/taskcluster/scripts/run_tests.sh"
+      ],
+      cycle: "standard",
+      kind: "test",
+      name: "FIPS tests",
+      symbol: "Tests-F",
+      tests: "fips",
+      group: "FIPS"
+    }));
+  }
+
   // The task that generates certificates.
   let task_cert = queue.scheduleTask(merge(build_base, {
     name: "Certificates",
@@ -703,6 +751,44 @@ async function scheduleWindows(name, base, build_script) {
     symbol: "B"
   });
 
+  // Make builds run FIPS tests, which need an extra FIPS build.
+  if (base.collection == "make") {
+    let extra_build = queue.scheduleTask(merge(build_base, {
+      env: { NSS_FORCE_FIPS: "1" },
+      group: "FIPS",
+      name: `${name} w/ NSS_FORCE_FIPS`
+    }));
+
+    // The task that generates certificates.
+    let task_cert = queue.scheduleTask(merge(build_base, {
+      name: "Certificates",
+      command: [
+        WINDOWS_CHECKOUT_CMD,
+        "bash -c nss/automation/taskcluster/windows/gen_certs.sh"
+      ],
+      parent: extra_build,
+      symbol: "Certs-F",
+      group: "FIPS",
+      env: { NSS_TEST_ENABLE_FIPS: "1" }
+    }));
+
+    // Schedule FIPS tests.
+    queue.scheduleTask(merge(base, {
+      parent: task_cert,
+      name: "FIPS",
+      command: [
+        WINDOWS_CHECKOUT_CMD,
+        "bash -c nss/automation/taskcluster/windows/run_tests.sh"
+      ],
+      cycle: "standard",
+      kind: "test",
+      name: "FIPS tests",
+      symbol: "Tests-F",
+      tests: "fips",
+      group: "FIPS"
+    }));
+  }
+
   // The task that builds NSPR+NSS.
   let task_build = queue.scheduleTask(merge(build_base, {name}));
 
@@ -781,9 +867,6 @@ function scheduleTests(task_build, task_cert, test_base) {
     name: "DB tests", symbol: "DB", tests: "dbtests"
   }));
   queue.scheduleTask(merge(cert_base, {
-    name: "FIPS tests", symbol: "FIPS", tests: "fips"
-  }));
-  queue.scheduleTask(merge(cert_base, {
     name: "Merge tests", symbol: "Merge", tests: "merge"
   }));
   queue.scheduleTask(merge(cert_base, {
diff --git a/build.sh b/build.sh
index 82c6ec56b..4960238ac 100755
--- a/build.sh
+++ b/build.sh
@@ -96,6 +96,7 @@ while [ $# -gt 0 ]; do
         --with-nspr=?*) set_nspr_path "${1#*=}"; no_local_nspr=1 ;;
         --system-nspr) set_nspr_path "/usr/include/nspr/:"; no_local_nspr=1 ;;
         --enable-libpkix) gyp_params+=(-Ddisable_libpkix=0) ;;
+        --enable-fips) gyp_params+=(-Ddisable_fips=0) ;;
         *) show_help; exit 2 ;;
     esac
     shift
diff --git a/coreconf/config.gypi b/coreconf/config.gypi
index bd53613f4..69f19fdf3 100644
--- a/coreconf/config.gypi
+++ b/coreconf/config.gypi
@@ -109,6 +109,7 @@
     'nss_public_dist_dir%': '<(nss_dist_dir)/public',
     'nss_private_dist_dir%': '<(nss_dist_dir)/private',
     'only_dev_random%': 1,
+    'disable_fips%': 1,
   },
   'target_defaults': {
     # Settings specific to targets should go here.
@@ -125,6 +126,11 @@
       '<(nss_dist_dir)/private/<(module)',
     ],
     'conditions': [
+      [ 'disable_fips==1', {
+        'defines': [
+          'NSS_FIPS_DISABLED',
+        ],
+      }],
       [ 'OS!="android" and OS!="mac" and OS!="win"', {
         'libraries': [
           '-lpthread',
diff --git a/help.txt b/help.txt
index d8899163c..15d0fe8ca 100644
--- a/help.txt
+++ b/help.txt
@@ -3,6 +3,7 @@ Usage: build.sh [-hcv] [-j <n>] [--nspr] [--gyp|-g] [--opt|-o] [-m32]
                 [--asan] [--ubsan] [--msan] [--sancov[=edge|bb|func|...]]
                 [--disable-tests] [--fuzz[=tls|oss]] [--system-sqlite]
                 [--no-zdefs] [--with-nspr] [--system-nspr] [--enable-libpkix]
+                [--enable-fips]
 
 This script builds NSS with gyp and ninja.
 
@@ -43,3 +44,4 @@ NSS build tool options:
     --system-nspr    use system nspr. This requires an installation of NSPR and
                      might not work on all systems.
     --enable-libpkix make libpkix part of the build.
+    --enable-fips    don't disable FIPS checks.
diff --git a/lib/freebl/blapii.h b/lib/freebl/blapii.h
index b1be7bedf..bcf62e9f3 100644
--- a/lib/freebl/blapii.h
+++ b/lib/freebl/blapii.h
@@ -22,8 +22,10 @@ typedef void (*freeblDestroyFunc)(void *cx, PRBool freeit);
 
 SEC_BEGIN_PROTOS
 
+#ifndef NSS_FIPS_DISABLED
 SECStatus BL_FIPSEntryOK(PRBool freeblOnly);
 PRBool BL_POSTRan(PRBool freeblOnly);
+#endif
 
 #if defined(XP_UNIX) && !defined(NO_FORK_CHECK)
 
diff --git a/lib/freebl/fipsfreebl.c b/lib/freebl/fipsfreebl.c
index 094513560..a57de05e0 100644
--- a/lib/freebl/fipsfreebl.c
+++ b/lib/freebl/fipsfreebl.c
@@ -6,6 +6,7 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 /* $Id: fipstest.c,v 1.31 2012/06/28 17:55:06 rrelyea%redhat.com Exp $ */
 
+#ifndef NSS_FIPS_DISABLED
 #ifdef FREEBL_NO_DEPEND
 #include "stubs.h"
 #endif
@@ -1589,9 +1590,6 @@ static PRBool self_tests_freebl_ran = PR_FALSE;
 static PRBool self_tests_ran = PR_FALSE;
 static PRBool self_tests_freebl_success = PR_FALSE;
 static PRBool self_tests_success = PR_FALSE;
-#if defined(DEBUG)
-static PRBool fips_mode_available = PR_FALSE;
-#endif
 
 /*
  * accessors for freebl
@@ -1644,7 +1642,6 @@ bl_startup_tests(void)
 
     PORT_Assert(self_tests_freebl_ran == PR_FALSE);
     PORT_Assert(self_tests_success == PR_FALSE);
-    PORT_Assert(fips_mode_available == PR_FALSE);
     self_tests_freebl_ran = PR_TRUE;      /* we are running the tests */
     self_tests_success = PR_FALSE;        /* force it just in case */
     self_tests_freebl_success = PR_FALSE; /* force it just in case */
@@ -1713,3 +1710,4 @@ BL_FIPSEntryOK(PRBool freebl_only)
     PORT_SetError(SEC_ERROR_LIBRARY_FAILURE);
     return SECFailure;
 }
+#endif
diff --git a/lib/freebl/ldvector.c b/lib/freebl/ldvector.c
index 2447a0c9f..6897c7c50 100644
--- a/lib/freebl/ldvector.c
+++ b/lib/freebl/ldvector.c
@@ -320,8 +320,12 @@ FREEBL_GetVector(void)
         return NULL;
     }
 #endif
-    /* make sure the Full self tests have been run before continuing */
+
+#ifndef NSS_FIPS_DISABLED
+    /* In FIPS mode make sure the Full self tests have been run before
+     * continuing. */
     BL_POSTRan(PR_FALSE);
+#endif
 
     return &vector;
 }
diff --git a/lib/freebl/nsslowhash.c b/lib/freebl/nsslowhash.c
index 5ed039689..22f97810f 100644
--- a/lib/freebl/nsslowhash.c
+++ b/lib/freebl/nsslowhash.c
@@ -22,6 +22,7 @@ struct NSSLOWHASHContextStr {
     void *hashCtxt;
 };
 
+#ifndef NSS_FIPS_DISABLED
 static int
 nsslow_GetFIPSEnabled(void)
 {
@@ -40,9 +41,10 @@ nsslow_GetFIPSEnabled(void)
         return 0;
     if (d != '1')
         return 0;
-#endif
+#endif /* LINUX */
     return 1;
 }
+#endif /* NSS_FIPS_DISABLED */
 
 static NSSLOWInitContext dummyContext = { 0 };
 static PRBool post_failed = PR_TRUE;
@@ -54,6 +56,7 @@ NSSLOW_Init(void)
     (void)FREEBL_InitStubs();
 #endif
 
+#ifndef NSS_FIPS_DISABLED
     /* make sure the FIPS product is installed if we are trying to
      * go into FIPS mode */
     if (nsslow_GetFIPSEnabled()) {
@@ -63,6 +66,7 @@ NSSLOW_Init(void)
             return NULL;
         }
     }
+#endif
     post_failed = PR_FALSE;
 
     return &dummyContext;
diff --git a/lib/freebl/shvfy.c b/lib/freebl/shvfy.c
index bd9cd1c94..98db4614b 100644
--- a/lib/freebl/shvfy.c
+++ b/lib/freebl/shvfy.c
@@ -19,6 +19,8 @@
 #include "pqg.h"
 #include "blapii.h"
 
+#ifndef NSS_FIPS_DISABLED
+
 /*
  * Most modern version of Linux support a speed optimization scheme where an
  * application called prelink modifies programs and shared libraries to quickly
@@ -537,3 +539,23 @@ BLAPI_VerifySelf(const char *name)
     }
     return blapi_SHVerify(name, (PRFuncPtr)decodeInt, PR_TRUE);
 }
+
+#else /* NSS_FIPS_DISABLED */
+
+PRBool
+BLAPI_SHVerifyFile(const char *shName)
+{
+    return PR_FALSE;
+}
+PRBool
+BLAPI_SHVerify(const char *name, PRFuncPtr addr)
+{
+    return PR_FALSE;
+}
+PRBool
+BLAPI_VerifySelf(const char *name)
+{
+    return PR_FALSE;
+}
+
+#endif /* NSS_FIPS_DISABLED */
diff --git a/lib/softoken/fipstest.c b/lib/softoken/fipstest.c
index 3563bd2d2..0cca74d6e 100644
--- a/lib/softoken/fipstest.c
+++ b/lib/softoken/fipstest.c
@@ -5,6 +5,7 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
+#ifndef NSS_FIPS_DISABLED
 #include "seccomon.h"
 #include "blapi.h"
 #include "softoken.h"
@@ -652,3 +653,11 @@ sftk_FIPSEntryOK()
     }
     return CKR_OK;
 }
+#else
+#include "pkcs11t.h"
+CK_RV
+sftk_FIPSEntryOK()
+{
+    return CKR_DEVICE_ERROR;
+}
+#endif /* NSS_FIPS_DISABLED */
diff --git a/lib/softoken/legacydb/lgfips.c b/lib/softoken/legacydb/lgfips.c
index b017424db..b991dcf8e 100644
--- a/lib/softoken/legacydb/lgfips.c
+++ b/lib/softoken/legacydb/lgfips.c
@@ -6,6 +6,8 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 /* $Id: fipstest.c,v 1.31 2012/06/28 17:55:06 rrelyea%redhat.com Exp $ */
 
+#ifndef NSS_FIPS_DISABLED
+
 #include "seccomon.h"
 #include "lgdb.h"
 #include "blapi.h"
@@ -113,3 +115,5 @@ lg_FIPSEntryOK()
 #endif
     return lg_self_tests_success;
 }
+
+#endif /* NSS_FIPS_DISABLED */
diff --git a/lib/softoken/legacydb/lginit.c b/lib/softoken/legacydb/lginit.c
index 6913eea50..4f0b53f52 100644
--- a/lib/softoken/legacydb/lginit.c
+++ b/lib/softoken/legacydb/lginit.c
@@ -586,11 +586,15 @@ legacy_Open(const char *configdir, const char *certPrefix,
 #define NSS_VERSION_VARIABLE __nss_dbm_version
 #include "verref.h"
 
+#ifndef NSS_FIPS_DISABLED
     if (flags & SDB_FIPS) {
+        /* We shouldn't get here when FIPS is not enabled on the database. But
+         * we also don't care when this NSS build doesn't support FIPS. */
         if (!lg_FIPSEntryOK()) {
             return CKR_DEVICE_ERROR;
         }
     }
+#endif
 
     rv = SECOID_Init();
     if (SECSuccess != rv) {
diff --git a/mach b/mach
index 3592299e6..5f32cb999 100755
--- a/mach
+++ b/mach
@@ -115,8 +115,10 @@ class testAction(argparse.Action):
             "DOMSUF": domsuf,
             "HOST": host
         }
+        os_env = os.environ
+        os_env.update(env)
         command = cwd + "/tests/all.sh"
-        subprocess.check_call(command, env=env)
+        subprocess.check_call(command, env=os_env)
 
     def __call__(self, parser, args, values, option_string=None):
         self.runTest(values)
diff --git a/nss.gyp b/nss.gyp
index 1727dbe0b..5a9baa537 100644
--- a/nss.gyp
+++ b/nss.gyp
@@ -131,7 +131,6 @@
             'cmd/digest/digest.gyp:digest',
             'cmd/ecperf/ecperf.gyp:ecperf',
             'cmd/fbectest/fbectest.gyp:fbectest',
-            'cmd/fipstest/fipstest.gyp:fipstest',
             'cmd/httpserv/httpserv.gyp:httpserv',
             'cmd/listsuites/listsuites.gyp:listsuites',
             'cmd/makepqg/makepqg.gyp:makepqg',
@@ -190,6 +189,11 @@
                 'gtests/freebl_gtest/freebl_gtest.gyp:freebl_gtest',
               ],
             }],
+            [ 'disable_fips==0', {
+              'dependencies': [
+                'cmd/fipstest/fipstest.gyp:fipstest',
+              ],
+            }],
           ],
         },
       ],
diff --git a/tests/cert/cert.sh b/tests/cert/cert.sh
index 9b3455747..be589ef7c 100755
--- a/tests/cert/cert.sh
+++ b/tests/cert/cert.sh
@@ -1975,19 +1975,19 @@ cert_test_ocspresp()
 cert_cleanup()
 {
   cert_log "$SCRIPTNAME: finished $SCRIPTNAME"
-  html "</TABLE><BR>" 
+  html "</TABLE><BR>"
   cd ${QADIR}
   . common/cleanup.sh
 }
 
 ################## main #################################################
 
-cert_init 
+cert_init
 cert_all_CA
-cert_extended_ssl 
-cert_ssl 
-cert_smime_client        
-if [ -z "$NSS_TEST_DISABLE_FIPS" ]; then
+cert_extended_ssl
+cert_ssl
+cert_smime_client
+if [[ -n "$NSS_TEST_ENABLE_FIPS" ]]; then
     cert_fips
 fi
 cert_eccurves
@@ -2004,7 +2004,7 @@ else
 fi
 
 if [ -n "$DO_DIST_ST" -a "$DO_DIST_ST" = "TRUE" ] ; then
-    cert_stresscerts 
+    cert_stresscerts
 fi
 
 cert_iopr_setup
diff --git a/tests/fips/fips.sh b/tests/fips/fips.sh
index 4153e61aa..11bd70b63 100755
--- a/tests/fips/fips.sh
+++ b/tests/fips/fips.sh
@@ -23,6 +23,7 @@
 ########################################################################
 fips_init()
 {
+  export NSS_TEST_ENABLE_FIPS=1
   SCRIPTNAME=fips.sh      # sourced - $0 would point to all.sh
 
   if [ -z "${CLEANUP}" ] ; then     # if nobody else is responsible for