Bug 488396 - DBM needs to be FIPS certifiable.

Add a check file to the legacydb
r=nelson
r=wtc

6 files changed, 56 insertions, 17 deletions

diff --git a/security/nss/cmd/shlibsign/Makefile b/security/nss/cmd/shlibsign/Makefile
index 63b1f8300..d62858548 100644
--- a/security/nss/cmd/shlibsign/Makefile
+++ b/security/nss/cmd/shlibsign/Makefile
@@ -80,6 +80,7 @@ endif
 
 CHECKLIBS = $(DIST)/lib/$(DLL_PREFIX)softokn3.$(DLL_SUFFIX)
 CHECKLIBS += $(wildcard $(DIST)/lib/$(DLL_PREFIX)freebl*3.$(DLL_SUFFIX))
+CHECKLIBS += $(DIST)/lib/$(DLL_PREFIX)nssdbm3.$(DLL_SUFFIX)
 CHECKLOC = $(CHECKLIBS:.$(DLL_SUFFIX)=.chk)
 
 MD_LIB_RELEASE_FILES = $(CHECKLOC)
diff --git a/security/nss/lib/softoken/lgglue.c b/security/nss/lib/softoken/lgglue.c
index b5112dd36..3047e4006 100644
--- a/security/nss/lib/softoken/lgglue.c
+++ b/security/nss/lib/softoken/lgglue.c
@@ -149,6 +149,7 @@ sftkdb_LoadFromPath(const char *path, const char *libname)
     return lib;
 }
 
+
 static PRLibrary *
 sftkdb_LoadLibrary(const char *libname)
 {
@@ -188,6 +189,7 @@ done:
 	libSpec.value.pathname = libname;
 	lib = PR_LoadLibraryWithFlags(libSpec, PR_LD_NOW | PR_LD_LOCAL);
     }
+
     return lib;
 }
 
@@ -270,18 +272,39 @@ sftkdb_decrypt_stub(SDB *sdb, SECItem *cipherText, SECItem **plainText)
     return rv;
 }
 
+static const char *LEGACY_LIB_NAME = 
+	SHLIB_PREFIX"nssdbm"SHLIB_VERSION"."SHLIB_SUFFIX;
+/*
+ * 2 bools to tell us if we've check the legacy library successfully or
+ * not. Initialize on startup to false by the C BSS segment;
+ */
+static PRBool legacy_glue_libCheckFailed;    /* set if we failed the check */
+static PRBool legacy_glue_libCheckSucceeded; /* set if we passed the check */
 static PRLibrary *legacy_glue_lib = NULL;
 static SECStatus 
-sftkdbLoad_Legacy()
+sftkdbLoad_Legacy(PRBool isFIPS)
 {
     PRLibrary *lib = NULL;
     LGSetCryptFunc setCryptFunction = NULL;
 
     if (legacy_glue_lib) {
+	/* this check is necessary because it's possible we loaded the
+	 * legacydb to read secmod.db, which told us whether we were in
+	 * FIPS mode or not. */
+	if (isFIPS && !legacy_glue_libCheckSucceeded) {
+	    if (legacy_glue_libCheckFailed || 
+		!BLAPI_SHVerify(LEGACY_LIB_NAME,(PRFuncPtr)legacy_glue_open)) {
+    	    	legacy_glue_libCheckFailed = PR_TRUE;
+		/* don't clobber legacy glue to avoid race. just let it
+		 * get cleared in shutdown */
+		return SECFailure;
+	    }
+    	    legacy_glue_libCheckSucceeded = PR_TRUE;
+	} 
 	return SECSuccess;
     }
 
-    lib = sftkdb_LoadLibrary(SHLIB_PREFIX"nssdbm"SHLIB_VERSION"."SHLIB_SUFFIX);
+    lib = sftkdb_LoadLibrary(LEGACY_LIB_NAME);
     if (lib == NULL) {
 	return SECFailure;
     }
@@ -306,6 +329,16 @@ sftkdbLoad_Legacy()
 	PR_UnloadLibrary(lib);
 	return SECFailure;
     }
+
+    /* verify the loaded library if we are in FIPS mode */
+    if (isFIPS) {
+	if (!BLAPI_SHVerify(LEGACY_LIB_NAME,(PRFuncPtr)legacy_glue_open)) {
+	    PR_UnloadLibrary(lib);
+	    return SECFailure;
+	}
+    	legacy_glue_libCheckSucceeded = PR_TRUE;
+    } 
+
     setCryptFunction(sftkdb_encrypt_stub,sftkdb_decrypt_stub);
     legacy_glue_lib = lib;
     return SECSuccess;
@@ -313,12 +346,12 @@ sftkdbLoad_Legacy()
 
 CK_RV
 sftkdbCall_open(const char *dir, const char *certPrefix, const char *keyPrefix, 
-		int certVersion, int keyVersion, int flags, 
+		int certVersion, int keyVersion, int flags, PRBool isFIPS,
 		SDB **certDB, SDB **keyDB)
 {
     SECStatus rv;
 
-    rv = sftkdbLoad_Legacy();
+    rv = sftkdbLoad_Legacy(isFIPS);
     if (rv != SECSuccess) {
 	return CKR_GENERAL_ERROR;
     }
@@ -337,7 +370,7 @@ sftkdbCall_ReadSecmodDB(const char *appName, const char *filename,
 {
     SECStatus rv;
 
-    rv = sftkdbLoad_Legacy();
+    rv = sftkdbLoad_Legacy(PR_FALSE);
     if (rv != SECSuccess) {
 	return NULL;
     }
@@ -355,7 +388,7 @@ sftkdbCall_ReleaseSecmodDBData(const char *appName,
 {
     SECStatus rv;
 
-    rv = sftkdbLoad_Legacy();
+    rv = sftkdbLoad_Legacy(PR_FALSE);
     if (rv != SECSuccess) {
 	return rv;
     }
@@ -374,7 +407,7 @@ sftkdbCall_DeleteSecmodDB(const char *appName,
 {
     SECStatus rv;
 
-    rv = sftkdbLoad_Legacy();
+    rv = sftkdbLoad_Legacy(PR_FALSE);
     if (rv != SECSuccess) {
 	return rv;
     }
@@ -392,7 +425,7 @@ sftkdbCall_AddSecmodDB(const char *appName,
 {
     SECStatus rv;
 
-    rv = sftkdbLoad_Legacy();
+    rv = sftkdbLoad_Legacy(PR_FALSE);
     if (rv != SECSuccess) {
 	return rv;
     }
@@ -427,6 +460,8 @@ sftkdbCall_Shutdown(void)
     legacy_glue_releaseSecmod = NULL;
     legacy_glue_deleteSecmod = NULL;
     legacy_glue_addSecmod = NULL;
+    legacy_glue_libCheckFailed    = PR_FALSE;
+    legacy_glue_libCheckSucceeded = PR_FALSE;
     return crv;
 }
     
diff --git a/security/nss/lib/softoken/lgglue.h b/security/nss/lib/softoken/lgglue.h
index 998f30a94..091ff82b6 100644
--- a/security/nss/lib/softoken/lgglue.h
+++ b/security/nss/lib/softoken/lgglue.h
@@ -75,7 +75,7 @@ typedef void (*LGSetCryptFunc)(LGEncryptFunc, LGDecryptFunc);
  */
 CK_RV sftkdbCall_open(const char *dir, const char *certPrefix, 
 		const char *keyPrefix, 
-		int certVersion, int keyVersion, int flags, 
+		int certVersion, int keyVersion, int flags, PRBool isFIPS,
 		SDB **certDB, SDB **keyDB);
 char ** sftkdbCall_ReadSecmodDB(const char *appName, const char *filename, 
 			const char *dbname, char *params, PRBool rw);
diff --git a/security/nss/lib/softoken/pkcs11.c b/security/nss/lib/softoken/pkcs11.c
index c5c2a8e2b..f9572476f 100644
--- a/security/nss/lib/softoken/pkcs11.c
+++ b/security/nss/lib/softoken/pkcs11.c
@@ -2151,7 +2151,9 @@ SFTK_SlotReInit(SFTKSlot *slot, char *configdir, char *updatedir,
 		params->updCertPrefix, params->updKeyPrefix,
 		params->updateID  ? params->updateID : updateID, 
 		params->readOnly, params->noCertDB, params->noKeyDB,
-		params->forceOpen, &certHandle, &keyHandle);
+		params->forceOpen, 
+		moduleIndex == NSC_FIPS_MODULE,
+		&certHandle, &keyHandle);
 	if (crv != CKR_OK) {
 	    goto loser;
 	}
diff --git a/security/nss/lib/softoken/sftkdb.c b/security/nss/lib/softoken/sftkdb.c
index 45100cc5c..655e19687 100644
--- a/security/nss/lib/softoken/sftkdb.c
+++ b/security/nss/lib/softoken/sftkdb.c
@@ -2547,7 +2547,7 @@ sftk_DBInit(const char *configdir, const char *certPrefix,
                 const char *keyPrefix, const char *updatedir,
 		const char *updCertPrefix, const char *updKeyPrefix, 
 		const char *updateID, PRBool readOnly, PRBool noCertDB,
-                PRBool noKeyDB, PRBool forceOpen,
+                PRBool noKeyDB, PRBool forceOpen, PRBool isFIPS,
                 SFTKDBHandle **certDB, SFTKDBHandle **keyDB)
 {
     const char *confdir;
@@ -2577,11 +2577,11 @@ sftk_DBInit(const char *configdir, const char *certPrefix,
     switch (dbType) {
     case SDB_LEGACY:
 	crv = sftkdbCall_open(confdir, certPrefix, keyPrefix, 8, 3, flags,
-		noCertDB? NULL : &certSDB, noKeyDB ? NULL: &keySDB);
+		 isFIPS, noCertDB? NULL : &certSDB, noKeyDB ? NULL: &keySDB);
 	break;
     case SDB_MULTIACCESS:
 	crv = sftkdbCall_open(configdir, certPrefix, keyPrefix, 8, 3, flags,
-		noCertDB? NULL : &certSDB, noKeyDB ? NULL: &keySDB);
+		isFIPS, noCertDB? NULL : &certSDB, noKeyDB ? NULL: &keySDB);
 	break;
     case SDB_SQL:
     case SDB_EXTERN: /* SHOULD open a loadable db */
@@ -2598,8 +2598,8 @@ sftk_DBInit(const char *configdir, const char *certPrefix,
 	    /* we have legacy databases, if we failed to open the new format 
 	     * DB's read only, just use the legacy ones */
 		crv = sftkdbCall_open(confdir, certPrefix, 
-			keyPrefix, 8, 3, flags, noCertDB? NULL : &certSDB,
-			noKeyDB ? NULL : &keySDB);
+			keyPrefix, 8, 3, flags, isFIPS, 
+			noCertDB? NULL : &certSDB, noKeyDB ? NULL : &keySDB);
 	    }
 	/* Handle the database merge case.
          *
@@ -2669,7 +2669,8 @@ sftk_DBInit(const char *configdir, const char *certPrefix,
 	CK_RV crv2;
 
 	crv2 = sftkdbCall_open(confdir, certPrefix, keyPrefix, 8, 3, flags,
-		noCertDB ? NULL : &updateCert, noKeyDB ? NULL : &updateKey);
+		isFIPS, noCertDB ? NULL : &updateCert, 
+		noKeyDB ? NULL : &updateKey);
 	if (crv2 == CKR_OK) {
 	    if (*certDB) {
 		(*certDB)->update = updateCert;
diff --git a/security/nss/lib/softoken/sftkdb.h b/security/nss/lib/softoken/sftkdb.h
index 36daddea0..2806d6f3c 100644
--- a/security/nss/lib/softoken/sftkdb.h
+++ b/security/nss/lib/softoken/sftkdb.h
@@ -107,7 +107,7 @@ CK_RV sftk_DBInit(const char *configdir, const char *certPrefix,
 	 	const char *keyPrefix, const char *updatedir, 
 		const char *updCertPrefix, const char *updKeyPrefix,
 		const char *updateID, PRBool readOnly, PRBool noCertDB, 
-		PRBool noKeyDB, PRBool forceOpen, 
+		PRBool noKeyDB, PRBool forceOpen, PRBool isFIPS,
 		SFTKDBHandle **certDB, SFTKDBHandle **keyDB);
 CK_RV sftkdb_Shutdown(void);