Bugzilla Bug 355297: changed the way we reseed the RNG so that we modifyNSS_3_11_4_BETA1

all the 256 bits of XKEY. Define struct SHA256ContextStr in the new header
sha256.h so that prng_fips1861.c can use it. r=nelsonb,jpierre.
Modified files: manifest.mn prng_fips1861.c sha512.c
Added file: sha256.h
 Tag: NSS_3_11_BRANCH

3 files changed, 30 insertions, 50 deletions

diff --git a/security/nss/lib/freebl/manifest.mn b/security/nss/lib/freebl/manifest.mn
index 00dac226d..8e172c426 100644
--- a/security/nss/lib/freebl/manifest.mn
+++ b/security/nss/lib/freebl/manifest.mn
@@ -160,6 +160,7 @@ ALL_HDRS =  \
 	secmpi.h \
 	sha.h \
 	sha_fast.h \
+	sha256.h \
 	shsign.h \
 	vis_proto.h \
 	$(NULL)
diff --git a/security/nss/lib/freebl/prng_fips1861.c b/security/nss/lib/freebl/prng_fips1861.c
index c06d32d07..50c29692e 100644
--- a/security/nss/lib/freebl/prng_fips1861.c
+++ b/security/nss/lib/freebl/prng_fips1861.c
@@ -46,6 +46,7 @@
 #include "nssilock.h"
 #include "secitem.h"
 #include "sha_fast.h"
+#include "sha256.h"
 #include "secrng.h"	/* for RNG_GetNoise() */
 #include "secmpi.h"
 
@@ -74,8 +75,10 @@
  * SHA-256, or SHA-512).
  */
 #define FIPS_B     256
-#define B_HASH_BUF SHA256_HashBuf
 #define BSIZE      (FIPS_B / PR_BITS_PER_BYTE)
+#if BSIZE != SHA256_LENGTH
+#error "this file requires that BSIZE and SHA256_LENGTH be equal"
+#endif
 
 /* Output size of the G function */
 #define FIPS_G     160
@@ -184,7 +187,7 @@ freeRNGContext()
     PZ_DestroyLock(globalrng->lock);
 
     /* zero global RNG context except for XKEY to preserve entropy */
-    rv = B_HASH_BUF(inputhash, globalrng->XKEY, BSIZE);
+    rv = SHA256_HashBuf(inputhash, globalrng->XKEY, BSIZE);
     PORT_Assert(SECSuccess == rv);
     memset(globalrng, 0, sizeof(*globalrng));
     memcpy(globalrng->XKEY, inputhash, BSIZE);
@@ -362,19 +365,7 @@ static PRStatus rng_init(void)
 	}
 	/* the RNG is in a valid state */
 	globalrng->isValid = PR_TRUE;
-	/*
-	 * Try to get some seed data for the RNG.
-	 *
-	 * The very first RNG_RandomUpdate call initializes all FIPS_B
-	 * bits of the RNG's seed-key.  Subsequent RNG_RandomUpdate calls
-	 * only modify the low FIPS_G bits of the seed-key.  So it's
-	 * important to pass high entropy to the first RNG_RandomUpdate
-	 * call.
-	 *
-	 * RNG_GetNoise only reads the high-resolution clocks, which
-	 * have low entropy.  So if RNG_SystemRNG fails, the RNG will
-	 * have at most slightly more than FIPS_G bits of entropy.
-	 */
+	/* Try to get some seed data for the RNG */
 	numBytes = RNG_SystemRNG(bytes, sizeof bytes);
 	PORT_Assert(numBytes == 0 || numBytes == sizeof bytes);
 	if (numBytes != 0) {
@@ -419,7 +410,6 @@ static SECStatus
 prng_RandomUpdate(RNGContext *rng, const void *data, size_t bytes)
 {
     SECStatus rv = SECSuccess;
-    unsigned char inputhash[BSIZE];
     /* check for a valid global RNG context */
     PORT_Assert(rng != NULL);
     if (rng == NULL) {
@@ -429,17 +419,6 @@ prng_RandomUpdate(RNGContext *rng, const void *data, size_t bytes)
     /* RNG_SystemInfoForRNG() sometimes does this, not really an error */
     if (bytes == 0)
 	return SECSuccess;
-    /* If received 20 bytes of input, use it, else hash the input before 
-     * locking.
-     */
-    if (bytes == BSIZE)
-	memcpy(inputhash, data, BSIZE);
-    else
-	rv = B_HASH_BUF(inputhash, data, bytes);
-    if (rv != SECSuccess) {
-	/* B_HASH_BUF set error */
-	return SECFailure;
-    }
     /* --- LOCKED --- */
     PZ_Lock(rng->lock);
     /*
@@ -449,20 +428,17 @@ prng_RandomUpdate(RNGContext *rng, const void *data, size_t bytes)
      * Algorithm 1 of FIPS 186-2 Change Notice 1, step 1 specifies that
      * a secret value for the seed-key must be chosen before the
      * generator can begin.  The size of XKEY is b bits, so fill it
-     * with the first b bits sent to RNG_RandomUpdate().
+     * with the b-bit hash of the input to the first RNG_RandomUpdate()
+     * call.
      */
     if (rng->seedCount == 0) {
 	/* This is the first call to RandomUpdate().  Use a hash
-	 * of the input to set the seed, XKEY.
+	 * of the input to set the seed-key, XKEY.
 	 *
-	 * <Step 1> copy seed bytes into context's XKEY
-	 */
-	memcpy(rng->XKEY, inputhash, BSIZE);
-	/* 
-	 * Now continue with algorithm.  Since the input was used to
-	 * initialize XKEY, the "optional user input" at this stage
-	 * will be a pad of zeros, XSEEDj = 0.
+	 * <Step 1> copy hash of seed bytes into context's XKEY
 	 */
+	SHA256_HashBuf(rng->XKEY, data, bytes);
+	/* Now continue with algorithm. */
 	rv = alg_fips186_2_cn_1(rng, NULL);
 	/* As per FIPS 140-2 continuous RNG test requirement, the first
 	 * iteration of output is discarded.  So here there is really
@@ -470,17 +446,28 @@ prng_RandomUpdate(RNGContext *rng, const void *data, size_t bytes)
 	 * before any bytes can be extracted from the generator.
 	 */
 	rng->avail = 0;
+    } else if (bytes == BSIZE && memcmp(rng->XKEY, data, BSIZE) == 0) {
+	/* Should we add the error code SEC_ERROR_BAD_RNG_SEED? */
+	PORT_SetError(SEC_ERROR_INVALID_ARGS);
+	rv = SECFailure;
     } else {
-	/* Execute the algorithm from FIPS 186-2 Change Notice 1 */
-	rv = alg_fips186_2_cn_1(rng, inputhash);
+	/*
+	 * FIPS 186-2 does not specify how to reseed the RNG.  We retrofit
+	 * our RNG with a reseed function from NIST SP 800-90.
+	 *
+	 * Use a hash of the seed-key and the input to reseed the RNG.
+	 */
+	SHA256Context ctx;
+	SHA256_Begin(&ctx);
+	SHA256_Update(&ctx, rng->XKEY, BSIZE);
+	SHA256_Update(&ctx, data, bytes);
+	SHA256_End(&ctx, rng->XKEY, NULL, BSIZE);
     }
     /* If got this far, have added bytes of seed data. */
     if (rv == SECSuccess)
 	rng->seedCount += bytes;
     PZ_Unlock(rng->lock);
     /* --- UNLOCKED --- */
-    /* housekeeping */
-    memset(inputhash, 0, BSIZE);
     return rv;
 }
 
@@ -498,7 +485,7 @@ RNG_RandomUpdate(const void *data, size_t bytes)
 ** Generate some random bytes, using the global random number generator
 ** object.
 */
-SECStatus 
+static SECStatus 
 prng_GenerateGlobalRandomBytes(RNGContext *rng,
                                void *dest, size_t len)
 {
diff --git a/security/nss/lib/freebl/sha512.c b/security/nss/lib/freebl/sha512.c
index 672aa9a0f..a25b83d6d 100644
--- a/security/nss/lib/freebl/sha512.c
+++ b/security/nss/lib/freebl/sha512.c
@@ -45,6 +45,7 @@
 #include "prtypes.h"	/* for PRUintXX */
 #include "secport.h"	/* for PORT_XXX */
 #include "blapi.h"
+#include "sha256.h"	/* for struct SHA256ContextStr */
 
 /* ============= Common constants and defines ======================= */
 
@@ -92,15 +93,6 @@ static const PRUint32 H256[8] = {
     0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19
 };
 
-struct SHA256ContextStr {
-    union {
-	PRUint32 w[64];	    /* message schedule, input buffer, plus 48 words */
-	PRUint8  b[256];
-    } u;
-    PRUint32 h[8];		/* 8 state variables */
-    PRUint32 sizeHi,sizeLo;	/* 64-bit count of hashed bytes. */
-};
-
 #if defined(_MSC_VER) && defined(_X86_)
 #ifndef FORCEINLINE
 #if (MSC_VER >= 1200)