New function BN_bntest_rand() to detect more BN library bugs.

The bn_cmp_part_words bug was only caught in the BN_mod_mul() test,
not in the BN_mul() test, so apparently the choice of parameters in
some cases is bad.

3 files changed, 63 insertions, 35 deletions

diff --git a/crypto/bn/bn.h b/crypto/bn/bn.h
index b55ac95642..3d5c91adb9 100644
--- a/crypto/bn/bn.h
+++ b/crypto/bn/bn.h
@@ -495,6 +495,8 @@ void bn_dump1(FILE *o, const char *a, const BN_ULONG *b,int n);
 # define bn_dump(a,b)
 #endif
 
+int BN_bntest_rand(BIGNUM *rnd, int bits, int top,int bottom);
+
 /* BEGIN ERROR CODES */
 /* The following lines are auto generated by the script mkerr.pl. Any changes
  * made after this point may be overwritten when the script is next run.
diff --git a/crypto/bn/bn_rand.c b/crypto/bn/bn_rand.c
index 21ecbc04ed..bab4510345 100644
--- a/crypto/bn/bn_rand.c
+++ b/crypto/bn/bn_rand.c
@@ -100,6 +100,27 @@ static int bnrand(int pseudorand, BIGNUM *rnd, int bits, int top, int bottom)
 			goto err;
 		}
 
+#if 1
+	if (pseudorand == 2)
+		{
+		/* generate patterns that are more likely to trigger BN
+		   library bugs */
+		int i;
+		unsigned char c;
+
+		for (i = 0; i < bytes; i++)
+			{
+			RAND_pseudo_bytes(&c, 1);
+			if (c >= 128 && i > 0)
+				buf[i] = buf[i-1];
+			else if (c < 42)
+				buf[i] = 0;
+			else if (c < 84)
+				buf[i] = 255;
+			}
+		}
+#endif
+
 	if (top)
 		{
 		if (bit == 0)
@@ -140,3 +161,10 @@ int     BN_pseudo_rand(BIGNUM *rnd, int bits, int top, int bottom)
 	{
 	return bnrand(1, rnd, bits, top, bottom);
 	}
+
+#if 1
+int     BN_bntest_rand(BIGNUM *rnd, int bits, int top, int bottom)
+	{
+	return bnrand(2, rnd, bits, top, bottom);
+	}
+#endif
diff --git a/crypto/bn/bntest.c b/crypto/bn/bntest.c
index 2df0d29b1a..7c9e5b4196 100644
--- a/crypto/bn/bntest.c
+++ b/crypto/bn/bntest.c
@@ -124,9 +124,7 @@ int main(int argc, char *argv[])
 
 	results = 0;
 
-	RAND_seed(rnd_seed, sizeof rnd_seed); /* or BN_rand may fail, and we don't
-	                                       * even check its return value
-	                                       * (which we should) */
+	RAND_seed(rnd_seed, sizeof rnd_seed); /* or BN_generate_prime may fail */
 
 	argc--;
 	argv++;
@@ -263,10 +261,10 @@ int test_add(BIO *bp)
 	BN_init(&b);
 	BN_init(&c);
 
-	BN_rand(&a,512,0,0);
+	BN_bntest_rand(&a,512,0,0);
 	for (i=0; i<num0; i++)
 		{
-		BN_rand(&b,450+i,0,0);
+		BN_bntest_rand(&b,450+i,0,0);
 		a.neg=rand_neg();
 		b.neg=rand_neg();
 		if (bp == NULL)
@@ -315,14 +313,14 @@ int test_sub(BIO *bp)
 		{
 		if (i < num1)
 			{
-			BN_rand(&a,512,0,0);
+			BN_bntest_rand(&a,512,0,0);
 			BN_copy(&b,&a);
 			if (BN_set_bit(&a,i)==0) return(0);
 			BN_add_word(&b,i);
 			}
 		else
 			{
-			BN_rand(&b,400+i-num1,0,0);
+			BN_bntest_rand(&b,400+i-num1,0,0);
 			a.neg=rand_neg();
 			b.neg=rand_neg();
 			}
@@ -372,13 +370,13 @@ int test_div(BIO *bp, BN_CTX *ctx)
 		{
 		if (i < num1)
 			{
-			BN_rand(&a,400,0,0);
+			BN_bntest_rand(&a,400,0,0);
 			BN_copy(&b,&a);
 			BN_lshift(&a,&a,i);
 			BN_add_word(&a,i);
 			}
 		else
-			BN_rand(&b,50+3*(i-num1),0,0);
+			BN_bntest_rand(&b,50+3*(i-num1),0,0);
 		a.neg=rand_neg();
 		b.neg=rand_neg();
 		if (bp == NULL)
@@ -442,13 +440,13 @@ int test_div_recp(BIO *bp, BN_CTX *ctx)
 		{
 		if (i < num1)
 			{
-			BN_rand(&a,400,0,0);
+			BN_bntest_rand(&a,400,0,0);
 			BN_copy(&b,&a);
 			BN_lshift(&a,&a,i);
 			BN_add_word(&a,i);
 			}
 		else
-			BN_rand(&b,50+3*(i-num1),0,0);
+			BN_bntest_rand(&b,50+3*(i-num1),0,0);
 		a.neg=rand_neg();
 		b.neg=rand_neg();
 		BN_RECP_CTX_set(&recp,&b,ctx);
@@ -519,11 +517,11 @@ int test_mul(BIO *bp)
 		{
 		if (i <= num1)
 			{
-			BN_rand(&a,100,0,0);
-			BN_rand(&b,100,0,0);
+			BN_bntest_rand(&a,100,0,0);
+			BN_bntest_rand(&b,100,0,0);
 			}
 		else
-			BN_rand(&b,i-num1,0,0);
+			BN_bntest_rand(&b,i-num1,0,0);
 		a.neg=rand_neg();
 		b.neg=rand_neg();
 		if (bp == NULL)
@@ -572,7 +570,7 @@ int test_sqr(BIO *bp, BN_CTX *ctx)
 
 	for (i=0; i<num0; i++)
 		{
-		BN_rand(&a,40+i*10,0,0);
+		BN_bntest_rand(&a,40+i*10,0,0);
 		a.neg=rand_neg();
 		if (bp == NULL)
 			for (j=0; j<100; j++)
@@ -623,15 +621,15 @@ int test_mont(BIO *bp, BN_CTX *ctx)
 
 	mont=BN_MONT_CTX_new();
 
-	BN_rand(&a,100,0,0); /**/
-	BN_rand(&b,100,0,0); /**/
+	BN_bntest_rand(&a,100,0,0); /**/
+	BN_bntest_rand(&b,100,0,0); /**/
 	for (i=0; i<num2; i++)
 		{
 		int bits = (200*(i+1))/num2;
 
 		if (bits == 0)
 			continue;
-		BN_rand(&n,bits,0,1);
+		BN_bntest_rand(&n,bits,0,1);
 		BN_MONT_CTX_set(mont,&n,ctx);
 
 		BN_nnmod(&a,&a,&n,ctx);
@@ -696,10 +694,10 @@ int test_mod(BIO *bp, BN_CTX *ctx)
 	d=BN_new();
 	e=BN_new();
 
-	BN_rand(a,1024,0,0); /**/
+	BN_bntest_rand(a,1024,0,0); /**/
 	for (i=0; i<num0; i++)
 		{
-		BN_rand(b,450+i*10,0,0); /**/
+		BN_bntest_rand(b,450+i*10,0,0); /**/
 		a->neg=rand_neg();
 		b->neg=rand_neg();
 		if (bp == NULL)
@@ -745,11 +743,11 @@ int test_mod_mul(BIO *bp, BN_CTX *ctx)
 	d=BN_new();
 	e=BN_new();
 
-	BN_rand(c,1024,0,0); /**/
+	BN_bntest_rand(c,1024,0,0); /**/
 	for (i=0; i<num0; i++)
 		{
-		BN_rand(a,475+i*10,0,0); /**/
-		BN_rand(b,425+i*11,0,0); /**/
+		BN_bntest_rand(a,475+i*10,0,0); /**/
+		BN_bntest_rand(b,425+i*11,0,0); /**/
 		a->neg=rand_neg();
 		b->neg=rand_neg();
 	/*	if (bp == NULL)
@@ -818,11 +816,11 @@ int test_mod_exp(BIO *bp, BN_CTX *ctx)
 	d=BN_new();
 	e=BN_new();
 
-	BN_rand(c,30,0,1); /* must be odd for montgomery */
+	BN_bntest_rand(c,30,0,1); /* must be odd for montgomery */
 	for (i=0; i<num2; i++)
 		{
-		BN_rand(a,20+i*5,0,0); /**/
-		BN_rand(b,2+i,0,0); /**/
+		BN_bntest_rand(a,20+i*5,0,0); /**/
+		BN_bntest_rand(b,2+i,0,0); /**/
 
 		if (!BN_mod_exp(d,a,b,c,ctx))
 			return(00);
@@ -872,8 +870,8 @@ int test_exp(BIO *bp, BN_CTX *ctx)
 
 	for (i=0; i<num2; i++)
 		{
-		BN_rand(a,20+i*5,0,0); /**/
-		BN_rand(b,2+i,0,0); /**/
+		BN_bntest_rand(a,20+i*5,0,0); /**/
+		BN_bntest_rand(b,2+i,0,0); /**/
 
 		if (!BN_exp(d,a,b,ctx))
 			return(00);
@@ -949,7 +947,7 @@ int test_kron(BIO *bp, BN_CTX *ctx)
 
 	for (i = 0; i < num0; i++)
 		{
-		if (!BN_rand(a, 512, 0, 0)) goto err;
+		if (!BN_bntest_rand(a, 512, 0, 0)) goto err;
 		a->neg = rand_neg();
 
 		/* t := (b-1)/2  (note that b is odd) */
@@ -1034,10 +1032,10 @@ int test_sqrt(BIO *bp, BN_CTX *ctx)
 			{
 			/* construct 'a' such that it is a square modulo p,
 			 * but in general not a proper square and not reduced modulo p */
-			if (!BN_rand(r, 256, 0, 3)) goto err;
+			if (!BN_bntest_rand(r, 256, 0, 3)) goto err;
 			if (!BN_nnmod(r, r, p, ctx)) goto err;
 			if (!BN_mod_sqr(r, r, p, ctx)) goto err;
-			if (!BN_rand(a, 256, 0, 3)) goto err;
+			if (!BN_bntest_rand(a, 256, 0, 3)) goto err;
 			if (!BN_nnmod(a, a, p, ctx)) goto err;
 			if (!BN_mod_sqr(a, a, p, ctx)) goto err;
 			if (!BN_mul(a, a, r, ctx)) goto err;
@@ -1089,7 +1087,7 @@ int test_lshift(BIO *bp,BN_CTX *ctx,BIGNUM *a_)
 	else
 	    {
 	    a=BN_new();
-	    BN_rand(a,200,0,0); /**/
+	    BN_bntest_rand(a,200,0,0); /**/
 	    a->neg=rand_neg();
 	    }
 	for (i=0; i<num0; i++)
@@ -1141,7 +1139,7 @@ int test_lshift1(BIO *bp)
 	b=BN_new();
 	c=BN_new();
 
-	BN_rand(a,200,0,0); /**/
+	BN_bntest_rand(a,200,0,0); /**/
 	a->neg=rand_neg();
 	for (i=0; i<num0; i++)
 		{
@@ -1185,7 +1183,7 @@ int test_rshift(BIO *bp,BN_CTX *ctx)
 	e=BN_new();
 	BN_one(c);
 
-	BN_rand(a,200,0,0); /**/
+	BN_bntest_rand(a,200,0,0); /**/
 	a->neg=rand_neg();
 	for (i=0; i<num0; i++)
 		{
@@ -1228,7 +1226,7 @@ int test_rshift1(BIO *bp)
 	b=BN_new();
 	c=BN_new();
 
-	BN_rand(a,200,0,0); /**/
+	BN_bntest_rand(a,200,0,0); /**/
 	a->neg=rand_neg();
 	for (i=0; i<num0; i++)
 		{