prime: Avoid rare assertion failure in gcry_prime_check.

* cipher/primegen.c (is_prime): Don't fail on the assert X > 1.
--

When using gcry_prime_check the function is_prime can be called with
quite small candidates so there is a real chance that the random X
values is indeed 0 or 1.  This would trigger the assert.  To avoid
this we now retry in this case.

Reported-by: Heiko Stamer
Signed-off-by: Werner Koch <wk@gnupg.org>

1 files changed, 17 insertions, 12 deletions

diff --git a/cipher/primegen.c b/cipher/primegen.c
index c7977d10..ce5ad3c3 100644
--- a/cipher/primegen.c
+++ b/cipher/primegen.c
@@ -935,20 +935,25 @@ is_prime (gcry_mpi_t n, int steps, unsigned int *count)
         }
       else
         {
-          _gcry_mpi_randomize( x, nbits, GCRY_WEAK_RANDOM );
-
-          /* Make sure that the number is smaller than the prime and
-             keep the randomness of the high bit. */
-          if ( mpi_test_bit ( x, nbits-2) )
+          /* We need to loop to avoid an X with value 0 or 1.  */
+          do
             {
-              mpi_set_highbit ( x, nbits-2); /* Clear all higher bits. */
-            }
-          else
-            {
-              mpi_set_highbit( x, nbits-2 );
-              mpi_clear_bit( x, nbits-2 );
+              _gcry_mpi_randomize (x, nbits, GCRY_WEAK_RANDOM);
+
+              /* Make sure that the number is smaller than the prime
+               * and keep the randomness of the high bit. */
+              if (mpi_test_bit (x, nbits-2))
+                {
+                  mpi_set_highbit (x, nbits-2); /* Clear all higher bits. */
+                }
+              else
+                {
+                  mpi_set_highbit (x, nbits-2);
+                  mpi_clear_bit (x, nbits-2);
+                }
             }
-          gcry_assert (mpi_cmp (x, nminus1) < 0 && mpi_cmp_ui (x, 1) > 0);
+          while (mpi_cmp_ui (x, 1) <= 0);
+          gcry_assert (mpi_cmp (x, nminus1) < 0);
 	}
       mpi_powm ( y, x, q, n);
       if ( mpi_cmp_ui(y, 1) && mpi_cmp( y, nminus1 ) )