Return proper error code if parameter check fails.

3 files changed, 47 insertions, 45 deletions

diff --git a/lib/auth/dh_common.c b/lib/auth/dh_common.c
index 172c7d45c0..f4bba1ae7a 100644
--- a/lib/auth/dh_common.c
+++ b/lib/auth/dh_common.c
@@ -74,14 +74,10 @@ _gnutls_proc_dh_common_client_kx (gnutls_session_t session,
 
   _gnutls_dh_set_peer_public (session, session->key->client_Y);
 
-  session->key->KEY =
-    gnutls_calc_dh_key (session->key->client_Y, session->key->dh_secret, p);
-
-  if (session->key->KEY == NULL)
-    {
-      gnutls_assert ();
-      return GNUTLS_E_MEMORY_ERROR;
-    }
+  ret =
+    gnutls_calc_dh_key (&session->key->KEY, session->key->client_Y, session->key->dh_secret, p);
+  if (ret < 0)
+    return gnutls_assert_val(ret);
 
   _gnutls_mpi_release (&session->key->client_Y);
   _gnutls_mpi_release (&session->key->dh_secret);
@@ -127,12 +123,11 @@ _gnutls_gen_dh_common_client_kx_int (gnutls_session_t session, gnutls_buffer_st*
   bigint_t x = NULL, X = NULL;
   int ret;
 
-  X = gnutls_calc_dh_secret (&x, session->key->client_g,
+  ret = gnutls_calc_dh_secret (&X, &x, session->key->client_g,
                              session->key->client_p, 0);
-  if (X == NULL || x == NULL)
+  if (ret < 0)
     {
       gnutls_assert ();
-      ret = GNUTLS_E_MEMORY_ERROR;
       goto error;
     }
 
@@ -146,13 +141,11 @@ _gnutls_gen_dh_common_client_kx_int (gnutls_session_t session, gnutls_buffer_st*
     }
 
   /* calculate the key after calculating the message */
-  session->key->KEY =
-    gnutls_calc_dh_key (session->key->client_Y, x, session->key->client_p);
-
-  if (session->key->KEY == NULL)
+  ret =
+    gnutls_calc_dh_key (&session->key->KEY, session->key->client_Y, x, session->key->client_p);
+  if (ret < 0)
     {
-      gnutls_assert ();
-      ret = GNUTLS_E_MEMORY_ERROR;
+      gnutls_assert();
       goto error;
     }
 
@@ -291,11 +284,11 @@ _gnutls_dh_common_print_server_kx (gnutls_session_t session,
   int ret;
 
   /* Y=g^x mod p */
-  Y = gnutls_calc_dh_secret (&x, g, p, q_bits);
-  if (Y == NULL || x == NULL)
+  ret = gnutls_calc_dh_secret (&Y, &x, g, p, q_bits);
+  if (ret < 0)
     {
       gnutls_assert ();
-      return GNUTLS_E_MEMORY_ERROR;
+      return ret;
     }
 
   session->key->dh_secret = x;
diff --git a/lib/gnutls_dh.c b/lib/gnutls_dh.c
index 48dd092e72..07e94441fe 100644
--- a/lib/gnutls_dh.c
+++ b/lib/gnutls_dh.c
@@ -45,28 +45,33 @@
 
 /* returns the public value (X), and the secret (ret_x).
  */
-bigint_t
-gnutls_calc_dh_secret (bigint_t * ret_x, bigint_t g, bigint_t prime, 
+int
+gnutls_calc_dh_secret (bigint_t* ret_y, bigint_t * ret_x, bigint_t g, bigint_t 
                        unsigned int q_bits)
 {
-  bigint_t e, x = NULL;
-  int x_size;
+  bigint_t e=NULL, x = NULL;
+  unsigned int x_size;
+  int ret;
   
   if (q_bits == 0)
-    x_size = _gnutls_mpi_get_nbits (prime) - 1;
+    {
+      x_size = _gnutls_mpi_get_nbits (prime);
+      if (x_size > 0) x_size--;
+    }
   else
     x_size = q_bits;
 
-  if (x_size > MAX_BITS || x_size <= 0)
+  if (x_size > MAX_BITS || x_size == 0)
     {
       gnutls_assert ();
-      return NULL;
+      return GNUTLS_E_RECEIVED_ILLEGAL_PARAMETER;
     }
 
   x = _gnutls_mpi_new(x_size);
   if (x == NULL)
     {
       gnutls_assert ();
+      ret = GNUTLS_E_MEMORY_ERROR;
       goto fail;
     }
 
@@ -74,6 +79,7 @@ gnutls_calc_dh_secret (bigint_t * ret_x, bigint_t g, bigint_t prime,
   if (e == NULL)
     {
       gnutls_assert ();
+      ret = GNUTLS_E_MEMORY_ERROR;
       goto fail;
     }
 
@@ -82,6 +88,7 @@ gnutls_calc_dh_secret (bigint_t * ret_x, bigint_t g, bigint_t prime,
       if (_gnutls_mpi_randomize (x, x_size, GNUTLS_RND_RANDOM) == NULL)
         {
           gnutls_assert();
+          ret = GNUTLS_E_INTERNAL_ERROR;
           goto fail;
         }
 
@@ -89,25 +96,26 @@ gnutls_calc_dh_secret (bigint_t * ret_x, bigint_t g, bigint_t prime,
     }
   while(_gnutls_mpi_cmp_ui(e, 1) == 0);
 
-  if (ret_x)
-    *ret_x = x;
-  else
-    _gnutls_mpi_release (&x);
-  return e;
+  *ret_x = x;
+  *ret_y = e;
+
+  return 0;
 
 fail:
   if (x) _gnutls_mpi_release (&x);
-  return NULL;
+  if (e) _gnutls_mpi_release (&e);
+  return ret;
 
 }
 
 /* returns f^x mod prime 
  */
-bigint_t
-gnutls_calc_dh_key (bigint_t f, bigint_t x, bigint_t prime)
+int
+gnutls_calc_dh_key (bigint_t *key, bigint_t f, bigint_t x, bigint_t prime)
 {
-  bigint_t k, ff, ret;
-  int bits;
+  bigint_t k, ff;
+  unsigned int bits;
+  int ret;
   
   ff = _gnutls_mpi_mod(f, prime);
   _gnutls_mpi_add_ui(ff, ff, 1);
@@ -118,15 +126,15 @@ gnutls_calc_dh_key (bigint_t f, bigint_t x, bigint_t prime)
       (_gnutls_mpi_cmp(ff,prime) == 0))
     {
       gnutls_assert();
-      ret = NULL;
+      ret = GNUTLS_E_RECEIVED_ILLEGAL_PARAMETER;
       goto cleanup;
     }
 
   bits = _gnutls_mpi_get_nbits (prime);
-  if (bits <= 0 || bits > MAX_BITS)
+  if (bits == 0 || bits > MAX_BITS)
     {
       gnutls_assert ();
-      ret = NULL;
+      ret = GNUTLS_E_RECEIVED_ILLEGAL_PARAMETER;
       goto cleanup;
     }
 
@@ -134,14 +142,15 @@ gnutls_calc_dh_key (bigint_t f, bigint_t x, bigint_t prime)
   if (k == NULL)
     {
       gnutls_assert();
-      ret = NULL;
+      ret = GNUTLS_E_MEMORY_ERROR;
       goto cleanup;
     }
 
   _gnutls_mpi_powm (k, f, x, prime);
 
-  ret = k;
-
+  *key = k;
+  
+  ret = 0;
 cleanup:
   _gnutls_mpi_release (&ff);
   
diff --git a/lib/gnutls_dh.h b/lib/gnutls_dh.h
index fdd659e254..e0e699ba3a 100644
--- a/lib/gnutls_dh.h
+++ b/lib/gnutls_dh.h
@@ -24,9 +24,9 @@
 #define GNUTLS_DH_H
 
 const bigint_t *_gnutls_dh_params_to_mpi (gnutls_dh_params_t);
-bigint_t gnutls_calc_dh_secret (bigint_t * ret_x, bigint_t g, bigint_t prime,
+int gnutls_calc_dh_secret (bigint_t *ret_y, bigint_t * ret_x, bigint_t g, bigin
                                 unsigned int q_bits);
-bigint_t gnutls_calc_dh_key (bigint_t f, bigint_t x, bigint_t prime);
+int gnutls_calc_dh_key (bigint_t* key, bigint_t f, bigint_t x, bigint_t prime);
 
 gnutls_dh_params_t
 _gnutls_get_dh_params (gnutls_dh_params_t dh_params,