From fbe778b60d9cf2f49a61d684df269f03cf1cc71b Mon Sep 17 00:00:00 2001 From: Nikos Mavrogiannopoulos Date: Thu, 19 Apr 2012 20:27:25 +0200 Subject: Return proper error code if parameter check fails. --- lib/auth/dh_common.c | 33 +++++++++++++------------------ lib/gnutls_dh.c | 55 ++++++++++++++++++++++++++++++---------------------- lib/gnutls_dh.h | 4 ++-- 3 files changed, 47 insertions(+), 45 deletions(-) diff --git a/lib/auth/dh_common.c b/lib/auth/dh_common.c index 172c7d45c0..f4bba1ae7a 100644 --- a/lib/auth/dh_common.c +++ b/lib/auth/dh_common.c @@ -74,14 +74,10 @@ _gnutls_proc_dh_common_client_kx (gnutls_session_t session, _gnutls_dh_set_peer_public (session, session->key->client_Y); - session->key->KEY = - gnutls_calc_dh_key (session->key->client_Y, session->key->dh_secret, p); - - if (session->key->KEY == NULL) - { - gnutls_assert (); - return GNUTLS_E_MEMORY_ERROR; - } + ret = + gnutls_calc_dh_key (&session->key->KEY, session->key->client_Y, session->key->dh_secret, p); + if (ret < 0) + return gnutls_assert_val(ret); _gnutls_mpi_release (&session->key->client_Y); _gnutls_mpi_release (&session->key->dh_secret); @@ -127,12 +123,11 @@ _gnutls_gen_dh_common_client_kx_int (gnutls_session_t session, gnutls_buffer_st* bigint_t x = NULL, X = NULL; int ret; - X = gnutls_calc_dh_secret (&x, session->key->client_g, + ret = gnutls_calc_dh_secret (&X, &x, session->key->client_g, session->key->client_p, 0); - if (X == NULL || x == NULL) + if (ret < 0) { gnutls_assert (); - ret = GNUTLS_E_MEMORY_ERROR; goto error; } @@ -146,13 +141,11 @@ _gnutls_gen_dh_common_client_kx_int (gnutls_session_t session, gnutls_buffer_st* } /* calculate the key after calculating the message */ - session->key->KEY = - gnutls_calc_dh_key (session->key->client_Y, x, session->key->client_p); - - if (session->key->KEY == NULL) + ret = + gnutls_calc_dh_key (&session->key->KEY, session->key->client_Y, x, session->key->client_p); + if (ret < 0) { - gnutls_assert (); - ret = GNUTLS_E_MEMORY_ERROR; + gnutls_assert(); goto error; } @@ -291,11 +284,11 @@ _gnutls_dh_common_print_server_kx (gnutls_session_t session, int ret; /* Y=g^x mod p */ - Y = gnutls_calc_dh_secret (&x, g, p, q_bits); - if (Y == NULL || x == NULL) + ret = gnutls_calc_dh_secret (&Y, &x, g, p, q_bits); + if (ret < 0) { gnutls_assert (); - return GNUTLS_E_MEMORY_ERROR; + return ret; } session->key->dh_secret = x; diff --git a/lib/gnutls_dh.c b/lib/gnutls_dh.c index 48dd092e72..07e94441fe 100644 --- a/lib/gnutls_dh.c +++ b/lib/gnutls_dh.c @@ -45,28 +45,33 @@ /* returns the public value (X), and the secret (ret_x). */ -bigint_t -gnutls_calc_dh_secret (bigint_t * ret_x, bigint_t g, bigint_t prime, +int +gnutls_calc_dh_secret (bigint_t* ret_y, bigint_t * ret_x, bigint_t g, bigint_t unsigned int q_bits) { - bigint_t e, x = NULL; - int x_size; + bigint_t e=NULL, x = NULL; + unsigned int x_size; + int ret; if (q_bits == 0) - x_size = _gnutls_mpi_get_nbits (prime) - 1; + { + x_size = _gnutls_mpi_get_nbits (prime); + if (x_size > 0) x_size--; + } else x_size = q_bits; - if (x_size > MAX_BITS || x_size <= 0) + if (x_size > MAX_BITS || x_size == 0) { gnutls_assert (); - return NULL; + return GNUTLS_E_RECEIVED_ILLEGAL_PARAMETER; } x = _gnutls_mpi_new(x_size); if (x == NULL) { gnutls_assert (); + ret = GNUTLS_E_MEMORY_ERROR; goto fail; } @@ -74,6 +79,7 @@ gnutls_calc_dh_secret (bigint_t * ret_x, bigint_t g, bigint_t prime, if (e == NULL) { gnutls_assert (); + ret = GNUTLS_E_MEMORY_ERROR; goto fail; } @@ -82,6 +88,7 @@ gnutls_calc_dh_secret (bigint_t * ret_x, bigint_t g, bigint_t prime, if (_gnutls_mpi_randomize (x, x_size, GNUTLS_RND_RANDOM) == NULL) { gnutls_assert(); + ret = GNUTLS_E_INTERNAL_ERROR; goto fail; } @@ -89,25 +96,26 @@ gnutls_calc_dh_secret (bigint_t * ret_x, bigint_t g, bigint_t prime, } while(_gnutls_mpi_cmp_ui(e, 1) == 0); - if (ret_x) - *ret_x = x; - else - _gnutls_mpi_release (&x); - return e; + *ret_x = x; + *ret_y = e; + + return 0; fail: if (x) _gnutls_mpi_release (&x); - return NULL; + if (e) _gnutls_mpi_release (&e); + return ret; } /* returns f^x mod prime */ -bigint_t -gnutls_calc_dh_key (bigint_t f, bigint_t x, bigint_t prime) +int +gnutls_calc_dh_key (bigint_t *key, bigint_t f, bigint_t x, bigint_t prime) { - bigint_t k, ff, ret; - int bits; + bigint_t k, ff; + unsigned int bits; + int ret; ff = _gnutls_mpi_mod(f, prime); _gnutls_mpi_add_ui(ff, ff, 1); @@ -118,15 +126,15 @@ gnutls_calc_dh_key (bigint_t f, bigint_t x, bigint_t prime) (_gnutls_mpi_cmp(ff,prime) == 0)) { gnutls_assert(); - ret = NULL; + ret = GNUTLS_E_RECEIVED_ILLEGAL_PARAMETER; goto cleanup; } bits = _gnutls_mpi_get_nbits (prime); - if (bits <= 0 || bits > MAX_BITS) + if (bits == 0 || bits > MAX_BITS) { gnutls_assert (); - ret = NULL; + ret = GNUTLS_E_RECEIVED_ILLEGAL_PARAMETER; goto cleanup; } @@ -134,14 +142,15 @@ gnutls_calc_dh_key (bigint_t f, bigint_t x, bigint_t prime) if (k == NULL) { gnutls_assert(); - ret = NULL; + ret = GNUTLS_E_MEMORY_ERROR; goto cleanup; } _gnutls_mpi_powm (k, f, x, prime); - ret = k; - + *key = k; + + ret = 0; cleanup: _gnutls_mpi_release (&ff); diff --git a/lib/gnutls_dh.h b/lib/gnutls_dh.h index fdd659e254..e0e699ba3a 100644 --- a/lib/gnutls_dh.h +++ b/lib/gnutls_dh.h @@ -24,9 +24,9 @@ #define GNUTLS_DH_H const bigint_t *_gnutls_dh_params_to_mpi (gnutls_dh_params_t); -bigint_t gnutls_calc_dh_secret (bigint_t * ret_x, bigint_t g, bigint_t prime, +int gnutls_calc_dh_secret (bigint_t *ret_y, bigint_t * ret_x, bigint_t g, bigin unsigned int q_bits); -bigint_t gnutls_calc_dh_key (bigint_t f, bigint_t x, bigint_t prime); +int gnutls_calc_dh_key (bigint_t* key, bigint_t f, bigint_t x, bigint_t prime); gnutls_dh_params_t _gnutls_get_dh_params (gnutls_dh_params_t dh_params, -- cgit v1.2.1