nettle: use rsa_*_key_prepare

Previously we calculated the size of the key directly, but
by using the rsa_*_key_prepare we benefit from any checks that
may be introduced in the future. Specifically any checks for invalid
public keys (e.g., keys that may crash the underlying gmp functions).

1 files changed, 32 insertions, 12 deletions

diff --git a/lib/nettle/pk.c b/lib/nettle/pk.c
index f6dc7399e0..954445f832 100644
--- a/lib/nettle/pk.c
+++ b/lib/nettle/pk.c
@@ -116,7 +116,8 @@ _dsa_params_to_privkey(const gnutls_pk_params_st * pk_params,
 }
 #endif
 
-static void
+/* returns 0 on invalid privkey */
+static unsigned
 _rsa_params_to_privkey(const gnutls_pk_params_st * pk_params,
 		       struct rsa_private_key *priv)
 {
@@ -126,18 +127,18 @@ _rsa_params_to_privkey(const gnutls_pk_params_st * pk_params,
 	memcpy(priv->c, pk_params->params[5], SIZEOF_MPZT);
 	memcpy(priv->a, pk_params->params[6], SIZEOF_MPZT);
 	memcpy(priv->b, pk_params->params[7], SIZEOF_MPZT);
-	priv->size =
-	    nettle_mpz_sizeinbase_256_u(TOMPZ
-					(pk_params->params[RSA_MODULUS]));
+	return (unsigned)rsa_private_key_prepare(priv);
 }
 
-static void
+/* returns 0 on invalid pubkey */
+static unsigned
 _rsa_params_to_pubkey(const gnutls_pk_params_st * pk_params,
 		      struct rsa_public_key *pub)
 {
 	memcpy(pub->n, pk_params->params[RSA_MODULUS], SIZEOF_MPZT);
 	memcpy(pub->e, pk_params->params[RSA_PUB], SIZEOF_MPZT);
-	pub->size = nettle_mpz_sizeinbase_256_u(pub->n);
+
+	return (unsigned)rsa_public_key_prepare(pub);
 }
 
 static int
@@ -340,7 +341,13 @@ _wrap_nettle_pk_encrypt(gnutls_pk_algorithm_t algo,
 		{
 			struct rsa_public_key pub;
 
-			_rsa_params_to_pubkey(pk_params, &pub);
+			ret = _rsa_params_to_pubkey(pk_params, &pub);
+			if (ret == 0) {
+				ret =
+				    gnutls_assert_val
+				    (GNUTLS_E_ENCRYPTION_FAILED);
+				goto cleanup;
+			}
 
 			ret =
 			    rsa_encrypt(&pub, NULL, rnd_func,
@@ -401,8 +408,13 @@ _wrap_nettle_pk_decrypt(gnutls_pk_algorithm_t algo,
 #endif
 			bigint_t c;
 
-			_rsa_params_to_privkey(pk_params, &priv);
-			_rsa_params_to_pubkey(pk_params, &pub);
+			ret = _rsa_params_to_privkey(pk_params, &priv);
+			if (ret == 0)
+				return gnutls_assert_val(GNUTLS_E_DECRYPTION_FAILED);
+
+			ret = _rsa_params_to_pubkey(pk_params, &pub);
+			if (ret == 0)
+				return gnutls_assert_val(GNUTLS_E_DECRYPTION_FAILED);
 
 			if (ciphertext->size != pub.size)
 				return
@@ -622,8 +634,13 @@ _wrap_nettle_pk_sign(gnutls_pk_algorithm_t algo,
 			struct rsa_public_key pub;
 			mpz_t s;
 
-			_rsa_params_to_privkey(pk_params, &priv);
-			_rsa_params_to_pubkey(pk_params, &pub);
+			ret = _rsa_params_to_privkey(pk_params, &priv);
+			if (ret == 0)
+				return gnutls_assert_val(GNUTLS_E_PK_SIGN_FAILED);
+
+			ret = _rsa_params_to_pubkey(pk_params, &pub);
+			if (ret == 0)
+				return gnutls_assert_val(GNUTLS_E_PK_SIGN_FAILED);
 
 			mpz_init(s);
 
@@ -795,7 +812,10 @@ _wrap_nettle_pk_verify(gnutls_pk_algorithm_t algo,
 		{
 			struct rsa_public_key pub;
 
-			_rsa_params_to_pubkey(pk_params, &pub);
+			ret = _rsa_params_to_pubkey(pk_params, &pub);
+			if (ret == 0) {
+				return gnutls_assert_val(GNUTLS_E_PK_SIG_VERIFY_FAILED);
+			}
 
 			if (signature->size != pub.size)
 				return