sign APIs: introduce RSA-RAW signing algorithm

This ensures that there is a signing algorithm for all the operations
we support. Previously, we required GNUTLS_SIGN_UNKNOWN to be acceptable
by signing functions to accomodate for raw RSA operations. Now we make
that explicit and in the process clean-up the API.

Signed-off-by: Nikos Mavrogiannopoulos <nmav@redhat.com>

10 files changed, 97 insertions, 41 deletions

diff --git a/lib/algorithms.h b/lib/algorithms.h
index f9cb6ff1d9..cd2082000d 100644
--- a/lib/algorithms.h
+++ b/lib/algorithms.h
@@ -336,6 +336,8 @@ const sign_algorithm_st *_gnutls_sign_to_tls_aid(gnutls_sign_algorithm_t
 unsigned int _gnutls_pk_bits_to_subgroup_bits(unsigned int pk_bits);
 gnutls_digest_algorithm_t _gnutls_pk_bits_to_sha_hash(unsigned int pk_bits);
 
+gnutls_digest_algorithm_t _gnutls_hash_size_to_sha_hash(unsigned int size);
+
 bool _gnutls_pk_is_not_prehashed(gnutls_pk_algorithm_t algorithm);
 
 /* ECC */
diff --git a/lib/algorithms/mac.c b/lib/algorithms/mac.c
index becc753776..ba982b72fc 100644
--- a/lib/algorithms/mac.c
+++ b/lib/algorithms/mac.c
@@ -408,3 +408,19 @@ const char *gnutls_digest_get_oid(gnutls_digest_algorithm_t algorithm)
 
 	return NULL;
 }
+
+gnutls_digest_algorithm_t _gnutls_hash_size_to_sha_hash(unsigned int size)
+{
+	if (size == 20)
+		return GNUTLS_DIG_SHA1;
+	else if (size == 28)
+		return GNUTLS_DIG_SHA224;
+	else if (size == 32)
+		return GNUTLS_DIG_SHA256;
+	else if (size == 48)
+		return GNUTLS_DIG_SHA384;
+	else if (size == 64)
+		return GNUTLS_DIG_SHA512;
+
+	return GNUTLS_DIG_UNKNOWN;
+}
diff --git a/lib/algorithms/sign.c b/lib/algorithms/sign.c
index 2809acf7ec..e920e15cc1 100644
--- a/lib/algorithms/sign.c
+++ b/lib/algorithms/sign.c
@@ -40,6 +40,13 @@
  * e.g., RSA-PSS-SHA256 can be generated by GNUTLS_PK_RSA or GNUTLS_PK_RSA_PSS.
  */
 static const gnutls_sign_entry_st sign_algorithms[] = {
+	{.name = "RSA-RAW",
+	 .oid = NULL,
+	 .id = GNUTLS_SIGN_RSA_RAW,
+	 .pk = GNUTLS_PK_RSA,
+	 .hash = GNUTLS_DIG_UNKNOWN,
+	 .aid = TLS_SIGN_AID_UNKNOWN
+	},
 	{.name = "RSA-SHA1",
 	 .oid = SIG_RSA_SHA1_OID, 
 	 .id = GNUTLS_SIGN_RSA_SHA1,
diff --git a/lib/includes/gnutls/abstract.h b/lib/includes/gnutls/abstract.h
index 98248d5b36..bf29f877cd 100644
--- a/lib/includes/gnutls/abstract.h
+++ b/lib/includes/gnutls/abstract.h
@@ -75,14 +75,11 @@ typedef int (*gnutls_privkey_decrypt_func) (gnutls_privkey_t key,
 					    const gnutls_datum_t *ciphertext,
 					    gnutls_datum_t * plaintext);
 
-#define GNUTLS_SIGN_CB_FLAG_RSA_DIGESTINFO (1<<1)
-
 /* to be called to sign pre-hashed data. The input will be
  * the output of the hash (such as SHA256) corresponding to
- * the signature algorithm. The flag GNUTLS_SIGN_CB_FLAG_RSA_DIGESTINFO
+ * the signature algorithm. The algorithm GNUTLS_SIGN_RSA_RAW
  * will be provided when RSA PKCS#1 DigestInfo structure is provided
  * as data (when this is called from a TLS 1.0 or 1.1 session).
- * In that case the signature algorithm will be set to %GNUTLS_SIGN_UNKNOWN
  */
 typedef int (*gnutls_privkey_sign_hash_func) (gnutls_privkey_t key,
 					      gnutls_sign_algorithm_t algo,
diff --git a/lib/includes/gnutls/gnutls.h.in b/lib/includes/gnutls/gnutls.h.in
index 9562785498..fd6f063cc4 100644
--- a/lib/includes/gnutls/gnutls.h.in
+++ b/lib/includes/gnutls/gnutls.h.in
@@ -729,6 +729,7 @@ const char *gnutls_pk_algorithm_get_name(gnutls_pk_algorithm_t algorithm);
 /**
  * gnutls_sign_algorithm_t:
  * @GNUTLS_SIGN_UNKNOWN: Unknown signature algorithm.
+ * @GNUTLS_SIGN_RSA_RAW: Digital signature algorithm RSA with DigestInfo formatted data
  * @GNUTLS_SIGN_RSA_SHA1: Digital signature algorithm RSA with SHA-1
  * @GNUTLS_SIGN_RSA_SHA: Same as %GNUTLS_SIGN_RSA_SHA1.
  * @GNUTLS_SIGN_DSA_SHA1: Digital signature algorithm DSA with SHA-1
@@ -808,7 +809,8 @@ typedef enum {
 	GNUTLS_SIGN_RSA_PSS_SHA384 = 33,
 	GNUTLS_SIGN_RSA_PSS_SHA512 = 34,
 	GNUTLS_SIGN_EDDSA_ED25519 = 35,
-	GNUTLS_SIGN_MAX = GNUTLS_SIGN_EDDSA_ED25519
+	GNUTLS_SIGN_RSA_RAW = 36,
+	GNUTLS_SIGN_MAX = GNUTLS_SIGN_RSA_RAW
 } gnutls_sign_algorithm_t;
 
 /**
diff --git a/lib/nettle/pk.c b/lib/nettle/pk.c
index cc258d5ca6..507d65ef1b 100644
--- a/lib/nettle/pk.c
+++ b/lib/nettle/pk.c
@@ -557,7 +557,13 @@ _rsa_pss_sign_digest_tr(gnutls_digest_algorithm_t dig,
 	return ret;
 }
 
-/* in case of DSA puts into data, r,s
+/* This is the lower-level part of privkey_sign_raw_data().
+ *
+ * It accepts data in the appropriate hash form, i.e., DigestInfo
+ * for PK_RSA, hash for PK_ECDSA, PK_DSA, PK_RSA_PSS, and raw data
+ * for Ed25519.
+ *
+ * in case of EC/DSA, signed data are encoded into r,s values
  */
 static int
 _wrap_nettle_pk_sign(gnutls_pk_algorithm_t algo,
diff --git a/lib/pkcs11_privkey.c b/lib/pkcs11_privkey.c
index 9e1d1de1fa..bb58de4da9 100644
--- a/lib/pkcs11_privkey.c
+++ b/lib/pkcs11_privkey.c
@@ -304,14 +304,15 @@ static const struct hash_mappings_st *hash_to_map(gnutls_digest_algorithm_t hash
 }
 
 /*-
- * _gnutls_pkcs11_privkey_sign_hash:
+ * _gnutls_pkcs11_privkey_sign:
  * @key: Holds the key
  * @hash: holds the data to be signed (should be output of a hash)
  * @signature: will contain the signature allocated with gnutls_malloc()
  *
  * This function will sign the given data using a signature algorithm
  * supported by the private key. It is assumed that the given data
- * are the output of a hash function.
+ * are the output of a hash function. Input is the same as in
+ * privkey_sign_raw_data().
  *
  * Returns: On success, %GNUTLS_E_SUCCESS (0) is returned, otherwise a
  *   negative error value.
diff --git a/lib/privkey.c b/lib/privkey.c
index 240413c077..cfd3341fd9 100644
--- a/lib/privkey.c
+++ b/lib/privkey.c
@@ -797,10 +797,9 @@ gnutls_privkey_import_ext3(gnutls_privkey_t pkey,
  *
  * The @sign_hash_fn is to be called to sign pre-hashed data. The input
  * to the callback is the output of the hash (such as SHA256) corresponding
- * to the signature algorithm. The flag %GNUTLS_SIGN_CB_FLAG_RSA_DIGESTINFO
- * will be provided when RSA PKCS#1 DigestInfo structure is given as
- * data (e.g., when this is called from a TLS 1.0 or 1.1 session).
- * In that case the signature algorithm will be set to %GNUTLS_SIGN_UNKNOWN.
+ * to the signature algorithm. For RSA PKCS#1 signatures, the signature
+ * algorithm can be set to %GNUTLS_SIGN_RSA_RAW, and in that case the data
+ * should be handled as if they were an RSA PKCS#1 DigestInfo structure.
  *
  * The @sign_data_fn is to be called to sign data. The input data will be
  * he data to be signed (and hashed), with the provided signature
@@ -1182,7 +1181,7 @@ gnutls_privkey_sign_hash2(gnutls_privkey_t signer,
 	const gnutls_sign_entry_st *se;
 
 	se = _gnutls_sign_to_entry(algo);
-	if (se == NULL)
+	if (unlikely(se == NULL))
 		return gnutls_assert_val(GNUTLS_E_INVALID_REQUEST);
 
 	ret = _gnutls_privkey_get_spki_params(signer, &params);
@@ -1212,7 +1211,7 @@ privkey_sign_and_hash_data(gnutls_privkey_t signer,
 	gnutls_datum_t digest;
 	const mac_entry_st *me;
 
-	if (se == NULL)
+	if (unlikely(se == NULL))
 		return gnutls_assert_val(GNUTLS_E_INVALID_REQUEST);
 
 	if (_gnutls_pk_is_not_prehashed(se->pk)) {
@@ -1288,6 +1287,7 @@ gnutls_privkey_sign_hash(gnutls_privkey_t signer,
 {
 	int ret;
 	gnutls_x509_spki_st params;
+	const gnutls_sign_entry_st *se;
 
 	ret = _gnutls_privkey_get_spki_params(signer, &params);
 	if (ret < 0) {
@@ -1302,7 +1302,25 @@ gnutls_privkey_sign_hash(gnutls_privkey_t signer,
 		return ret;
 	}
 
-	return privkey_sign_prehashed(signer, _gnutls_pk_to_sign_entry(params.pk, hash_algo),
+	/* legacy callers of this API could use a hash algorithm of 0 (unknown)
+	 * to indicate raw hashing. As we now always want to know the signing
+	 * algorithm involved, we try discovering the hash algorithm. */
+	if (hash_algo == 0 && (params.pk == GNUTLS_PK_DSA || params.pk == GNUTLS_PK_ECDSA)) {
+		hash_algo = _gnutls_hash_size_to_sha_hash(hash_data->size);
+	}
+
+	if (params.pk == GNUTLS_PK_RSA && (flags & GNUTLS_PRIVKEY_SIGN_FLAG_TLS1_RSA)) {
+		/* the corresponding signature algorithm is SIGN_RSA_RAW,
+		 * irrespective of hash algorithm. */
+		se = _gnutls_sign_to_entry(GNUTLS_SIGN_RSA_RAW);
+	} else {
+		se = _gnutls_pk_to_sign_entry(params.pk, hash_algo);
+	}
+
+	if (unlikely(se == NULL))
+		return gnutls_assert_val(GNUTLS_E_INVALID_REQUEST);
+
+	return privkey_sign_prehashed(signer, se,
 				      hash_data, signature, &params, flags);
 }
 
@@ -1317,19 +1335,20 @@ privkey_sign_prehashed(gnutls_privkey_t signer,
 	int ret;
 	gnutls_datum_t digest;
 
-	if (flags & GNUTLS_PRIVKEY_SIGN_FLAG_TLS1_RSA)
+	if (unlikely(se == NULL))
+		return gnutls_assert_val(GNUTLS_E_INVALID_REQUEST);
+
+	if (se->id == GNUTLS_SIGN_RSA_RAW) {
 		return privkey_sign_raw_data(signer,
 					     se,
 					     hash_data, signature,
 					     params);
+	}
 
 	if (_gnutls_pk_is_not_prehashed(signer->pk_algorithm)) {
 		return gnutls_assert_val(GNUTLS_E_INVALID_REQUEST);
 	}
 
-	if (se == NULL)
-		return gnutls_assert_val(GNUTLS_E_INVALID_REQUEST);
-
 	digest.data = gnutls_malloc(hash_data->size);
 	if (digest.data == NULL) {
 		gnutls_assert();
@@ -1371,7 +1390,8 @@ privkey_sign_prehashed(gnutls_privkey_t signer,
  * supported by the private key. Note that this is a low-level function
  * and does not apply any preprocessing or hash on the signed data. 
  * For example on an RSA key the input @data should be of the DigestInfo
- * PKCS #1 1.5 format. Use it only if you know what are you doing.
+ * PKCS #1 1.5 format, on RSA-PSS, DSA or ECDSA the input should be a hash output
+ * and on Ed25519 the raw data to be signed.
  *
  * Note this function is equivalent to using the %GNUTLS_PRIVKEY_SIGN_FLAG_TLS1_RSA
  * flag with gnutls_privkey_sign_hash().
@@ -1388,12 +1408,8 @@ privkey_sign_raw_data(gnutls_privkey_t key,
 		      gnutls_datum_t * signature,
 		      gnutls_x509_spki_st * params)
 {
-	gnutls_pk_algorithm_t pk;
-
-	if (se == NULL) /* it can be null when signing raw-rsa */
-		pk = params->pk;
-	else
-		pk = se->pk;
+	if (unlikely(se == NULL))
+		return gnutls_assert_val(GNUTLS_E_INVALID_REQUEST);
 
 	switch (key->type) {
 #ifdef ENABLE_PKCS11
@@ -1403,7 +1419,7 @@ privkey_sign_raw_data(gnutls_privkey_t key,
 						   params);
 #endif
 	case GNUTLS_PRIVKEY_X509:
-		return _gnutls_pk_sign(pk, signature, data,
+		return _gnutls_pk_sign(se->pk, signature, data,
 				       &key->key.x509->params, params);
 	case GNUTLS_PRIVKEY_EXT:
 		if (unlikely(key->key.ext.sign_data_func == NULL &&
@@ -1411,7 +1427,7 @@ privkey_sign_raw_data(gnutls_privkey_t key,
 			key->key.ext.sign_func == NULL))
 			return gnutls_assert_val(GNUTLS_E_INVALID_REQUEST);
 
-		if (_gnutls_pk_is_not_prehashed(pk)) {
+		if (_gnutls_pk_is_not_prehashed(se->pk)) {
 			if (!key->key.ext.sign_data_func)
 				return gnutls_assert_val(GNUTLS_E_INVALID_REQUEST);
 
@@ -1422,16 +1438,17 @@ privkey_sign_raw_data(gnutls_privkey_t key,
 		} else if (key->key.ext.sign_hash_func) {
 			unsigned int flags = 0;
 
-			if (pk == GNUTLS_PK_RSA)
-				flags |= GNUTLS_SIGN_CB_FLAG_RSA_DIGESTINFO;
+			if (se->pk == GNUTLS_PK_RSA) {
+				se = _gnutls_sign_to_entry(GNUTLS_SIGN_RSA_RAW);
+			}
 
 			/* se may not be set here if we are doing legacy RSA */
-			return key->key.ext.sign_hash_func(key, se?se->id:GNUTLS_SIGN_UNKNOWN,
+			return key->key.ext.sign_hash_func(key, se->id,
 							   key->key.ext.userdata,
 							   flags,
 							   data, signature);
 		} else {
-			if (!PK_IS_OK_FOR_EXT2(pk))
+			if (!PK_IS_OK_FOR_EXT2(se->pk))
 				return gnutls_assert_val(GNUTLS_E_INVALID_REQUEST);
 
 			return key->key.ext.sign_func(key, key->key.ext.userdata,
diff --git a/lib/tls-sig.c b/lib/tls-sig.c
index 4b124627b7..26b36e6115 100644
--- a/lib/tls-sig.c
+++ b/lib/tls-sig.c
@@ -1,5 +1,6 @@
 /*
  * Copyright (C) 2001-2012 Free Software Foundation, Inc.
+ * Copyright (C) 2017 Red Hat, Inc.
  *
  * Author: Nikos Mavrogiannopoulos
  *
@@ -117,15 +118,15 @@ _gnutls_handshake_sign_data10(gnutls_session_t session,
 	int ret;
 	digest_hd_st td_sha;
 	uint8_t concat[MAX_SIG_SIZE];
-	const mac_entry_st *hash_algo;
+	const mac_entry_st *me;
 	gnutls_pk_algorithm_t pk_algo;
 
 	if (gnutls_privkey_get_pk_algorithm(pkey, NULL) == GNUTLS_PK_RSA)
-		hash_algo = hash_to_entry(GNUTLS_DIG_MD5_SHA1);
+		me = hash_to_entry(GNUTLS_DIG_MD5_SHA1);
 	else
-		hash_algo = hash_to_entry(
+		me = hash_to_entry(
 				gnutls_sign_get_hash_algorithm(sign_algo));
-	if (hash_algo == NULL)
+	if (me == NULL)
 		return gnutls_assert_val(GNUTLS_E_UNKNOWN_HASH_ALGORITHM);
 
 	pk_algo = gnutls_sign_get_pk_algorithm(sign_algo);
@@ -136,7 +137,7 @@ _gnutls_handshake_sign_data10(gnutls_session_t session,
 	    ("HSK[%p]: signing handshake data: using %s\n", session,
 	     gnutls_sign_algorithm_get_name(sign_algo));
 
-	ret = _gnutls_hash_init(&td_sha, hash_algo);
+	ret = _gnutls_hash_init(&td_sha, me);
 	if (ret < 0) {
 		gnutls_assert();
 		return ret;
@@ -151,9 +152,10 @@ _gnutls_handshake_sign_data10(gnutls_session_t session,
 	_gnutls_hash_deinit(&td_sha, concat);
 
 	dconcat.data = concat;
-	dconcat.size = _gnutls_hash_get_algo_len(hash_algo);
+	dconcat.size = _gnutls_hash_get_algo_len(me);
 
-	ret = gnutls_privkey_sign_raw_data(pkey, 0, &dconcat, signature);
+	ret = gnutls_privkey_sign_hash(pkey, me->id, GNUTLS_PRIVKEY_SIGN_FLAG_TLS1_RSA,
+				       &dconcat, signature);
 	if (ret < 0) {
 		gnutls_assert();
 	}
@@ -641,7 +643,9 @@ _gnutls_handshake_sign_crt_vrfy3(gnutls_session_t session,
 
 	dconcat.size += 20;
 
-	ret = gnutls_privkey_sign_raw_data(pkey, 0, &dconcat, signature);
+	ret = gnutls_privkey_sign_hash(pkey, GNUTLS_DIG_SHA1,
+				       GNUTLS_PRIVKEY_SIGN_FLAG_TLS1_RSA,
+				       &dconcat, signature);
 	if (ret < 0)
 		return gnutls_assert_val(ret);
 
@@ -723,7 +727,8 @@ _gnutls_handshake_sign_crt_vrfy(gnutls_session_t session,
 	dconcat.data = concat;
 	dconcat.size = _gnutls_hash_get_algo_len(me);
 
-	ret = gnutls_privkey_sign_raw_data(pkey, 0, &dconcat, signature);
+	ret = gnutls_privkey_sign_hash(pkey, me->id, GNUTLS_PRIVKEY_SIGN_FLAG_TLS1_RSA,
+				       &dconcat, signature);
 	if (ret < 0) {
 		gnutls_assert();
 		return ret;
diff --git a/tests/sign-verify-ext4.c b/tests/sign-verify-ext4.c
index 272b0a784f..7f5e9c6c39 100644
--- a/tests/sign-verify-ext4.c
+++ b/tests/sign-verify-ext4.c
@@ -111,6 +111,9 @@ int key_cb_sign_hash_func (gnutls_privkey_t key, gnutls_sign_algorithm_t sig,
 	struct key_cb_data *p = userdata;
 
 	if (flags & GNUTLS_SIGN_CB_FLAG_RSA_DIGESTINFO) {
+		if (sig != GNUTLS_SIGN_RSA_RAW)
+			fail("unexpected signature algorithm with DigestInfo\n");
+
 		if (debug)
 			fprintf(stderr, "signing digestinfo with: raw RSA\n");
 		return gnutls_privkey_sign_hash(p->rkey, 0, GNUTLS_PRIVKEY_SIGN_FLAG_TLS1_RSA, data, signature);