gnutls_privkey_sign_hash2: accept the GNUTLS_PRIVKEY_SIGN_FLAG_TLS1_RSA flagtmp-fix-raw-flag-in-newapi

Previously this flag was ignored, although documented not to.
This patch also enables the tests sign-verify-newapi and sign-verify-data-newapi
which were supposed to test this interface, but were never enabled.

This was caught by Andreas Metzler.

Signed-off-by: Nikos Mavrogiannopoulos <nmav@gnutls.org>

4 files changed, 24 insertions, 12 deletions

diff --git a/NEWS b/NEWS
index 9268a0ee24..77407a3017 100644
--- a/NEWS
+++ b/NEWS
@@ -5,6 +5,15 @@ Copyright (C) 2000-2016 Free Software Foundation, Inc.
 Copyright (C) 2013-2019 Nikos Mavrogiannopoulos
 See the end for copying conditions.
 
+* Version 3.6.9 (unreleased)
+
+** libgnutls: gnutls_privkey_sign_hash2 now accepts the GNUTLS_PRIVKEY_SIGN_FLAG_TLS1_RSA
+   flag as documented. This makes it a complete replacement of gnutls_privkey_sign_hash().
+
+** API and ABI modifications:
+No changes since last version.
+
+
 * Version 3.6.8 (released 2019-05-28)
 
 ** libgnutls: Added gnutls_prf_early() function to retrieve early keying
diff --git a/lib/privkey.c b/lib/privkey.c
index 8b3e3557c2..8e353c5e5f 100644
--- a/lib/privkey.c
+++ b/lib/privkey.c
@@ -1207,7 +1207,8 @@ gnutls_privkey_sign_data2(gnutls_privkey_t signer,
  *
  * The flags may be %GNUTLS_PRIVKEY_SIGN_FLAG_TLS1_RSA or %GNUTLS_PRIVKEY_SIGN_FLAG_RSA_PSS.
  * In the former case this function will ignore @hash_algo and perform a raw PKCS1 signature,
- * and in the latter an RSA-PSS signature will be generated.
+ * and in the latter an RSA-PSS signature will be generated. Note that the flag
+ * %GNUTLS_PRIVKEY_SIGN_FLAG_TLS1_RSA is supported since 3.6.9.
  *
  * Note that, not all algorithm support signing already hashed data. When
  * signing with Ed25519, gnutls_privkey_sign_data() should be used.
@@ -1228,9 +1229,16 @@ gnutls_privkey_sign_hash2(gnutls_privkey_t signer,
 	gnutls_x509_spki_st params;
 	const gnutls_sign_entry_st *se;
 
-	se = _gnutls_sign_to_entry(algo);
-	if (unlikely(se == NULL))
-		return gnutls_assert_val(GNUTLS_E_INVALID_REQUEST);
+	if (flags & GNUTLS_PRIVKEY_SIGN_FLAG_TLS1_RSA) {
+		/* the corresponding signature algorithm is SIGN_RSA_RAW,
+		 * irrespective of hash algorithm. */
+		se = _gnutls_sign_to_entry(GNUTLS_SIGN_RSA_RAW);
+	} else {
+		se = _gnutls_sign_to_entry(algo);
+		if (unlikely(se == NULL))
+			return gnutls_assert_val(GNUTLS_E_INVALID_REQUEST);
+
+	}
 
 	ret = _gnutls_privkey_get_spki_params(signer, &params);
 	if (ret < 0) {
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 4ffa698253..a67f1549c2 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -211,7 +211,8 @@ ctests += mini-record-2 simple gnutls_hmac_fast set_pkcs12_cred cert certuniquei
 	 tls13-server-kx-neg gnutls_ext_raw_parse_dtls key-export-pkcs8 \
 	 null_retrieve_function tls-record-size-limit tls-crt_type-neg \
 	 resume-with-stek-expiration resume-with-previous-stek rawpk-api \
-	 tls-record-size-limit-asym dh-compute ecdh-compute
+	 tls-record-size-limit-asym dh-compute ecdh-compute sign-verify-data-newapi \
+	 sign-verify-newapi
 
 if HAVE_SECCOMP_TESTS
 ctests += dtls-with-seccomp tls-with-seccomp dtls-client-with-seccomp tls-client-with-seccomp
diff --git a/tests/sign-verify-newapi.c b/tests/sign-verify-newapi.c
index aa284006aa..7dae1b18a2 100644
--- a/tests/sign-verify-newapi.c
+++ b/tests/sign-verify-newapi.c
@@ -227,19 +227,13 @@ void doit(void)
 				testfail("gnutls_privkey_sign_hash: %s\n",
 				     gnutls_strerror(ret));
 
-			sign_algo =
-			    gnutls_pk_to_sign
-			    (gnutls_pubkey_get_pk_algorithm(pubkey, NULL),
-			     tests[i].digest);
-
 			ret =
-			    gnutls_pubkey_verify_hash2(pubkey, sign_algo,
+			    gnutls_pubkey_verify_hash2(pubkey, tests[i].sigalgo,
 							GNUTLS_PUBKEY_VERIFY_FLAG_TLS1_RSA,
 							hash_data,
 							&signature);
 			if (ret < 0)
 				testfail("gnutls_pubkey_verify_hash-3 (raw hashed data)\n");
-
 			gnutls_free(signature.data);
 			/* test the legacy API */
 			ret =