handshake: select a signature algorithm early

That is, select the signature algorithm at the point the certificate and
ciphersuites are decided. Also ensure that a compatible signature algorithm
with the ciphersuite and the key is selected.

That prevents situations where a ciphersuite and a certificate are
negotiated, but later on the handshake we figure that there are no
common signature algorithms.

Signed-off-by: Nikos Mavrogiannopoulos <nmav@redhat.com>

3 files changed, 60 insertions, 6 deletions

diff --git a/lib/auth/cert.c b/lib/auth/cert.c
index 70183d8785..bcf7ffff4b 100644
--- a/lib/auth/cert.c
+++ b/lib/auth/cert.c
@@ -1441,6 +1441,39 @@ unsigned pubkey_is_compat_with_cs(gnutls_session_t session,
 	return 1;
 }
 
+/* Selects a signature algorithm (if required by the ciphersuite and TLS
+ * version), appropriate for the certificate. If none can be selected
+ * returns an error.
+ */
+static
+int select_sign_algorithm(gnutls_session_t session,
+			  gnutls_pcert_st * cert,
+			  const gnutls_cipher_suite_entry_st *cs)
+{
+	gnutls_sign_algorithm_t algo;
+	const version_entry_st *ver = get_version(session);
+
+	if (_gnutls_kx_encipher_type(cs->kx_algorithm) != CIPHER_SIGN)
+		return 0;
+
+	if (!_gnutls_version_has_selectable_sighash(ver)) {
+		/* For SSL3.0 and TLS1.0 we lie as we cannot express md5-sha1 as
+		 * signature algorithm. */
+		algo = gnutls_pk_to_sign(cert->pubkey->params.algo, GNUTLS_DIG_SHA1);
+		gnutls_sign_algorithm_set_server(session, algo);
+		return 0;
+	}
+
+	algo = _gnutls_session_get_sign_algo(session, cert, 0);
+	if (algo == GNUTLS_SIGN_UNKNOWN)
+		return gnutls_assert_val(GNUTLS_E_INCOMPATIBLE_SIG_WITH_KEY);
+
+	gnutls_sign_algorithm_set_server(session, algo);
+	_gnutls_handshake_log("Selected signature algorithm: %s\n", gnutls_sign_algorithm_get_name(algo));
+
+	return 0;
+}
+
 /* finds the most appropriate certificate in the cert list.
  * The 'appropriate' is defined by the user.
  *
@@ -1493,6 +1526,11 @@ _gnutls_server_select_cert(gnutls_session_t session, const gnutls_cipher_suite_e
 			return gnutls_assert_val(GNUTLS_E_INSUFFICIENT_CREDENTIALS);
 		}
 
+		ret = select_sign_algorithm(session, &session->internals.selected_cert_list[0], cs);
+		if (ret < 0) {
+			return gnutls_assert_val(ret);
+		}
+
 		return 0;
 	}
 
@@ -1516,18 +1554,26 @@ _gnutls_server_select_cert(gnutls_session_t session, const gnutls_cipher_suite_e
 						       server_name) != 0) {
 				/* if requested algorithms are also compatible select it */
 
-				if (pubkey_is_compat_with_cs(session,
+				if (!pubkey_is_compat_with_cs(session,
 							     cred->certs[i].cert_list[0].pubkey,
 							     cred->certs[i].cert_list[0].type,
 							     cs)) {
+					continue;
+				}
+
+				ret = select_sign_algorithm(session, &cred->certs[i].cert_list[0], cs);
+				if (ret >= 0) {
 					idx = i;
 					_gnutls_debug_log("Selected (%s) cert based on ciphersuite %x.%x: %s\n",
 						  gnutls_pk_get_name(cred->certs[i].cert_list[0].pubkey->params.algo),
 						  (unsigned)cs->id[0],
 						  (unsigned)cs->id[1],
 						  cs->name);
+					/* found */
 					goto finished;
 				}
+
+
 			}
 		}
 	}
@@ -1542,16 +1588,22 @@ _gnutls_server_select_cert(gnutls_session_t session, const gnutls_cipher_suite_e
 						      [i].cert_list
 						      [0].type));
 
-		if (pubkey_is_compat_with_cs(session,
+		if (!pubkey_is_compat_with_cs(session,
 					     cred->certs[i].cert_list[0].pubkey,
 					     cred->certs[i].cert_list[0].type,
 					     cs)) {
+			continue;
+		}
+
+		ret = select_sign_algorithm(session, &cred->certs[i].cert_list[0], cs);
+		if (ret >= 0) {
 			idx = i;
 			_gnutls_debug_log("Selected (%s) cert based on ciphersuite %x.%x: %s\n",
 					  gnutls_pk_get_name(cred->certs[i].cert_list[0].pubkey->params.algo),
 					  (unsigned)cs->id[0],
 					  (unsigned)cs->id[1],
 					  cs->name);
+			/* found */
 			goto finished;
 		}
 	}
diff --git a/lib/gnutls_int.h b/lib/gnutls_int.h
index fa00ad234d..eb5fab1506 100644
--- a/lib/gnutls_int.h
+++ b/lib/gnutls_int.h
@@ -598,8 +598,12 @@ typedef struct {
 	/* The selected (after server hello EC or DH group */
 	const gnutls_group_entry_st *grp;
 
-	/* Holds the signature algorithm used in this session - If any */
+	/* Holds the signature algorithm that will be used in this session,
+	 * selected by the server at the time of Ciphersuite/certificate
+	 * selection - see select_sign_algorithm() */
 	gnutls_sign_algorithm_t server_sign_algo;
+
+	/* Holds the signature algorithm used in this session - If any */
 	gnutls_sign_algorithm_t client_sign_algo;
 
 	/* Whether the master secret negotiation will be according to
diff --git a/lib/tls-sig.c b/lib/tls-sig.c
index 4ebab54f88..95a7b3ea64 100644
--- a/lib/tls-sig.c
+++ b/lib/tls-sig.c
@@ -175,14 +175,12 @@ _gnutls_handshake_sign_data(gnutls_session_t session,
 	unsigned key_usage = 0;
 	int ret;
 
-	*sign_algo = _gnutls_session_get_sign_algo(session, cert, 0);
+	*sign_algo = session->security_parameters.server_sign_algo;
 	if (*sign_algo == GNUTLS_SIGN_UNKNOWN) {
 		gnutls_assert();
 		return GNUTLS_E_UNWANTED_ALGORITHM;
 	}
 
-	gnutls_sign_algorithm_set_server(session, *sign_algo);
-
 	gnutls_pubkey_get_key_usage(cert->pubkey, &key_usage);
 
 	ret = check_key_usage_for_sig(session, key_usage, 1);