Corrected signature generation and verification

in the Certificate Verify message when in TLS 1.2. Reported
by Todd A. Ouska.

3 files changed, 35 insertions, 96 deletions

diff --git a/NEWS b/NEWS
index c021ce2f44..f60bb71251 100644
--- a/NEWS
+++ b/NEWS
@@ -5,6 +5,10 @@ See the end for copying conditions.
 
 * Version 2.11.7 (unreleased)
 
+** libgnutls: Corrected signature generation and verification
+in the Certificate Verify message when in TLS 1.2. Reported
+by Todd A. Ouska.
+
 ** libgnutls: Corrected issue in DHE-PSK ciphersuites that ignored
 the PSK callback.
 
diff --git a/lib/gnutls_extensions.c b/lib/gnutls_extensions.c
index 05ca7e2e2e..bf474ec84a 100644
--- a/lib/gnutls_extensions.c
+++ b/lib/gnutls_extensions.c
@@ -709,7 +709,7 @@ _gnutls_ext_get_session_data (gnutls_session_t session,
           return 0;
         }
     }
-  return GNUTLS_E_INVALID_REQUEST;
+  return GNUTLS_E_REQUESTED_DATA_NOT_AVAILABLE;
 }
 
 int
diff --git a/lib/gnutls_sig.c b/lib/gnutls_sig.c
index 58ecedce13..63a209c07d 100644
--- a/lib/gnutls_sig.c
+++ b/lib/gnutls_sig.c
@@ -43,7 +43,7 @@
 #include <x509/common.h>
 
 static int
-_gnutls_tls_sign (gnutls_session_t session,
+sign_tls_hash (gnutls_session_t session, gnutls_digest_algorithm_t hash_algo,
                   gnutls_cert * cert, gnutls_privkey_t pkey,
                   const gnutls_datum_t * hash_concat,
                   gnutls_datum_t * signature);
@@ -55,72 +55,6 @@ _gnutls_tls_sign (gnutls_session_t session,
  */
 #define MAX_SIG_SIZE 19 + MAX_HASH_SIZE
 
-/* Create a DER-encoded value as a opaque signature when RSA is used.
- * See RFC 5246 DigitallySigned for the actual format.
- */
-static int
-_gnutls_rsa_encode_sig (gnutls_mac_algorithm_t algo,
-                        const gnutls_datum_t * hash,
-                        gnutls_datum_t * signature)
-{
-  ASN1_TYPE di;
-  const char *oid;
-  int result, signature_size;
-
-  oid = _gnutls_x509_mac_to_oid (algo);
-  if (!oid)
-    {
-      gnutls_assert ();
-      return GNUTLS_E_UNKNOWN_HASH_ALGORITHM;
-    }
-
-  if ((result = asn1_create_element
-       (_gnutls_get_gnutls_asn (), "GNUTLS.DigestInfo", &di)) != ASN1_SUCCESS)
-    {
-      gnutls_assert ();
-      return _gnutls_asn2err (result);
-    }
-
-  if ((result = asn1_write_value (di, "digestAlgorithm.algorithm",
-                                  oid, strlen (oid))) != ASN1_SUCCESS)
-    {
-      gnutls_assert ();
-      asn1_delete_structure (&di);
-      return _gnutls_asn2err (result);
-    }
-
-  /* Use NULL parameters. */
-  if ((result = asn1_write_value (di, "digestAlgorithm.parameters",
-                                  ASN1_NULL, ASN1_NULL_SIZE)) != ASN1_SUCCESS)
-    {
-      gnutls_assert ();
-      asn1_delete_structure (&di);
-      return _gnutls_asn2err (result);
-    }
-
-  if ((result = asn1_write_value (di, "digest",
-                                  hash->data, hash->size)) != ASN1_SUCCESS)
-    {
-      gnutls_assert ();
-      asn1_delete_structure (&di);
-      return _gnutls_asn2err (result);
-    }
-
-  signature_size = signature->size;
-  result = asn1_der_coding (di, "", signature->data, &signature_size, NULL);
-  asn1_delete_structure (&di);
-
-  if (result != ASN1_SUCCESS)
-    {
-      gnutls_assert ();
-      return _gnutls_asn2err (result);
-    }
-
-  signature->size = signature_size;
-
-  return 0;
-}
-
 static int 
 get_hash_algo(gnutls_session_t session, gnutls_cert* cert, 
   gnutls_sign_algorithm_t sign_algo,
@@ -215,17 +149,12 @@ _gnutls_handshake_sign_data (gnutls_session_t session, gnutls_cert * cert,
           dconcat.size = 36;
         }
       else
-        {                       /* TLS 1.2 way */
-          gnutls_datum_t hash;
+        { /* TLS 1.2 way */
 
           _gnutls_hash_deinit (&td_sha, concat);
 
-          hash.data = concat;
-          hash.size = _gnutls_hash_get_algo_len (hash_algo);
           dconcat.data = concat;
-          dconcat.size = sizeof concat;
-
-          _gnutls_rsa_encode_sig (hash_algo, &hash, &dconcat);
+          dconcat.size = _gnutls_hash_get_algo_len (hash_algo);
         }
       break;
     case GNUTLS_PK_DSA:
@@ -246,7 +175,8 @@ _gnutls_handshake_sign_data (gnutls_session_t session, gnutls_cert * cert,
       _gnutls_hash_deinit (&td_sha, NULL);
       return GNUTLS_E_INTERNAL_ERROR;
     }
-  ret = _gnutls_tls_sign (session, cert, pkey, &dconcat, signature);
+
+  ret = sign_tls_hash (session, hash_algo, cert, pkey, &dconcat, signature);
   if (ret < 0)
     {
       gnutls_assert ();
@@ -301,11 +231,12 @@ _gnutls_soft_sign (gnutls_pk_algorithm_t algo, bigint_t * params,
  * it supports signing.
  */
 static int
-_gnutls_tls_sign (gnutls_session_t session,
+sign_tls_hash (gnutls_session_t session, gnutls_digest_algorithm_t hash_algo,
                   gnutls_cert * cert, gnutls_privkey_t pkey,
                   const gnutls_datum_t * hash_concat,
                   gnutls_datum_t * signature)
 {
+  gnutls_protocol_t ver = gnutls_protocol_get_version (session);
 
   /* If our certificate supports signing
    */
@@ -331,11 +262,14 @@ _gnutls_tls_sign (gnutls_session_t session,
         }
     }
 
-  return _gnutls_privkey_sign_hash (pkey, hash_concat, signature);
+   if (!_gnutls_version_has_selectable_sighash (ver))
+    return _gnutls_privkey_sign_hash (pkey, hash_concat, signature);
+  else
+    return gnutls_privkey_sign_hash2 (pkey, hash_algo, 0, hash_concat, signature);
 }
 
 static int
-_gnutls_verify_sig (gnutls_cert * cert,
+verify_tls_hash (gnutls_protocol_t ver, gnutls_cert * cert,
                     const gnutls_datum_t * hash_concat,
                     gnutls_datum_t * signature, size_t sha1pos,
                     gnutls_pk_algorithm_t pk_algo)
@@ -370,8 +304,14 @@ _gnutls_verify_sig (gnutls_cert * cert,
       vdata.size = hash_concat->size;
 
       /* verify signature */
-      if ((ret = _gnutls_rsa_verify (&vdata, signature, cert->params,
-                                     cert->params_size, 1)) < 0)
+      if (!_gnutls_version_has_selectable_sighash (ver))
+        ret = _gnutls_rsa_verify (&vdata, signature, cert->params,
+                                     cert->params_size, 1);
+      else
+        ret = pubkey_verify_sig( NULL, &vdata, signature, pk_algo, 
+          cert->params, cert->params_size);
+
+      if (ret < 0)
         {
           gnutls_assert ();
           return ret;
@@ -383,16 +323,16 @@ _gnutls_verify_sig (gnutls_cert * cert,
       vdata.data = &hash_concat->data[sha1pos];
       vdata.size = hash_concat->size - sha1pos;
 
+      ret = pubkey_verify_sig( NULL, &vdata, signature, pk_algo, 
+        cert->params, cert->params_size);
       /* verify signature */
-      if ((ret = _gnutls_dsa_verify (&vdata, signature, cert->params,
-                                     cert->params_size)) < 0)
+      if (ret < 0)
         {
           gnutls_assert ();
           return ret;
         }
 
       break;
-
     default:
       gnutls_assert ();
       return GNUTLS_E_INTERNAL_ERROR;
@@ -472,19 +412,13 @@ _gnutls_handshake_verify_data (gnutls_session_t session, gnutls_cert * cert,
     }
   else
     {
-      gnutls_datum_t hash;
-
       _gnutls_hash_deinit (&td_sha, concat);
 
-      hash.data = concat;
-      hash.size = _gnutls_hash_get_algo_len (hash_algo);
       dconcat.data = concat;
-      dconcat.size = sizeof concat;
-
-      _gnutls_rsa_encode_sig (hash_algo, &hash, &dconcat);
+      dconcat.size = _gnutls_hash_get_algo_len (hash_algo);
     }
 
-  ret = _gnutls_verify_sig (cert, &dconcat, signature,
+  ret = verify_tls_hash (ver, cert, &dconcat, signature,
                             dconcat.size -
                             _gnutls_hash_get_algo_len (hash_algo),
                             _gnutls_sign_get_pk_algorithm (algo));
@@ -516,6 +450,7 @@ _gnutls_handshake_verify_cert_vrfy12 (gnutls_session_t session,
   gnutls_sign_algorithm_t _sign_algo;
   gnutls_digest_algorithm_t hash_algo;
   digest_hd_st *handshake_td;
+  gnutls_protocol_t ver = gnutls_protocol_get_version (session);
 
   handshake_td = &session->internals.handshake_mac_handle.tls12.sha1;
   hash_algo = handshake_td->algorithm;
@@ -548,7 +483,7 @@ _gnutls_handshake_verify_cert_vrfy12 (gnutls_session_t session,
   dconcat.size = _gnutls_hash_get_algo_len (hash_algo);
 
   ret =
-    _gnutls_verify_sig (cert, &dconcat, signature, 0,
+    verify_tls_hash (ver, cert, &dconcat, signature, 0,
                         cert->subject_pk_algorithm);
   if (ret < 0)
     {
@@ -636,7 +571,7 @@ _gnutls_handshake_verify_cert_vrfy (gnutls_session_t session,
   dconcat.size = 20 + 16;       /* md5+ sha */
 
   ret =
-    _gnutls_verify_sig (cert, &dconcat, signature, 16,
+    verify_tls_hash (ver, cert, &dconcat, signature, 16,
                         cert->subject_pk_algorithm);
   if (ret < 0)
     {
@@ -713,7 +648,7 @@ _gnutls_handshake_sign_cert_vrfy12 (gnutls_session_t session,
   dconcat.data = concat;
   dconcat.size = _gnutls_hash_get_algo_len (hash_algo);
 
-  ret = _gnutls_tls_sign (session, cert, pkey, &dconcat, signature);
+  ret = sign_tls_hash (session, hash_algo, cert, pkey, &dconcat, signature);
   if (ret < 0)
     {
       gnutls_assert ();
@@ -816,7 +751,7 @@ _gnutls_handshake_sign_cert_vrfy (gnutls_session_t session,
       gnutls_assert ();
       return GNUTLS_E_INTERNAL_ERROR;
     }
-  ret = _gnutls_tls_sign (session, cert, pkey, &dconcat, signature);
+  ret = sign_tls_hash (session, GNUTLS_DIG_NULL, cert, pkey, &dconcat, signature);
   if (ret < 0)
     {
       gnutls_assert ();