ssl: Fix handling of RSASSA-PSS certificates

Decouple the RSASSA-PSS signature algorithms used in the Certificate from
the algorithms used for signing the CertificateVerify message.

2 files changed, 17 insertions, 44 deletions

diff --git a/lib/ssl/src/ssl_handshake.erl b/lib/ssl/src/ssl_handshake.erl
index 0dc927b43a..e5f6798a50 100644
--- a/lib/ssl/src/ssl_handshake.erl
+++ b/lib/ssl/src/ssl_handshake.erl
@@ -410,8 +410,6 @@ verify_signature({3, 3}, Hash, {HashAlgo, SignAlgo}, Signature,
                                                   SignAlgo == rsa_pss_pss ->
     Options = verify_options(SignAlgo, HashAlgo, PubKeyParams),
     public_key:verify({digest, Hash}, HashAlgo, Signature, PubKey, Options);
-
-
 verify_signature({3, Minor}, Hash, {HashAlgo, SignAlgo}, Signature, {?rsaEncryption, PubKey, PubKeyParams})
   when Minor >= 3 ->
     Options = verify_options(SignAlgo, HashAlgo, PubKeyParams),
@@ -1856,8 +1854,9 @@ do_digitally_signed({3, Minor}, Hash, HashAlgo, #{algorithm := Alg} = Engine, Si
 do_digitally_signed({3, 3}, Hash, HashAlgo, #{algorithm := Alg} = Engine, SignAlgo) ->
     Options = signature_options(SignAlgo, HashAlgo),
     crypto:sign(Alg, HashAlgo, {digest, Hash}, maps:remove(algorithm, Engine), Options);
-do_digitally_signed({3, 4}, Hash, HashAlgo, {#'RSAPrivateKey'{} = Key, #'RSASSA-PSS-params'{} = Params}, SignAlgo) ->
-    Options = signature_options(SignAlgo, HashAlgo, Params),
+do_digitally_signed({3, 4}, Hash, HashAlgo, {#'RSAPrivateKey'{} = Key,
+                                             #'RSASSA-PSS-params'{}}, SignAlgo) ->
+    Options = signature_options(SignAlgo, HashAlgo),
     public_key:sign(Hash, HashAlgo, Key, Options);
 do_digitally_signed({3, 4}, Hash, HashAlgo, Key, SignAlgo) ->
     Options = signature_options(SignAlgo, HashAlgo),
@@ -1871,38 +1870,25 @@ do_digitally_signed({3, Minor}, Hash, _HashAlgo, #'RSAPrivateKey'{} = Key, rsa)
 do_digitally_signed(_Version, Hash, HashAlgo, Key, _SignAlgo) ->
     public_key:sign({digest, Hash}, HashAlgo, Key).
     
-signature_options(SignAlgo, HashAlgo) ->
-    signature_options(SignAlgo, HashAlgo, undefined).
-signature_options(rsa_pss_pss, HashAlgo, #'RSASSA-PSS-params'{} = Params) ->
-    pss_pss_options(HashAlgo, Params);
-signature_options(rsa_pss_rsae, HashAlgo, _) ->
-    pss_rsae_options(HashAlgo);
-signature_options(_, _, _) ->
+signature_options(SignAlgo, HashAlgo) when SignAlgo =:= rsa_pss_rsae orelse
+                                           SignAlgo =:= rsa_pss_pss ->
+    pss_options(HashAlgo);
+signature_options(_, _) ->
     [].
 
-verify_options(rsa_pss_rsae, HashAlgo, _KeyParams) -> 
-    pss_rsae_options(HashAlgo);
-verify_options(rsa_pss_pss, HashAlgo, #'RSASSA-PSS-params'{} = Params) ->
-    pss_pss_options(HashAlgo, Params);
+verify_options(SignAlgo, HashAlgo, _KeyParams)
+  when SignAlgo =:= rsa_pss_rsae orelse
+       SignAlgo =:= rsa_pss_pss ->
+    pss_options(HashAlgo);
 verify_options(_, _, _) ->
     [].
 
-pss_rsae_options(HashAlgo) ->
+pss_options(HashAlgo) ->
     %% of the digest algorithm: rsa_pss_saltlen = -1
     [{rsa_padding, rsa_pkcs1_pss_padding},
      {rsa_pss_saltlen, -1},
      {rsa_mgf1_md, HashAlgo}].
 
-pss_pss_options(HashAlgo, #'RSASSA-PSS-params'{saltLength = SaltLen,
-                                               maskGenAlgorithm = 
-                                                   #'MaskGenAlgorithm'{algorithm = ?'id-mgf1',
-                                                                       parameters = #'HashAlgorithm'{algorithm = HashOid}}}) ->
-
-    HashAlgo = public_key:pkix_hash_type(HashOid),
-    [{rsa_padding, rsa_pkcs1_pss_padding},
-     {rsa_pss_saltlen, SaltLen},
-     {rsa_mgf1_md, HashAlgo}].
-
 bad_key(#'DSAPrivateKey'{}) ->
     unacceptable_dsa_key;
 bad_key(#'RSAPrivateKey'{}) ->
diff --git a/lib/ssl/src/tls_handshake_1_3.erl b/lib/ssl/src/tls_handshake_1_3.erl
index 07942363b6..508cb80938 100644
--- a/lib/ssl/src/tls_handshake_1_3.erl
+++ b/lib/ssl/src/tls_handshake_1_3.erl
@@ -557,7 +557,7 @@ do_start(#client_hello{cipher_suites = ClientCiphers,
     {Ref,Maybe} = maybe(),
 
     try
-        
+
         %% Handle ALPN extension if ALPN is configured
         ALPNProtocol = Maybe(handle_alpn(ALPNPreferredProtocols, ClientALPN)),
 
@@ -574,16 +574,14 @@ do_start(#client_hello{cipher_suites = ClientCiphers,
         Maybe(check_cert_sign_algo(SignAlgo, SignHash, ClientSignAlgs, ClientSignAlgsCert)),
 
         %% Select signature algorithm (used in CertificateVerify message).
-        SelectedSignAlg = Maybe(select_sign_algo(PublicKeyAlgo, ClientSignAlgs, 
-                                                 handle_pss(PublicKeyAlgo,
-                                                            SignAlgo, SignHash, ServerSignAlgs))),
-        
+        SelectedSignAlg = Maybe(select_sign_algo(PublicKeyAlgo, ClientSignAlgs, ServerSignAlgs)),
+
         %% Select client public key. If no public key found in ClientShares or
         %% ClientShares is empty, trigger HelloRetryRequest as we were able
         %% to find an acceptable set of parameters but the ClientHello does not
         %% contain sufficient information.
         {Group, ClientPubKey} = get_client_public_key(Groups, ClientShares),
-        
+
         %% Generate server_share
         KeyShare = ssl_cipher:generate_server_share(Group),
 
@@ -605,7 +603,7 @@ do_start(#client_hello{cipher_suites = ClientCiphers,
                                       sign_alg => SelectedSignAlg,
                                       peer_public_key => ClientPubKey,
                                       alpn => ALPNProtocol}),
-        
+
         %% 4.1.4.  Hello Retry Request
         %%
         %% The server will send this message in response to a ClientHello
@@ -2416,14 +2414,3 @@ process_user_tickets([H|T], Acc, N) ->
 %% (see Section 4.6.1), modulo 2^32.
 obfuscate_ticket_age(TicketAge, AgeAdd) ->
     (TicketAge + AgeAdd) rem round(math:pow(2,32)).
-
-handle_pss(rsa_pss_pss, rsa_pss_pss, sha256, Algs) ->
-    Algs -- [rsa_pss_pss_sha384, rsa_pss_pss_sha512];
-handle_pss(rsa_pss_pss, rsa_pss_pss, sha384, Algs) ->
-    Algs -- [rsa_pss_pss_sha256, rsa_pss_pss_sha512];
-handle_pss(rsa_pss_pss, rsa_pss_pss, sha512, Algs) ->
-    Algs -- [rsa_pss_pss_sha256, rsa_pss_pss_sha384];
-handle_pss(_,_,_, Algs) ->
-    Algs -- [rsa_pss_pss_sha256, rsa_pss_pss_sha384, rsa_pss_pss_sha512].
-
-