upstream: make the PKCS#11 RSA code more like the new PKCS#11

ECDSA code: use a single custom RSA_METHOD instead of a method per key

suggested by me, but markus@ did all the work.
ok djm@

OpenBSD-Commit-ID: 8aafcebe923dc742fc5537a995cee549d07e4b2e

1 files changed, 29 insertions, 16 deletions

diff --git a/ssh-pkcs11.c b/ssh-pkcs11.c
index c36d3160..d7b3a65f 100644
--- a/ssh-pkcs11.c
+++ b/ssh-pkcs11.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: ssh-pkcs11.c,v 1.32 2019/01/20 23:05:52 djm Exp $ */
+/* $OpenBSD: ssh-pkcs11.c,v 1.33 2019/01/20 23:08:24 djm Exp $ */
 /*
  * Copyright (c) 2010 Markus Friedl.  All rights reserved.
  * Copyright (c) 2014 Pedro Martelletto. All rights reserved.
@@ -334,8 +334,8 @@ pkcs11_rsa_private_encrypt(int flen, const u_char *from, u_char *to, RSA *rsa,
 	CK_RV			rv;
 	int			rval = -1;
 
-	if ((k11 = RSA_get_app_data(rsa)) == NULL) {
-		error("RSA_get_app_data failed for rsa %p", rsa);
+	if ((k11 = RSA_get_ex_data(rsa, 0)) == NULL) {
+		error("RSA_get_ex_data failed for rsa %p", rsa);
 		return (-1);
 	}
 
@@ -365,13 +365,35 @@ pkcs11_rsa_private_decrypt(int flen, const u_char *from, u_char *to, RSA *rsa,
 	return (-1);
 }
 
+static RSA_METHOD *rsa_method;
+
+static int
+pkcs11_rsa_start_wrapper(void)
+{
+	if (rsa_method != NULL)
+		return (0);
+	rsa_method = RSA_meth_dup(RSA_get_default_method());
+	if (rsa_method == NULL)
+		return (-1);
+	if (!RSA_meth_set1_name(rsa_method, "pkcs11") ||
+	    !RSA_meth_set_priv_enc(rsa_method, pkcs11_rsa_private_encrypt) ||
+	    !RSA_meth_set_priv_dec(rsa_method, pkcs11_rsa_private_decrypt) ||
+	    !RSA_meth_set_finish(rsa_method, pkcs11_rsa_finish)) {
+		error("%s: setup pkcs11 method failed", __func__);
+		return (-1);
+	}
+	return (0);
+}
+
 /* redirect private key operations for rsa key to pkcs11 token */
 static int
 pkcs11_rsa_wrap(struct pkcs11_provider *provider, CK_ULONG slotidx,
     CK_ATTRIBUTE *keyid_attrib, RSA *rsa)
 {
 	struct pkcs11_key	*k11;
-	const RSA_METHOD	*def = RSA_get_default_method();
+
+	if (pkcs11_rsa_start_wrapper() == -1)
+		return (-1);
 
 	k11 = xcalloc(1, sizeof(*k11));
 	k11->provider = provider;
@@ -383,19 +405,10 @@ pkcs11_rsa_wrap(struct pkcs11_provider *provider, CK_ULONG slotidx,
 		k11->keyid = xmalloc(k11->keyid_len);
 		memcpy(k11->keyid, keyid_attrib->pValue, k11->keyid_len);
 	}
-	k11->rsa_method = RSA_meth_dup(def);
-	if (k11->rsa_method == NULL)
-		fatal("%s: RSA_meth_dup failed", __func__);
-	k11->orig_finish = RSA_meth_get_finish(def);
-	if (!RSA_meth_set1_name(k11->rsa_method, "pkcs11") ||
-	    !RSA_meth_set_priv_enc(k11->rsa_method,
-	    pkcs11_rsa_private_encrypt) ||
-	    !RSA_meth_set_priv_dec(k11->rsa_method,
-	    pkcs11_rsa_private_decrypt) ||
-	    !RSA_meth_set_finish(k11->rsa_method, pkcs11_rsa_finish))
-		fatal("%s: setup pkcs11 method failed", __func__);
+
+	k11->rsa_method = rsa_method;
 	RSA_set_method(rsa, k11->rsa_method);
-	RSA_set_app_data(rsa, k11);
+	RSA_set_ex_data(rsa, 0, k11);
 	return (0);
 }