Add new functions PK11_PrivDecryptPKCS1 and PK11_PubEncryptPKCS1

which do PKCS1 based RSA data encryption/decryption (as opposed to key
wrapping).  This facilitates the migration of some legacy applications
from NSS 2.x to NSS 3.x.  Bugscape 57169.  bugzillla is still down.

3 files changed, 56 insertions, 13 deletions

diff --git a/security/nss/lib/nss/nss.def b/security/nss/lib/nss/nss.def
index e14a52d24..63a03d961 100644
--- a/security/nss/lib/nss/nss.def
+++ b/security/nss/lib/nss/nss.def
@@ -799,6 +799,8 @@ PK11_UnlinkGenericObject;
 ;+};
 ;+NSS_3.9.3 { 	# NSS 3.9.3 release
 ;+    global:
+PK11_PrivDecryptPKCS1;
+PK11_PubEncryptPKCS1;
 SECMOD_CancelWait;
 SECMOD_HasRemovableSlots;
 SECMOD_UpdateSlotList;
diff --git a/security/nss/lib/pk11wrap/pk11func.h b/security/nss/lib/pk11wrap/pk11func.h
index acf33f251..7d92e32f8 100644
--- a/security/nss/lib/pk11wrap/pk11func.h
+++ b/security/nss/lib/pk11wrap/pk11func.h
@@ -415,11 +415,21 @@ SECKEYPrivateKey * PK11_FindKeyByKeyID(PK11SlotInfo *slot, SECItem *keyID,
 CK_OBJECT_HANDLE PK11_FindObjectForCert(CERTCertificate *cert,
 					void *wincx, PK11SlotInfo **pSlot);
 int PK11_GetPrivateModulusLen(SECKEYPrivateKey *key); 
+
+/* note: despite the name, this function takes a private key. */
 SECStatus PK11_PubDecryptRaw(SECKEYPrivateKey *key, unsigned char *data,
    unsigned *outLen, unsigned int maxLen, unsigned char *enc, unsigned encLen);
-/* The encrypt version of the above function */
+#define PK11_PrivDecryptRaw PK11_PubDecryptRaw
+/* The encrypt function that complements the above decrypt function. */
 SECStatus PK11_PubEncryptRaw(SECKEYPublicKey *key, unsigned char *enc,
                 unsigned char *data, unsigned dataLen, void *wincx);
+
+SECStatus PK11_PrivDecryptPKCS1(SECKEYPrivateKey *key, unsigned char *data,
+   unsigned *outLen, unsigned int maxLen, unsigned char *enc, unsigned encLen);
+/* The encrypt function that complements the above decrypt function. */
+SECStatus PK11_PubEncryptPKCS1(SECKEYPublicKey *key, unsigned char *enc,
+                unsigned char *data, unsigned dataLen, void *wincx);
+
 SECStatus PK11_ImportPrivateKeyInfo(PK11SlotInfo *slot, 
 		SECKEYPrivateKeyInfo *pki, SECItem *nickname,
 		SECItem *publicValue, PRBool isPerm, PRBool isPrivate,
diff --git a/security/nss/lib/pk11wrap/pk11skey.c b/security/nss/lib/pk11wrap/pk11skey.c
index 38e304294..d316f5d5e 100644
--- a/security/nss/lib/pk11wrap/pk11skey.c
+++ b/security/nss/lib/pk11wrap/pk11skey.c
@@ -3623,13 +3623,12 @@ PK11_Sign(SECKEYPrivateKey *key, SECItem *sig, SECItem *hash)
  * then we need to move this check into some of PK11_PubDecrypt callers,
  * (namely SSL 2.0).
  */
-SECStatus
-PK11_PubDecryptRaw(SECKEYPrivateKey *key, unsigned char *data, 
+static SECStatus
+pk11_PrivDecryptRaw(SECKEYPrivateKey *key, unsigned char *data, 
 	unsigned *outLen, unsigned int maxLen, unsigned char *enc,
-							 unsigned encLen)
+			     unsigned encLen, CK_MECHANISM_PTR mech)
 {
     PK11SlotInfo *slot = key->pkcs11Slot;
-    CK_MECHANISM mech = {CKM_RSA_X_509, NULL, 0 };
     CK_ULONG out = maxLen;
     PRBool owner = PR_TRUE;
     CK_SESSION_HANDLE session;
@@ -3649,7 +3648,7 @@ PK11_PubDecryptRaw(SECKEYPrivateKey *key, unsigned char *data,
     }
     session = pk11_GetNewSession(slot,&owner);
     if (!owner || !(slot->isThreadSafe)) PK11_EnterSlotMonitor(slot);
-    crv = PK11_GETTAB(slot)->C_DecryptInit(session,&mech,key->pkcs11ID);
+    crv = PK11_GETTAB(slot)->C_DecryptInit(session, mech, key->pkcs11ID);
     if (crv != CKR_OK) {
 	if (!owner || !(slot->isThreadSafe)) PK11_ExitSlotMonitor(slot);
 	pk11_CloseSession(slot,session,owner);
@@ -3668,13 +3667,30 @@ PK11_PubDecryptRaw(SECKEYPrivateKey *key, unsigned char *data,
     return SECSuccess;
 }
 
-/* The encrypt version of the above function */
 SECStatus
-PK11_PubEncryptRaw(SECKEYPublicKey *key, unsigned char *enc,
-		unsigned char *data, unsigned dataLen, void *wincx)
+PK11_PubDecryptRaw(SECKEYPrivateKey *key, unsigned char *data, 
+	unsigned *outLen, unsigned int maxLen, unsigned char *enc,
+							 unsigned encLen)
 {
-    PK11SlotInfo *slot;
     CK_MECHANISM mech = {CKM_RSA_X_509, NULL, 0 };
+    return pk11_PrivDecryptRaw(key, data, outLen, maxLen, enc, encLen, &mech);
+}
+
+SECStatus
+PK11_PrivDecryptPKCS1(SECKEYPrivateKey *key, unsigned char *data, 
+	unsigned *outLen, unsigned int maxLen, unsigned char *enc,
+				    unsigned encLen)
+{
+    CK_MECHANISM mech = {CKM_RSA_PKCS, NULL, 0 };
+    return pk11_PrivDecryptRaw(key, data, outLen, maxLen, enc, encLen, &mech);
+}
+
+SECStatus
+pk11_PubEncryptRaw(SECKEYPublicKey *key, unsigned char *enc,
+		unsigned char *data, unsigned dataLen, 
+		CK_MECHANISM_PTR mech, void *wincx)
+{
+    PK11SlotInfo *slot;
     CK_OBJECT_HANDLE id;
     CK_ULONG out = dataLen;
     PRBool owner = PR_TRUE;
@@ -3686,7 +3702,7 @@ PK11_PubEncryptRaw(SECKEYPublicKey *key, unsigned char *enc,
 	return SECFailure;
     }
 
-    slot = PK11_GetBestSlot(mech.mechanism, wincx);
+    slot = PK11_GetBestSlot(mech->mechanism, wincx);
     if (slot == NULL) {
 	PORT_SetError( SEC_ERROR_NO_MODULE );
 	return SECFailure;
@@ -3696,7 +3712,7 @@ PK11_PubEncryptRaw(SECKEYPublicKey *key, unsigned char *enc,
 
     session = pk11_GetNewSession(slot,&owner);
     if (!owner || !(slot->isThreadSafe)) PK11_EnterSlotMonitor(slot);
-    crv = PK11_GETTAB(slot)->C_EncryptInit(session,&mech,id);
+    crv = PK11_GETTAB(slot)->C_EncryptInit(session, mech, id);
     if (crv != CKR_OK) {
 	if (!owner || !(slot->isThreadSafe)) PK11_ExitSlotMonitor(slot);
 	pk11_CloseSession(slot,session,owner);
@@ -3715,7 +3731,22 @@ PK11_PubEncryptRaw(SECKEYPublicKey *key, unsigned char *enc,
     return SECSuccess;
 }
 
-	
+SECStatus
+PK11_PubEncryptRaw(SECKEYPublicKey *key, unsigned char *enc,
+		unsigned char *data, unsigned dataLen, void *wincx)
+{
+    CK_MECHANISM mech = {CKM_RSA_X_509, NULL, 0 };
+    return pk11_PubEncryptRaw(key, enc, data, dataLen, &mech, wincx);
+}
+
+SECStatus
+PK11_PubEncryptPKCS1(SECKEYPublicKey *key, unsigned char *enc,
+		unsigned char *data, unsigned dataLen, void *wincx)
+{
+    CK_MECHANISM mech = {CKM_RSA_PKCS, NULL, 0 };
+    return pk11_PubEncryptRaw(key, enc, data, dataLen, &mech, wincx);
+}
+
 /**********************************************************************
  *
  *                   Now Deal with Crypto Contexts