Bug 1340103, Introduction of SECKEYECPublicKey.encoding in NSS 3.28 broke ABI, r=rrelyea/mt

3 files changed, 27 insertions, 55 deletions

diff --git a/lib/cryptohi/keyi.h b/lib/cryptohi/keyi.h
index 374a4ad9b..f8f5f7f7d 100644
--- a/lib/cryptohi/keyi.h
+++ b/lib/cryptohi/keyi.h
@@ -17,13 +17,6 @@ KeyType seckey_GetKeyType(SECOidTag pubKeyOid);
 SECStatus sec_DecodeSigAlg(const SECKEYPublicKey *key, SECOidTag sigAlg,
                            const SECItem *param, SECOidTag *encalg, SECOidTag *hashalg);
 
-/*
- * Set the point encoding of a SECKEYPublicKey from the OID.
- * This has to be called on any SECKEYPublicKey holding a SECKEYECPublicKey
- * before it can be used. The encoding is used to dermine the public key size.
- */
-SECStatus seckey_SetPointEncoding(PLArenaPool *arena, SECKEYPublicKey *pubKey);
-
 SEC_END_PROTOS
 
 #endif /* _KEYHI_H_ */
diff --git a/lib/cryptohi/keythi.h b/lib/cryptohi/keythi.h
index 1555ce2e2..36896540f 100644
--- a/lib/cryptohi/keythi.h
+++ b/lib/cryptohi/keythi.h
@@ -125,9 +125,9 @@ typedef SECItem SECKEYECParams;
 
 struct SECKEYECPublicKeyStr {
     SECKEYECParams DEREncodedParams;
-    int size;            /* size in bits */
-    SECItem publicValue; /* encoded point */
-    ECPointEncoding encoding;
+    int size;                 /* size in bits */
+    SECItem publicValue;      /* encoded point */
+    ECPointEncoding encoding; /* deprecated, ignored */
 };
 typedef struct SECKEYECPublicKeyStr SECKEYECPublicKey;
 
diff --git a/lib/cryptohi/seckey.c b/lib/cryptohi/seckey.c
index df976f46c..9ea48b767 100644
--- a/lib/cryptohi/seckey.c
+++ b/lib/cryptohi/seckey.c
@@ -547,6 +547,23 @@ CERT_GetCertKeyType(const CERTSubjectPublicKeyInfo *spki)
     return seckey_GetKeyType(SECOID_GetAlgorithmTag(&spki->algorithm));
 }
 
+/* Ensure pubKey contains an OID */
+static SECStatus
+seckey_HasCurveOID(const SECKEYPublicKey *pubKey)
+{
+    SECItem oid;
+    SECStatus rv;
+    PORTCheapArenaPool tmpArena;
+
+    PORT_InitCheapArena(&tmpArena, DER_DEFAULT_CHUNKSIZE);
+    /* If we can decode it, an OID is available. */
+    rv = SEC_QuickDERDecodeItem(&tmpArena.arena, &oid,
+                                SEC_ASN1_GET(SEC_ObjectIDTemplate),
+                                &pubKey->u.ec.DEREncodedParams);
+    PORT_DestroyCheapArena(&tmpArena);
+    return rv;
+}
+
 static SECKEYPublicKey *
 seckey_ExtractPublicKey(const CERTSubjectPublicKeyInfo *spki)
 {
@@ -639,7 +656,8 @@ seckey_ExtractPublicKey(const CERTSubjectPublicKeyInfo *spki)
                 if (rv != SECSuccess) {
                     break;
                 }
-                rv = seckey_SetPointEncoding(arena, pubk);
+                pubk->u.ec.encoding = ECPoint_Undefined;
+                rv = seckey_HasCurveOID(pubk);
                 if (rv == SECSuccess) {
                     return pubk;
                 }
@@ -1162,16 +1180,16 @@ SECKEY_CopyPublicKey(const SECKEYPublicKey *pubk)
             break;
         case ecKey:
             copyk->u.ec.size = pubk->u.ec.size;
-            rv = SECITEM_CopyItem(arena, &copyk->u.ec.DEREncodedParams,
-                                  &pubk->u.ec.DEREncodedParams);
+            rv = seckey_HasCurveOID(pubk);
             if (rv != SECSuccess) {
                 break;
             }
-            rv = seckey_SetPointEncoding(arena, copyk);
+            rv = SECITEM_CopyItem(arena, &copyk->u.ec.DEREncodedParams,
+                                  &pubk->u.ec.DEREncodedParams);
             if (rv != SECSuccess) {
                 break;
             }
-            PORT_Assert(copyk->u.ec.encoding == pubk->u.ec.encoding);
+            copyk->u.ec.encoding = ECPoint_Undefined;
             rv = SECITEM_CopyItem(arena, &copyk->u.ec.publicValue,
                                   &pubk->u.ec.publicValue);
             break;
@@ -1253,10 +1271,7 @@ SECKEY_ConvertToPublicKey(SECKEYPrivateKey *privk)
             if (rv != SECSuccess || pubk->u.ec.publicValue.len == 0) {
                 break;
             }
-            rv = seckey_SetPointEncoding(arena, pubk);
-            if (rv != SECSuccess) {
-                break;
-            }
+            pubk->u.ec.encoding = ECPoint_Undefined;
             return pubk;
         default:
             break;
@@ -1959,39 +1974,3 @@ SECKEY_GetECCOid(const SECKEYECParams *params)
 
     return oidData->offset;
 }
-
-/* Set curve encoding in SECKEYECPublicKey in pubKey from OID.
- * If the encoding is not set, determining the key size of EC public keys will
- * fail.
- */
-SECStatus
-seckey_SetPointEncoding(PLArenaPool *arena, SECKEYPublicKey *pubKey)
-{
-    SECItem oid;
-    SECOidTag tag;
-    SECStatus rv;
-
-    /* decode the OID tag */
-    rv = SEC_QuickDERDecodeItem(arena, &oid, SEC_ASN1_GET(SEC_ObjectIDTemplate),
-                                &pubKey->u.ec.DEREncodedParams);
-    if (rv != SECSuccess) {
-        return SECFailure;
-    }
-
-    tag = SECOID_FindOIDTag(&oid);
-    switch (tag) {
-        case SEC_OID_CURVE25519:
-            pubKey->u.ec.encoding = ECPoint_XOnly;
-            break;
-        case SEC_OID_SECG_EC_SECP256R1:
-        /* fall through */
-        case SEC_OID_SECG_EC_SECP384R1:
-        /* fall through */
-        case SEC_OID_SECG_EC_SECP521R1:
-        /* fall through */
-        default:
-            /* unknown curve, default to uncompressed */
-            pubKey->u.ec.encoding = ECPoint_Uncompressed;
-    }
-    return SECSuccess;
-}