Fixup ages old confusion wrt OpenPGP fingerprint vs Key ID

Originally introduced in commit f5203aea8bd83dc18e48dda4a564429c0e48bab4
in 2004, pgpPubkeyFingerprint() has been returning the 64 bits long
Key ID from the tail of 160 bits long fingerprint, not the actual
fingerprint.

Add a new public API for retrieving the Key ID specifically, adjust
the handful of internal users to use it and make pgpPubkeyFingerprint()
return the actual fingerprint. It's an API break sure but there are
unlikely to be any callers outside rpm and we're breaking the API + ABI
left and right in this release so doesn't matter...

3 files changed, 39 insertions, 14 deletions

diff --git a/rpmio/rpmkeyring.c b/rpmio/rpmkeyring.c
index ac3e3bc5a..a5c223e4b 100644
--- a/rpmio/rpmkeyring.c
+++ b/rpmio/rpmkeyring.c
@@ -133,7 +133,7 @@ rpmPubkey rpmPubkeyNew(const uint8_t *pkt, size_t pktlen)
     if (pkt == NULL || pktlen == 0)
 	goto exit;
 
-    if (pgpPubkeyFingerprint(pkt, pktlen, keyid))
+    if (pgpPubkeyKeyID(pkt, pktlen, keyid))
 	goto exit;
 
     if (pgpPrtParams(pkt, pktlen, PGPTAG_PUBLIC_KEY, &pgpkey))
diff --git a/rpmio/rpmpgp.c b/rpmio/rpmpgp.c
index 0cd55dd77..c14340a2b 100644
--- a/rpmio/rpmpgp.c
+++ b/rpmio/rpmpgp.c
@@ -765,7 +765,8 @@ static int pgpPrtUserID(pgpTag tag, const uint8_t *h, size_t hlen,
     return 0;
 }
 
-static int getFingerprint(const uint8_t *h, size_t hlen, pgpKeyID_t keyid)
+int pgpPubkeyFingerprint(const uint8_t *h, size_t hlen,
+			  uint8_t **fp, size_t *fplen)
 {
     int rc = -1; /* assume failure */
     const uint8_t *se;
@@ -800,8 +801,8 @@ static int getFingerprint(const uint8_t *h, size_t hlen, pgpKeyID_t keyid)
 	/* Does the size and number of MPI's match our expectations? */
 	if (se == pend && mpis == 0) {
 	    DIGEST_CTX ctx = rpmDigestInit(PGPHASHALGO_SHA1, RPMDIGEST_NONE);
-	    uint8_t * d = NULL;
-	    size_t dlen;
+	    uint8_t *d = NULL;
+	    size_t dlen = 0;
 	    int i = se - h;
 	    uint8_t in[3] = { 0x99, (i >> 8), i };
 
@@ -809,10 +810,12 @@ static int getFingerprint(const uint8_t *h, size_t hlen, pgpKeyID_t keyid)
 	    (void) rpmDigestUpdate(ctx, h, i);
 	    (void) rpmDigestFinal(ctx, (void **)&d, &dlen, 0);
 
-	    if (d) {
-		memcpy(keyid, (d + (dlen-8)), 8);
-		free(d);
+	    if (dlen == 20) {
 		rc = 0;
+		*fp = d;
+		*fplen = dlen;
+	    } else {
+		free(d);
 	    }
 	}
 
@@ -823,14 +826,26 @@ static int getFingerprint(const uint8_t *h, size_t hlen, pgpKeyID_t keyid)
     return rc;
 }
 
-int pgpPubkeyFingerprint(const uint8_t * pkt, size_t pktlen, pgpKeyID_t keyid)
+static int getKeyID(const uint8_t *h, size_t hlen, pgpKeyID_t keyid)
+{
+    uint8_t *fp = NULL;
+    size_t fplen = 0;
+    int rc = pgpPubkeyFingerprint(h, hlen, &fp, &fplen);
+    if (fp && fplen > 8) {
+	memcpy(keyid, (fp + (fplen-8)), 8);
+	free(fp);
+    }
+    return rc;
+}
+
+int pgpPubkeyKeyID(const uint8_t * pkt, size_t pktlen, pgpKeyID_t keyid)
 {
     struct pgpPkt p;
 
     if (decodePkt(pkt, pktlen, &p))
 	return -1;
     
-    return getFingerprint(p.body, p.blen, keyid);
+    return getKeyID(p.body, p.blen, keyid);
 }
 
 static int pgpPrtPkt(struct pgpPkt *p, pgpDigParams _digp)
@@ -842,8 +857,8 @@ static int pgpPrtPkt(struct pgpPkt *p, pgpDigParams _digp)
 	rc = pgpPrtSig(p->tag, p->body, p->blen, _digp);
 	break;
     case PGPTAG_PUBLIC_KEY:
-	/* Get the public key fingerprint. */
-	if (!getFingerprint(p->body, p->blen, _digp->signid))
+	/* Get the public key Key ID. */
+	if (!getKeyID(p->body, p->blen, _digp->signid))
 	    _digp->saved |= PGPDIG_SAVED_ID;
 	else
 	    memset(_digp->signid, 0, sizeof(_digp->signid));
@@ -1047,7 +1062,7 @@ int pgpPrtParamsSubkeys(const uint8_t *pkts, size_t pktlen,
 	    /* Copy UID from main key to subkey */
 	    digps[count]->userid = xstrdup(mainkey->userid);
 
-	    if (getFingerprint(pkt.body, pkt.blen, digps[count]->signid)) {
+	    if (getKeyID(pkt.body, pkt.blen, digps[count]->signid)) {
 		pgpDigParamsFree(digps[count]);
 		continue;
 	    }
diff --git a/rpmio/rpmpgp.h b/rpmio/rpmpgp.h
index 04e5d6606..97e43fa13 100644
--- a/rpmio/rpmpgp.h
+++ b/rpmio/rpmpgp.h
@@ -971,11 +971,21 @@ char * pgpHexStr(const uint8_t *p, size_t plen);
  * Calculate OpenPGP public key fingerprint.
  * @param pkt		OpenPGP packet (i.e. PGPTAG_PUBLIC_KEY)
  * @param pktlen	OpenPGP packet length (no. of bytes)
- * @retval keyid	public key fingerprint
+ * @retval fp		public key fingerprint
+ * @retval fplen	public key fingerprint length
  * @return		0 on success, else -1
  */
 int pgpPubkeyFingerprint(const uint8_t * pkt, size_t pktlen,
-		pgpKeyID_t keyid);
+			 uint8_t **fp, size_t *fplen);
+
+/** \ingroup rpmpgp
+ * Calculate OpenPGP public key Key ID
+ * @param pkt		OpenPGP packet (i.e. PGPTAG_PUBLIC_KEY)
+ * @param pktlen	OpenPGP packet length (no. of bytes)
+ * @retval keyid	public key Key ID
+ * @return		0 on success, else -1
+ */
+int pgpPubkeyKeyID(const uint8_t * pkt, size_t pktlen, pgpKeyID_t keyid);
 
 /** \ingroup rpmpgp
  * Parse a OpenPGP packet(s).