From 5d8d2c1be01d1e55cfa172cfe49ca97893366725 Mon Sep 17 00:00:00 2001
From: Demi Marie Obenour <demi@invisiblethingslab.com>
Date: Sun, 7 Feb 2021 16:46:31 -0500
Subject: Fix return value checks in OpenSSL code

According to `man 3ssl` the only successful return value for
EVP_PKEY_verify_init() is 1, and EVP_PKEY_CTX_set_rsa_padding() and
EVP_PKEY_CTX_set_signature_md() can both return 0 or a negative number
on failure or any positive number on success.  BN_bn2binpad() returns -1
on error, but 0 (an empty key or signature) is also not valid.
Therefore use != 1 to check the return value of EVP_PKEY_verify_init(),
<= 0 to check the return values of the other three functions mentioned
above.  Also delete a bunch of cruft.

(cherry picked from commit 1ddaeddffa52f02db198417ebf73cb6c5d432250)
---
 rpmio/digest_openssl.c | 55 +++++++++++---------------------------------------
 1 file changed, 12 insertions(+), 43 deletions(-)

diff --git a/rpmio/digest_openssl.c b/rpmio/digest_openssl.c
index a28a13acc..646933eed 100644
--- a/rpmio/digest_openssl.c
+++ b/rpmio/digest_openssl.c
@@ -450,7 +450,7 @@ static void pgpFreeSigRSA(pgpDigAlg pgpsig)
 static int pgpVerifySigRSA(pgpDigAlg pgpkey, pgpDigAlg pgpsig,
                            uint8_t *hash, size_t hashlen, int hash_algo)
 {
-    int rc, ret;
+    int rc = 1; /* assume failure */
     EVP_PKEY_CTX *pkey_ctx = NULL;
     struct pgpDigSigRSA_s *sig = pgpsig->data;
 
@@ -458,53 +458,32 @@ static int pgpVerifySigRSA(pgpDigAlg pgpkey, pgpDigAlg pgpsig,
 
     struct pgpDigKeyRSA_s *key = pgpkey->data;
 
-    if (!constructRSASigningKey(key)) {
-        rc = 1;
+    if (!constructRSASigningKey(key))
         goto done;
-    }
 
     pkey_ctx = EVP_PKEY_CTX_new(key->evp_pkey, NULL);
-    if (!pkey_ctx) {
-        rc = 1;
+    if (!pkey_ctx)
         goto done;
-    }
 
-    ret = EVP_PKEY_verify_init(pkey_ctx);
-    if (ret < 0) {
-        rc = 1;
+    if (EVP_PKEY_verify_init(pkey_ctx) != 1)
         goto done;
-    }
 
-    ret = EVP_PKEY_CTX_set_rsa_padding(pkey_ctx, RSA_PKCS1_PADDING);
-    if (ret < 0) {
-        rc = 1;
+    if (EVP_PKEY_CTX_set_rsa_padding(pkey_ctx, RSA_PKCS1_PADDING) <= 0)
         goto done;
-    }
 
-    ret = EVP_PKEY_CTX_set_signature_md(pkey_ctx, getEVPMD(hash_algo));
-    if (ret < 0) {
-        rc = 1;
+    if (EVP_PKEY_CTX_set_signature_md(pkey_ctx, getEVPMD(hash_algo)) <= 0)
         goto done;
-    }
 
     int pkey_len = EVP_PKEY_size(key->evp_pkey);
     padded_sig = xcalloc(1, pkey_len);
-    if (!BN_bn2binpad(sig->bn, padded_sig, pkey_len)) {
-        rc = 1;
+    if (BN_bn2binpad(sig->bn, padded_sig, pkey_len) <= 0)
         goto done;
-    }
 
-    ret = EVP_PKEY_verify(pkey_ctx, padded_sig, pkey_len, hash, hashlen);
-    if (ret == 1)
+    if (EVP_PKEY_verify(pkey_ctx, padded_sig, pkey_len, hash, hashlen) == 1)
     {
         /* Success */
         rc = 0;
     }
-    else
-    {
-        /* Failure */
-        rc = 1;
-    }
 
 done:
     EVP_PKEY_CTX_free(pkey_ctx);
@@ -735,32 +714,22 @@ static void pgpFreeSigDSA(pgpDigAlg pgpsig)
 static int pgpVerifySigDSA(pgpDigAlg pgpkey, pgpDigAlg pgpsig,
                            uint8_t *hash, size_t hashlen, int hash_algo)
 {
-    int rc, ret;
+    int rc = 1; /* assume failure */
     struct pgpDigSigDSA_s *sig = pgpsig->data;
 
     struct pgpDigKeyDSA_s *key = pgpkey->data;
 
-    if (!constructDSASigningKey(key)) {
-        rc = 1;
+    if (!constructDSASigningKey(key))
         goto done;
-    }
 
-    if (!constructDSASignature(sig)) {
-        rc = 1;
+    if (!constructDSASignature(sig))
         goto done;
-    }
 
-    ret = DSA_do_verify(hash, hashlen, sig->dsa_sig, key->dsa_key);
-    if (ret == 1)
+    if (DSA_do_verify(hash, hashlen, sig->dsa_sig, key->dsa_key) == 1)
     {
         /* Success */
         rc = 0;
     }
-    else
-    {
-        /* Failure */
-        rc = 1;
-    }
 
 done:
     return rc;
-- 
cgit v1.2.1