Fix malloc failure handling of X509_ALGOR_set0()

Also update and slightly extend the respective documentation and simplify some code.

Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/16251)

6 files changed, 36 insertions, 36 deletions

diff --git a/crypto/cms/cms_cd.c b/crypto/cms/cms_cd.c
index 6de6d55e58..a7f47a6a3d 100644
--- a/crypto/cms/cms_cd.c
+++ b/crypto/cms/cms_cd.c
@@ -50,8 +50,9 @@ CMS_ContentInfo *ossl_cms_CompressedData_create(int comp_nid,
 
     cd->version = 0;
 
-    X509_ALGOR_set0(cd->compressionAlgorithm,
-                    OBJ_nid2obj(NID_zlib_compression), V_ASN1_UNDEF, NULL);
+    (void)X509_ALGOR_set0(cd->compressionAlgorithm,
+                          OBJ_nid2obj(NID_zlib_compression),
+                          V_ASN1_UNDEF, NULL); /* cannot fail */
 
     cd->encapContentInfo->eContentType = OBJ_nid2obj(NID_pkcs7_data);
 
diff --git a/crypto/cms/cms_dh.c b/crypto/cms/cms_dh.c
index f14546c703..2ff85d979b 100644
--- a/crypto/cms/cms_dh.c
+++ b/crypto/cms/cms_dh.c
@@ -238,8 +238,8 @@ static int dh_cms_encrypt(CMS_RecipientInfo *ri)
         pubkey->flags |= ASN1_STRING_FLAG_BITS_LEFT;
 
         penc = NULL;
-        X509_ALGOR_set0(talg, OBJ_nid2obj(NID_dhpublicnumber),
-                        V_ASN1_UNDEF, NULL);
+        (void)X509_ALGOR_set0(talg, OBJ_nid2obj(NID_dhpublicnumber),
+                              V_ASN1_UNDEF, NULL); /* cannot fail */
     }
 
     /* See if custom parameters set */
@@ -316,10 +316,10 @@ static int dh_cms_encrypt(CMS_RecipientInfo *ri)
         goto err;
     ASN1_STRING_set0(wrap_str, penc, penclen);
     penc = NULL;
-    X509_ALGOR_set0(talg, OBJ_nid2obj(NID_id_smime_alg_ESDH),
-                    V_ASN1_SEQUENCE, wrap_str);
-
-    rv = 1;
+    rv = X509_ALGOR_set0(talg, OBJ_nid2obj(NID_id_smime_alg_ESDH),
+                         V_ASN1_SEQUENCE, wrap_str);
+    if (!rv)
+        ASN1_STRING_free(wrap_str);
 
  err:
     OPENSSL_free(penc);
diff --git a/crypto/cms/cms_ec.c b/crypto/cms/cms_ec.c
index b07af92bad..fd6c5d7077 100644
--- a/crypto/cms/cms_ec.c
+++ b/crypto/cms/cms_ec.c
@@ -281,8 +281,8 @@ static int ecdh_cms_encrypt(CMS_RecipientInfo *ri)
         pubkey->flags |= ASN1_STRING_FLAG_BITS_LEFT;
 
         penc = NULL;
-        X509_ALGOR_set0(talg, OBJ_nid2obj(NID_X9_62_id_ecPublicKey),
-                        V_ASN1_UNDEF, NULL);
+        (void)X509_ALGOR_set0(talg, OBJ_nid2obj(NID_X9_62_id_ecPublicKey),
+                              V_ASN1_UNDEF, NULL); /* cannot fail */
     }
 
     /* See if custom parameters set */
@@ -365,9 +365,9 @@ static int ecdh_cms_encrypt(CMS_RecipientInfo *ri)
         goto err;
     ASN1_STRING_set0(wrap_str, penc, penclen);
     penc = NULL;
-    X509_ALGOR_set0(talg, OBJ_nid2obj(kdf_nid), V_ASN1_SEQUENCE, wrap_str);
-
-    rv = 1;
+    rv = X509_ALGOR_set0(talg, OBJ_nid2obj(kdf_nid), V_ASN1_SEQUENCE, wrap_str);
+    if (!rv)
+        ASN1_STRING_free(wrap_str);
 
  err:
     OPENSSL_free(penc);
@@ -394,7 +394,7 @@ int ossl_cms_ecdsa_dsa_sign(CMS_SignerInfo *si, int verify)
 {
     assert(verify == 0 || verify == 1);
 
-    if (verify == 0) {
+    if (!verify) {
         int snid, hnid;
         X509_ALGOR *alg1, *alg2;
         EVP_PKEY *pkey = si->pkey;
@@ -407,7 +407,7 @@ int ossl_cms_ecdsa_dsa_sign(CMS_SignerInfo *si, int verify)
             return -1;
         if (!OBJ_find_sigid_by_algs(&snid, hnid, EVP_PKEY_get_id(pkey)))
             return -1;
-        X509_ALGOR_set0(alg2, OBJ_nid2obj(snid), V_ASN1_UNDEF, 0);
+        return X509_ALGOR_set0(alg2, OBJ_nid2obj(snid), V_ASN1_UNDEF, NULL);
     }
     return 1;
 }
diff --git a/crypto/cms/cms_env.c b/crypto/cms/cms_env.c
index ca8f84f14a..6374e20c4f 100644
--- a/crypto/cms/cms_env.c
+++ b/crypto/cms/cms_env.c
@@ -730,8 +730,8 @@ CMS_RecipientInfo *CMS_add0_recipient_key(CMS_ContentInfo *cms, int nid,
         kekri->kekid->other->keyAttr = otherType;
     }
 
-    X509_ALGOR_set0(kekri->keyEncryptionAlgorithm,
-                    OBJ_nid2obj(nid), V_ASN1_UNDEF, NULL);
+    (void)X509_ALGOR_set0(kekri->keyEncryptionAlgorithm, OBJ_nid2obj(nid),
+                          V_ASN1_UNDEF, NULL); /* cannot fail */
 
     return ri;
 
diff --git a/crypto/cms/cms_rsa.c b/crypto/cms/cms_rsa.c
index eafa1788de..64fab3e392 100644
--- a/crypto/cms/cms_rsa.c
+++ b/crypto/cms/cms_rsa.c
@@ -123,10 +123,10 @@ static int rsa_cms_encrypt(CMS_RecipientInfo *ri)
         if (EVP_PKEY_CTX_get_rsa_padding(pkctx, &pad_mode) <= 0)
             return 0;
     }
-    if (pad_mode == RSA_PKCS1_PADDING) {
-        X509_ALGOR_set0(alg, OBJ_nid2obj(NID_rsaEncryption), V_ASN1_NULL, 0);
-        return 1;
-    }
+    if (pad_mode == RSA_PKCS1_PADDING)
+        return X509_ALGOR_set0(alg, OBJ_nid2obj(NID_rsaEncryption),
+                               V_ASN1_NULL, NULL);
+
     /* Not supported */
     if (pad_mode != RSA_PKCS1_OAEP_PADDING)
         return 0;
@@ -160,8 +160,9 @@ static int rsa_cms_encrypt(CMS_RecipientInfo *ri)
     }
     /* create string with pss parameter encoding. */
     if (!ASN1_item_pack(oaep, ASN1_ITEM_rptr(RSA_OAEP_PARAMS), &os))
-         goto err;
-    X509_ALGOR_set0(alg, OBJ_nid2obj(NID_rsaesOaep), V_ASN1_SEQUENCE, os);
+        goto err;
+    if (!X509_ALGOR_set0(alg, OBJ_nid2obj(NID_rsaesOaep), V_ASN1_SEQUENCE, os))
+        goto err;
     os = NULL;
     rv = 1;
  err:
@@ -196,18 +197,21 @@ static int rsa_cms_sign(CMS_SignerInfo *si)
         if (EVP_PKEY_CTX_get_rsa_padding(pkctx, &pad_mode) <= 0)
             return 0;
     }
-    if (pad_mode == RSA_PKCS1_PADDING) {
-        X509_ALGOR_set0(alg, OBJ_nid2obj(NID_rsaEncryption), V_ASN1_NULL, 0);
-        return 1;
-    }
+    if (pad_mode == RSA_PKCS1_PADDING)
+        return X509_ALGOR_set0(alg, OBJ_nid2obj(NID_rsaEncryption),
+                               V_ASN1_NULL, NULL);
+
     /* We don't support it */
     if (pad_mode != RSA_PKCS1_PSS_PADDING)
         return 0;
     os = ossl_rsa_ctx_to_pss_string(pkctx);
     if (os == NULL)
         return 0;
-    X509_ALGOR_set0(alg, OBJ_nid2obj(EVP_PKEY_RSA_PSS), V_ASN1_SEQUENCE, os);
-    return 1;
+    if (X509_ALGOR_set0(alg, OBJ_nid2obj(EVP_PKEY_RSA_PSS),
+                        V_ASN1_SEQUENCE, os))
+        return 1;
+    ASN1_STRING_free(os);
+    return 0;
 }
 
 static int rsa_cms_verify(CMS_SignerInfo *si)
diff --git a/crypto/cms/cms_sd.c b/crypto/cms/cms_sd.c
index 7a77a0870a..8985be4fb4 100644
--- a/crypto/cms/cms_sd.c
+++ b/crypto/cms/cms_sd.c
@@ -354,6 +354,7 @@ CMS_SignerInfo *CMS_add1_signer(CMS_ContentInfo *cms,
 
     if (md == NULL) {
         int def_nid;
+
         if (EVP_PKEY_get_default_digest_nid(pk, &def_nid) <= 0)
             goto err;
         md = EVP_get_digestbynid(def_nid);
@@ -363,11 +364,6 @@ CMS_SignerInfo *CMS_add1_signer(CMS_ContentInfo *cms,
         }
     }
 
-    if (!md) {
-        ERR_raise(ERR_LIB_CMS, CMS_R_NO_DIGEST_SET);
-        goto err;
-    }
-
     if (md == NULL) {
         ERR_raise(ERR_LIB_CMS, CMS_R_NO_DIGEST_SET);
         goto err;
@@ -388,8 +384,7 @@ CMS_SignerInfo *CMS_add1_signer(CMS_ContentInfo *cms,
     }
 
     if (i == sk_X509_ALGOR_num(sd->digestAlgorithms)) {
-        alg = X509_ALGOR_new();
-        if (alg == NULL)
+        if ((alg = X509_ALGOR_new()) == NULL)
             goto merr;
         X509_ALGOR_set_md(alg, md);
         if (!sk_X509_ALGOR_push(sd->digestAlgorithms, alg)) {