Adapt i2d_PrivateKey for provider only keys

It uses EVP_PKEY serializers to get the desired results.

One might think that ddoing this might make things a bit dicy for
existing serializers, as they should obviously use i2d functions.
However, since our serializers use much more primitive functions
such as i2d_ASN1_INTEGER(), or keytype specific ones such as
i2d_RSAPrivateKey(), there is no clash.

Reviewed-by: Shane Lontis <shane.lontis@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/11056)

1 files changed, 32 insertions, 0 deletions

diff --git a/crypto/asn1/i2d_pr.c b/crypto/asn1/i2d_pr.c
index 4decdb1dd7..dd2a82da74 100644
--- a/crypto/asn1/i2d_pr.c
+++ b/crypto/asn1/i2d_pr.c
@@ -10,6 +10,8 @@
 #include <stdio.h>
 #include "internal/cryptlib.h"
 #include <openssl/evp.h>
+#include <openssl/serializer.h>
+#include <openssl/buffer.h>
 #include <openssl/x509.h>
 #include "crypto/asn1.h"
 #include "crypto/evp.h"
@@ -28,6 +30,36 @@ int i2d_PrivateKey(const EVP_PKEY *a, unsigned char **pp)
         }
         return ret;
     }
+    if (a->pkeys[0].keymgmt != NULL) {
+        const char *serprop = OSSL_SERIALIZER_PrivateKey_TO_DER_PQ;
+        OSSL_SERIALIZER_CTX *ctx =
+            OSSL_SERIALIZER_CTX_new_by_EVP_PKEY(a, serprop);
+        BIO *out = BIO_new(BIO_s_mem());
+        BUF_MEM *buf = NULL;
+        int ret = -1;
+
+        if (ctx != NULL
+            && out != NULL
+            && OSSL_SERIALIZER_CTX_get_serializer(ctx) != NULL
+            && OSSL_SERIALIZER_to_bio(ctx, out)
+            && BIO_get_mem_ptr(out, &buf) > 0) {
+            ret = buf->length;
+
+            if (pp != NULL) {
+                if (*pp == NULL) {
+                    *pp = (unsigned char *)buf->data;
+                    buf->length = 0;
+                    buf->data = NULL;
+                } else {
+                    memcpy(*pp, buf->data, ret);
+                    *pp += ret;
+                }
+            }
+        }
+        BIO_free(out);
+        OSSL_SERIALIZER_CTX_free(ctx);
+        return ret;
+    }
     ASN1err(ASN1_F_I2D_PRIVATEKEY, ASN1_R_UNSUPPORTED_PUBLIC_KEY_TYPE);
     return -1;
 }