ENCODER: Refactor provider implementations, and some cleanup

The encoder implementations were implemented by unnecessarily copying
code into numerous topical source files, making them hard to maintain.
This changes merges all those into two source files, one that encodes
into DER and PEM, the other to text.

Diverse small cleanups are included.

Reviewed-by: Shane Lontis <shane.lontis@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/12803)

1 files changed, 132 insertions, 73 deletions

diff --git a/providers/encoders.inc b/providers/encoders.inc
index 4d894a73e5..284703c6e8 100644
--- a/providers/encoders.inc
+++ b/providers/encoders.inc
@@ -11,102 +11,161 @@
 # error Macro ENCODER undefined
 #endif
 
-    ENCODER("RSA", "yes", "text", "private", rsa_priv_text_encoder_functions),
-    ENCODER("RSA", "yes", "text", "public", rsa_pub_text_encoder_functions),
-    ENCODER("RSA", "yes", "der", "private", rsa_priv_der_encoder_functions),
-    ENCODER("RSA", "yes", "der", "public", rsa_pub_der_encoder_functions),
-    ENCODER("RSA", "yes", "pem", "private", rsa_priv_pem_encoder_functions),
-    ENCODER("RSA", "yes", "pem", "public", rsa_pub_pem_encoder_functions),
+    ENCODER("RSA", "yes", "text", "private",
+            rsa_priv_to_text_encoder_functions),
+    ENCODER("RSA", "yes", "text", "public",
+            rsa_pub_to_text_encoder_functions),
+    ENCODER("RSA", "yes", "der", "private",
+            rsa_priv_to_der_encoder_functions),
+    ENCODER("RSA", "yes", "der", "public",
+            rsa_pub_to_der_encoder_functions),
+    ENCODER("RSA", "yes", "pem", "private",
+            rsa_priv_to_pem_encoder_functions),
+    ENCODER("RSA", "yes", "pem", "public",
+            rsa_pub_to_pem_encoder_functions),
     ENCODER("RSA-PSS", "yes", "text", "private",
-        rsa_priv_text_encoder_functions),
-    ENCODER("RSA-PSS", "yes", "text", "public", rsa_pub_text_encoder_functions),
-    ENCODER("RSA-PSS", "yes", "der", "private", rsa_priv_der_encoder_functions),
-    ENCODER("RSA-PSS", "yes", "der", "public", rsa_pub_der_encoder_functions),
-    ENCODER("RSA-PSS", "yes", "pem", "private", rsa_priv_pem_encoder_functions),
-    ENCODER("RSA-PSS", "yes", "pem", "public", rsa_pub_pem_encoder_functions),
+            rsa_priv_to_text_encoder_functions),
+    ENCODER("RSA-PSS", "yes", "text", "public",
+            rsa_pub_to_text_encoder_functions),
+    ENCODER("RSA-PSS", "yes", "der", "private",
+            rsa_priv_to_der_encoder_functions),
+    ENCODER("RSA-PSS", "yes", "der", "public",
+            rsa_pub_to_der_encoder_functions),
+    ENCODER("RSA-PSS", "yes", "pem", "private",
+            rsa_priv_to_pem_encoder_functions),
+    ENCODER("RSA-PSS", "yes", "pem", "public",
+            rsa_pub_to_pem_encoder_functions),
 
 #ifndef OPENSSL_NO_DH
-    ENCODER("DH", "yes", "text", "private", dh_priv_text_encoder_functions),
-    ENCODER("DH", "yes", "text", "public", dh_pub_text_encoder_functions),
-    ENCODER("DH", "yes", "text", "parameters", dh_param_text_encoder_functions),
-    ENCODER("DH", "yes", "der", "private", dh_priv_der_encoder_functions),
-    ENCODER("DH", "yes", "der", "public", dh_pub_der_encoder_functions),
-    ENCODER("DH", "yes", "der", "parameters", dh_param_der_encoder_functions),
-    ENCODER("DH", "yes", "pem", "private", dh_priv_pem_encoder_functions),
-    ENCODER("DH", "yes", "pem", "public", dh_pub_pem_encoder_functions),
-    ENCODER("DH", "yes", "pem", "parameters", dh_param_pem_encoder_functions),
+    ENCODER("DH", "yes", "text", "private",
+            dh_priv_to_text_encoder_functions),
+    ENCODER("DH", "yes", "text", "public",
+            dh_pub_to_text_encoder_functions),
+    ENCODER("DH", "yes", "text", "parameters",
+            dh_param_to_text_encoder_functions),
+    ENCODER("DH", "yes", "der", "private",
+            dh_priv_to_der_encoder_functions),
+    ENCODER("DH", "yes", "der", "public",
+            dh_pub_to_der_encoder_functions),
+    ENCODER("DH", "yes", "der", "parameters",
+            dh_param_to_der_encoder_functions),
+    ENCODER("DH", "yes", "pem", "private",
+            dh_priv_to_pem_encoder_functions),
+    ENCODER("DH", "yes", "pem", "public",
+            dh_pub_to_pem_encoder_functions),
+    ENCODER("DH", "yes", "pem", "parameters",
+            dh_param_to_pem_encoder_functions),
 
-    ENCODER("DHX", "yes", "text", "private", dh_priv_text_encoder_functions),
-    ENCODER("DHX", "yes", "text", "public", dh_pub_text_encoder_functions),
-    ENCODER("DHX", "yes", "text", "parameters", dh_param_text_encoder_functions),
-    ENCODER("DHX", "yes", "der", "private", dh_priv_der_encoder_functions),
-    ENCODER("DHX", "yes", "der", "public", dh_pub_der_encoder_functions),
-    ENCODER("DHX", "yes", "der", "parameters", dh_param_der_encoder_functions),
-    ENCODER("DHX", "yes", "pem", "private", dh_priv_pem_encoder_functions),
-    ENCODER("DHX", "yes", "pem", "public", dh_pub_pem_encoder_functions),
-    ENCODER("DHX", "yes", "pem", "parameters", dh_param_pem_encoder_functions),
+    ENCODER("DHX", "yes", "text", "private",
+            dh_priv_to_text_encoder_functions),
+    ENCODER("DHX", "yes", "text", "public",
+            dh_pub_to_text_encoder_functions),
+    ENCODER("DHX", "yes", "text", "parameters",
+            dh_param_to_text_encoder_functions),
+    ENCODER("DHX", "yes", "der", "private",
+            dh_priv_to_der_encoder_functions),
+    ENCODER("DHX", "yes", "der", "public",
+            dh_pub_to_der_encoder_functions),
+    ENCODER("DHX", "yes", "der", "parameters",
+            dh_param_to_der_encoder_functions),
+    ENCODER("DHX", "yes", "pem", "private",
+            dh_priv_to_pem_encoder_functions),
+    ENCODER("DHX", "yes", "pem", "public",
+            dh_pub_to_pem_encoder_functions),
+    ENCODER("DHX", "yes", "pem", "parameters",
+            dh_param_to_pem_encoder_functions),
 #endif
 
 #ifndef OPENSSL_NO_DSA
-    ENCODER("DSA", "yes", "text", "private", dsa_priv_text_encoder_functions),
-    ENCODER("DSA", "yes", "text", "public", dsa_pub_text_encoder_functions),
+    ENCODER("DSA", "yes", "text", "private",
+            dsa_priv_to_text_encoder_functions),
+    ENCODER("DSA", "yes", "text", "public",
+            dsa_pub_to_text_encoder_functions),
     ENCODER("DSA", "yes", "text", "parameters",
-        dsa_param_text_encoder_functions),
-    ENCODER("DSA", "yes", "der", "private", dsa_priv_der_encoder_functions),
-    ENCODER("DSA", "yes", "der", "public", dsa_pub_der_encoder_functions),
-    ENCODER("DSA", "yes", "der", "parameters", dsa_param_der_encoder_functions),
-    ENCODER("DSA", "yes", "pem", "private", dsa_priv_pem_encoder_functions),
-    ENCODER("DSA", "yes", "pem", "public", dsa_pub_pem_encoder_functions),
-    ENCODER("DSA", "yes", "pem", "parameters", dsa_param_pem_encoder_functions),
+            dsa_param_to_text_encoder_functions),
+    ENCODER("DSA", "yes", "der", "private",
+            dsa_priv_to_der_encoder_functions),
+    ENCODER("DSA", "yes", "der", "public",
+            dsa_pub_to_der_encoder_functions),
+    ENCODER("DSA", "yes", "der", "parameters",
+            dsa_param_to_der_encoder_functions),
+    ENCODER("DSA", "yes", "pem", "private",
+            dsa_priv_to_pem_encoder_functions),
+    ENCODER("DSA", "yes", "pem", "public",
+            dsa_pub_to_pem_encoder_functions),
+    ENCODER("DSA", "yes", "pem", "parameters",
+            dsa_param_to_pem_encoder_functions),
 #endif
 
 #ifndef OPENSSL_NO_EC
     ENCODER("X25519", "yes", "text", "private",
-        x25519_priv_print_encoder_functions),
+            x25519_priv_to_text_encoder_functions),
     ENCODER("X25519", "yes", "text", "public",
-        x25519_pub_print_encoder_functions),
+            x25519_pub_to_text_encoder_functions),
     ENCODER("X25519", "yes", "der", "private",
-        x25519_priv_der_encoder_functions),
-    ENCODER("X25519", "yes", "der", "public", x25519_pub_der_encoder_functions),
+            x25519_priv_to_der_encoder_functions),
+    ENCODER("X25519", "yes", "der", "public",
+            x25519_pub_to_der_encoder_functions),
     ENCODER("X25519", "yes", "pem", "private",
-        x25519_priv_pem_encoder_functions),
-    ENCODER("X25519", "yes", "pem", "public", x25519_pub_pem_encoder_functions),
+            x25519_priv_to_pem_encoder_functions),
+    ENCODER("X25519", "yes", "pem", "public",
+            x25519_pub_to_pem_encoder_functions),
 
-    ENCODER("X448", "no", "text", "private", x448_priv_print_encoder_functions),
-    ENCODER("X448", "no", "text", "public", x448_pub_print_encoder_functions),
-    ENCODER("X448", "no", "der", "private", x448_priv_der_encoder_functions),
-    ENCODER("X448", "no", "der", "public", x448_pub_der_encoder_functions),
-    ENCODER("X448", "no", "pem", "private", x448_priv_pem_encoder_functions),
-    ENCODER("X448", "no", "pem", "public", x448_pub_pem_encoder_functions),
+    ENCODER("X448", "yes", "text", "private",
+            x448_priv_to_text_encoder_functions),
+    ENCODER("X448", "yes", "text", "public",
+            x448_pub_to_text_encoder_functions),
+    ENCODER("X448", "yes", "der", "private",
+            x448_priv_to_der_encoder_functions),
+    ENCODER("X448", "yes", "der", "public",
+            x448_pub_to_der_encoder_functions),
+    ENCODER("X448", "yes", "pem", "private",
+            x448_priv_to_pem_encoder_functions),
+    ENCODER("X448", "yes", "pem", "public",
+            x448_pub_to_pem_encoder_functions),
 
     ENCODER("ED25519", "yes", "text", "private",
-        ed25519_priv_print_encoder_functions),
+            ed25519_priv_to_text_encoder_functions),
     ENCODER("ED25519", "yes", "text", "public",
-        ed25519_pub_print_encoder_functions),
+            ed25519_pub_to_text_encoder_functions),
     ENCODER("ED25519", "yes", "der", "private",
-        ed25519_priv_der_encoder_functions),
+            ed25519_priv_to_der_encoder_functions),
     ENCODER("ED25519", "yes", "der", "public",
-        ed25519_pub_der_encoder_functions),
+            ed25519_pub_to_der_encoder_functions),
     ENCODER("ED25519", "yes", "pem", "private",
-        ed25519_priv_pem_encoder_functions),
+            ed25519_priv_to_pem_encoder_functions),
     ENCODER("ED25519", "yes", "pem", "public",
-        ed25519_pub_pem_encoder_functions),
+            ed25519_pub_to_pem_encoder_functions),
 
-    ENCODER("ED448", "no", "text", "private",
-        ed448_priv_print_encoder_functions),
-    ENCODER("ED448", "no", "text", "public", ed448_pub_print_encoder_functions),
-    ENCODER("ED448", "no", "der", "private", ed448_priv_der_encoder_functions),
-    ENCODER("ED448", "no", "der", "public", ed448_pub_der_encoder_functions),
-    ENCODER("ED448", "no", "pem", "private", ed448_priv_pem_encoder_functions),
-    ENCODER("ED448", "no", "pem", "public", ed448_pub_pem_encoder_functions),
+    ENCODER("ED448", "yes", "text", "private",
+            ed448_priv_to_text_encoder_functions),
+    ENCODER("ED448", "yes", "text", "public",
+            ed448_pub_to_text_encoder_functions),
+    ENCODER("ED448", "yes", "der", "private",
+            ed448_priv_to_der_encoder_functions),
+    ENCODER("ED448", "yes", "der", "public",
+            ed448_pub_to_der_encoder_functions),
+    ENCODER("ED448", "yes", "pem", "private",
+            ed448_priv_to_pem_encoder_functions),
+    ENCODER("ED448", "yes", "pem", "public",
+            ed448_pub_to_pem_encoder_functions),
 
-    ENCODER("EC", "yes", "text", "private", ec_priv_text_encoder_functions),
-    ENCODER("EC", "yes", "text", "public", ec_pub_text_encoder_functions),
-    ENCODER("EC", "yes", "text", "parameters", ec_param_text_encoder_functions),
-    ENCODER("EC", "yes", "der", "private", ec_priv_der_encoder_functions),
-    ENCODER("EC", "yes", "der", "public", ec_pub_der_encoder_functions),
-    ENCODER("EC", "yes", "der", "parameters", ec_param_der_encoder_functions),
-    ENCODER("EC", "yes", "pem", "private", ec_priv_pem_encoder_functions),
-    ENCODER("EC", "yes", "pem", "public", ec_pub_pem_encoder_functions),
-    ENCODER("EC", "yes", "pem", "parameters", ec_param_pem_encoder_functions),
+    ENCODER("EC", "yes", "text", "private",
+            ec_priv_to_text_encoder_functions),
+    ENCODER("EC", "yes", "text", "public",
+            ec_pub_to_text_encoder_functions),
+    ENCODER("EC", "yes", "text", "parameters",
+            ec_param_to_text_encoder_functions),
+    ENCODER("EC", "yes", "der", "private",
+            ec_priv_to_der_encoder_functions),
+    ENCODER("EC", "yes", "der", "public",
+            ec_pub_to_der_encoder_functions),
+    ENCODER("EC", "yes", "der", "parameters",
+            ec_param_to_der_encoder_functions),
+    ENCODER("EC", "yes", "pem", "private",
+            ec_priv_to_pem_encoder_functions),
+    ENCODER("EC", "yes", "pem", "public",
+            ec_pub_to_pem_encoder_functions),
+    ENCODER("EC", "yes", "pem", "parameters",
+            ec_param_to_pem_encoder_functions),
 #endif