MDEV-26015: Set DH param automaticallybb-10.6-MDEV-26015

So far  MariaDB Server creates a DH (Diffie Hellman) parameter with a
fixed length (= 2048). This leads to the limitation that, for example, the
use of a DHE cipher suite requires a server certificate with an RSA key
of the same length.

To remedy this, the DH parameter is now automatically set by OpenSSL or WolfSSL
during the TLS handshake (taking into account the key length used
of the server certificate).

While WoldSSL supports this by default, OpenSSL has to activate this using the
SSL_CTX_set_dh_auto macro. OpenSSL versions < 1.0.2  doesn't support this feature
and use the previous implementation with a DH parameter with a fixed size.

4 files changed, 42 insertions, 7 deletions

diff --git a/libmariadb b/libmariadb
-Subproject 74a405d9770720e000c391672fca9e67421fa4b
+Subproject 903c3ef3fc707a56e9a48d4ffb4282ce0177b1f
diff --git a/mysql-test/main/MDEV-26015.result b/mysql-test/main/MDEV-26015.result
new file mode 100644
index 00000000000..5798b3baa4d
--- /dev/null
+++ b/mysql-test/main/MDEV-26015.result
@@ -0,0 +1,2 @@
+Variable_name	Value
+Ssl_cipher	DHE-RSA-AES128-SHA256
diff --git a/mysql-test/main/MDEV-26015.test b/mysql-test/main/MDEV-26015.test
new file mode 100644
index 00000000000..ed2f9b38345
--- /dev/null
+++ b/mysql-test/main/MDEV-26015.test
@@ -0,0 +1,11 @@
+# Tests for SSL connections, only run if mysqld is compiled
+# with support for SSL.
+
+-- source include/have_ssl_communication.inc
+
+#
+# MDEV-26015 - using DHE cipher will fail if DH public key size doesn't match
+# RSA key size of server certificate
+#
+--exec $MYSQL -uroot --ssl-cipher=DHE-RSA-AES128-SHA256 --tls_version=TLSv1.2 -e"show status like 'ssl_cipher'" 2>&1
+
diff --git a/vio/viosslfactories.c b/vio/viosslfactories.c
index d9fcc942a71..f44ec6ccdfe 100644
--- a/vio/viosslfactories.c
+++ b/vio/viosslfactories.c
@@ -24,7 +24,13 @@
 static my_bool     ssl_algorithms_added    = FALSE;
 static my_bool     ssl_error_strings_loaded= FALSE;
 
-/* the function below was generated with "openssl dhparam -2 -C 2048" */
+#if !defined(WOLFSSL) && !defined(SSL_CTRL_SET_DH_AUTO)
+
+/* Older OpenSSL versions (< 1.0.2) don't handle dhparam
+   setting automatically during TLS handshake.
+
+   The function below was generated with "openssl dhparam -2 -C 2048"
+*/
 
 static
 DH *get_dh2048()
@@ -72,6 +78,7 @@ DH *get_dh2048()
     }
     return dh;
 }
+#endif
 
 static const char*
 ssl_error_string[] =
@@ -228,7 +235,6 @@ new_VioSSLFd(const char *key_file, const char *cert_file,
              enum enum_ssl_init_error *error,
              const char *crl_file, const char *crl_path, ulonglong tls_version)
 {
-  DH *dh;
   struct st_VioSSLFd *ssl_fd;
   long ssl_ctx_options;
   DBUG_ENTER("new_VioSSLFd");
@@ -334,18 +340,36 @@ new_VioSSLFd(const char *key_file, const char *cert_file,
     goto err2;
   }
 
-  /* DH stuff */
+#if !defined(WOLFSSL)
+  /* DH stuff:
+
+     WolfSSL chooses right DH public key automatically depending
+     on RSA key size of server certificate during TLS handshake.
+     For OpenSSL >=  1.0.2 this feature needs to be explicitly turned on.
+     OpenSSL versions < 1.0.2 will still use the old get_dh2048() method
+     to obtain and set DH public key (see also MDEV-26015).
+  */
   if (!is_client_method)
   {
-    dh=get_dh2048();
+#if !defined(SSL_CTRL_SET_DH_AUTO)
+    DH *dh=get_dh2048();
     if (!SSL_CTX_set_tmp_dh(ssl_fd->ssl_context, dh))
     {
       *error= SSL_INITERR_DH;
-      goto err3;
+      DH_free(dh);
+      goto err2;
     }
 
     DH_free(dh);
+#else
+    if (!SSL_CTX_set_dh_auto(ssl_fd->ssl_context, 1))
+    {
+      *error= SSL_INITERR_DH;
+      goto err2;
+    }
+#endif
   }
+#endif
 
 #ifdef HAVE_WOLFSSL
   /* set IO functions used by wolfSSL */
@@ -357,8 +381,6 @@ new_VioSSLFd(const char *key_file, const char *cert_file,
 
   DBUG_RETURN(ssl_fd);
 
-err3:
-  DH_free(dh);
 err2:
   SSL_CTX_free(ssl_fd->ssl_context);
 err1: