MDEV-10332 support for OpenSSL 1.1 and LibreSSL

post-review fixes:
* move all ssl implementation related ifdefs/defines to one file
  (ssl_compat.h)
* work around OpenSSL-1.1 desire to malloc every EVP context by
  run-time checking that context allocated on the stack is big enough
  (openssl.c)
* use newer version of the AWS SDK for OpenSSL 1.1
* use get_dh2048() function as generated by openssl 1.1
  (viosslfactories.c)

2 files changed, 47 insertions, 62 deletions

diff --git a/vio/vio.c b/vio/vio.c
index 77a58a8272c..2c5f37a75a4 100644
--- a/vio/vio.c
+++ b/vio/vio.c
@@ -22,6 +22,7 @@
 */
 
 #include "vio_priv.h"
+#include "ssl_compat.h"
 
 #ifdef _WIN32
 
diff --git a/vio/viosslfactories.c b/vio/viosslfactories.c
index 497047cac72..71ef2879464 100644
--- a/vio/viosslfactories.c
+++ b/vio/viosslfactories.c
@@ -15,20 +15,12 @@
    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA */
 
 #include "vio_priv.h"
+#include <ssl_compat.h>
 
 #ifdef HAVE_OPENSSL
-#if defined(HAVE_YASSL) || defined(LIBRESSL_VERSION_NUMBER)
-#define OPENSSL_init_ssl(X,Y) SSL_library_init()
-#else
+#ifndef HAVE_YASSL
 #include <openssl/dh.h>
 #include <openssl/bn.h>
-
-#if OPENSSL_VERSION_NUMBER >= 0x10100000L
-#define ERR_remove_state(X)
-#else
-#define OPENSSL_init_ssl(X,Y) SSL_library_init()
-#endif
-
 #endif
 
 static my_bool     ssl_algorithms_added    = FALSE;
@@ -36,59 +28,51 @@ static my_bool     ssl_error_strings_loaded= FALSE;
 
 /* the function below was generated with "openssl dhparam -2 -C 2048" */
 
-/* {{{ get_dh_2048 */
-static DH *get_dh_2048()
+static
+DH *get_dh2048()
 {
-  static unsigned char dh2048_p[]={
-    0xA1,0xBB,0x7C,0x20,0xC5,0x5B,0xC0,0x7B,0x21,0x8B,0xD6,0xA8,
-    0x15,0xFC,0x3B,0xBA,0xAB,0x9F,0xDF,0x68,0xC4,0x79,0x78,0x0D,
-    0xC1,0x12,0x64,0xE4,0x15,0xC9,0x66,0xDB,0xF6,0xCB,0xB3,0x39,
-    0x02,0x5B,0x78,0x62,0xFB,0x09,0xAE,0x09,0x6B,0xDD,0xD4,0x5D,
-    0x97,0xBC,0xDC,0x7F,0xE6,0xD6,0xF1,0xCB,0xF5,0xEB,0xDA,0xA7,
-    0x2E,0x5A,0x43,0x2B,0xE9,0x40,0xE2,0x85,0x00,0x1C,0xC0,0x0A,
-    0x98,0x77,0xA9,0x31,0xDE,0x0B,0x75,0x4D,0x1E,0x1F,0x16,0x83,
-    0xCA,0xDE,0xBD,0x21,0xFC,0xC1,0x82,0x37,0x36,0x33,0x0B,0x66,
-    0x06,0x3C,0xF3,0xAF,0x21,0x57,0x57,0x80,0xF6,0x94,0x1B,0xA9,
-    0xD4,0xF6,0x8F,0x18,0x62,0x0E,0xC4,0x22,0xF9,0x5B,0x62,0xCC,
-    0x3F,0x19,0x95,0xCF,0x4B,0x00,0xA6,0x6C,0x0B,0xAF,0x9F,0xD5,
-    0xFA,0x3D,0x6D,0xDA,0x30,0x83,0x07,0x91,0xAC,0x15,0xFF,0x8F,
-    0x59,0x54,0xEA,0x25,0xBC,0x4E,0xEB,0x6A,0x54,0xDF,0x75,0x09,
-    0x72,0x0F,0xEF,0x23,0x70,0xE0,0xA8,0x04,0xEA,0xFF,0x90,0x54,
-    0xCD,0x84,0x18,0xC0,0x75,0x91,0x99,0x0F,0xA1,0x78,0x0C,0x07,
-    0xB7,0xC5,0xDE,0x55,0x06,0x7B,0x95,0x68,0x2C,0x33,0x39,0xBC,
-    0x2C,0xD0,0x6D,0xDD,0xFA,0xDC,0xB5,0x8F,0x82,0x39,0xF8,0x67,
-    0x44,0xF1,0xD8,0xF7,0x78,0x11,0x9A,0x77,0x9B,0x53,0x47,0xD6,
-    0x2B,0x5D,0x67,0xB8,0xB7,0xBC,0xC1,0xD7,0x79,0x62,0x15,0xC2,
-    0xC5,0x83,0x97,0xA7,0xF8,0xB4,0x9C,0xF6,0x8F,0x9A,0xC7,0xDA,
-    0x1B,0xBB,0x87,0x07,0xA7,0x71,0xAD,0xB2,0x8A,0x50,0xF8,0x26,
-    0x12,0xB7,0x3E,0x0B,
-  };
-  static unsigned char dh2048_g[]={
-    0x02,
-  };
-  DH *dh;
-  if ((dh=DH_new()) == NULL)
-    return(NULL);
-#if OPENSSL_VERSION_NUMBER < 0x10100000L || defined(LIBRESSL_VERSION_NUMBER)
-  (dh)->p=BN_bin2bn(dh2048_p,sizeof(dh2048_p),NULL);
-  (dh)->g=BN_bin2bn(dh2048_g,sizeof(dh2048_g),NULL);
-  if ((dh)->p == NULL || (dh)->g == NULL)
-  { DH_free(dh); return NULL; }
-#else
-  {
-    BIGNUM *dhp_bn= BN_bin2bn(dh2048_p,sizeof(dh2048_p),NULL),
-           *dhg_bn= BN_bin2bn(dh2048_g,sizeof(dh2048_g),NULL);
-    if (dhp_bn == NULL || dhg_bn == NULL ||
-        !DH_set0_pqg(dh, dhp_bn, NULL, dhg_bn))
-    {
-      DH_free(dh);
-      BN_free(dhp_bn);
-      BN_free(dhg_bn);
-      return NULL;
+    static unsigned char dhp_2048[] = {
+        0xA1,0xBB,0x7C,0x20,0xC5,0x5B,0xC0,0x7B,0x21,0x8B,0xD6,0xA8,
+        0x15,0xFC,0x3B,0xBA,0xAB,0x9F,0xDF,0x68,0xC4,0x79,0x78,0x0D,
+        0xC1,0x12,0x64,0xE4,0x15,0xC9,0x66,0xDB,0xF6,0xCB,0xB3,0x39,
+        0x02,0x5B,0x78,0x62,0xFB,0x09,0xAE,0x09,0x6B,0xDD,0xD4,0x5D,
+        0x97,0xBC,0xDC,0x7F,0xE6,0xD6,0xF1,0xCB,0xF5,0xEB,0xDA,0xA7,
+        0x2E,0x5A,0x43,0x2B,0xE9,0x40,0xE2,0x85,0x00,0x1C,0xC0,0x0A,
+        0x98,0x77,0xA9,0x31,0xDE,0x0B,0x75,0x4D,0x1E,0x1F,0x16,0x83,
+        0xCA,0xDE,0xBD,0x21,0xFC,0xC1,0x82,0x37,0x36,0x33,0x0B,0x66,
+        0x06,0x3C,0xF3,0xAF,0x21,0x57,0x57,0x80,0xF6,0x94,0x1B,0xA9,
+        0xD4,0xF6,0x8F,0x18,0x62,0x0E,0xC4,0x22,0xF9,0x5B,0x62,0xCC,
+        0x3F,0x19,0x95,0xCF,0x4B,0x00,0xA6,0x6C,0x0B,0xAF,0x9F,0xD5,
+        0xFA,0x3D,0x6D,0xDA,0x30,0x83,0x07,0x91,0xAC,0x15,0xFF,0x8F,
+        0x59,0x54,0xEA,0x25,0xBC,0x4E,0xEB,0x6A,0x54,0xDF,0x75,0x09,
+        0x72,0x0F,0xEF,0x23,0x70,0xE0,0xA8,0x04,0xEA,0xFF,0x90,0x54,
+        0xCD,0x84,0x18,0xC0,0x75,0x91,0x99,0x0F,0xA1,0x78,0x0C,0x07,
+        0xB7,0xC5,0xDE,0x55,0x06,0x7B,0x95,0x68,0x2C,0x33,0x39,0xBC,
+        0x2C,0xD0,0x6D,0xDD,0xFA,0xDC,0xB5,0x8F,0x82,0x39,0xF8,0x67,
+        0x44,0xF1,0xD8,0xF7,0x78,0x11,0x9A,0x77,0x9B,0x53,0x47,0xD6,
+        0x2B,0x5D,0x67,0xB8,0xB7,0xBC,0xC1,0xD7,0x79,0x62,0x15,0xC2,
+        0xC5,0x83,0x97,0xA7,0xF8,0xB4,0x9C,0xF6,0x8F,0x9A,0xC7,0xDA,
+        0x1B,0xBB,0x87,0x07,0xA7,0x71,0xAD,0xB2,0x8A,0x50,0xF8,0x26,
+        0x12,0xB7,0x3E,0x0B,
+    };
+    static unsigned char dhg_2048[] = {
+        0x02
+    };
+    DH *dh = DH_new();
+    BIGNUM *dhp_bn, *dhg_bn;
+
+    if (dh == NULL)
+        return NULL;
+    dhp_bn = BN_bin2bn(dhp_2048, sizeof (dhp_2048), NULL);
+    dhg_bn = BN_bin2bn(dhg_2048, sizeof (dhg_2048), NULL);
+    if (dhp_bn == NULL || dhg_bn == NULL
+            || !DH_set0_pqg(dh, dhp_bn, NULL, dhg_bn)) {
+        DH_free(dh);
+        BN_free(dhp_bn);
+        BN_free(dhg_bn);
+        return NULL;
     }
-  }
-#endif
-  return dh;
+    return dh;
 }
 
 static const char*
@@ -287,7 +271,7 @@ new_VioSSLFd(const char *key_file, const char *cert_file,
   /* DH stuff */
   if (!is_client_method)
   {
-    dh=get_dh_2048();
+    dh=get_dh2048();
     if (!SSL_CTX_set_tmp_dh(ssl_fd->ssl_context, dh))
     {
       *error= SSL_INITERR_DH;