MDEV-10332 support for OpenSSL 1.1 and LibreSSL

post-review fixes:
* move all ssl implementation related ifdefs/defines to one file
  (ssl_compat.h)
* work around OpenSSL-1.1 desire to malloc every EVP context by
  run-time checking that context allocated on the stack is big enough
  (openssl.c)
* use newer version of the AWS SDK for OpenSSL 1.1
* use get_dh2048() function as generated by openssl 1.1
  (viosslfactories.c)

1 files changed, 29 insertions, 56 deletions

diff --git a/mysys_ssl/my_md5.cc b/mysys_ssl/my_md5.cc
index 02c01dd7148..0105082b7e1 100644
--- a/mysys_ssl/my_md5.cc
+++ b/mysys_ssl/my_md5.cc
@@ -1,5 +1,5 @@
 /* Copyright (c) 2012, Oracle and/or its affiliates.
-   Copyright (c) 2014, SkySQL Ab.
+   Copyright (c) 2017, MariaDB Corporation
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -27,50 +27,34 @@
 #include <my_md5.h>
 #include <stdarg.h>
 
-#define MA_HASH_CTX_SIZE 512
-
 #if defined(HAVE_YASSL)
 #include "md5.hpp"
+#include <ssl_compat.h>
 
-typedef TaoCrypt::MD5 MD5_CONTEXT;
+typedef TaoCrypt::MD5 EVP_MD_CTX;
 
-static void md5_init(MD5_CONTEXT *context)
+static void md5_init(EVP_MD_CTX *context)
 {
-  context= new(context) MD5_CONTEXT;
+  context= new(context) EVP_MD_CTX;
   context->Init();
 }
 
-/*
-  this is a variant of md5_init to be used in this file only.
-  does nothing for yassl, because the context's constructor was called automatically.
-*/
-static void md5_init_fast(MD5_CONTEXT *context)
-{
-}
-
-static void md5_input(MD5_CONTEXT *context, const uchar *buf, unsigned len)
+static void md5_input(EVP_MD_CTX *context, const uchar *buf, unsigned len)
 {
   context->Update((const TaoCrypt::byte *) buf, len);
 }
 
-static void md5_result(MD5_CONTEXT *context, uchar digest[MD5_HASH_SIZE])
+static void md5_result(EVP_MD_CTX *context, uchar digest[MD5_HASH_SIZE])
 {
     context->Final((TaoCrypt::byte *) digest);
 }
 
 #elif defined(HAVE_OPENSSL)
-
-
 #include <openssl/evp.h>
+#include <ssl_compat.h>
 
-#if OPENSSL_VERSION_NUMBER < 0x10100000L || defined(LIBRESSL_VERSION_NUMBER)
-#define EVP_MD_CTX_reset(X) EVP_MD_CTX_cleanup(X)
-#endif
-typedef EVP_MD_CTX MD5_CONTEXT;
-
-static void md5_init(MD5_CONTEXT *context)
+static void md5_init(EVP_MD_CTX *context)
 {
-  memset(context, 0, my_md5_context_size());
   EVP_MD_CTX_init(context);
 #ifdef EVP_MD_CTX_FLAG_NON_FIPS_ALLOW
   /* Ok to ignore FIPS: MD5 is not used for crypto here */
@@ -79,20 +63,15 @@ static void md5_init(MD5_CONTEXT *context)
   EVP_DigestInit_ex(context, EVP_md5(), NULL);
 }
 
-static void md5_init_fast(MD5_CONTEXT *context)
-{
-  md5_init(context);
-}
-
-static void md5_input(MD5_CONTEXT *context, const uchar *buf, unsigned len)
+static void md5_input(EVP_MD_CTX *context, const uchar *buf, unsigned len)
 {
   EVP_DigestUpdate(context, buf, len);
 }
 
-static void md5_result(MD5_CONTEXT *context, uchar digest[MD5_HASH_SIZE])
+static void md5_result(EVP_MD_CTX *context, uchar digest[MD5_HASH_SIZE])
 {
   EVP_DigestFinal_ex(context, digest, NULL);
-  EVP_MD_CTX_reset(context);
+  EVP_MD_CTX_cleanup(context);
 }
 
 #endif /* HAVE_YASSL */
@@ -108,26 +87,23 @@ static void md5_result(MD5_CONTEXT *context, uchar digest[MD5_HASH_SIZE])
 */
 void my_md5(uchar *digest, const char *buf, size_t len)
 {
-#ifdef HAVE_YASSL
-  MD5_CONTEXT md5_context;
-#else
-  unsigned char md5_context[MA_HASH_CTX_SIZE];
-#endif
-  md5_init_fast((MD5_CONTEXT *)&md5_context);
-  md5_input((MD5_CONTEXT *)&md5_context, (const uchar *)buf, len);
-  md5_result((MD5_CONTEXT *)&md5_context, digest);
+  char ctx_buf[EVP_MD_CTX_SIZE];
+  EVP_MD_CTX * const ctx= (EVP_MD_CTX*)ctx_buf;
+  md5_init(ctx);
+  md5_input(ctx, (const uchar *)buf, len);
+  md5_result(ctx, digest);
 }
 
 
 /**
   Wrapper function to compute MD5 message digest for
-  two messages in order to emulate md5(msg1, msg2).
+  many messages, concatenated.
 
   @param digest [out]  Computed MD5 digest
   @param buf1   [in]   First message
   @param len1   [in]   Length of first message
-  @param buf2   [in]   Second message
-  @param len2   [in]   Length of second message
+         ...
+  @param bufN   [in]   NULL terminates the list of buf,len pairs.
 
   @return              void
 */
@@ -135,37 +111,34 @@ void my_md5_multi(uchar *digest, ...)
 {
   va_list args;
   const uchar *str;
-#ifdef HAVE_YASSL
-  MD5_CONTEXT md5_context;
-#else
-  unsigned char md5_context[MA_HASH_CTX_SIZE];
-#endif
+  char ctx_buf[EVP_MD_CTX_SIZE];
+  EVP_MD_CTX * const ctx= (EVP_MD_CTX*)ctx_buf;
   va_start(args, digest);
 
-  md5_init_fast((MD5_CONTEXT *)&md5_context);
+  md5_init(ctx);
   for (str= va_arg(args, const uchar*); str; str= va_arg(args, const uchar*))
-    md5_input((MD5_CONTEXT *)&md5_context, str, va_arg(args, size_t));
+    md5_input(ctx, str, va_arg(args, size_t));
 
-  md5_result((MD5_CONTEXT *)&md5_context, digest);
+  md5_result(ctx, digest);
   va_end(args);
 }
 
 size_t my_md5_context_size()
 {
-  return MA_HASH_CTX_SIZE;
+  return EVP_MD_CTX_SIZE;
 }
 
 void my_md5_init(void *context)
 {
-  md5_init((MD5_CONTEXT *)context);
+  md5_init((EVP_MD_CTX *)context);
 }
 
 void my_md5_input(void *context, const uchar *buf, size_t len)
 {
-  md5_input((MD5_CONTEXT *)context, buf, len);
+  md5_input((EVP_MD_CTX *)context, buf, len);
 }
 
 void my_md5_result(void *context, uchar *digest)
 {
-  md5_result((MD5_CONTEXT *)context, digest);
+  md5_result((EVP_MD_CTX *)context, digest);
 }