MDEV-10332 support for OpenSSL 1.1 and LibreSSL

post-review fixes:
* move all ssl implementation related ifdefs/defines to one file
  (ssl_compat.h)
* work around OpenSSL-1.1 desire to malloc every EVP context by
  run-time checking that context allocated on the stack is big enough
  (openssl.c)
* use newer version of the AWS SDK for OpenSSL 1.1
* use get_dh2048() function as generated by openssl 1.1
  (viosslfactories.c)

5 files changed, 140 insertions, 138 deletions

diff --git a/mysys_ssl/CMakeLists.txt b/mysys_ssl/CMakeLists.txt
index 4f6f7458c5b..f8a767ed6f3 100644
--- a/mysys_ssl/CMakeLists.txt
+++ b/mysys_ssl/CMakeLists.txt
@@ -28,6 +28,7 @@ SET(MYSYS_SSL_HIDDEN_SOURCES
     my_sha384.cc
     my_sha512.cc
     my_md5.cc
+    openssl.c
    )
 
 SET(MYSYS_SSL_SOURCES
diff --git a/mysys_ssl/my_crypt.cc b/mysys_ssl/my_crypt.cc
index 9cacee4b450..6ad9171bfbc 100644
--- a/mysys_ssl/my_crypt.cc
+++ b/mysys_ssl/my_crypt.cc
@@ -1,6 +1,6 @@
 /*
  Copyright (c) 2014 Google Inc.
- Copyright (c) 2014, 2015 MariaDB Corporation
+ Copyright (c) 2014, 2017 MariaDB Corporation
 
  This program is free software; you can redistribute it and/or modify
  it under the terms of the GNU General Public License as published by
@@ -21,30 +21,31 @@
 #ifdef HAVE_YASSL
 #include "yassl.cc"
 #else
-
 #include <openssl/evp.h>
 #include <openssl/aes.h>
 #include <openssl/err.h>
 #include <openssl/rand.h>
-
 #endif
-#include <my_crypt.h>
 
-#define MY_CIPHER_CTX_SIZE 384
+#include <my_crypt.h>
+#include <ssl_compat.h>
 
 class MyCTX
 {
 public:
+  char ctx_buf[EVP_CIPHER_CTX_SIZE];
   EVP_CIPHER_CTX *ctx;
-  const uchar *key;
-  unsigned int klen;
-  MyCTX() {
-            ctx= EVP_CIPHER_CTX_new();
-          }
-  virtual ~MyCTX() {
-                     EVP_CIPHER_CTX_free(ctx);
-                     ERR_remove_state(0);
-                    }
+
+  MyCTX()
+  {
+    ctx= (EVP_CIPHER_CTX *)ctx_buf;
+    EVP_CIPHER_CTX_init(ctx);
+  }
+  virtual ~MyCTX()
+  {
+    EVP_CIPHER_CTX_cleanup(ctx);
+    ERR_remove_state(0);
+  }
 
   virtual int init(const EVP_CIPHER *cipher, int encrypt, const uchar *key,
                    uint klen, const uchar *iv, uint ivlen)
@@ -78,9 +79,12 @@ public:
 class MyCTX_nopad : public MyCTX
 {
 public:
+  const uchar *key;
+  uint klen, buf_len;
+  uchar oiv[MY_AES_BLOCK_SIZE];
+
   MyCTX_nopad() : MyCTX() { }
   ~MyCTX_nopad() { }
-  unsigned int buf_len;
 
   int init(const EVP_CIPHER *cipher, int encrypt, const uchar *key, uint klen,
            const uchar *iv, uint ivlen)
@@ -89,19 +93,8 @@ public:
     this->key= key;
     this->klen= klen;
     this->buf_len= 0;
-    /* FIX-ME:
-       For the sake of backward compatibility we do some strange hack here:
-       Since ECB doesn't need an IV (and therefore is considered kind of
-       insecure) we need to store the specified iv.
-       The last nonpadding block will be encrypted with an additional
-       expensive crypt_call in ctr mode instead
-       of encrypting the entire plain text in ctr-mode */
-#if OPENSSL_VERSION_NUMBER >= 0x10100000L && !defined(LIBRESSL_VERSION_NUMBER)
-    const unsigned char *oiv= EVP_CIPHER_CTX_original_iv(ctx);
-#else
-    const unsigned char *oiv= ctx->oiv;
-#endif
-    memcpy((char *)oiv, iv, ivlen);
+    memcpy(oiv, iv, ivlen);
+    DBUG_ASSERT(ivlen == 0 || ivlen == sizeof(oiv));
 
     int res= MyCTX::init(cipher, encrypt, key, klen, iv, ivlen);
 
@@ -111,34 +104,30 @@ public:
 
   int update(const uchar *src, uint slen, uchar *dst, uint *dlen)
   {
-    buf_len= slen % MY_AES_BLOCK_SIZE;
+    buf_len+= slen;
     return MyCTX::update(src, slen, dst, dlen);
   }
 
   int finish(uchar *dst, uint *dlen)
   {
+    buf_len %= MY_AES_BLOCK_SIZE;
     if (buf_len)
     {
-      const uchar *org_iv;
-      unsigned char *buf;
+      uchar *buf= EVP_CIPHER_CTX_buf_noconst(ctx);
       /*
         Not much we can do, block ciphers cannot encrypt data that aren't
         a multiple of the block length. At least not without padding.
         Let's do something CTR-like for the last partial block.
+
+        NOTE this assumes that there are only buf_len bytes in the buf.
+        If OpenSSL will change that, we'll need to change the implementation
+        of this class too.
       */
       uchar mask[MY_AES_BLOCK_SIZE];
       uint mlen;
 
-#if OPENSSL_VERSION_NUMBER >= 0x10100000L && !defined(LIBRESSL_VERSION_NUMBER)
-      org_iv= EVP_CIPHER_CTX_original_iv(ctx);
-      buf= EVP_CIPHER_CTX_buf_noconst(ctx);
-#else
-      org_iv= ctx->oiv;
-      buf= ctx->buf;
-#endif
-
       my_aes_crypt(MY_AES_ECB, ENCRYPTION_FLAG_ENCRYPT | ENCRYPTION_FLAG_NOPAD,
-                   org_iv, sizeof(mask), mask, &mlen, key, klen, 0, 0);
+                   oiv, sizeof(mask), mask, &mlen, key, klen, 0, 0);
       DBUG_ASSERT(mlen == sizeof(mask));
 
       for (uint i=0; i < buf_len; i++)
@@ -178,9 +167,8 @@ make_aes_dispatcher(gcm)
 class MyCTX_gcm : public MyCTX
 {
 public:
-  const uchar *aad= NULL;
+  const uchar *aad;
   int aadlen;
-  my_bool encrypt;
   MyCTX_gcm() : MyCTX() { }
   ~MyCTX_gcm() { }
 
@@ -192,7 +180,6 @@ public:
     int real_ivlen= EVP_CIPHER_CTX_iv_length(ctx);
     aad= iv + real_ivlen;
     aadlen= ivlen - real_ivlen;
-    this->encrypt= encrypt;
     return res;
   }
 
@@ -204,7 +191,7 @@ public:
       before decrypting the data. it can encrypt data piecewise, like, first
       half, then the second half, but it must decrypt all at once
     */
-    if (!this->encrypt)
+    if (!EVP_CIPHER_CTX_encrypting(ctx))
     {
       /* encrypted string must contain authenticaton tag (see MDEV-11174) */
       if (slen < MY_AES_BLOCK_SIZE)
@@ -214,7 +201,7 @@ public:
                               (void*)(src + slen)))
         return MY_AES_OPENSSL_ERROR;
     }
-    int unused= 0;
+    int unused;
     if (aadlen && !EVP_CipherUpdate(ctx, NULL, &unused, aad, aadlen))
       return MY_AES_OPENSSL_ERROR;
     aadlen= 0;
@@ -223,12 +210,12 @@ public:
 
   int finish(uchar *dst, uint *dlen)
   {
-    int fin= 0;
+    int fin;
     if (!EVP_CipherFinal_ex(ctx, dst, &fin))
       return MY_AES_BAD_DATA;
     DBUG_ASSERT(fin == 0);
 
-    if (this->encrypt)
+    if (EVP_CIPHER_CTX_encrypting(ctx))
     {
       if(!EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, MY_AES_BLOCK_SIZE, dst))
         return MY_AES_OPENSSL_ERROR;
@@ -298,20 +285,15 @@ int my_aes_crypt(enum my_aes_mode mode, int flags,
 {
   void *ctx= alloca(MY_AES_CTX_SIZE);
   int res1, res2;
-  uint d1= 0, d2= 0;
+  uint d1= 0, d2;
   if ((res1= my_aes_crypt_init(ctx, mode, flags, key, klen, iv, ivlen)))
     return res1;
   res1= my_aes_crypt_update(ctx, src, slen, dst, &d1);
   res2= my_aes_crypt_finish(ctx, dst + d1, &d2);
-  *dlen= d1 + d2;
-  /* in case of failure clear error queue */
-#ifndef HAVE_YASSL
-  /* since we don't check the crypto error messages we need to
-     clear the error queue - otherwise subsequent crypto or tls/ssl
-     calls will fail */
-  if (!*dlen)
-    ERR_clear_error();
-#endif
+  if (res1 || res2)
+    ERR_remove_state(0); /* in case of failure clear error queue */
+  else
+    *dlen= d1 + d2;
   return res1 ? res1 : res2;
 }
 
@@ -353,12 +335,6 @@ int my_random_bytes(uchar* buf, int num)
 
 int my_random_bytes(uchar *buf, int num)
 {
-  /*
-    Unfortunately RAND_bytes manual page does not provide any guarantees
-    in relation to blocking behavior. Here we explicitly use SSLeay random
-    instead of whatever random engine is currently set in OpenSSL. That way
-    we are guaranteed to have a non-blocking random.
-  */
   RAND_METHOD *rand = RAND_OpenSSL();
   if (rand == NULL || rand->bytes(buf, num) != 1)
     return MY_AES_OPENSSL_ERROR;
diff --git a/mysys_ssl/my_md5.cc b/mysys_ssl/my_md5.cc
index 02c01dd7148..0105082b7e1 100644
--- a/mysys_ssl/my_md5.cc
+++ b/mysys_ssl/my_md5.cc
@@ -1,5 +1,5 @@
 /* Copyright (c) 2012, Oracle and/or its affiliates.
-   Copyright (c) 2014, SkySQL Ab.
+   Copyright (c) 2017, MariaDB Corporation
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -27,50 +27,34 @@
 #include <my_md5.h>
 #include <stdarg.h>
 
-#define MA_HASH_CTX_SIZE 512
-
 #if defined(HAVE_YASSL)
 #include "md5.hpp"
+#include <ssl_compat.h>
 
-typedef TaoCrypt::MD5 MD5_CONTEXT;
+typedef TaoCrypt::MD5 EVP_MD_CTX;
 
-static void md5_init(MD5_CONTEXT *context)
+static void md5_init(EVP_MD_CTX *context)
 {
-  context= new(context) MD5_CONTEXT;
+  context= new(context) EVP_MD_CTX;
   context->Init();
 }
 
-/*
-  this is a variant of md5_init to be used in this file only.
-  does nothing for yassl, because the context's constructor was called automatically.
-*/
-static void md5_init_fast(MD5_CONTEXT *context)
-{
-}
-
-static void md5_input(MD5_CONTEXT *context, const uchar *buf, unsigned len)
+static void md5_input(EVP_MD_CTX *context, const uchar *buf, unsigned len)
 {
   context->Update((const TaoCrypt::byte *) buf, len);
 }
 
-static void md5_result(MD5_CONTEXT *context, uchar digest[MD5_HASH_SIZE])
+static void md5_result(EVP_MD_CTX *context, uchar digest[MD5_HASH_SIZE])
 {
     context->Final((TaoCrypt::byte *) digest);
 }
 
 #elif defined(HAVE_OPENSSL)
-
-
 #include <openssl/evp.h>
+#include <ssl_compat.h>
 
-#if OPENSSL_VERSION_NUMBER < 0x10100000L || defined(LIBRESSL_VERSION_NUMBER)
-#define EVP_MD_CTX_reset(X) EVP_MD_CTX_cleanup(X)
-#endif
-typedef EVP_MD_CTX MD5_CONTEXT;
-
-static void md5_init(MD5_CONTEXT *context)
+static void md5_init(EVP_MD_CTX *context)
 {
-  memset(context, 0, my_md5_context_size());
   EVP_MD_CTX_init(context);
 #ifdef EVP_MD_CTX_FLAG_NON_FIPS_ALLOW
   /* Ok to ignore FIPS: MD5 is not used for crypto here */
@@ -79,20 +63,15 @@ static void md5_init(MD5_CONTEXT *context)
   EVP_DigestInit_ex(context, EVP_md5(), NULL);
 }
 
-static void md5_init_fast(MD5_CONTEXT *context)
-{
-  md5_init(context);
-}
-
-static void md5_input(MD5_CONTEXT *context, const uchar *buf, unsigned len)
+static void md5_input(EVP_MD_CTX *context, const uchar *buf, unsigned len)
 {
   EVP_DigestUpdate(context, buf, len);
 }
 
-static void md5_result(MD5_CONTEXT *context, uchar digest[MD5_HASH_SIZE])
+static void md5_result(EVP_MD_CTX *context, uchar digest[MD5_HASH_SIZE])
 {
   EVP_DigestFinal_ex(context, digest, NULL);
-  EVP_MD_CTX_reset(context);
+  EVP_MD_CTX_cleanup(context);
 }
 
 #endif /* HAVE_YASSL */
@@ -108,26 +87,23 @@ static void md5_result(MD5_CONTEXT *context, uchar digest[MD5_HASH_SIZE])
 */
 void my_md5(uchar *digest, const char *buf, size_t len)
 {
-#ifdef HAVE_YASSL
-  MD5_CONTEXT md5_context;
-#else
-  unsigned char md5_context[MA_HASH_CTX_SIZE];
-#endif
-  md5_init_fast((MD5_CONTEXT *)&md5_context);
-  md5_input((MD5_CONTEXT *)&md5_context, (const uchar *)buf, len);
-  md5_result((MD5_CONTEXT *)&md5_context, digest);
+  char ctx_buf[EVP_MD_CTX_SIZE];
+  EVP_MD_CTX * const ctx= (EVP_MD_CTX*)ctx_buf;
+  md5_init(ctx);
+  md5_input(ctx, (const uchar *)buf, len);
+  md5_result(ctx, digest);
 }
 
 
 /**
   Wrapper function to compute MD5 message digest for
-  two messages in order to emulate md5(msg1, msg2).
+  many messages, concatenated.
 
   @param digest [out]  Computed MD5 digest
   @param buf1   [in]   First message
   @param len1   [in]   Length of first message
-  @param buf2   [in]   Second message
-  @param len2   [in]   Length of second message
+         ...
+  @param bufN   [in]   NULL terminates the list of buf,len pairs.
 
   @return              void
 */
@@ -135,37 +111,34 @@ void my_md5_multi(uchar *digest, ...)
 {
   va_list args;
   const uchar *str;
-#ifdef HAVE_YASSL
-  MD5_CONTEXT md5_context;
-#else
-  unsigned char md5_context[MA_HASH_CTX_SIZE];
-#endif
+  char ctx_buf[EVP_MD_CTX_SIZE];
+  EVP_MD_CTX * const ctx= (EVP_MD_CTX*)ctx_buf;
   va_start(args, digest);
 
-  md5_init_fast((MD5_CONTEXT *)&md5_context);
+  md5_init(ctx);
   for (str= va_arg(args, const uchar*); str; str= va_arg(args, const uchar*))
-    md5_input((MD5_CONTEXT *)&md5_context, str, va_arg(args, size_t));
+    md5_input(ctx, str, va_arg(args, size_t));
 
-  md5_result((MD5_CONTEXT *)&md5_context, digest);
+  md5_result(ctx, digest);
   va_end(args);
 }
 
 size_t my_md5_context_size()
 {
-  return MA_HASH_CTX_SIZE;
+  return EVP_MD_CTX_SIZE;
 }
 
 void my_md5_init(void *context)
 {
-  md5_init((MD5_CONTEXT *)context);
+  md5_init((EVP_MD_CTX *)context);
 }
 
 void my_md5_input(void *context, const uchar *buf, size_t len)
 {
-  md5_input((MD5_CONTEXT *)context, buf, len);
+  md5_input((EVP_MD_CTX *)context, buf, len);
 }
 
 void my_md5_result(void *context, uchar *digest)
 {
-  md5_result((MD5_CONTEXT *)context, digest);
+  md5_result((EVP_MD_CTX *)context, digest);
 }
diff --git a/mysys_ssl/openssl.c b/mysys_ssl/openssl.c
new file mode 100644
index 00000000000..a3f1ca29ec1
--- /dev/null
+++ b/mysys_ssl/openssl.c
@@ -0,0 +1,71 @@
+/*
+ Copyright (c) 2017, MariaDB Corporation.
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; version 2 of the License.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA */
+
+#include <my_global.h>
+#include <ssl_compat.h>
+
+#ifdef HAVE_YASSL
+
+int check_openssl_compatibility()
+{
+  return 0;
+}
+#else
+#include <openssl/evp.h>
+
+#ifdef HAVE_OPENSSL11
+typedef void *(*CRYPTO_malloc_t)(size_t, const char *, int);
+#endif
+
+#ifdef HAVE_OPENSSL10
+typedef void *(*CRYPTO_malloc_t)(size_t);
+#define CRYPTO_malloc   malloc
+#define CRYPTO_realloc  realloc
+#define CRYPTO_free     free
+#endif
+
+static uint allocated_size, allocated_count;
+
+static void *coc_malloc(size_t size)
+{
+  allocated_size+= size;
+  allocated_count++;
+  return malloc(size);
+}
+
+int check_openssl_compatibility()
+{
+  EVP_CIPHER_CTX *evp_ctx;
+  EVP_MD_CTX     *md5_ctx;
+
+  CRYPTO_set_mem_functions((CRYPTO_malloc_t)coc_malloc, CRYPTO_realloc, CRYPTO_free);
+
+  allocated_size= allocated_count= 0;
+  evp_ctx= EVP_CIPHER_CTX_new();
+  EVP_CIPHER_CTX_free(evp_ctx);
+  if (allocated_count != 1 || allocated_size > EVP_CIPHER_CTX_SIZE)
+    return 1;
+
+  allocated_size= allocated_count= 0;
+  md5_ctx= EVP_MD_CTX_create();
+  EVP_MD_CTX_destroy(md5_ctx);
+  if (allocated_count != 1 || allocated_size > EVP_MD_CTX_SIZE)
+    return 1;
+
+  CRYPTO_set_mem_functions(CRYPTO_malloc, CRYPTO_realloc, CRYPTO_free);
+  return 0;
+}
+#endif
diff --git a/mysys_ssl/yassl.cc b/mysys_ssl/yassl.cc
index 86acd315861..23376c82b4f 100644
--- a/mysys_ssl/yassl.cc
+++ b/mysys_ssl/yassl.cc
@@ -24,7 +24,6 @@
 
 #include <openssl/ssl.h>
 #include "aes.hpp"
-#include <my_sys.h>
 
 using yaSSL::yaERR_remove_state;
 
@@ -45,7 +44,6 @@ typedef struct
   int buf_len;
   int final_used;
   uchar tao_buf[sizeof(TaoCrypt::AES)];   // TaoCrypt::AES object
-  uchar oiv[TaoCrypt::AES::BLOCK_SIZE];   // original IV
   uchar buf[TaoCrypt::AES::BLOCK_SIZE];   // last partial input block
   uchar final[TaoCrypt::AES::BLOCK_SIZE]; // last decrypted (output) block
 } EVP_CIPHER_CTX;
@@ -76,26 +74,12 @@ static void EVP_CIPHER_CTX_init(EVP_CIPHER_CTX *ctx)
   ctx->final_used= ctx->buf_len= ctx->flags= 0;
 }
 
-static EVP_CIPHER_CTX *EVP_CIPHER_CTX_new()
-{
-  EVP_CIPHER_CTX *ctx= (EVP_CIPHER_CTX *)my_malloc(sizeof(EVP_CIPHER_CTX), MYF(0));
-  if (ctx)
-    EVP_CIPHER_CTX_init(ctx);
-  return ctx;
-}
-
 static int EVP_CIPHER_CTX_cleanup(EVP_CIPHER_CTX *ctx)
 {
   TAO(ctx)->~AES();
   return 1;
 }
 
-static void EVP_CIPHER_CTX_free(EVP_CIPHER_CTX *ctx)
-{
-  EVP_CIPHER_CTX_cleanup(ctx);
-  my_free(ctx);
-}
-
 static int EVP_CIPHER_CTX_set_padding(EVP_CIPHER_CTX *ctx, int pad)
 {
   if (pad)
@@ -112,10 +96,7 @@ static int EVP_CipherInit_ex(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *cipher,
                                        : TaoCrypt::DECRYPTION, cipher->mode);
   TAO(ctx)->SetKey(key, cipher->key_len);
   if (iv)
-  {
     TAO(ctx)->SetIV(iv);
-    memcpy(ctx->oiv, iv, TaoCrypt::AES::BLOCK_SIZE);
-  }
   ctx->encrypt= enc;
   ctx->key_len= cipher->key_len;
   ctx->flags|= cipher->mode == TaoCrypt::CBC ? EVP_CIPH_CBC_MODE : EVP_CIPH_ECB_MODE;