MDEV-18531 : Use WolfSSL instead of YaSSL as "bundled" SSL/encryption library

- Add new submodule for WolfSSL
- Build and use wolfssl and wolfcrypt instead of yassl/taocrypt
- Use HAVE_WOLFSSL instead of HAVE_YASSL
- Increase MY_AES_CTX_SIZE, to avoid compile time asserts in my_crypt.cc
(sizeof(EVP_CIPHER_CTX) is larger on WolfSSL)

4 files changed, 54 insertions, 31 deletions

diff --git a/mysys_ssl/CMakeLists.txt b/mysys_ssl/CMakeLists.txt
index 86749128664..1c3f60b5bb0 100644
--- a/mysys_ssl/CMakeLists.txt
+++ b/mysys_ssl/CMakeLists.txt
@@ -36,7 +36,6 @@ SET(MYSYS_SSL_SOURCES
     my_crypt.cc
    )
 
-# We do RESTRICT_SYMBOL_EXPORTS(yassl) elsewhere.
 # In order to get correct symbol visibility, these files
 # must be compiled with "-fvisibility=hidden"
 IF(WITH_SSL STREQUAL "bundled" AND HAVE_VISIBILITY_HIDDEN)
diff --git a/mysys_ssl/my_crypt.cc b/mysys_ssl/my_crypt.cc
index 65dd5cd769e..e83c949f21e 100644
--- a/mysys_ssl/my_crypt.cc
+++ b/mysys_ssl/my_crypt.cc
@@ -18,14 +18,10 @@
 #include <my_global.h>
 #include <string.h>
 
-#ifdef HAVE_YASSL
-#include "yassl.cc"
-#else
 #include <openssl/evp.h>
 #include <openssl/aes.h>
 #include <openssl/err.h>
 #include <openssl/rand.h>
-#endif
 
 #include <my_crypt.h>
 #include <ssl_compat.h>
@@ -54,7 +50,7 @@ public:
     if (unlikely(!cipher))
       return MY_AES_BAD_KEYSIZE;
 
-    if (!EVP_CipherInit_ex(ctx, cipher, NULL, key, iv, encrypt))
+    if (EVP_CipherInit_ex(ctx, cipher, NULL, key, iv, encrypt) != 1)
       return MY_AES_OPENSSL_ERROR;
 
     DBUG_ASSERT(EVP_CIPHER_CTX_key_length(ctx) == (int)klen);
@@ -64,14 +60,30 @@ public:
   }
   virtual int update(const uchar *src, uint slen, uchar *dst, uint *dlen)
   {
-    if (!EVP_CipherUpdate(ctx, dst, (int*)dlen, src, slen))
+    if (EVP_CipherUpdate(ctx, dst, (int*)dlen, src, slen) != 1)
       return MY_AES_OPENSSL_ERROR;
     return MY_AES_OK;
   }
   virtual int finish(uchar *dst, uint *dlen)
   {
-    if (!EVP_CipherFinal_ex(ctx, dst, (int*)dlen))
+#ifdef HAVE_WOLFSSL
+     /*
+       Bug in WolfSSL - sometimes EVP_CipherFinal_ex
+       returns success without setting destination length
+       when it should return error.
+       We catch it by presetting invalid value for length,
+       and checking if it has changed after the call.
+
+       See https://github.com/wolfSSL/wolfssl/issues/2224
+     */
+    *dlen= UINT_MAX;
+#endif
+    if (EVP_CipherFinal_ex(ctx, dst, (int*)dlen) != 1)
       return MY_AES_BAD_DATA;
+#ifdef HAVE_WOLFSSL
+    if (*dlen == UINT_MAX)
+      return MY_AES_BAD_DATA;
+#endif
     return MY_AES_OK;
   }
 };
diff --git a/mysys_ssl/my_md5.cc b/mysys_ssl/my_md5.cc
index 85490c1de77..407dee3bc69 100644
--- a/mysys_ssl/my_md5.cc
+++ b/mysys_ssl/my_md5.cc
@@ -27,26 +27,23 @@
 #include <my_md5.h>
 #include <stdarg.h>
 
-#if defined(HAVE_YASSL)
-#include "md5.hpp"
+#if defined(HAVE_WOLFSSL)
+#include <wolfssl/wolfcrypt/md5.h>
 #include <ssl_compat.h>
-
-typedef TaoCrypt::MD5 EVP_MD_CTX;
-
+typedef wc_Md5 EVP_MD_CTX;
 static void md5_init(EVP_MD_CTX *context)
 {
-  context= new(context) EVP_MD_CTX;
-  context->Init();
+  wc_InitMd5(context);;
 }
 
 static void md5_input(EVP_MD_CTX *context, const uchar *buf, unsigned len)
 {
-  context->Update((const TaoCrypt::byte *) buf, len);
+  wc_Md5Update(context, buf, len);
 }
 
 static void md5_result(EVP_MD_CTX *context, uchar digest[MD5_HASH_SIZE])
 {
-    context->Final((TaoCrypt::byte *) digest);
+  wc_Md5Final(context,digest);
 }
 
 #elif defined(HAVE_OPENSSL)
@@ -74,7 +71,7 @@ static void md5_result(EVP_MD_CTX *context, uchar digest[MD5_HASH_SIZE])
   EVP_MD_CTX_reset(context);
 }
 
-#endif /* HAVE_YASSL */
+#endif /* HAVE_WOLFSSL */
 
 /**
   Wrapper function to compute MD5 message digest.
diff --git a/mysys_ssl/my_sha.ic b/mysys_ssl/my_sha.ic
index 97344dc0415..6bba614765e 100644
--- a/mysys_ssl/my_sha.ic
+++ b/mysys_ssl/my_sha.ic
@@ -28,35 +28,50 @@
 
 #define HASH_SIZE (NUM > 1 ? NUM/8 : 20)
 
-#if defined(HAVE_YASSL)
-#include "sha.hpp"
-
-#define xCONTEXT(x)     TaoCrypt::SHA ## x
+#if defined(HAVE_WOLFSSL)
+#define WOLFSSL_SHA512
+#define WOLFSSL_SHA384
+#define WOLFSSL_SHA224
+#include <wolfcrypt/sha.h>
+#include <wolfcrypt/sha256.h>
+#include <wolfcrypt/sha512.h>
+#define xCONTEXT(x)     wc_Sha ## x
 #define yCONTEXT(y)     xCONTEXT(y)
 #define CONTEXT         yCONTEXT(NUM)
-#define SHA1            SHA
+#define wc_InitSha1     wc_InitSha
+#define wc_Sha1Final    wc_ShaFinal
+#define wc_Sha1Update   wc_ShaUpdate
+#define wc_Sha1         wc_Sha
+#define SHA224_CTX      SHA256_CTX
+#define SHA384_CTX      SHA512_CTX
 
+#define xSHA_Init(x)    wc_InitSha ## x
+#define xSHA_Update(x)  wc_Sha ## x ## Update
+#define xSHA_Final(x)   wc_Sha ## x ## Final
+#define ySHA_Init(y)    xSHA_Init(y)
+#define ySHA_Update(y)  xSHA_Update(y)
+#define ySHA_Final(y)   xSHA_Final(y)
+#define SHA_Init        ySHA_Init(NUM)
+#define SHA_Update      ySHA_Update(NUM)
+#define SHA_Final       ySHA_Final(NUM)
 static void sha_init(CONTEXT *context)
 {
-  context->Init();
+  SHA_Init(context);
 }
 
-/*
-  this is a variant of sha_init to be used in this file only.
-  does nothing for yassl, because the context's constructor was called automatically.
-*/
 static void sha_init_fast(CONTEXT *context)
 {
+  sha_init(context);
 }
 
 static void sha_input(CONTEXT *context, const uchar *buf, unsigned len)
 {
-  context->Update((const TaoCrypt::byte *) buf, len);
+  SHA_Update(context, buf, len);
 }
 
 static void sha_result(CONTEXT *context, uchar digest[HASH_SIZE])
 {
-    context->Final((TaoCrypt::byte *) digest);
+  SHA_Final(context, digest);
 }
 
 #elif defined(HAVE_OPENSSL)
@@ -99,7 +114,7 @@ static void sha_result(CONTEXT *context, uchar digest[HASH_SIZE])
   SHA_Final(digest, context);
 }
 
-#endif /* HAVE_YASSL */
+#endif /* HAVE_WOLFSSL */
 
 #define xmy_sha_multi(x)         my_sha ## x ## _multi
 #define xmy_sha_context_size(x)  my_sha ## x ## _context_size