MDEV-10332 support for OpenSSL 1.1 and LibreSSL

post-review fixes:
* move all ssl implementation related ifdefs/defines to one file
  (ssl_compat.h)
* work around OpenSSL-1.1 desire to malloc every EVP context by
  run-time checking that context allocated on the stack is big enough
  (openssl.c)
* use newer version of the AWS SDK for OpenSSL 1.1
* use get_dh2048() function as generated by openssl 1.1
  (viosslfactories.c)

3 files changed, 75 insertions, 27 deletions

diff --git a/include/my_crypt.h b/include/my_crypt.h
index e7dd9d80100..719e349bfb9 100644
--- a/include/my_crypt.h
+++ b/include/my_crypt.h
@@ -21,19 +21,4 @@
 #include <my_config.h> /* HAVE_EncryptAes128{Ctr,Gcm} */
 #include <mysql/service_my_crypt.h>
 
-/* OpenSSL version specific definitions */
-#if !defined(HAVE_YASSL) && defined(OPENSSL_VERSION_NUMBER)
-#if OPENSSL_VERSION_NUMBER >= 0x10100000L && !defined(LIBRESSL_VERSION_NUMBER)
-#define ERR_remove_state(X)
-#else
-#define EVP_CIPHER_CTX_reset(X) EVP_CIPHER_CTX_cleanup(X)
-#define RAND_OpenSSL() RAND_SSLeay();
-#if defined(HAVE_ERR_remove_thread_state)
-#define ERR_remove_state(X) ERR_remove_thread_state(NULL)
-#endif
-#endif
-#elif defined(HAVE_YASSL)
-#define EVP_CIPHER_CTX_reset(X) EVP_CIPHER_CTX_cleanup(X)
-#endif /* !defined(HAVE_YASSL) */
-
 #endif /* MY_CRYPT_INCLUDED */
diff --git a/include/ssl_compat.h b/include/ssl_compat.h
new file mode 100644
index 00000000000..b0e3ed497cd
--- /dev/null
+++ b/include/ssl_compat.h
@@ -0,0 +1,75 @@
+/*
+ Copyright (c) 2016, 2017 MariaDB Corporation
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; version 2 of the License.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA */
+
+#include <openssl/opensslv.h>
+
+/* OpenSSL version specific definitions */
+#if !defined(HAVE_YASSL) && defined(OPENSSL_VERSION_NUMBER)
+
+#if OPENSSL_VERSION_NUMBER >= 0x10002000L && !defined(LIBRESSL_VERSION_NUMBER)
+#define HAVE_X509_check_host 1
+#endif
+
+#if OPENSSL_VERSION_NUMBER >= 0x10100000L && !defined(LIBRESSL_VERSION_NUMBER)
+#define HAVE_OPENSSL11 1
+#define ERR_remove_state(X) ERR_clear_error()
+#define EVP_MD_CTX_cleanup(X) EVP_MD_CTX_reset(X)
+#define EVP_CIPHER_CTX_SIZE 168
+#define EVP_MD_CTX_SIZE 48
+#undef EVP_MD_CTX_init
+#define EVP_MD_CTX_init(X) do { bzero((X), EVP_MD_CTX_SIZE); EVP_MD_CTX_reset(X); } while(0)
+#undef EVP_CIPHER_CTX_init
+#define EVP_CIPHER_CTX_init(X) do { bzero((X), EVP_CIPHER_CTX_SIZE); EVP_CIPHER_CTX_reset(X); } while(0)
+
+#else
+#define HAVE_OPENSSL10 1
+/*
+  Unfortunately RAND_bytes manual page does not provide any guarantees
+  in relation to blocking behavior. Here we explicitly use SSLeay random
+  instead of whatever random engine is currently set in OpenSSL. That way
+  we are guaranteed to have a non-blocking random.
+*/
+#define RAND_OpenSSL() RAND_SSLeay()
+
+#ifdef HAVE_ERR_remove_thread_state
+#define ERR_remove_state(X) ERR_remove_thread_state(NULL)
+#endif /* HAVE_ERR_remove_thread_state */
+
+#endif /* HAVE_OPENSSL11 */
+
+#elif defined(HAVE_YASSL)
+#define BN_free(X) do { } while(0)
+#endif /* !defined(HAVE_YASSL) */
+
+#ifndef HAVE_OPENSSL11
+#define ASN1_STRING_get0_data(X)        ASN1_STRING_data(X)
+#define OPENSSL_init_ssl(X,Y)           SSL_library_init()
+#define DH_set0_pqg(D,P,Q,G)            ((D)->p= (P), (D)->g= (G))
+#define EVP_CIPHER_CTX_buf_noconst(ctx) ((ctx)->buf)
+#define EVP_CIPHER_CTX_encrypting(ctx)  ((ctx)->encrypt)
+#define EVP_CIPHER_CTX_SIZE             sizeof(EVP_CIPHER_CTX)
+#define EVP_MD_CTX_SIZE                 sizeof(EVP_MD_CTX)
+#endif
+
+#ifdef	__cplusplus
+extern "C" {
+#endif /* __cplusplus */
+
+int check_openssl_compatibility();
+
+#ifdef	__cplusplus
+}
+#endif
diff --git a/include/violite.h b/include/violite.h
index 78ca45da6f1..5dcf27dbab1 100644
--- a/include/violite.h
+++ b/include/violite.h
@@ -123,13 +123,6 @@ int vio_getnameinfo(const struct sockaddr *sa,
                     int flags);
 
 #ifdef HAVE_OPENSSL
-#include <openssl/opensslv.h>
-#if OPENSSL_VERSION_NUMBER < 0x0090700f
-#define DES_cblock des_cblock
-#define DES_key_schedule des_key_schedule
-#define DES_set_key_unchecked(k,ks) des_set_key_unchecked((k),*(ks))
-#define DES_ede3_cbc_encrypt(i,o,l,k1,k2,k3,iv,e) des_ede3_cbc_encrypt((i),(o),(l),*(k1),*(k2),*(k3),(iv),(e))
-#endif
 /* apple deprecated openssl in MacOSX Lion */
 #ifdef __APPLE__
 #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
@@ -146,11 +139,6 @@ typedef my_socket YASSL_SOCKET_T;
 #include <openssl/ssl.h>
 #include <openssl/err.h>
 
-#if OPENSSL_VERSION_NUMBER >= 0x10100000L && !defined(LIBRESSL_VERSION_NUMBER)
-#define ERR_remove_state(X)
-#elif defined(HAVE_ERR_remove_thread_state)
-#define ERR_remove_state(X) ERR_remove_thread_state(NULL)
-#endif
 enum enum_ssl_init_error
 {
   SSL_INITERR_NOERROR= 0, SSL_INITERR_CERT, SSL_INITERR_KEY,