MDEV-10594 SSL hostname verification fails for SubjectAltNames

use X509_check_host for OpenSSL 1.0.2+
This adds:
* support for subjectAltNames
* wildcards
* sub-domain matching

6 files changed, 103 insertions, 10 deletions

diff --git a/mysql-test/lib/generate-ssl-certs.sh b/mysql-test/lib/generate-ssl-certs.sh
index e5e995489a0..8f15ba9d521 100755
--- a/mysql-test/lib/generate-ssl-certs.sh
+++ b/mysql-test/lib/generate-ssl-certs.sh
@@ -29,4 +29,11 @@ openssl req -newkey rsa:1024 -keyout client-key.pem -out demoCA/client-req.pem -
 openssl rsa -in client-key.pem -out client-key.pem
 openssl ca -keyfile cakey.pem -days 7300 -batch -cert cacert.pem -policy policy_anything -out client-cert.pem -infiles demoCA/client-req.pem
 
+# with SubjectAltName, only for OpenSSL 1.0.2+
+cat > demoCA/sanext.conf <<EOF
+subjectAltName=DNS:localhost
+EOF
+openssl req -newkey rsa:1024 -keyout serversan-key.pem -out demoCA/serversan-req.pem -days 7300 -nodes -subj '/CN=server/C=FI/ST=Helsinki/L=Helsinki/O=MariaDB'
+openssl ca -keyfile cakey.pem -extfile demoCA/sanext.conf -days 7300 -batch -cert cacert.pem -policy policy_anything -out serversan-cert.pem -infiles demoCA/serversan-req.pem
+
 rm -rf demoCA
diff --git a/mysql-test/std_data/serversan-cert.pem b/mysql-test/std_data/serversan-cert.pem
new file mode 100644
index 00000000000..e47779f420d
--- /dev/null
+++ b/mysql-test/std_data/serversan-cert.pem
@@ -0,0 +1,60 @@
+Certificate:
+    Data:
+        Version: 3 (0x2)
+        Serial Number: 4 (0x4)
+    Signature Algorithm: sha256WithRSAEncryption
+        Issuer: CN=cacert, C=FI, ST=Helsinki, L=Helsinki, O=MariaDB
+        Validity
+            Not Before: Apr 25 20:52:33 2017 GMT
+            Not After : Apr 20 20:52:33 2037 GMT
+        Subject: C=FI, ST=Helsinki, L=Helsinki, O=MariaDB, CN=server
+        Subject Public Key Info:
+            Public Key Algorithm: rsaEncryption
+                Public-Key: (1024 bit)
+                Modulus:
+                    00:a7:74:d4:2b:80:cb:96:08:2a:b9:c2:87:18:0d:
+                    69:2b:da:cf:ef:21:cb:05:d4:80:2c:f3:85:bc:78:
+                    b2:42:d9:9f:f1:dc:47:68:c5:af:5a:c9:01:f0:dd:
+                    91:cb:3a:b9:38:b2:36:6b:a3:66:ef:cd:44:0f:8f:
+                    39:57:60:ad:3b:44:33:51:c2:7f:cb:5c:8d:55:b8:
+                    1e:e8:80:e0:ed:9d:8d:10:7a:42:68:73:06:63:83:
+                    ce:db:05:5b:e1:7b:f9:0e:87:20:38:b8:11:6a:b7:
+                    59:3d:4a:ca:cb:60:e6:e1:73:d9:a2:24:4a:70:93:
+                    5e:cf:d5:04:d5:ad:ac:96:a5
+                Exponent: 65537 (0x10001)
+        X509v3 extensions:
+            X509v3 Subject Alternative Name: 
+                DNS:localhost
+    Signature Algorithm: sha256WithRSAEncryption
+         4b:78:d9:09:4c:25:cc:fb:17:8f:31:13:ac:d7:36:2d:5f:d4:
+         ce:94:84:d2:a7:fa:e2:1e:ae:b6:72:1f:01:56:0f:89:80:c0:
+         01:ba:ad:d7:cb:24:c5:25:ec:f8:35:ac:52:1b:4f:af:7c:26:
+         8d:d4:d4:91:05:21:b7:ba:3f:6b:1b:8d:1d:a5:6b:7e:7d:be:
+         2f:6a:09:83:c2:c3:6c:2f:8a:31:fa:7b:36:3f:6d:e1:62:ca:
+         a0:3c:43:b8:53:5a:4a:b3:4d:7a:cb:9c:6e:db:a4:ce:a1:95:
+         5e:26:d8:22:39:8c:34:0e:92:bd:87:a2:b1:7a:68:25:57:17:
+         b2:d8:43:3b:98:e4:80:6b:7d:3e:ab:32:82:6d:b8:80:45:83:
+         d6:55:f8:cd:31:74:17:8c:42:75:09:71:66:b9:e0:94:16:ca:
+         1d:db:1e:89:12:a1:9f:00:cb:83:99:5d:5d:28:7a:df:2a:87:
+         b5:8d:f1:9c:b9:89:2a:0d:6c:af:61:00:41:cb:03:df:99:4a:
+         fe:93:81:88:ff:47:4e:2a:b5:2b:bf:85:0f:9a:21:7b:20:58:
+         7a:1c:67:b5:8b:da:db:03:69:25:db:76:0e:f9:23:57:8d:8a:
+         47:dc:15:16:7c:2d:66:8f:6a:10:f3:b2:ea:2e:31:c6:d4:2c:
+         90:15:56:f4
+-----BEGIN CERTIFICATE-----
+MIICuzCCAaOgAwIBAgIBBDANBgkqhkiG9w0BAQsFADBWMQ8wDQYDVQQDDAZjYWNl
+cnQxCzAJBgNVBAYTAkZJMREwDwYDVQQIDAhIZWxzaW5raTERMA8GA1UEBwwISGVs
+c2lua2kxEDAOBgNVBAoMB01hcmlhREIwHhcNMTcwNDI1MjA1MjMzWhcNMzcwNDIw
+MjA1MjMzWjBWMQswCQYDVQQGEwJGSTERMA8GA1UECAwISGVsc2lua2kxETAPBgNV
+BAcMCEhlbHNpbmtpMRAwDgYDVQQKDAdNYXJpYURCMQ8wDQYDVQQDDAZzZXJ2ZXIw
+gZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAKd01CuAy5YIKrnChxgNaSvaz+8h
+ywXUgCzzhbx4skLZn/HcR2jFr1rJAfDdkcs6uTiyNmujZu/NRA+POVdgrTtEM1HC
+f8tcjVW4HuiA4O2djRB6QmhzBmODztsFW+F7+Q6HIDi4EWq3WT1Kystg5uFz2aIk
+SnCTXs/VBNWtrJalAgMBAAGjGDAWMBQGA1UdEQQNMAuCCWxvY2FsaG9zdDANBgkq
+hkiG9w0BAQsFAAOCAQEAS3jZCUwlzPsXjzETrNc2LV/UzpSE0qf64h6utnIfAVYP
+iYDAAbqt18skxSXs+DWsUhtPr3wmjdTUkQUht7o/axuNHaVrfn2+L2oJg8LDbC+K
+Mfp7Nj9t4WLKoDxDuFNaSrNNesucbtukzqGVXibYIjmMNA6SvYeisXpoJVcXsthD
+O5jkgGt9Pqsygm24gEWD1lX4zTF0F4xCdQlxZrnglBbKHdseiRKhnwDLg5ldXSh6
+3yqHtY3xnLmJKg1sr2EAQcsD35lK/pOBiP9HTiq1K7+FD5oheyBYehxntYva2wNp
+Jdt2DvkjV42KR9wVFnwtZo9qEPOy6i4xxtQskBVW9A==
+-----END CERTIFICATE-----
diff --git a/mysql-test/std_data/serversan-key.pem b/mysql-test/std_data/serversan-key.pem
new file mode 100644
index 00000000000..393c0bc9c1a
--- /dev/null
+++ b/mysql-test/std_data/serversan-key.pem
@@ -0,0 +1,16 @@
+-----BEGIN PRIVATE KEY-----
+MIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBAKd01CuAy5YIKrnC
+hxgNaSvaz+8hywXUgCzzhbx4skLZn/HcR2jFr1rJAfDdkcs6uTiyNmujZu/NRA+P
+OVdgrTtEM1HCf8tcjVW4HuiA4O2djRB6QmhzBmODztsFW+F7+Q6HIDi4EWq3WT1K
+ystg5uFz2aIkSnCTXs/VBNWtrJalAgMBAAECgYBReSgZmmpzLroK8zhjXXMEIUv1
+3w02YvOR61HwJxEkMVn+hNxBf50XoKDPHh5nMMUZbqvHpxLYLZilsVuGxcTCPVzw
+YxTooPcJY8x61oUclI2Ls5czu/OfzoJhA9ESaFn6e4xReUFmNi8ygTMuPReZZ90T
+ZvDikonKtCCk99MSaQJBANrmlPtfY57KJ18f1TqLvqy73I1vQjffSOrK3deYbvvB
+jUJ79G9Wzj8Hje2y+XkkK+OIPcND1DnoTCTuqVazn+cCQQDD1jy8zrVg/JEPhQkS
+BM7nvm4PIb0cgTPrOhsHDIF4hbaAZnA0N4ZEJ2q7YitXfOeR98x+aH/WJOrzzhmE
+VXOTAkBQ4lK6b4zH57qUk5aeg3R5LxFX0XyOWJsA5uUB/PlFXUdtAZBYc6LR92Ci
+LDeyY4M0F+t6c12/5+3615UKzGSRAkA+SGV6utcOqGTOJcZTt7nCFFtWbqmBZkoH
+1qv/2udWWFhJj8rBoKMQC+UzAS69nVjcoI2l6kA17/nVXkfZQYAHAkEAmOHCZCVQ
+9CCYTJICvoZR2euUYdnatLN8d2/ARWjzcRDTdS82P2oscATwAsvJxsphDmbOmVWP
+Hfy1t8OOCHKYAQ==
+-----END PRIVATE KEY-----
diff --git a/mysql-test/suite.pm b/mysql-test/suite.pm
index ea07af7376c..4d921d1b049 100644
--- a/mysql-test/suite.pm
+++ b/mysql-test/suite.pm
@@ -66,6 +66,10 @@ sub skip_combinations {
     unless $::mysqld_variables{'version-ssl-library'} =~ /OpenSSL (\S+)/
        and $1 ge "1.0.1d";
 
+  $skip{'t/ssl_7937.combinations'} = [ 'x509v3' ]
+    unless $::mysqld_variables{'version-ssl-library'} =~ /OpenSSL (\S+)/
+       and $1 ge "1.0.2";
+
   %skip;
 }
 
diff --git a/mysql-test/t/ssl_7937.combinations b/mysql-test/t/ssl_7937.combinations
index 46a45686a9b..71b134e229a 100644
--- a/mysql-test/t/ssl_7937.combinations
+++ b/mysql-test/t/ssl_7937.combinations
@@ -1,3 +1,8 @@
+[x509v3]
+--loose-enable-ssl
+--loose-ssl-cert=$MYSQL_TEST_DIR/std_data/serversan-cert.pem
+--loose-ssl-key=$MYSQL_TEST_DIR/std_data/serversan-key.pem
+
 [ssl]
 --loose-enable-ssl
 
diff --git a/sql-common/client.c b/sql-common/client.c
index 42b6667b1bf..332e60947e6 100644
--- a/sql-common/client.c
+++ b/sql-common/client.c
@@ -1768,15 +1768,22 @@ mysql_get_ssl_cipher(MYSQL *mysql __attribute__((unused)))
 
 #if defined(HAVE_OPENSSL)
 
+#if OPENSSL_VERSION_NUMBER >= 0x10002000L && !defined(HAVE_YASSL)
+#include <openssl/x509v3.h>
+#define HAVE_X509_check_host
+#endif
+
 static int ssl_verify_server_cert(Vio *vio, const char* server_hostname, const char **errptr)
 {
   SSL *ssl;
   X509 *server_cert= NULL;
+#ifndef HAVE_X509_check_host
   char *cn= NULL;
   int cn_loc= -1;
   ASN1_STRING *cn_asn1= NULL;
   X509_NAME_ENTRY *cn_entry= NULL;
   X509_NAME *subject= NULL;
+#endif
   int ret_validation= 1;
 
   DBUG_ENTER("ssl_verify_server_cert");
@@ -1811,14 +1818,9 @@ static int ssl_verify_server_cert(Vio *vio, const char* server_hostname, const c
     are what we expect.
   */
 
-  /*
-   Some notes for future development
-   We should check host name in alternative name first and then if needed check in common name.
-   Currently yssl doesn't support alternative name.
-   openssl 1.0.2 support X509_check_host method for host name validation, we may need to start using
-   X509_check_host in the future.
-  */
-
+#ifdef HAVE_X509_check_host
+  ret_validation= X509_check_host(server_cert, server_hostname, 0, 0, 0) != 1;
+#else
   subject= X509_get_subject_name(server_cert);
   cn_loc= X509_NAME_get_index_by_NID(subject, NID_commonName, -1);
   if (cn_loc < 0)
@@ -1826,7 +1828,6 @@ static int ssl_verify_server_cert(Vio *vio, const char* server_hostname, const c
     *errptr= "Failed to get CN location in the certificate subject";
     goto error;
   }
-
   cn_entry= X509_NAME_get_entry(subject, cn_loc);
   if (cn_entry == NULL)
   {
@@ -1855,7 +1856,7 @@ static int ssl_verify_server_cert(Vio *vio, const char* server_hostname, const c
     /* Success */
     ret_validation= 0;
   }
-
+#endif
   *errptr= "SSL certificate validation failure";
 
 error: