MDEV-7772: SIGSEGV on my_aes_encrypt_cbc when -DWITH_SSL=bundled

Two problems:
- Read/Write outside of buffer at memcpy() because of incorrect parameters
. OPENSSL_assert(EVP_CIPHER_CTX_iv_length(&ctx.ctx) == iv_length); // ECB does not use IV, thus incorrect assertion

Added:
  mysql-test/include/encryption_algorithms.combinations to run tests with methods cbc, ecb and ctr in
  those systems where they are available (see suite.pm).

13 files changed, 35 insertions, 24 deletions

diff --git a/mysql-test/include/encryption_algorithms.combinations b/mysql-test/include/encryption_algorithms.combinations
new file mode 100644
index 00000000000..09c1e91e1cd
--- /dev/null
+++ b/mysql-test/include/encryption_algorithms.combinations
@@ -0,0 +1,8 @@
+[cbc]
+encryption-algorithm=aes_cbc
+
+[ecb]
+encryption-algorithm=aes_ecb
+
+[ctr]
+encryption-algorithm=aes_ctr
diff --git a/mysql-test/include/have_openssl_ctr.inc b/mysql-test/include/encryption_algorithms.inc
index 546cc1b9b93..546cc1b9b93 100644
--- a/mysql-test/include/have_openssl_ctr.inc
+++ b/mysql-test/include/encryption_algorithms.inc
diff --git a/mysql-test/include/have_example_key_management_plugin.inc b/mysql-test/include/have_example_key_management_plugin.inc
index 622e9a5286c..3bb4529b623 100644
--- a/mysql-test/include/have_example_key_management_plugin.inc
+++ b/mysql-test/include/have_example_key_management_plugin.inc
@@ -1,3 +1,5 @@
+-- source encryption_algorithms.inc
+
 if (`select count(*) = 0 from information_schema.plugins
      where plugin_name = 'example_key_management_plugin' and plugin_status='active'`)
 {
diff --git a/mysql-test/include/have_file_key_management_plugin.inc b/mysql-test/include/have_file_key_management_plugin.inc
index fc682a77f35..3b3eb036ede 100644
--- a/mysql-test/include/have_file_key_management_plugin.inc
+++ b/mysql-test/include/have_file_key_management_plugin.inc
@@ -1,3 +1,5 @@
+-- source encryption_algorithms.inc
+
 if (`select count(*) = 0 from information_schema.plugins
      where plugin_name = 'file_key_management_plugin' and plugin_status='active'`)
 {
diff --git a/mysql-test/include/have_file_key_management_plugin.opt b/mysql-test/include/have_file_key_management_plugin.opt
index 599847be1fe..5aa1ce18b00 100644
--- a/mysql-test/include/have_file_key_management_plugin.opt
+++ b/mysql-test/include/have_file_key_management_plugin.opt
@@ -1,4 +1,4 @@
 --plugin-load-add=$FILE_KEY_MANAGEMENT_PLUGIN_SO
 --loose-file-key-management-plugin
 --loose-file-key-management-plugin-filename=$MYSQL_TEST_DIR/std_data/keys.txt
---encryption-algorithm=aes_cbc
+
diff --git a/mysql-test/include/have_openssl_ctr.opt b/mysql-test/include/have_openssl_ctr.opt
deleted file mode 100644
index a88a4224c9c..00000000000
--- a/mysql-test/include/have_openssl_ctr.opt
+++ /dev/null
@@ -1 +0,0 @@
---encryption-algorithm=aes_ctr
diff --git a/mysql-test/suite.pm b/mysql-test/suite.pm
index 8101173b508..5eafd4e2ffd 100644
--- a/mysql-test/suite.pm
+++ b/mysql-test/suite.pm
@@ -66,7 +66,7 @@ sub skip_combinations {
     unless $::mysqld_variables{'version-ssl-library'} =~ /OpenSSL (\S+)/
        and $1 ge "1.0.1";
 
-  $skip{'include/have_openssl_ctr.inc'} = 'no or too old openssl'
+  $skip{'include/encryption_algorithms.combinations'} = [ 'ctr' ]
     unless $::mysqld_variables{'version-ssl-library'} =~ /OpenSSL (\S+)/
        and $1 ge "1.0.1";
 
diff --git a/mysql-test/suite/innodb/t/innodb-page_encryption_log_encryption.opt b/mysql-test/suite/innodb/t/innodb-page_encryption_log_encryption.opt
index fe33453e8fa..f14c376c3b9 100644
--- a/mysql-test/suite/innodb/t/innodb-page_encryption_log_encryption.opt
+++ b/mysql-test/suite/innodb/t/innodb-page_encryption_log_encryption.opt
@@ -1,2 +1 @@
---encryption-algorithm=aes_cbc
 --innodb-encrypt-log
diff --git a/mysql-test/suite/innodb/t/innodb_encryption-page-compression.test b/mysql-test/suite/innodb/t/innodb_encryption-page-compression.test
index ce59cf9547a..48a9a780e0d 100644
--- a/mysql-test/suite/innodb/t/innodb_encryption-page-compression.test
+++ b/mysql-test/suite/innodb/t/innodb_encryption-page-compression.test
@@ -1,5 +1,4 @@
 -- source include/have_innodb.inc
--- source include/have_openssl_ctr.inc
 -- source include/have_example_key_management_plugin.inc
 
 # embedded does not support restart
diff --git a/mysql-test/suite/innodb/t/innodb_encryption.opt b/mysql-test/suite/innodb/t/innodb_encryption.opt
index 306e3f95660..f7f583d18ef 100644
--- a/mysql-test/suite/innodb/t/innodb_encryption.opt
+++ b/mysql-test/suite/innodb/t/innodb_encryption.opt
@@ -4,4 +4,4 @@
 --innodb-encryption-rotate-key-age=15
 --innodb-encryption-threads=4
 --innodb-tablespaces-encryption
---encryption-algorithm=aes_ctr
+
diff --git a/mysql-test/suite/innodb/t/innodb_encryption.test b/mysql-test/suite/innodb/t/innodb_encryption.test
index ca3a68444a4..50aca2a7260 100644
--- a/mysql-test/suite/innodb/t/innodb_encryption.test
+++ b/mysql-test/suite/innodb/t/innodb_encryption.test
@@ -3,7 +3,6 @@
 #
 -- source include/have_innodb.inc
 -- source include/have_example_key_management_plugin.inc
--- source include/have_openssl_ctr.inc
 
 # embedded does not support restart
 -- source include/not_embedded.inc
diff --git a/mysql-test/suite/innodb/t/innodb_encryption_tables.test b/mysql-test/suite/innodb/t/innodb_encryption_tables.test
index c630b7d57b0..d192a16ffd4 100644
--- a/mysql-test/suite/innodb/t/innodb_encryption_tables.test
+++ b/mysql-test/suite/innodb/t/innodb_encryption_tables.test
@@ -1,6 +1,5 @@
 -- source include/have_innodb.inc
 -- source include/have_example_key_management_plugin.inc
--- source include/have_openssl_ctr.inc
 --source include/not_embedded.inc
 
 --disable_query_log
diff --git a/mysys_ssl/my_aes.cc b/mysys_ssl/my_aes.cc
index 20bd03551c2..6486f7447c3 100644
--- a/mysys_ssl/my_aes.cc
+++ b/mysys_ssl/my_aes.cc
@@ -255,12 +255,12 @@ static int my_aes_encrypt_cbc(const uchar* source, uint32 source_length,
   }
 
   if (noPadding) {
-	if (remaining_bytes!=0) {
-	   memcpy(dest + source_length, source + source_length, remaining_bytes);
-	}
+    if (remaining_bytes!=0) {
+      /* Note that we moved the original pointers above */
+      memcpy(dest, source, remaining_bytes);
+    }
     *dest_length = MY_AES_BLOCK_SIZE * (num_blocks) + remaining_bytes;
     return AES_OK;
-
   }
 
   /* Encode the rest. We always have incomplete block */
@@ -383,12 +383,12 @@ static int my_aes_encrypt_ecb(const uchar* source, uint32 source_length,
   }
 
   if (noPadding) {
-	if (remaining_bytes!=0) {
-	   memcpy(dest + source_length, source + source_length, remaining_bytes);
-	}
-	*dest_length = MY_AES_BLOCK_SIZE * (num_blocks) + remaining_bytes;
+    if (remaining_bytes!=0) {
+      /* Note that we moved the original pointers above */
+      memcpy(dest, source, remaining_bytes);
+    }
+    *dest_length = MY_AES_BLOCK_SIZE * (num_blocks) + remaining_bytes;
     return AES_OK;
-
   }
 
   /* Encode the rest. We always have incomplete block */
@@ -430,7 +430,8 @@ static int my_aes_encrypt_ecb(const uchar* source, uint32 source_length,
   }
   EVP_CIPHER_CTX_key_length(&ctx.ctx);
   OPENSSL_assert(EVP_CIPHER_CTX_key_length(&ctx.ctx) == key_length);
-  OPENSSL_assert(EVP_CIPHER_CTX_iv_length(&ctx.ctx) == iv_length);
+  // ECB does not use IV
+  OPENSSL_assert(EVP_CIPHER_CTX_iv_length(&ctx.ctx) == 0);
   OPENSSL_assert(EVP_CIPHER_CTX_block_size(&ctx.ctx) == 16);
   if (! EVP_EncryptUpdate(&ctx.ctx, (unsigned char *) dest, &u_len,
                           (unsigned const char *) source, source_length))
@@ -438,9 +439,9 @@ static int my_aes_encrypt_ecb(const uchar* source, uint32 source_length,
   if (! EVP_EncryptFinal_ex(&ctx.ctx, (unsigned char *) dest + u_len, &f_len))
     return AES_BAD_DATA;                        /* Error */
 
-  if (remaining_bytes!=0) {
-   	  memcpy(dest + source_length, source + source_length, remaining_bytes);
-     }
+  if (remaining_bytes!=0)
+    memcpy(dest + source_length, source + source_length, remaining_bytes);
+
   *dest_length = (unsigned long int) (u_len + f_len + remaining_bytes);
 
   return AES_OK;
@@ -524,7 +525,8 @@ static int my_aes_decrypt_cbc(const uchar* source, uint32 source_length,
   if (noPadding) {
     memcpy(dest, block, MY_AES_BLOCK_SIZE);
     if (remaining_bytes!=0) {
-            	  memcpy(dest + source_length, source + source_length, remaining_bytes);
+      /* Note that we have moved dest and source */
+      memcpy(dest + MY_AES_BLOCK_SIZE, source + MY_AES_BLOCK_SIZE, remaining_bytes);
     }
     *dest_length = MY_AES_BLOCK_SIZE * num_blocks + remaining_bytes;
     return AES_OK;
@@ -656,7 +658,8 @@ static int my_aes_decrypt_ecb(const uchar* source, uint32 source_length,
   if (noPadding) {
     memcpy(dest, block, MY_AES_BLOCK_SIZE);
     if (remaining_bytes!=0) {
-      memcpy(dest + source_length, source + source_length, remaining_bytes);
+      /* Note that we have moved dest and source */
+      memcpy(dest + MY_AES_BLOCK_SIZE, source + MY_AES_BLOCK_SIZE, remaining_bytes);
     }
     *dest_length = MY_AES_BLOCK_SIZE * num_blocks + remaining_bytes;
     return AES_OK;
@@ -699,7 +702,8 @@ static int my_aes_decrypt_ecb(const uchar* source, uint32 source_length,
     EVP_CIPHER_CTX_set_padding(&ctx.ctx, 0);
   }
   OPENSSL_assert(EVP_CIPHER_CTX_key_length(&ctx.ctx) == key_length);
-  OPENSSL_assert(EVP_CIPHER_CTX_iv_length(&ctx.ctx) == iv_length);
+  // ECB does not use IV
+  OPENSSL_assert(EVP_CIPHER_CTX_iv_length(&ctx.ctx) == 0);
   OPENSSL_assert(EVP_CIPHER_CTX_block_size(&ctx.ctx) == 16);
   if (! EVP_DecryptUpdate(&ctx.ctx, (unsigned char *) dest, &u_len,
                           (unsigned char *)source, source_length))