Bug#11765108 (Bug#58036) client utf32, utf16, ucs2 should be disallowed, they crash server

A separate fix for 5.1 (as 5.1 and 5.5 have seriously
differged in the related pieces of the code).
A patch for 5.5 was approved earlier.

Problem: ucs2 was correctly disallowed in "SET NAMES" only,
while mysql_real_connect() and mysql_change_user() still allowed
to use ucs2, which made server crash.

Fix: disallow ucs2 in mysql_real_connect() and mysql_change_user().

  @ sql/sql_priv.h
    - changing return type for thd_init_client_charset() to bool,
      to return errors to the caller

  @ sql/sql_var.cc
    - using new function

  @ sql/sql_connect.cc
    - thd_client_charset_init:
      in case of unsupported client character set send error and return true;
      in case of success return false
    - check_connection:
      Return error if character set initialization failed

  @ sql/sql_parse.cc
    - check charset in the very beginnig of the CMD_CHANGE_USER handling code

  @ tests/mysql_client_test.c
    - adding tests

1 files changed, 67 insertions, 0 deletions

diff --git a/tests/mysql_client_test.c b/tests/mysql_client_test.c
index 80c7be64e94..5a45c132a5f 100644
--- a/tests/mysql_client_test.c
+++ b/tests/mysql_client_test.c
@@ -18399,6 +18399,72 @@ static void test_bug47485()
 
 
 /*
+  Bug#58036 client utf32, utf16, ucs2 should be disallowed, they crash server
+*/
+static void test_bug58036()
+{
+  MYSQL *conn;
+  DBUG_ENTER("test_bug47485");
+  myheader("test_bug58036");
+
+  /* Part1: try to connect with ucs2 client character set */
+  conn= mysql_client_init(NULL);
+  mysql_options(conn, MYSQL_SET_CHARSET_NAME, "ucs2");
+  if (mysql_real_connect(conn, opt_host, opt_user,
+                         opt_password,  opt_db ? opt_db : "test",
+                         opt_port, opt_unix_socket, 0))
+  {
+    if (!opt_silent)
+      printf("mysql_real_connect() succeeded (failure expected)\n");
+    mysql_close(conn);
+    DIE();
+  }
+
+  if (!opt_silent)
+    printf("Got mysql_real_connect() error (expected): %s (%d)\n",
+           mysql_error(conn), mysql_errno(conn));  
+  DIE_UNLESS(mysql_errno(conn) == ER_WRONG_VALUE_FOR_VAR);
+  mysql_close(conn);
+
+
+  /*
+    Part2:
+    - connect with latin1
+    - then change client character set to ucs2
+    - then try mysql_change_user()
+  */
+  conn= mysql_client_init(NULL);
+  mysql_options(conn, MYSQL_SET_CHARSET_NAME, "latin1");
+  if (!mysql_real_connect(conn, opt_host, opt_user,
+                         opt_password, opt_db ? opt_db : "test",
+                         opt_port, opt_unix_socket, 0))
+  {
+    if (!opt_silent)
+      printf("mysql_real_connect() failed: %s (%d)\n",
+             mysql_error(conn), mysql_errno(conn));
+    mysql_close(conn);
+    DIE();
+  }
+
+  mysql_options(conn, MYSQL_SET_CHARSET_NAME, "ucs2");
+  if (!mysql_change_user(conn, opt_user, opt_password, NULL))
+  {
+    if (!opt_silent)
+      printf("mysql_change_user() succedded, error expected!");
+    mysql_close(conn);
+    DIE();
+  }
+
+  if (!opt_silent)
+    printf("Got mysql_change_user() error (expected): %s (%d)\n",
+           mysql_error(conn), mysql_errno(conn));
+  mysql_close(conn);
+
+  DBUG_VOID_RETURN;
+}
+
+
+/*
   Read and parse arguments and MySQL options from my.cnf
 */
 
@@ -18724,6 +18790,7 @@ static struct my_tests_st my_tests[]= {
   { "test_bug42373", test_bug42373 },
   { "test_bug54041", test_bug54041 },
   { "test_bug47485", test_bug47485 },
+  { "test_bug58036", test_bug58036 },
   { 0, 0 }
 };