MDEV-10823 Certain unicode characters in hostname prevent mysqld from starting

Server uses gethostname() for the default base name for pid/log files.
If a character is not representable in current ANSI encoding, gethostname
replaces it with question mark. Thus, generated log file name would also
contain a question mark. However, Windows forbids certain characters in
filenames, among them '?'.

This is described in MSDN article https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx
At attempts to create the file via freopen() fails, thus server would not
be able to start.

The fix is to verify hostname and fall back to  "mysql", if
invalid characters are found.

2 files changed, 15 insertions, 4 deletions

diff --git a/mysys/my_access.c b/mysys/my_access.c
index 68cd01d33e6..0da5e7f0bf0 100644
--- a/mysys/my_access.c
+++ b/mysys/my_access.c
@@ -173,6 +173,11 @@ static my_bool does_drive_exists(char drive_letter)
   file names with a colon (:) are not allowed because such file names
   store data in Alternate Data Streams which can be used to hide 
   the data.
+  Apart from colon, other characters that are not allowed in filenames
+  on Windows are greater/less sign, double quotes, forward slash, backslash,
+  pipe and star characters.
+
+  See MSDN documentation on filename restrictions.
 
   @param name contains the file name with or without path
   @param length contains the length of file name
@@ -181,6 +186,8 @@ static my_bool does_drive_exists(char drive_letter)
  
   @return TRUE if the file name is allowed, FALSE otherwise.
 */
+#define ILLEGAL_FILENAME_CHARS "<>:\"/\|?*"
+
 my_bool is_filename_allowed(const char *name __attribute__((unused)),
                             size_t length __attribute__((unused)),
                             my_bool allow_current_dir __attribute__((unused)))
@@ -205,6 +212,8 @@ my_bool is_filename_allowed(const char *name __attribute__((unused)),
       return (allow_current_dir && (ch - name == 1) && 
               does_drive_exists(*name));
     }
+    else if (strchr(ILLEGAL_FILENAME_CHARS, *ch))
+      return FALSE;
   }
   return TRUE;
 } /* is_filename_allowed */
diff --git a/sql/mysqld.cc b/sql/mysqld.cc
index a71d6d10042..41f6def8e08 100644
--- a/sql/mysqld.cc
+++ b/sql/mysqld.cc
@@ -4167,6 +4167,8 @@ static int init_common_variables()
     return 1;
   }
 
+  opt_log_basename= const_cast<char *>("mysql");
+
   if (gethostname(glob_hostname,sizeof(glob_hostname)) < 0)
   {
     /*
@@ -4176,9 +4178,8 @@ static int init_common_variables()
     strmake(glob_hostname, STRING_WITH_LEN("localhost"));
     sql_print_warning("gethostname failed, using '%s' as hostname",
                         glob_hostname);
-    opt_log_basename= const_cast<char *>("mysql");
   }
-  else
+  else if (is_filename_allowed(glob_hostname, strlen(glob_hostname), FALSE))
     opt_log_basename= glob_hostname;
 
   strmake(pidfile_name, opt_log_basename, sizeof(pidfile_name)-5);
@@ -8970,9 +8971,10 @@ mysqld_get_one_option(int optid, const struct my_option *opt, char *argument)
   case (int) OPT_LOG_BASENAME:
   {
     if (opt_log_basename[0] == 0 || strchr(opt_log_basename, FN_EXTCHAR) ||
-        strchr(opt_log_basename,FN_LIBCHAR))
+        strchr(opt_log_basename,FN_LIBCHAR) ||
+        !is_filename_allowed(opt_log_basename, strlen(opt_log_basename), FALSE))
     {
-      sql_print_error("Wrong argument for --log-basename. It can't be empty or contain '.' or '" FN_DIRSEP "'");
+      sql_print_error("Wrong argument for --log-basename. It can't be empty or contain '.' or '" FN_DIRSEP "'. It must be valid filename.");
       return 1;
     }
     if (log_error_file_ptr != disabled_my_option)