Merge 10.0 into 10.1

Also, implement MDEV-11027 a little differently from 5.5 and 10.0:

recv_apply_hashed_log_recs(): Change the return type back to void
(DB_SUCCESS was always returned).

Report progress also via systemd using sd_notifyf().

13 files changed, 205 insertions, 85 deletions

diff --git a/mysys/mf_format.c b/mysys/mf_format.c
index 91354db0b64..6672a4386e4 100644
--- a/mysys/mf_format.c
+++ b/mysys/mf_format.c
@@ -97,13 +97,8 @@ char * fn_format(char * to, const char *name, const char *dir,
     pos=strmake(strmov(to,dev),name,length);
     (void) strmov(pos,ext);			/* Don't convert extension */
   }
-  /*
-    If MY_RETURN_REAL_PATH and MY_RESOLVE_SYMLINK is given, only do
-    realpath if the file is a symbolic link
-  */
   if (flag & MY_RETURN_REAL_PATH)
-    (void) my_realpath(to, to, MYF(flag & MY_RESOLVE_SYMLINKS ?
-				   MY_RESOLVE_LINK: 0));
+    (void) my_realpath(to, to, MYF(0));
   else if (flag & MY_RESOLVE_SYMLINKS)
   {
     strmov(buff,to);
diff --git a/mysys/my_context.c b/mysys/my_context.c
index 017cebdc110..29ba9f79c72 100644
--- a/mysys/my_context.c
+++ b/mysys/my_context.c
@@ -206,7 +206,8 @@ my_context_spawn(struct my_context *c, void (*f)(void *), void *d)
     (
      "movq %%rsp, (%[save])\n\t"
      "movq %[stack], %%rsp\n\t"
-#if (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 4)) && !defined(__INTEL_COMPILER)
+#if (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 4) || __clang__) && \
+     !defined(__INTEL_COMPILER)
      /*
        This emits a DWARF DW_CFA_undefined directive to make the return address
        undefined. This indicates that this is the top of the stack frame, and
diff --git a/mysys/my_create.c b/mysys/my_create.c
index 6a3bcd63557..014b65c4e14 100644
--- a/mysys/my_create.c
+++ b/mysys/my_create.c
@@ -36,7 +36,7 @@
 File my_create(const char *FileName, int CreateFlags, int access_flags,
 	       myf MyFlags)
 {
-  int fd, rc;
+  int fd;
   DBUG_ENTER("my_create");
   DBUG_PRINT("my",("Name: '%s' CreateFlags: %d  AccessFlags: %d  MyFlags: %lu",
 		   FileName, CreateFlags, access_flags, MyFlags));
@@ -54,21 +54,7 @@ File my_create(const char *FileName, int CreateFlags, int access_flags,
     fd= -1;
   }
 
-  rc= my_register_filename(fd, FileName, FILE_BY_CREATE,
+  fd= my_register_filename(fd, FileName, FILE_BY_CREATE,
                            EE_CANTCREATEFILE, MyFlags);
-  /*
-    my_register_filename() may fail on some platforms even if the call to
-    *open() above succeeds. In this case, don't leave the stale file because
-    callers assume the file to not exist if my_create() fails, so they don't
-    do any cleanups.
-  */
-  if (unlikely(fd >= 0 && rc < 0))
-  {
-    int tmp= my_errno;
-    my_close(fd, MyFlags);
-    my_delete(FileName, MyFlags);
-    my_errno= tmp;
-  }
-  
-  DBUG_RETURN(rc);
+  DBUG_RETURN(fd);
 } /* my_create */
diff --git a/mysys/my_delete.c b/mysys/my_delete.c
index 3dfe290dabe..0faf6079d98 100644
--- a/mysys/my_delete.c
+++ b/mysys/my_delete.c
@@ -21,6 +21,12 @@
 static int my_win_unlink(const char *name);
 #endif
 
+CREATE_NOSYMLINK_FUNCTION(
+  unlink_nosymlinks(const char *pathname),
+  unlinkat(dfd, filename, 0),
+  unlink(pathname)
+);
+
 int my_delete(const char *name, myf MyFlags)
 {
   int err;
@@ -30,7 +36,10 @@ int my_delete(const char *name, myf MyFlags)
 #ifdef _WIN32
   err = my_win_unlink(name);
 #else
-  err = unlink(name);
+  if (MyFlags & MY_NOSYMLINKS)
+    err= unlink_nosymlinks(name);
+  else
+    err= unlink(name);
 #endif
 
   if(err)
diff --git a/mysys/my_div.c b/mysys/my_div.c
index 660b87e5ab4..44eb5392421 100644
--- a/mysys/my_div.c
+++ b/mysys/my_div.c
@@ -27,7 +27,7 @@
 char * my_filename(File fd)
 {
   DBUG_ENTER("my_filename");
-  if ((uint) fd >= (uint) my_file_limit)
+  if ((uint) fd >= (uint) my_file_limit || !my_file_info[fd].name)
     DBUG_RETURN((char*) "UNKNOWN");
   if (fd >= 0 && my_file_info[fd].type != UNOPEN)
   {
diff --git a/mysys/my_fopen.c b/mysys/my_fopen.c
index 2ac033c8264..0bb431dd560 100644
--- a/mysys/my_fopen.c
+++ b/mysys/my_fopen.c
@@ -69,19 +69,13 @@ FILE *my_fopen(const char *filename, int flags, myf MyFlags)
       DBUG_RETURN(fd);				/* safeguard */
     }
     mysql_mutex_lock(&THR_LOCK_open);
-    if ((my_file_info[filedesc].name= (char*)
-	 my_strdup(filename,MyFlags)))
-    {
-      my_stream_opened++;
-      my_file_total_opened++;
-      my_file_info[filedesc].type= STREAM_BY_FOPEN;
-      mysql_mutex_unlock(&THR_LOCK_open);
-      DBUG_PRINT("exit",("stream: 0x%lx", (long) fd));
-      DBUG_RETURN(fd);
-    }
+    my_file_info[filedesc].name= (char*) my_strdup(filename,MyFlags);
+    my_stream_opened++;
+    my_file_total_opened++;
+    my_file_info[filedesc].type= STREAM_BY_FOPEN;
     mysql_mutex_unlock(&THR_LOCK_open);
-    (void) my_fclose(fd,MyFlags);
-    my_errno=ENOMEM;
+    DBUG_PRINT("exit",("stream: 0x%lx", (long) fd));
+    DBUG_RETURN(fd);
   }
   else
     my_errno=errno;
diff --git a/mysys/my_init.c b/mysys/my_init.c
index 6ec9552c528..7f0f7a8cc44 100644
--- a/mysys/my_init.c
+++ b/mysys/my_init.c
@@ -227,7 +227,7 @@ Voluntary context switches %ld, Involuntary context switches %ld\n",
  
   /* At very last, delete mysys key, it is used everywhere including DBUG */
   pthread_key_delete(THR_KEY_mysys);
-  my_init_done=0;
+  my_init_done= my_thr_key_mysys_exists= 0;
 } /* my_end */
 
 #ifndef DBUG_OFF
diff --git a/mysys/my_open.c b/mysys/my_open.c
index b6d8f08bfc1..3999810eb2e 100644
--- a/mysys/my_open.c
+++ b/mysys/my_open.c
@@ -15,9 +15,14 @@
 
 #include "mysys_priv.h"
 #include "mysys_err.h"
-#include <my_dir.h>
+#include <m_string.h>
 #include <errno.h>
 
+CREATE_NOSYMLINK_FUNCTION(
+  open_nosymlinks(const char *pathname, int flags, int mode),
+  openat(dfd, filename, O_NOFOLLOW | flags, mode),
+  open(pathname, O_NOFOLLOW | flags, mode)
+);
 
 /*
   Open a file
@@ -45,10 +50,11 @@ File my_open(const char *FileName, int Flags, myf MyFlags)
     MyFlags|= my_global_flags;
 #if defined(_WIN32)
   fd= my_win_open(FileName, Flags);
-#elif !defined(NO_OPEN_3)
-  fd = open(FileName, Flags | O_CLOEXEC, my_umask);     /* Normal unix */
 #else
-  fd = open((char *) FileName, Flags | O_CLOEXEC);
+  if (MyFlags & MY_NOSYMLINKS)
+    fd = open_nosymlinks(FileName, Flags | O_CLOEXEC, my_umask);
+  else
+    fd = open(FileName, Flags | O_CLOEXEC, my_umask);
 #endif
 
   fd= my_register_filename(fd, FileName, FILE_BY_OPEN,
@@ -131,25 +137,16 @@ File my_register_filename(File fd, const char *FileName, enum file_type
       thread_safe_increment(my_file_opened,&THR_LOCK_open);
       DBUG_RETURN(fd);				/* safeguard */
     }
-    else
-    {
-      mysql_mutex_lock(&THR_LOCK_open);
-      if ((my_file_info[fd].name = (char*) my_strdup(FileName,MyFlags)))
-      {
-        my_file_opened++;
-        my_file_total_opened++;
-        my_file_info[fd].type = type_of_file;
-        mysql_mutex_unlock(&THR_LOCK_open);
-        DBUG_PRINT("exit",("fd: %d",fd));
-        DBUG_RETURN(fd);
-      }
-      mysql_mutex_unlock(&THR_LOCK_open);
-      my_errno= ENOMEM;
-    }
-    (void) my_close(fd, MyFlags);
+    mysql_mutex_lock(&THR_LOCK_open);
+    my_file_info[fd].name = (char*) my_strdup(FileName, MyFlags);
+    my_file_opened++;
+    my_file_total_opened++;
+    my_file_info[fd].type = type_of_file;
+    mysql_mutex_unlock(&THR_LOCK_open);
+    DBUG_PRINT("exit",("fd: %d",fd));
+    DBUG_RETURN(fd);
   }
-  else
-    my_errno= errno;
+  my_errno= errno;
 
   DBUG_PRINT("error",("Got error %d on open", my_errno));
   if (MyFlags & (MY_FFNF | MY_FAE | MY_WME))
diff --git a/mysys/my_symlink.c b/mysys/my_symlink.c
index b0e910f7ba0..72648d4c9a8 100644
--- a/mysys/my_symlink.c
+++ b/mysys/my_symlink.c
@@ -1,5 +1,6 @@
 /*
    Copyright (c) 2001, 2011, Oracle and/or its affiliates
+   Copyright (c) 2010, 2017, MariaDB
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -23,6 +24,14 @@
 #include <sys/stat.h>
 #endif
 
+static int always_valid(const char *filename __attribute__((unused)))
+{
+  return 0;
+}
+
+int (*mysys_test_invalid_symlink)(const char *filename)= always_valid;
+
+
 /*
   Reads the content of a symbolic link
   If the file is not a symbolic link, return the original file name in to.
@@ -120,6 +129,11 @@ int my_is_symlink(const char *filename __attribute__((unused)))
 
   to is guaranteed to never set to a string longer than FN_REFLEN
   (including the end \0)
+
+  On error returns -1, unless error is file not found, in which case it
+  is 1.
+
+  Sets my_errno to specific error number.
 */
 
 int my_realpath(char *to, const char *filename, myf MyFlags)
@@ -145,7 +159,10 @@ int my_realpath(char *to, const char *filename, myf MyFlags)
     if (MyFlags & MY_WME)
       my_error(EE_REALPATH, MYF(0), filename, my_errno);
     my_load_path(to, filename, NullS);
-    result= -1;
+    if (my_errno == ENOENT)
+      result= 1;
+    else
+      result= -1;
   }
   DBUG_RETURN(result);
 #elif defined(_WIN32)
@@ -168,3 +185,78 @@ int my_realpath(char *to, const char *filename, myf MyFlags)
 #endif
   return 0;
 }
+
+#ifdef HAVE_OPEN_PARENT_DIR_NOSYMLINKS
+/** opens the parent dir. walks the path, and does not resolve symlinks
+
+   returns the pointer to the file name (basename) within the pathname
+   or NULL in case of an error
+
+   stores the parent dir (dirname) file descriptor in pdfd.
+   It can be -1 even if there was no error!
+
+   This is used for symlinked tables for DATA/INDEX DIRECTORY.
+   The paths there have been realpath()-ed. So, we can assume here that
+
+    * `pathname` is an absolute path
+    * no '.', '..', and '//' in the path
+    * file exists
+*/
+
+const char *my_open_parent_dir_nosymlinks(const char *pathname, int *pdfd)
+{
+  char buf[PATH_MAX+1];
+  char *s= buf, *e= buf+1, *end= strnmov(buf, pathname, sizeof(buf));
+  int fd, dfd= -1;
+
+  if (*end)
+  {
+    errno= ENAMETOOLONG;
+    return NULL;
+  }
+
+  if (*s != '/') /* not an absolute path */
+  {
+    errno= ENOENT;
+    return NULL;
+  }
+
+  for (;;)
+  {
+    if (*e == '/') /* '//' in the path */
+    {
+      errno= ENOENT;
+      goto err;
+    }
+    while (*e && *e != '/')
+      e++;
+    *e= 0;
+
+    if (!memcmp(s, ".", 2) || !memcmp(s, "..", 3))
+    {
+      errno= ENOENT;
+      goto err;
+    }
+
+    if (++e >= end)
+    {
+      *pdfd= dfd;
+      return pathname + (s - buf);
+    }
+
+    fd = openat(dfd, s, O_NOFOLLOW | O_PATH);
+    if (fd < 0)
+      goto err;
+
+    if (dfd >= 0)
+      close(dfd);
+
+    dfd= fd;
+    s= e;
+  }
+err:
+  if (dfd >= 0)
+    close(dfd);
+  return NULL;
+}
+#endif
diff --git a/mysys/my_symlink2.c b/mysys/my_symlink2.c
index e2cc641ebec..a2f7b6a0741 100644
--- a/mysys/my_symlink2.c
+++ b/mysys/my_symlink2.c
@@ -92,27 +92,6 @@ File my_create_with_symlink(const char *linkname, const char *filename,
 }
 
 /*
-  If the file was a symlink, delete both symlink and the file which the
-  symlink pointed to.
-*/
-
-int my_delete_with_symlink(const char *name, myf MyFlags)
-{
-  char link_name[FN_REFLEN];
-  int was_symlink= (!my_disable_symlinks &&
-		    !my_readlink(link_name, name, MYF(0)));
-  int result;
-  DBUG_ENTER("my_delete_with_symlink");
-
-  if (!(result=my_delete(name, MyFlags)))
-  {
-    if (was_symlink)
-      result=my_delete(link_name, MyFlags);
-  }
-  DBUG_RETURN(result);
-}
-
-/*
   If the file is a normal file, just rename it.
   If the file is a symlink:
    - Create a new file with the name 'to' that points at
@@ -182,3 +161,31 @@ int my_rename_with_symlink(const char *from, const char *to, myf MyFlags)
   DBUG_RETURN(result);
 #endif /* HAVE_READLINK */
 }
+
+/** delete a - possibly symlinked - table file
+
+  This is used to delete a file that is part of a table (e.g. MYI or MYD
+  file of MyISAM) when dropping a table. A file might be a symlink -
+  if the table was created with DATA DIRECTORY or INDEX DIRECTORY -
+  in this case both the symlink and the symlinked file are deleted,
+  but only if the symlinked file is not in the datadir.
+*/
+int my_handler_delete_with_symlink(PSI_file_key key, const char *name,
+                                   const char *ext, myf sync_dir)
+{
+  char orig[FN_REFLEN], real[FN_REFLEN];
+  int res= 0;
+  DBUG_ENTER("my_handler_delete_with_symlink");
+
+  fn_format(orig, name, "", ext, MY_UNPACK_FILENAME | MY_APPEND_EXT);
+  if (my_is_symlink(orig))
+  {
+    /*
+      Delete the symlinked file only if the symlink is not
+      pointing into datadir.
+    */
+    if (!(my_realpath(real, orig, MYF(0)) || mysys_test_invalid_symlink(real)))
+      res= mysql_file_delete(key, real, MYF(MY_NOSYMLINKS | sync_dir));
+  }
+  DBUG_RETURN(mysql_file_delete(key, orig, MYF(sync_dir)) || res);
+}
diff --git a/mysys/my_sync.c b/mysys/my_sync.c
index c0afd587ada..d1e239692f1 100644
--- a/mysys/my_sync.c
+++ b/mysys/my_sync.c
@@ -196,7 +196,7 @@ int my_sync_dir_by_file(const char *file_name __attribute__((unused)),
   char dir_name[FN_REFLEN];
   size_t dir_name_length;
   dirname_part(dir_name, file_name, &dir_name_length);
-  return my_sync_dir(dir_name, my_flags);
+  return my_sync_dir(dir_name, my_flags & ~MY_NOSYMLINKS);
 #else
   return 0;
 #endif
diff --git a/mysys/my_thr_init.c b/mysys/my_thr_init.c
index 5234010ac38..1d3c3059bd5 100644
--- a/mysys/my_thr_init.c
+++ b/mysys/my_thr_init.c
@@ -44,6 +44,8 @@ static uint get_thread_lib(void);
 
 /** True if @c my_thread_global_init() has been called. */
 static my_bool my_thread_global_init_done= 0;
+/* True if THR_KEY_mysys is created */
+my_bool my_thr_key_mysys_exists= 0;
 
 
 /*
@@ -167,11 +169,20 @@ my_bool my_thread_global_init(void)
     return 0;
   my_thread_global_init_done= 1;
 
-  if ((pth_ret= pthread_key_create(&THR_KEY_mysys, NULL)) != 0)
+  /*
+    THR_KEY_mysys is deleted in my_end() as DBUG libraries are using it even
+    after my_thread_global_end() is called.
+    my_thr_key_mysys_exist is used to protect against application like QT
+    that calls my_thread_global_init() + my_thread_global_end() multiple times
+    without calling my_init() + my_end().
+  */
+  if (!my_thr_key_mysys_exists &&
+      (pth_ret= pthread_key_create(&THR_KEY_mysys, NULL)) != 0)
   {
     fprintf(stderr, "Can't initialize threads: error %d\n", pth_ret);
     return 1;
   }
+  my_thr_key_mysys_exists= 1;
 
   /* Mutex used by my_thread_init() and after my_thread_destroy_mutex() */
   my_thread_init_internal_mutex();
diff --git a/mysys/mysys_priv.h b/mysys/mysys_priv.h
index d080aca7404..471862a6d0b 100644
--- a/mysys/mysys_priv.h
+++ b/mysys/mysys_priv.h
@@ -108,6 +108,34 @@ size_t sf_malloc_usable_size(void *ptr, my_bool *is_thread_specific);
 
 void my_error_unregister_all(void);
 
+#if !defined(O_PATH) && defined(O_EXEC) /* FreeBSD */
+#define O_PATH O_EXEC
+#endif
+
+#ifdef O_PATH
+#define HAVE_OPEN_PARENT_DIR_NOSYMLINKS
+const char *my_open_parent_dir_nosymlinks(const char *pathname, int *pdfd);
+#define NOSYMLINK_FUNCTION_BODY(AT,NOAT)                                \
+  int dfd, res;                                                         \
+  const char *filename= my_open_parent_dir_nosymlinks(pathname, &dfd);  \
+  if (filename == NULL) return -1;                                      \
+  res= AT;                                                              \
+  if (dfd >= 0) close(dfd);                                             \
+  return res;
+#elif defined(HAVE_REALPATH)
+#define NOSYMLINK_FUNCTION_BODY(AT,NOAT)                                \
+  char buf[PATH_MAX+1];                                                 \
+  if (realpath(pathname, buf) == NULL) return -1;                       \
+  if (strcmp(pathname, buf)) { errno= ENOTDIR; return -1; }             \
+  return NOAT;
+#else
+#define NOSYMLINK_FUNCTION_BODY(AT,NOAT)                                \
+  return NOAT;
+#endif
+
+#define CREATE_NOSYMLINK_FUNCTION(PROTO,AT,NOAT)                        \
+static int PROTO { NOSYMLINK_FUNCTION_BODY(AT,NOAT) }
+
 #ifdef _WIN32
 #include <sys/stat.h>
 /* my_winfile.c exports, should not be used outside mysys */