MDEV-14229: Stack trace is not resolved for shared objects

Resolving a stacktrace including functions in dynamic libraries requires
us to look inside the libraries for the symbols. Addr2line needs to be
started with the correct binary for each address on the stack. To do this,
figure out which library it is using dladdr, then if the addr2line
binary was started with a different binary, fork it again with the
correct one.

We only have one addr2line process running at any point during the
stacktrace resolving step. The maximum number of forks for addr2line should
generally be around 6.

One for server stacktrace code, one for plugin code, one when going back
into server code, one for pthread library, one for libc, one for the
_start function in the server. More can come up if plugin calls server
function which goes back to a plugin, etc.

5 files changed, 73 insertions, 32 deletions

diff --git a/config.h.cmake b/config.h.cmake
index dfdd4215e4a..99a2ebdd093 100644
--- a/config.h.cmake
+++ b/config.h.cmake
@@ -148,6 +148,7 @@
 #cmakedefine HAVE_CUSERID 1
 #cmakedefine HAVE_CXX_NEW 1
 #cmakedefine HAVE_DIRECTIO 1
+#cmakedefine HAVE_DLADDR 1
 #cmakedefine HAVE_DLERROR 1
 #cmakedefine HAVE_DLOPEN 1
 #cmakedefine HAVE_DOPRNT 1
diff --git a/configure.cmake b/configure.cmake
index 470fb719fb8..43f8f71390d 100644
--- a/configure.cmake
+++ b/configure.cmake
@@ -346,6 +346,7 @@ CHECK_FUNCTION_EXISTS (ftruncate HAVE_FTRUNCATE)
 CHECK_FUNCTION_EXISTS (getline HAVE_GETLINE)
 CHECK_FUNCTION_EXISTS (compress HAVE_COMPRESS)
 CHECK_FUNCTION_EXISTS (crypt HAVE_CRYPT)
+CHECK_FUNCTION_EXISTS (dladdr HAVE_DLADDR)
 CHECK_FUNCTION_EXISTS (dlerror HAVE_DLERROR)
 CHECK_FUNCTION_EXISTS (dlopen HAVE_DLOPEN)
 CHECK_FUNCTION_EXISTS (fchmod HAVE_FCHMOD)
diff --git a/include/my_global.h b/include/my_global.h
index cf140cf54ce..767ac3e459e 100644
--- a/include/my_global.h
+++ b/include/my_global.h
@@ -1373,11 +1373,15 @@ static inline char *dlerror(void)
 #ifndef HAVE_DLERROR
 #define dlerror() ""
 #endif
+#ifndef HAVE_DLADDR
+#define dladdr(A, B) 0
+#endif
 #else
 #define dlerror() "No support for dynamic loading (static build?)"
 #define dlopen(A,B) 0
 #define dlsym(A,B) 0
 #define dlclose(A) 0
+#define dladdr(A, B) 0
 #endif
 
 /*
diff --git a/mysys/CMakeLists.txt b/mysys/CMakeLists.txt
index cb86850c2de..15f3fbad9bf 100644
--- a/mysys/CMakeLists.txt
+++ b/mysys/CMakeLists.txt
@@ -67,7 +67,7 @@ ENDIF()
 
 ADD_CONVENIENCE_LIBRARY(mysys ${MYSYS_SOURCES})
 TARGET_LINK_LIBRARIES(mysys dbug strings ${ZLIB_LIBRARY} 
- ${LIBNSL} ${LIBM} ${LIBRT} ${LIBSOCKET} ${LIBEXECINFO})
+  ${LIBNSL} ${LIBM} ${LIBRT} ${LIBSOCKET} ${LIBEXECINFO} ${LIBDL})
 DTRACE_INSTRUMENT(mysys)
 
 IF(HAVE_BFD_H)
diff --git a/mysys/my_addr_resolve.c b/mysys/my_addr_resolve.c
index 90e6f43f390..f831ad5121f 100644
--- a/mysys/my_addr_resolve.c
+++ b/mysys/my_addr_resolve.c
@@ -132,15 +132,79 @@ err:
 
 #include <m_string.h>
 #include <ctype.h>
+
+#include <sys/wait.h>
+
 static int in[2], out[2];
-static int initialized= 0;
+static pid_t pid;
+static char addr2line_binary[1024];
 static char output[1024];
+
+int start_addr2line_fork(const char *binary_path)
+{
+
+  if (pid > 0)
+  {
+    /* Don't leak FDs */
+    close(in[1]);
+    close(out[0]);
+    /* Don't create zombie processes. */
+    waitpid(pid, NULL, 0);
+  }
+
+  if (pipe(in) < 0)
+    return 1;
+  if (pipe(out) < 0)
+    return 1;
+
+  pid = fork();
+  if (pid == -1)
+    return 1;
+
+  if (!pid) /* child */
+  {
+    dup2(in[0], 0);
+    dup2(out[1], 1);
+    close(in[0]);
+    close(in[1]);
+    close(out[0]);
+    close(out[1]);
+    execlp("addr2line", "addr2line", "-C", "-f", "-e", binary_path, NULL);
+    exit(1);
+  }
+
+  close(in[0]);
+  close(out[1]);
+
+  return 0;
+}
+
 int my_addr_resolve(void *ptr, my_addr_loc *loc)
 {
   char input[32], *s;
   size_t len;
 
-  len= my_snprintf(input, sizeof(input), "%p\n", ptr);
+  Dl_info info;
+  void *offset;
+
+  if (!dladdr(ptr, &info))
+    return 1;
+
+  if (strcmp(addr2line_binary, info.dli_fname))
+  {
+    /* We use dli_fname in case the path is longer than the length of our static
+       string. We don't want to allocate anything dynamicaly here as we are in
+       a "crashed" state. */
+    if (start_addr2line_fork(info.dli_fname))
+    {
+      addr2line_binary[0] = '\0';
+      return 1;
+    }
+    /* Save result for future comparisons. */
+    strnmov(addr2line_binary, info.dli_fname, sizeof(addr2line_binary));
+  }
+  offset = info.dli_fbase;
+  len= my_snprintf(input, sizeof(input), "%p\n", ptr - offset);
   if (write(in[1], input, len) <= 0)
     return 1;
   if (read(out[0], output, sizeof(output)) <= 0)
@@ -168,35 +232,6 @@ int my_addr_resolve(void *ptr, my_addr_loc *loc)
 
 const char *my_addr_resolve_init()
 {
-  if (!initialized)
-  {
-    pid_t pid;
-
-    if (pipe(in) < 0)
-      return "pipe(in)";
-    if (pipe(out) < 0)
-      return "pipe(out)";
-
-    pid = fork();
-    if (pid == -1)
-      return "fork";
-
-    if (!pid) /* child */
-    {
-      dup2(in[0], 0);
-      dup2(out[1], 1);
-      close(in[0]);
-      close(in[1]);
-      close(out[0]);
-      close(out[1]);
-      execlp("addr2line", "addr2line", "-C", "-f", "-e", my_progname, NULL);
-      exit(1);
-    }
-    
-    close(in[0]);
-    close(out[1]);
-    initialized= 1;
-  }
   return 0;
 }
 #endif