[GWP-ASan] Add generic unwinders and structure backtrace output.

Summary:
Adds two flavours of generic unwinder and all the supporting cruft. If the
supporting allocator is okay with bringing in sanitizer_common, they can use
the fast frame-pointer based unwinder from sanitizer_common. Otherwise, we also
provide the backtrace() libc-based unwinder as well. Of course, the allocator
can always specify its own unwinder and unwinder-symbolizer.

The slightly changed output format is exemplified in the first comment on this
patch. It now better incorporates backtrace information, and displays
allocation details on the second line.

Reviewers: eugenis, vlad.tsyrklevich

Reviewed By: eugenis, vlad.tsyrklevich

Subscribers: srhines, kubamracek, mgorny, cryptoad, #sanitizers, llvm-commits, morehouse

Tags: #sanitizers, #llvm

Differential Revision: https://reviews.llvm.org/D63841

git-svn-id: https://llvm.org/svn/llvm-project/compiler-rt/trunk@364941 91177308-0d34-0410-b5e6-96231b3b80d8

24 files changed, 434 insertions, 137 deletions

diff --git a/lib/gwp_asan/CMakeLists.txt b/lib/gwp_asan/CMakeLists.txt
index 33db1109b..952acb183 100644
--- a/lib/gwp_asan/CMakeLists.txt
+++ b/lib/gwp_asan/CMakeLists.txt
@@ -23,6 +23,10 @@ set(GWP_ASAN_HEADERS
 # parts of the C++ standard library.
 set(GWP_ASAN_CFLAGS -fno-rtti -fno-exceptions -nostdinc++ -pthread)
 append_list_if(COMPILER_RT_HAS_FPIC_FLAG -fPIC GWP_ASAN_CFLAGS)
+append_list_if(COMPILER_RT_HAS_OMIT_FRAME_POINTER_FLAG -fno-omit-frame-pointer
+               GWP_ASAN_CFLAGS)
+append_list_if(COMPILER_RT_HAS_OMIT_FRAME_POINTER_FLAG
+               -mno-omit-leaf-frame-pointer GWP_ASAN_CFLAGS)
 
 # Remove -stdlib= which is unused when passing -nostdinc++.
 string(REGEX REPLACE "-stdlib=[a-zA-Z+]*" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
@@ -40,6 +44,12 @@ set(GWP_ASAN_OPTIONS_PARSER_HEADERS
   options.h
   options.inc
 )
+set(GWP_ASAN_BACKTRACE_HEADERS
+  optional/backtrace.h
+  options.h
+  options.inc
+)
+
 set(GWP_ASAN_OPTIONS_PARSER_CFLAGS
     ${GWP_ASAN_CFLAGS}
     ${SANITIZER_COMMON_CFLAGS})
@@ -68,29 +78,26 @@ if (COMPILER_RT_HAS_GWP_ASAN)
 
   # Note: If you choose to add this as an object library, ensure you also
   # include the sanitizer_common flag parsing object lib (generally
-  # 'RTSanitizerCommonNoTermination').
+  # 'RTSanitizerCommonNoTermination'). Also, you'll need to either implement
+  # your own backtrace support (see optional/backtrace.h), or choose between one
+  # of the pre-implemented backtrace support options (see below).
   add_compiler_rt_object_libraries(RTGwpAsanOptionsParser
       ARCHS ${GWP_ASAN_SUPPORTED_ARCH}
       SOURCES ${GWP_ASAN_OPTIONS_PARSER_SOURCES}
       ADDITIONAL_HEADERS ${GWP_ASAN_OPTIONS_PARSER_HEADERS}
       CFLAGS ${GWP_ASAN_OPTIONS_PARSER_CFLAGS})
 
-  # Ensure that the build for the options parser succeeds, as
-  # 'RTGwpAsanOptionsParser' may not be built if it's not needed. This library
-  # has only a very small amount of logic, all of the testable components are
-  # exercised in the sanitizer_common test suite.
-  foreach(arch ${GWP_ASAN_SUPPORTED_ARCH})
-    add_compiler_rt_runtime(
-      clang_rt.gwp_asan_options_parser
-      STATIC
-      ARCHS ${arch}
-      SOURCES ${GWP_ASAN_OPTIONS_PARSER_SOURCES}
-      ADDITIONAL_HEADERS ${GWP_ASAN_OPTIONS_PARSER_HEADERS}
-      CFLAGS ${GWP_ASAN_OPTIONS_PARSER_CFLAGS}
-      OBJECT_LIBS ${GWP_ASAN_OPTIONS_PARSER_OBJECT_LIBS}
-      PARENT_TARGET gwp_asan
-    )
-  endforeach()
+  # As above, build the pre-implemented optional backtrace support libraries.
+  add_compiler_rt_object_libraries(RTGwpAsanBacktraceLibc
+      ARCHS ${GWP_ASAN_SUPPORTED_ARCH}
+      SOURCES optional/backtrace_linux_libc.cpp
+      ADDITIONAL_HEADERS ${GWP_ASAN_BACKTRACE_HEADERS}
+      CFLAGS ${GWP_ASAN_CFLAGS})
+  add_compiler_rt_object_libraries(RTGwpAsanBacktraceSanitizerCommon
+      ARCHS ${GWP_ASAN_SUPPORTED_ARCH}
+      SOURCES optional/backtrace_sanitizer_common.cpp
+      ADDITIONAL_HEADERS ${GWP_ASAN_BACKTRACE_HEADERS}
+      CFLAGS ${GWP_ASAN_CFLAGS} ${SANITIZER_COMMON_CFLAGS})
 endif()
 
 if(COMPILER_RT_INCLUDE_TESTS)
diff --git a/lib/gwp_asan/guarded_pool_allocator.cpp b/lib/gwp_asan/guarded_pool_allocator.cpp
index 68ec00ddd..318068404 100644
--- a/lib/gwp_asan/guarded_pool_allocator.cpp
+++ b/lib/gwp_asan/guarded_pool_allocator.cpp
@@ -11,6 +11,8 @@
 #include "gwp_asan/options.h"
 
 #include <assert.h>
+#include <inttypes.h>
+#include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <time.h>
@@ -26,6 +28,25 @@ namespace {
 // referenced by users outside this translation unit, in order to avoid
 // init-order-fiasco.
 GuardedPoolAllocator *SingletonPtr = nullptr;
+
+class ScopedBoolean {
+public:
+  ScopedBoolean(bool &B) : Bool(B) { Bool = true; }
+  ~ScopedBoolean() { Bool = false; }
+
+private:
+  bool &Bool;
+};
+
+void defaultPrintStackTrace(uintptr_t *Trace, options::Printf_t Printf) {
+  if (Trace[0] == 0)
+    Printf("  <unknown (does your allocator support backtracing?)>\n");
+
+  for (size_t i = 0; Trace[i] != 0; ++i) {
+    Printf("  #%zu 0x%zx in <unknown>\n", i, Trace[i]);
+  }
+  Printf("\n");
+}
 } // anonymous namespace
 
 // Gets the singleton implementation of this class. Thread-compatible until
@@ -33,25 +54,33 @@ GuardedPoolAllocator *SingletonPtr = nullptr;
 GuardedPoolAllocator *getSingleton() { return SingletonPtr; }
 
 void GuardedPoolAllocator::AllocationMetadata::RecordAllocation(
-    uintptr_t AllocAddr, size_t AllocSize) {
+    uintptr_t AllocAddr, size_t AllocSize, options::Backtrace_t Backtrace) {
   Addr = AllocAddr;
   Size = AllocSize;
   IsDeallocated = false;
 
-  // TODO(hctim): Implement stack trace collection.
   // TODO(hctim): Ask the caller to provide the thread ID, so we don't waste
   // other thread's time getting the thread ID under lock.
   AllocationTrace.ThreadID = getThreadID();
   DeallocationTrace.ThreadID = kInvalidThreadID;
-  AllocationTrace.Trace[0] = 0;
+  if (Backtrace)
+    Backtrace(AllocationTrace.Trace, kMaximumStackFrames);
+  else
+    AllocationTrace.Trace[0] = 0;
   DeallocationTrace.Trace[0] = 0;
 }
 
-void GuardedPoolAllocator::AllocationMetadata::RecordDeallocation() {
+void GuardedPoolAllocator::AllocationMetadata::RecordDeallocation(
+    options::Backtrace_t Backtrace) {
   IsDeallocated = true;
-  // TODO(hctim): Implement stack trace collection. Ensure that the unwinder is
-  // not called if we have our recursive flag called, otherwise non-reentrant
-  // unwinders may deadlock.
+  // Ensure that the unwinder is not called if the recursive flag is set,
+  // otherwise non-reentrant unwinders may deadlock.
+  if (Backtrace && !ThreadLocals.RecursiveGuard) {
+    ScopedBoolean B(ThreadLocals.RecursiveGuard);
+    Backtrace(DeallocationTrace.Trace, kMaximumStackFrames);
+  } else {
+    DeallocationTrace.Trace[0] = 0;
+  }
   DeallocationTrace.ThreadID = getThreadID();
 }
 
@@ -93,6 +122,11 @@ void GuardedPoolAllocator::init(const options::Options &Opts) {
 
   PerfectlyRightAlign = Opts.PerfectlyRightAlign;
   Printf = Opts.Printf;
+  Backtrace = Opts.Backtrace;
+  if (Opts.PrintBacktrace)
+    PrintBacktrace = Opts.PrintBacktrace;
+  else
+    PrintBacktrace = defaultPrintStackTrace;
 
   size_t PoolBytesRequired =
       PageSize * (1 + MaxSimultaneousAllocations) +
@@ -126,17 +160,6 @@ void GuardedPoolAllocator::init(const options::Options &Opts) {
     installSignalHandlers();
 }
 
-namespace {
-class ScopedBoolean {
-public:
-  ScopedBoolean(bool &B) : Bool(B) { Bool = true; }
-  ~ScopedBoolean() { Bool = false; }
-
-private:
-  bool &Bool;
-};
-} // anonymous namespace
-
 void *GuardedPoolAllocator::allocate(size_t Size) {
   // GuardedPagePoolEnd == 0 when GWP-ASan is disabled. If we are disabled, fall
   // back to the supporting allocator.
@@ -169,7 +192,7 @@ void *GuardedPoolAllocator::allocate(size_t Size) {
   // unmapped.
   markReadWrite(reinterpret_cast<void *>(getPageAddr(Ptr)), Size);
 
-  Meta->RecordAllocation(Ptr, Size);
+  Meta->RecordAllocation(Ptr, Size, Backtrace);
 
   return reinterpret_cast<void *>(Ptr);
 }
@@ -196,7 +219,7 @@ void GuardedPoolAllocator::deallocate(void *Ptr) {
     // Ensure that the deallocation is recorded before marking the page as
     // inaccessible. Otherwise, a racy use-after-free will have inconsistent
     // metadata.
-    Meta->RecordDeallocation();
+    Meta->RecordDeallocation(Backtrace);
   }
 
   markInaccessible(reinterpret_cast<void *>(SlotStart),
@@ -328,78 +351,101 @@ Error GuardedPoolAllocator::diagnoseUnknownError(uintptr_t AccessPtr,
 
   // If we have reached here, the error is still unknown. There is no metadata
   // available.
+  *Meta = nullptr;
   return Error::UNKNOWN;
 }
 
-// Prints the provided error and metadata information. Returns true if there is
-// additional context that can be provided, false otherwise (i.e. returns false
-// if Error == {UNKNOWN, INVALID_FREE without metadata}).
-bool printErrorType(Error E, uintptr_t AccessPtr, AllocationMetadata *Meta,
-                    options::Printf_t Printf) {
+namespace {
+// Prints the provided error and metadata information.
+void printErrorType(Error E, uintptr_t AccessPtr, AllocationMetadata *Meta,
+                    options::Printf_t Printf, uint64_t ThreadID) {
+  // Print using intermediate strings. Platforms like Android don't like when
+  // you print multiple times to the same line, as there may be a newline
+  // appended to a log file automatically per Printf() call.
+  const char *ErrorString;
   switch (E) {
   case Error::UNKNOWN:
-    Printf("GWP-ASan couldn't automatically determine the source of the "
-           "memory error when accessing 0x%zx. It was likely caused by a wild "
-           "memory access into the GWP-ASan pool.\n",
-           AccessPtr);
-    return false;
+    ErrorString = "GWP-ASan couldn't automatically determine the source of "
+                  "the memory error. It was likely caused by a wild memory "
+                  "access into the GWP-ASan pool. The error occured";
+    break;
   case Error::USE_AFTER_FREE:
-    Printf("Use after free occurred when accessing memory at: 0x%zx\n",
-           AccessPtr);
+    ErrorString = "Use after free";
     break;
   case Error::DOUBLE_FREE:
-    Printf("Double free occurred when trying to free memory at: 0x%zx\n",
-           AccessPtr);
+    ErrorString = "Double free";
     break;
   case Error::INVALID_FREE:
-    Printf(
-        "Invalid (wild) free occurred when trying to free memory at: 0x%zx\n",
-        AccessPtr);
-    // It's possible for an invalid free to fall onto a slot that has never been
-    // allocated. If this is the case, there is no valid metadata.
-    if (Meta == nullptr)
-      return false;
+    ErrorString = "Invalid (wild) free";
     break;
   case Error::BUFFER_OVERFLOW:
-    Printf("Buffer overflow occurred when accessing memory at: 0x%zx\n",
-           AccessPtr);
+    ErrorString = "Buffer overflow";
     break;
   case Error::BUFFER_UNDERFLOW:
-    Printf("Buffer underflow occurred when accessing memory at: 0x%zx\n",
-           AccessPtr);
+    ErrorString = "Buffer underflow";
     break;
   }
 
-  Printf("0x%zx is ", AccessPtr);
-  if (AccessPtr < Meta->Addr)
-    Printf("located %zu bytes to the left of a %zu-byte allocation located at "
-           "0x%zx\n",
-           Meta->Addr - AccessPtr, Meta->Size, Meta->Addr);
-  else if (AccessPtr > Meta->Addr)
-    Printf("located %zu bytes to the right of a %zu-byte allocation located at "
-           "0x%zx\n",
-           AccessPtr - Meta->Addr, Meta->Size, Meta->Addr);
+  constexpr size_t kDescriptionBufferLen = 128;
+  char DescriptionBuffer[kDescriptionBufferLen];
+  if (Meta) {
+    if (E == Error::USE_AFTER_FREE) {
+      snprintf(DescriptionBuffer, kDescriptionBufferLen,
+               "(%zu byte%s into a %zu-byte allocation at 0x%zx)",
+               AccessPtr - Meta->Addr, (AccessPtr - Meta->Addr == 1) ? "" : "s",
+               Meta->Size, Meta->Addr);
+    } else if (AccessPtr < Meta->Addr) {
+      snprintf(DescriptionBuffer, kDescriptionBufferLen,
+               "(%zu byte%s to the left of a %zu-byte allocation at 0x%zx)",
+               Meta->Addr - AccessPtr, (Meta->Addr - AccessPtr == 1) ? "" : "s",
+               Meta->Size, Meta->Addr);
+    } else if (AccessPtr > Meta->Addr) {
+      snprintf(DescriptionBuffer, kDescriptionBufferLen,
+               "(%zu byte%s to the right of a %zu-byte allocation at 0x%zx)",
+               AccessPtr - Meta->Addr, (AccessPtr - Meta->Addr == 1) ? "" : "s",
+               Meta->Size, Meta->Addr);
+    } else {
+      snprintf(DescriptionBuffer, kDescriptionBufferLen,
+               "(a %zu-byte allocation)", Meta->Size);
+    }
+  }
+
+  // Possible number of digits of a 64-bit number: ceil(log10(2^64)) == 20. Add
+  // a null terminator, and round to the nearest 8-byte boundary.
+  constexpr size_t kThreadBufferLen = 24;
+  char ThreadBuffer[kThreadBufferLen];
+  if (ThreadID == GuardedPoolAllocator::kInvalidThreadID)
+    snprintf(ThreadBuffer, kThreadBufferLen, "<unknown>");
   else
-    Printf("a %zu-byte allocation\n", Meta->Size);
-  return true;
+    snprintf(ThreadBuffer, kThreadBufferLen, "%" PRIu64, ThreadID);
+
+  Printf("%s at 0x%zx %s by thread %s here:\n", ErrorString, AccessPtr,
+         DescriptionBuffer, ThreadBuffer);
 }
 
-void printThreadInformation(Error E, uintptr_t AccessPtr,
-                            AllocationMetadata *Meta,
-                            options::Printf_t Printf) {
-  Printf("0x%zx was allocated by thread ", AccessPtr);
-  if (Meta->AllocationTrace.ThreadID == UINT64_MAX)
-    Printf("UNKNOWN.\n");
-  else
-    Printf("%zu.\n", Meta->AllocationTrace.ThreadID);
+void printAllocDeallocTraces(uintptr_t AccessPtr, AllocationMetadata *Meta,
+                             options::Printf_t Printf,
+                             options::PrintBacktrace_t PrintBacktrace) {
+  assert(Meta != nullptr && "Metadata is non-null for printAllocDeallocTraces");
 
-  if (E == Error::USE_AFTER_FREE || E == Error::DOUBLE_FREE) {
-    Printf("0x%zx was freed by thread ", AccessPtr);
-    if (Meta->AllocationTrace.ThreadID == UINT64_MAX)
-      Printf("UNKNOWN.\n");
+  if (Meta->IsDeallocated) {
+    if (Meta->DeallocationTrace.ThreadID ==
+        GuardedPoolAllocator::kInvalidThreadID)
+      Printf("0x%zx was deallocated by thread <unknown> here:\n", AccessPtr);
     else
-      Printf("%zu.\n", Meta->AllocationTrace.ThreadID);
+      Printf("0x%zx was deallocated by thread %zu here:\n", AccessPtr,
+             Meta->DeallocationTrace.ThreadID);
+
+    PrintBacktrace(Meta->DeallocationTrace.Trace, Printf);
   }
+
+  if (Meta->AllocationTrace.ThreadID == GuardedPoolAllocator::kInvalidThreadID)
+    Printf("0x%zx was allocated by thread <unknown> here:\n", Meta->Addr);
+  else
+    Printf("0x%zx was allocated by thread %zu here:\n", Meta->Addr,
+           Meta->AllocationTrace.ThreadID);
+
+  PrintBacktrace(Meta->AllocationTrace.Trace, Printf);
 }
 
 struct ScopedEndOfReportDecorator {
@@ -407,6 +453,7 @@ struct ScopedEndOfReportDecorator {
   ~ScopedEndOfReportDecorator() { Printf("*** End GWP-ASan report ***\n"); }
   options::Printf_t Printf;
 };
+} // anonymous namespace
 
 void GuardedPoolAllocator::reportErrorInternal(uintptr_t AccessPtr, Error E) {
   if (!pointerIsMine(reinterpret_cast<void *>(AccessPtr))) {
@@ -434,22 +481,21 @@ void GuardedPoolAllocator::reportErrorInternal(uintptr_t AccessPtr, Error E) {
       Meta = nullptr;
   }
 
-  // Print the error information, and if there is no valid metadata, stop here.
-  if (!printErrorType(E, AccessPtr, Meta, Printf)) {
-    return;
-  }
+  // Print the error information.
+  uint64_t ThreadID = getThreadID();
+  printErrorType(E, AccessPtr, Meta, Printf, ThreadID);
+  if (Backtrace) {
+    static constexpr unsigned kMaximumStackFramesForCrashTrace = 128;
+    uintptr_t Trace[kMaximumStackFramesForCrashTrace];
+    Backtrace(Trace, kMaximumStackFramesForCrashTrace);
 
-  // Ensure that we have a valid metadata pointer from this point forward.
-  if (Meta == nullptr) {
-    Printf("GWP-ASan internal unreachable error. Metadata is not null.\n");
-    return;
+    PrintBacktrace(Trace, Printf);
+  } else {
+    Printf("  <unknown (does your allocator support backtracing?)>\n\n");
   }
 
-  printThreadInformation(E, AccessPtr, Meta, Printf);
-  // TODO(hctim): Implement stack unwinding here. Ask the caller to provide us
-  // with the base pointer, and we unwind the stack to give a stack trace for
-  // the access.
-  // TODO(hctim): Implement dumping here of allocation/deallocation traces.
+  if (Meta)
+    printAllocDeallocTraces(AccessPtr, Meta, Printf, PrintBacktrace);
 }
 
 TLS_INITIAL_EXEC
diff --git a/lib/gwp_asan/guarded_pool_allocator.h b/lib/gwp_asan/guarded_pool_allocator.h
index 9b77c007b..400d50c0b 100644
--- a/lib/gwp_asan/guarded_pool_allocator.h
+++ b/lib/gwp_asan/guarded_pool_allocator.h
@@ -41,16 +41,14 @@ public:
   struct AllocationMetadata {
     // Maximum number of stack trace frames to collect for allocations + frees.
     // TODO(hctim): Implement stack frame compression, a-la Chromium.
-    // Currently the maximum stack frames is one, as we don't collect traces.
-    static constexpr size_t kMaximumStackFrames = 1;
+    static constexpr size_t kMaximumStackFrames = 64;
 
-    // Records the given allocation metadata into this struct. In the future,
-    // this will collect the allocation trace as well.
-    void RecordAllocation(uintptr_t Addr, size_t Size);
+    // Records the given allocation metadata into this struct.
+    void RecordAllocation(uintptr_t Addr, size_t Size,
+                          options::Backtrace_t Backtrace);
 
-    // Record that this allocation is now deallocated. In future, this will
-    // collect the deallocation trace as well.
-    void RecordDeallocation();
+    // Record that this allocation is now deallocated.
+    void RecordDeallocation(options::Backtrace_t Backtrace);
 
     struct CallSiteInfo {
       // The backtrace to the allocation/deallocation. If the first value is
@@ -234,6 +232,8 @@ private:
   // general) use printf() from the cstdlib as it may malloc(), causing infinite
   // recursion.
   options::Printf_t Printf = nullptr;
+  options::Backtrace_t Backtrace = nullptr;
+  options::PrintBacktrace_t PrintBacktrace = nullptr;
 
   // The adjusted sample rate for allocation sampling. Default *must* be
   // nonzero, as dynamic initialisation may call malloc (e.g. from libstdc++)
diff --git a/lib/gwp_asan/optional/backtrace.h b/lib/gwp_asan/optional/backtrace.h
new file mode 100644
index 000000000..2700970e5
--- /dev/null
+++ b/lib/gwp_asan/optional/backtrace.h
@@ -0,0 +1,23 @@
+//===-- backtrace.h ---------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef GWP_ASAN_OPTIONAL_BACKTRACE_H_
+#define GWP_ASAN_OPTIONAL_BACKTRACE_H_
+
+#include "gwp_asan/options.h"
+
+namespace gwp_asan {
+namespace options {
+// Functions to get the platform-specific and implementation-specific backtrace
+// and backtrace printing functions.
+Backtrace_t getBacktraceFunction();
+PrintBacktrace_t getPrintBacktraceFunction();
+} // namespace options
+} // namespace gwp_asan
+
+#endif // GWP_ASAN_OPTIONAL_BACKTRACE_H_
diff --git a/lib/gwp_asan/optional/backtrace_linux_libc.cpp b/lib/gwp_asan/optional/backtrace_linux_libc.cpp
new file mode 100644
index 000000000..f20a31009
--- /dev/null
+++ b/lib/gwp_asan/optional/backtrace_linux_libc.cpp
@@ -0,0 +1,64 @@
+//===-- backtrace_linux_libc.cpp --------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <assert.h>
+#include <execinfo.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "gwp_asan/optional/backtrace.h"
+#include "gwp_asan/options.h"
+
+namespace {
+void Backtrace(uintptr_t *TraceBuffer, size_t Size) {
+  // Grab (what seems to be) one more trace than we need. TraceBuffer needs to
+  // be null-terminated, but we wish to remove the frame of this function call.
+  static_assert(sizeof(uintptr_t) == sizeof(void *), "uintptr_t is not void*");
+  int NumTraces =
+      backtrace(reinterpret_cast<void **>(TraceBuffer), Size);
+
+  // Now shift the entire trace one place to the left and null-terminate.
+  memmove(TraceBuffer, TraceBuffer + 1, NumTraces * sizeof(void *));
+  TraceBuffer[NumTraces - 1] = 0;
+}
+
+static void PrintBacktrace(uintptr_t *Trace,
+                           gwp_asan::options::Printf_t Printf) {
+  size_t NumTraces = 0;
+  for (; Trace[NumTraces] != 0; ++NumTraces) {
+  }
+
+  if (NumTraces == 0) {
+    Printf("  <not found (does your allocator support backtracing?)>\n\n");
+    return;
+  }
+
+  char **BacktraceSymbols =
+      backtrace_symbols(reinterpret_cast<void **>(Trace), NumTraces);
+
+  for (size_t i = 0; i < NumTraces; ++i) {
+    if (!BacktraceSymbols)
+      Printf("  #%zu %p\n", i, Trace[i]);
+    else
+      Printf("  #%zu %s\n", i, BacktraceSymbols[i]);
+  }
+
+  Printf("\n");
+  if (BacktraceSymbols)
+    free(BacktraceSymbols);
+}
+} // anonymous namespace
+
+namespace gwp_asan {
+namespace options {
+Backtrace_t getBacktraceFunction() { return Backtrace; }
+PrintBacktrace_t getPrintBacktraceFunction() { return PrintBacktrace; }
+} // namespace options
+} // namespace gwp_asan
diff --git a/lib/gwp_asan/optional/backtrace_sanitizer_common.cpp b/lib/gwp_asan/optional/backtrace_sanitizer_common.cpp
new file mode 100644
index 000000000..7d17eec0d
--- /dev/null
+++ b/lib/gwp_asan/optional/backtrace_sanitizer_common.cpp
@@ -0,0 +1,69 @@
+//===-- backtrace_sanitizer_common.cpp --------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <assert.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <string.h>
+
+#include "gwp_asan/optional/backtrace.h"
+#include "gwp_asan/options.h"
+#include "sanitizer_common/sanitizer_stacktrace.h"
+
+void __sanitizer::BufferedStackTrace::UnwindImpl(uptr pc, uptr bp,
+                                                 void *context,
+                                                 bool request_fast,
+                                                 u32 max_depth) {
+  if (!StackTrace::WillUseFastUnwind(request_fast)) {
+    return Unwind(max_depth, pc, bp, context, 0, 0, request_fast);
+  }
+  Unwind(max_depth, pc, 0, context, 0, 0, false);
+}
+
+namespace {
+void Backtrace(uintptr_t *TraceBuffer, size_t Size) {
+  __sanitizer::BufferedStackTrace Trace;
+  Trace.Reset();
+  if (Size > __sanitizer::kStackTraceMax)
+    Size = __sanitizer::kStackTraceMax;
+
+  Trace.Unwind((__sanitizer::uptr)__builtin_return_address(0),
+               (__sanitizer::uptr)__builtin_frame_address(0),
+               /* ucontext */ nullptr,
+               /* fast unwind */ true, Size - 1);
+
+  memcpy(TraceBuffer, Trace.trace, Trace.size * sizeof(uintptr_t));
+  TraceBuffer[Trace.size] = 0;
+}
+
+static void PrintBacktrace(uintptr_t *Trace,
+                           gwp_asan::options::Printf_t Printf) {
+  __sanitizer::StackTrace StackTrace;
+  StackTrace.trace = reinterpret_cast<__sanitizer::uptr *>(Trace);
+
+  for (StackTrace.size = 0; StackTrace.size < __sanitizer::kStackTraceMax;
+       ++StackTrace.size) {
+    if (Trace[StackTrace.size] == 0)
+      break;
+  }
+
+  if (StackTrace.size == 0) {
+    Printf("  <unknown (does your allocator support backtracing?)>\n\n");
+    return;
+  }
+
+  StackTrace.Print();
+}
+} // anonymous namespace
+
+namespace gwp_asan {
+namespace options {
+Backtrace_t getBacktraceFunction() { return Backtrace; }
+PrintBacktrace_t getPrintBacktraceFunction() { return PrintBacktrace; }
+} // namespace options
+} // namespace gwp_asan
diff --git a/lib/gwp_asan/optional/options_parser.cpp b/lib/gwp_asan/optional/options_parser.cpp
index ba9af4930..6c2167288 100644
--- a/lib/gwp_asan/optional/options_parser.cpp
+++ b/lib/gwp_asan/optional/options_parser.cpp
@@ -47,6 +47,8 @@ Options *getOptionsInternal() {
 } // anonymous namespace
 
 void initOptions() {
+  __sanitizer::SetCommonFlagsDefaults();
+
   Options *o = getOptionsInternal();
   o->setDefaults();
 
@@ -85,7 +87,7 @@ void initOptions() {
   o->Printf = __sanitizer::Printf;
 }
 
-const Options &getOptions() { return *getOptionsInternal(); }
+Options &getOptions() { return *getOptionsInternal(); }
 
 } // namespace options
 } // namespace gwp_asan
diff --git a/lib/gwp_asan/optional/options_parser.h b/lib/gwp_asan/optional/options_parser.h
index 7a1d3b098..7a6bfaf0c 100644
--- a/lib/gwp_asan/optional/options_parser.h
+++ b/lib/gwp_asan/optional/options_parser.h
@@ -9,18 +9,17 @@
 #ifndef GWP_ASAN_OPTIONAL_OPTIONS_PARSER_H_
 #define GWP_ASAN_OPTIONAL_OPTIONS_PARSER_H_
 
+#include "gwp_asan/optional/backtrace.h"
 #include "gwp_asan/options.h"
 #include "sanitizer_common/sanitizer_common.h"
 
 namespace gwp_asan {
 namespace options {
-
 // Parse the options from the GWP_ASAN_FLAGS environment variable.
 void initOptions();
-// Returns a pointer to the initialised options. Call initOptions() prior to
-// calling this function.
-const Options &getOptions();
-
+// Returns the initialised options. Call initOptions() prior to calling this
+// function.
+Options &getOptions();
 } // namespace options
 } // namespace gwp_asan
 
diff --git a/lib/gwp_asan/options.h b/lib/gwp_asan/options.h
index c1b6e6719..6423e1652 100644
--- a/lib/gwp_asan/options.h
+++ b/lib/gwp_asan/options.h
@@ -9,6 +9,9 @@
 #ifndef GWP_ASAN_OPTIONS_H_
 #define GWP_ASAN_OPTIONS_H_
 
+#include <stddef.h>
+#include <stdint.h>
+
 namespace gwp_asan {
 namespace options {
 // The function pointer type for printf(). Follows the standard format from the
@@ -17,8 +20,21 @@ namespace options {
 // printf() signature, and pass the wrapper instead.
 typedef void (*Printf_t)(const char *Format, ...);
 
+// The function pointer type for backtrace information. Required to be
+// implemented by the supporting allocator. The callee should elide itself and
+// all frames below itself from TraceBuffer, i.e. the caller's frame should be
+// in TraceBuffer[0], and subsequent frames 1..n into TraceBuffer[1..n], where a
+// maximum of `MaximumDepth - 1` frames are stored. TraceBuffer should be
+// nullptr-terminated (i.e. if there are 5 frames; TraceBuffer[5] == nullptr).
+// If the allocator cannot supply backtrace information, it should set
+// TraceBuffer[0] == nullptr.
+typedef void (*Backtrace_t)(uintptr_t *TraceBuffer, size_t Size);
+typedef void (*PrintBacktrace_t)(uintptr_t *TraceBuffer, Printf_t Print);
+
 struct Options {
   Printf_t Printf = nullptr;
+  Backtrace_t Backtrace = nullptr;
+  PrintBacktrace_t PrintBacktrace = nullptr;
 
   // Read the options from the included definitions file.
 #define GWP_ASAN_OPTION(Type, Name, DefaultValue, Description)                 \
@@ -33,6 +49,8 @@ struct Options {
 #undef GWP_ASAN_OPTION
 
     Printf = nullptr;
+    Backtrace = nullptr;
+    PrintBacktrace = nullptr;
   }
 };
 } // namespace options
diff --git a/lib/gwp_asan/tests/CMakeLists.txt b/lib/gwp_asan/tests/CMakeLists.txt
index d87061f72..68cbdfe1b 100644
--- a/lib/gwp_asan/tests/CMakeLists.txt
+++ b/lib/gwp_asan/tests/CMakeLists.txt
@@ -4,7 +4,8 @@ set(GWP_ASAN_UNITTEST_CFLAGS
   ${COMPILER_RT_UNITTEST_CFLAGS}
   ${COMPILER_RT_GTEST_CFLAGS}
   -I${COMPILER_RT_SOURCE_DIR}/lib/
-  -O2)
+  -O2
+  -g)
 
 file(GLOB GWP_ASAN_HEADERS ../*.h)
 file(GLOB GWP_ASAN_UNITTESTS *.cpp)
@@ -15,7 +16,7 @@ set(GWP_ASAN_UNIT_TEST_HEADERS
 add_custom_target(GwpAsanUnitTests)
 set_target_properties(GwpAsanUnitTests PROPERTIES FOLDER "Compiler-RT Tests")
 
-set(GWP_ASAN_UNITTEST_LINK_FLAGS ${COMPILER_RT_UNITTEST_LINK_FLAGS})
+set(GWP_ASAN_UNITTEST_LINK_FLAGS ${COMPILER_RT_UNITTEST_LINK_FLAGS} -ldl)
 list(APPEND GWP_ASAN_UNITTEST_LINK_FLAGS --driver-mode=g++)
 if(NOT WIN32)
   list(APPEND GWP_ASAN_UNITTEST_LINK_FLAGS -pthread)
@@ -30,8 +31,11 @@ if(COMPILER_RT_DEFAULT_TARGET_ARCH IN_LIST GWP_ASAN_SUPPORTED_ARCH)
   # RTSanitizerCommonNoTermination(NoLibc) required for __sanitizer::Printf.
   set(GWP_ASAN_TEST_RUNTIME_OBJECTS
     $<TARGET_OBJECTS:RTGwpAsan.${arch}>
+    $<TARGET_OBJECTS:RTGwpAsanBacktraceSanitizerCommon.${arch}>
+    $<TARGET_OBJECTS:RTGwpAsanOptionsParser.${arch}>
     $<TARGET_OBJECTS:RTSanitizerCommon.${arch}>
-    $<TARGET_OBJECTS:RTSanitizerCommonNoLibc.${arch}>)
+    $<TARGET_OBJECTS:RTSanitizerCommonLibc.${arch}>
+    $<TARGET_OBJECTS:RTSanitizerCommonSymbolizer.${arch}>)
 
   add_library(${GWP_ASAN_TEST_RUNTIME} STATIC
     ${GWP_ASAN_TEST_RUNTIME_OBJECTS})
diff --git a/lib/gwp_asan/tests/backtrace.cpp b/lib/gwp_asan/tests/backtrace.cpp
new file mode 100644
index 000000000..9ed16a571
--- /dev/null
+++ b/lib/gwp_asan/tests/backtrace.cpp
@@ -0,0 +1,41 @@
+//===-- backtrace.cc --------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <string>
+
+#include "gwp_asan/tests/harness.h"
+
+TEST_F(BacktraceGuardedPoolAllocator, DoubleFree) {
+  void *Ptr = GPA.allocate(1);
+  GPA.deallocate(Ptr);
+
+  std::string DeathRegex = "Double free.*";
+  DeathRegex.append("backtrace\\.cpp:25.*");
+
+  DeathRegex.append("was deallocated.*");
+  DeathRegex.append("backtrace\\.cpp:15.*");
+
+  DeathRegex.append("was allocated.*");
+  DeathRegex.append("backtrace\\.cpp:14.*");
+  ASSERT_DEATH(GPA.deallocate(Ptr), DeathRegex);
+}
+
+TEST_F(BacktraceGuardedPoolAllocator, UseAfterFree) {
+  char *Ptr = static_cast<char *>(GPA.allocate(1));
+  GPA.deallocate(Ptr);
+
+  std::string DeathRegex = "Use after free.*";
+  DeathRegex.append("backtrace\\.cpp:40.*");
+
+  DeathRegex.append("was deallocated.*");
+  DeathRegex.append("backtrace\\.cpp:30.*");
+
+  DeathRegex.append("was allocated.*");
+  DeathRegex.append("backtrace\\.cpp:29.*");
+  ASSERT_DEATH({ *Ptr = 7; }, DeathRegex);
+}
diff --git a/lib/gwp_asan/tests/harness.h b/lib/gwp_asan/tests/harness.h
index 987564dd9..136e566d8 100644
--- a/lib/gwp_asan/tests/harness.h
+++ b/lib/gwp_asan/tests/harness.h
@@ -17,6 +17,8 @@
 #include "sanitizer_common/sanitizer_common.h"
 
 #include "gwp_asan/guarded_pool_allocator.h"
+#include "gwp_asan/optional/backtrace.h"
+#include "gwp_asan/optional/options_parser.h"
 #include "gwp_asan/options.h"
 
 class DefaultGuardedPoolAllocator : public ::testing::Test {
@@ -57,4 +59,25 @@ protected:
       MaxSimultaneousAllocations;
 };
 
+class BacktraceGuardedPoolAllocator : public ::testing::Test {
+public:
+  BacktraceGuardedPoolAllocator() {
+    // Call initOptions to initialise the internal sanitizer_common flags. These
+    // flags are referenced by the sanitizer_common unwinder, and if left
+    // uninitialised, they'll unintentionally crash the program.
+    gwp_asan::options::initOptions();
+
+    gwp_asan::options::Options Opts;
+    Opts.setDefaults();
+
+    Opts.Printf = __sanitizer::Printf;
+    Opts.Backtrace = gwp_asan::options::getBacktraceFunction();
+    Opts.PrintBacktrace = gwp_asan::options::getPrintBacktraceFunction();
+    GPA.init(Opts);
+  }
+
+protected:
+  gwp_asan::GuardedPoolAllocator GPA;
+};
+
 #endif // GWP_ASAN_TESTS_HARNESS_H_
diff --git a/test/gwp_asan/double_delete.cpp b/test/gwp_asan/double_delete.cpp
index 6231b70e3..7a1bb867f 100644
--- a/test/gwp_asan/double_delete.cpp
+++ b/test/gwp_asan/double_delete.cpp
@@ -3,7 +3,7 @@
 // RUN: not %run %t 2>&1 | FileCheck %s
 
 // CHECK: GWP-ASan detected a memory error
-// CHECK: Double free occurred when trying to free memory at:
+// CHECK: Double free at 0x{{[a-f0-9]+}} (a 1-byte allocation)
 
 #include <cstdlib>
 
diff --git a/test/gwp_asan/double_deletea.cpp b/test/gwp_asan/double_deletea.cpp
index 0a207cfa4..c9b048a02 100644
--- a/test/gwp_asan/double_deletea.cpp
+++ b/test/gwp_asan/double_deletea.cpp
@@ -3,7 +3,7 @@
 // RUN: not %run %t 2>&1 | FileCheck %s
 
 // CHECK: GWP-ASan detected a memory error
-// CHECK: Double free occurred when trying to free memory at:
+// CHECK: Double free at 0x{{[a-f0-9]+}} (a 50-byte allocation)
 
 #include <cstdlib>
 
diff --git a/test/gwp_asan/double_free.cpp b/test/gwp_asan/double_free.cpp
index ece87892c..e69da3472 100644
--- a/test/gwp_asan/double_free.cpp
+++ b/test/gwp_asan/double_free.cpp
@@ -2,13 +2,13 @@
 // RUN: %clangxx_gwp_asan %s -o %t
 // RUN: not %run %t 2>&1 | FileCheck %s
 
-// CHECK: GWP-ASan detected a memory error
-// CHECK: Double free occurred when trying to free memory at:
-
 #include <cstdlib>
 
 int main() {
+  // CHECK: GWP-ASan detected a memory error
+  // CHECK: Double free at 0x{{[a-f0-9]+}} (a 10-byte allocation)
   void *Ptr = malloc(10);
+
   free(Ptr);
   free(Ptr);
   return 0;
diff --git a/test/gwp_asan/heap_buffer_overflow.cpp b/test/gwp_asan/heap_buffer_overflow.cpp
index 1f88ff380..5e190266c 100644
--- a/test/gwp_asan/heap_buffer_overflow.cpp
+++ b/test/gwp_asan/heap_buffer_overflow.cpp
@@ -3,8 +3,8 @@
 // RUN: %expect_crash %run %t 2>&1 | FileCheck %s
 
 // CHECK: GWP-ASan detected a memory error
-// CHECK: Buffer overflow occurred when accessing memory at:
-// CHECK: is located {{[0-9]+}} bytes to the right
+// CHECK: Buffer overflow at 0x{{[a-f0-9]+}} ({{[1-9][0-9]*}} bytes to the right
+// CHECK-SAME: of a {{[1-9][0-9]*}}-byte allocation
 
 #include <cstdlib>
 
diff --git a/test/gwp_asan/heap_buffer_underflow.cpp b/test/gwp_asan/heap_buffer_underflow.cpp
index e529f756a..4ba89949a 100644
--- a/test/gwp_asan/heap_buffer_underflow.cpp
+++ b/test/gwp_asan/heap_buffer_underflow.cpp
@@ -3,8 +3,8 @@
 // RUN: %expect_crash %run %t 2>&1 | FileCheck %s
 
 // CHECK: GWP-ASan detected a memory error
-// CHECK: Buffer underflow occurred when accessing memory at:
-// CHECK: is located 1 bytes to the left
+// CHECK: Buffer underflow at 0x{{[a-f0-9]+}} (1 byte to the left
+// CHECK-SAME: of a {{[1-9][0-9]*}}-byte allocation
 
 #include <cstdlib>
 
diff --git a/test/gwp_asan/invalid_free_left.cpp b/test/gwp_asan/invalid_free_left.cpp
index 2a9511472..9bfb04d6e 100644
--- a/test/gwp_asan/invalid_free_left.cpp
+++ b/test/gwp_asan/invalid_free_left.cpp
@@ -3,8 +3,8 @@
 // RUN: not %run %t 2>&1 | FileCheck %s
 
 // CHECK: GWP-ASan detected a memory error
-// CHECK: Invalid (wild) free occurred when trying to free memory at:
-// CHECK: is located 1 bytes to the left of
+// CHECK: Invalid (wild) free at 0x{{[a-f0-9]+}} (1 byte to the left of a
+// CHECK-SAME: 1-byte allocation
 
 #include <cstdlib>
 
diff --git a/test/gwp_asan/invalid_free_right.cpp b/test/gwp_asan/invalid_free_right.cpp
index 7f0276d21..f707dfa07 100644
--- a/test/gwp_asan/invalid_free_right.cpp
+++ b/test/gwp_asan/invalid_free_right.cpp
@@ -3,8 +3,8 @@
 // RUN: not %run %t 2>&1 | FileCheck %s
 
 // CHECK: GWP-ASan detected a memory error
-// CHECK: Invalid (wild) free occurred when trying to free memory at:
-// CHECK: is located 1 bytes to the right
+// CHECK: Invalid (wild) free at 0x{{[a-f0-9]+}} (1 byte to the right of a
+// CHECK-SAME: 1-byte allocation
 
 #include <cstdlib>
 
diff --git a/test/gwp_asan/lit.cfg.py b/test/gwp_asan/lit.cfg.py
index c568bf645..669072f7e 100644
--- a/test/gwp_asan/lit.cfg.py
+++ b/test/gwp_asan/lit.cfg.py
@@ -20,7 +20,8 @@ if not config.android:
 
 cxx_flags = (c_flags + config.cxx_mode_flags + ["-std=c++11"])
 
-gwp_asan_flags = ["-fsanitize=scudo"]
+gwp_asan_flags = ["-fsanitize=scudo", "-g", "-fno-omit-frame-pointer",
+                  "-mno-omit-leaf-frame-pointer"]
 
 def build_invocation(compile_flags):
   return " " + " ".join([config.clang] + compile_flags) + " "
diff --git a/test/gwp_asan/realloc.cpp b/test/gwp_asan/realloc.cpp
index 3b92f6eda..774a5b3b2 100644
--- a/test/gwp_asan/realloc.cpp
+++ b/test/gwp_asan/realloc.cpp
@@ -23,8 +23,8 @@ int main() {
   free(Ptr + 1);
 
   // CHECK-MALLOC: GWP-ASan detected a memory error
-  // CHECK-MALLOC: Invalid (wild) free occurred when trying to free memory at:
-  // CHECK-MALLOC: is located 1 bytes to the right of a 1-byte allocation
+  // CHECK-MALLOC: Invalid (wild) free at 0x{{[a-f0-9]+}} (1 byte to the right
+  // CHECK-MALLOC-SAME: of a 1-byte allocation
 #elif defined(TEST_FREE)
   char *Ptr = (char *) malloc(1);
   // realloc(ptr, 0) is equivalent to free(ptr) and must return nullptr. Note
@@ -36,8 +36,8 @@ int main() {
   *Ptr = 0;
 
   // CHECK-FREE: GWP-ASan detected a memory error
-  // CHECK-FREE: Use after free occurred when accessing memory at:
-  // CHECK-FREE: is a 1-byte allocation
+  // CHECK-FREE: Use after free at 0x{{[a-f0-9]+}} (0 bytes into a 1-byte
+  // CHECK-FREE-SAME: allocation
 #endif
 
   return 0;
diff --git a/test/gwp_asan/use_after_delete.cpp b/test/gwp_asan/use_after_delete.cpp
index 79df4ad0a..a08b7769d 100644
--- a/test/gwp_asan/use_after_delete.cpp
+++ b/test/gwp_asan/use_after_delete.cpp
@@ -3,7 +3,7 @@
 // RUN: %expect_crash %run %t 2>&1 | FileCheck %s
 
 // CHECK: GWP-ASan detected a memory error
-// CHECK: Use after free occurred when accessing memory at:
+// CHECK: Use after free at 0x{{[a-f0-9]+}} (0 bytes into a 1-byte allocation
 
 #include <cstdlib>
 
diff --git a/test/gwp_asan/use_after_deletea.cpp b/test/gwp_asan/use_after_deletea.cpp
index 7db6a84c9..998a6a706 100644
--- a/test/gwp_asan/use_after_deletea.cpp
+++ b/test/gwp_asan/use_after_deletea.cpp
@@ -3,7 +3,7 @@
 // RUN: %expect_crash %run %t 2>&1 | FileCheck %s
 
 // CHECK: GWP-ASan detected a memory error
-// CHECK: Use after free occurred when accessing memory at:
+// CHECK: Use after free at 0x{{[a-f0-9]+}} (0 bytes into a 10-byte allocation
 
 #include <cstdlib>
 
diff --git a/test/gwp_asan/use_after_free.cpp b/test/gwp_asan/use_after_free.cpp
index 8d5a6ff28..828cebf8e 100644
--- a/test/gwp_asan/use_after_free.cpp
+++ b/test/gwp_asan/use_after_free.cpp
@@ -3,7 +3,7 @@
 // RUN: %expect_crash %run %t 2>&1 | FileCheck %s
 
 // CHECK: GWP-ASan detected a memory error
-// CHECK: Use after free occurred when accessing memory at:
+// CHECK: Use after free at 0x{{[a-f0-9]+}} (0 bytes into a 10-byte allocation
 
 #include <cstdlib>