[scudo][standalone] Introduce the C & C++ wrappers

Summary:
This CL adds C & C++ wrappers and associated tests. Those use default
configurations for a Scudo combined allocator that will likely be
tweaked in the future.

This is the final CL required to have a functional C & C++ allocator
based on Scudo.

The structure I have chosen is to define the core C allocation
primitives in an `.inc` file that can be customized through defines.
This allows to easily have 2 (or more) sets of wrappers backed by
different combined allocators, as demonstrated by the `Bionic`
wrappers: one set for the "default" allocator, one set for the "svelte"
allocator.

Currently all the tests added have been gtests, but I am planning to
add some more lit tests as well.

Reviewers: morehouse, eugenis, vitalybuka, hctim, rengolin

Reviewed By: morehouse

Subscribers: srhines, mgorny, delcypher, jfb, #sanitizers, llvm-commits

Tags: #llvm, #sanitizers

Differential Revision: https://reviews.llvm.org/D63612

git-svn-id: https://llvm.org/svn/llvm-project/compiler-rt/trunk@364332 91177308-0d34-0410-b5e6-96231b3b80d8

11 files changed, 909 insertions, 36 deletions

diff --git a/lib/scudo/standalone/CMakeLists.txt b/lib/scudo/standalone/CMakeLists.txt
index 953acd8f4..027b04105 100644
--- a/lib/scudo/standalone/CMakeLists.txt
+++ b/lib/scudo/standalone/CMakeLists.txt
@@ -33,29 +33,6 @@ if(ANDROID)
   append_list_if(COMPILER_RT_HAS_Z_GLOBAL -Wl,-z,global SCUDO_LINK_FLAGS)
 endif()
 
-set(SCUDO_SOURCES
-  checksum.cc
-  crc32_hw.cc
-  common.cc
-  flags.cc
-  flags_parser.cc
-  fuchsia.cc
-  linux.cc
-  report.cc
-  secondary.cc
-  string_utils.cc)
-
-# Enable the SSE 4.2 instruction set for crc32_hw.cc, if available.
-if (COMPILER_RT_HAS_MSSE4_2_FLAG)
-  set_source_files_properties(crc32_hw.cc PROPERTIES COMPILE_FLAGS -msse4.2)
-endif()
-
-# Enable the AArch64 CRC32 feature for crc32_hw.cc, if available.
-# Note that it is enabled by default starting with armv8.1-a.
-if (COMPILER_RT_HAS_MCRC_FLAG)
-  set_source_files_properties(crc32_hw.cc PROPERTIES COMPILE_FLAGS -mcrc)
-endif()
-
 set(SCUDO_HEADERS
   allocator_config.h
   atomic_helpers.h
@@ -85,7 +62,38 @@ set(SCUDO_HEADERS
   tsd.h
   tsd_exclusive.h
   tsd_shared.h
-  vector.h)
+  vector.h
+  wrappers_c_checks.h
+  wrappers_c.h)
+
+set(SCUDO_SOURCES
+  checksum.cc
+  crc32_hw.cc
+  common.cc
+  flags.cc
+  flags_parser.cc
+  fuchsia.cc
+  linux.cc
+  report.cc
+  secondary.cc
+  string_utils.cc)
+
+# Enable the SSE 4.2 instruction set for crc32_hw.cc, if available.
+if (COMPILER_RT_HAS_MSSE4_2_FLAG)
+  set_source_files_properties(crc32_hw.cc PROPERTIES COMPILE_FLAGS -msse4.2)
+endif()
+
+# Enable the AArch64 CRC32 feature for crc32_hw.cc, if available.
+# Note that it is enabled by default starting with armv8.1-a.
+if (COMPILER_RT_HAS_MCRC_FLAG)
+  set_source_files_properties(crc32_hw.cc PROPERTIES COMPILE_FLAGS -mcrc)
+endif()
+
+set(SCUDO_SOURCES_C_WRAPPERS
+  wrappers_c.cc)
+
+set(SCUDO_SOURCES_CXX_WRAPPERS
+  wrappers_cpp.cc)
 
 if(COMPILER_RT_HAS_SCUDO_STANDALONE)
   add_compiler_rt_object_libraries(RTScudoStandalone
@@ -93,11 +101,28 @@ if(COMPILER_RT_HAS_SCUDO_STANDALONE)
     SOURCES ${SCUDO_SOURCES}
     ADDITIONAL_HEADERS ${SCUDO_HEADERS}
     CFLAGS ${SCUDO_CFLAGS})
+  add_compiler_rt_object_libraries(RTScudoStandaloneCWrappers
+    ARCHS ${SCUDO_STANDALONE_SUPPORTED_ARCH}
+    SOURCES ${SCUDO_SOURCES_C_WRAPPERS}
+    ADDITIONAL_HEADERS ${SCUDO_HEADERS}
+    CFLAGS ${SCUDO_CFLAGS})
+  add_compiler_rt_object_libraries(RTScudoStandaloneCxxWrappers
+    ARCHS ${SCUDO_STANDALONE_SUPPORTED_ARCH}
+    SOURCES ${SCUDO_SOURCES_CXX_WRAPPERS}
+    ADDITIONAL_HEADERS ${SCUDO_HEADERS}
+    CFLAGS ${SCUDO_CFLAGS})
 
   add_compiler_rt_runtime(clang_rt.scudo_standalone
     STATIC
     ARCHS ${SCUDO_STANDALONE_SUPPORTED_ARCH}
-    SOURCES ${SCUDO_SOURCES}
+    SOURCES ${SCUDO_SOURCES} ${SCUDO_SOURCES_C_WRAPPERS}
+    ADDITIONAL_HEADERS ${SCUDO_HEADERS}
+    CFLAGS ${SCUDO_CFLAGS}
+    PARENT_TARGET scudo_standalone)
+  add_compiler_rt_runtime(clang_rt.scudo_standalone_cxx
+    STATIC
+    ARCHS ${SCUDO_STANDALONE_SUPPORTED_ARCH}
+    SOURCES ${SCUDO_SOURCES_CXX_WRAPPERS}
     ADDITIONAL_HEADERS ${SCUDO_HEADERS}
     CFLAGS ${SCUDO_CFLAGS}
     PARENT_TARGET scudo_standalone)
diff --git a/lib/scudo/standalone/tests/CMakeLists.txt b/lib/scudo/standalone/tests/CMakeLists.txt
index 9167ba7c4..f0c259a90 100644
--- a/lib/scudo/standalone/tests/CMakeLists.txt
+++ b/lib/scudo/standalone/tests/CMakeLists.txt
@@ -10,7 +10,10 @@ set(SCUDO_UNITTEST_CFLAGS
   -I${COMPILER_RT_SOURCE_DIR}/include
   -I${COMPILER_RT_SOURCE_DIR}/lib
   -I${COMPILER_RT_SOURCE_DIR}/lib/scudo/standalone
-  -DGTEST_HAS_RTTI=0)
+  -DGTEST_HAS_RTTI=0
+  # Extra flags for the C++ tests
+  -fsized-deallocation
+  -Wno-mismatched-new-delete)
 
 set(SCUDO_TEST_ARCH ${SCUDO_STANDALONE_SUPPORTED_ARCH})
 
@@ -21,27 +24,30 @@ foreach(lib ${SANITIZER_TEST_CXX_LIBRARIES})
 endforeach()
 list(APPEND LINK_FLAGS -pthread)
 
-set(TEST_HEADERS)
+set(SCUDO_TEST_HEADERS)
 foreach (header ${SCUDO_HEADERS})
-  list(APPEND TEST_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/../${header})
+  list(APPEND SCUDO_TEST_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/../${header})
 endforeach()
 
-# add_scudo_unittest(<name>
-#                   SOURCES <sources list>
-#                   HEADERS <extra headers list>)
 macro(add_scudo_unittest testname)
-  cmake_parse_arguments(TEST "" "" "SOURCES;HEADERS" ${ARGN})
+  cmake_parse_arguments(TEST "" "" "SOURCES;ADDITIONAL_RTOBJECTS" ${ARGN})
   if(COMPILER_RT_HAS_SCUDO_STANDALONE)
     foreach(arch ${SCUDO_TEST_ARCH})
+      # Additional runtime objects get added along RTScudoStandalone
+      set(SCUDO_TEST_RTOBJECTS $<TARGET_OBJECTS:RTScudoStandalone.${arch}>)
+      foreach(rtobject ${TEST_ADDITIONAL_RTOBJECTS})
+        list(APPEND SCUDO_TEST_RTOBJECTS $<TARGET_OBJECTS:${rtobject}.${arch}>)
+      endforeach()
+      # Add the static runtime library made of all the runtime objects
+      set(RUNTIME RT${testname}.${arch})
+      add_library(${RUNTIME} STATIC ${SCUDO_TEST_RTOBJECTS})
       set(ScudoUnitTestsObjects)
-      add_library("RTScudoStandalone.test.${arch}" STATIC
-        $<TARGET_OBJECTS:RTScudoStandalone.${arch}>)
       generate_compiler_rt_tests(ScudoUnitTestsObjects ScudoUnitTests
         "${testname}-${arch}-Test" ${arch}
         SOURCES ${TEST_SOURCES} ${COMPILER_RT_GTEST_SOURCE}
-        COMPILE_DEPS ${TEST_HEADERS}
+        COMPILE_DEPS ${SCUDO_TEST_HEADERS}
         DEPS gtest scudo_standalone
-        RUNTIME RTScudoStandalone.test.${arch}
+        RUNTIME ${RUNTIME}
         CFLAGS ${SCUDO_UNITTEST_CFLAGS}
         LINK_FLAGS ${LINK_FLAGS})
     endforeach()
@@ -72,3 +78,19 @@ set(SCUDO_UNIT_TEST_SOURCES
 
 add_scudo_unittest(ScudoUnitTest
   SOURCES ${SCUDO_UNIT_TEST_SOURCES})
+
+set(SCUDO_C_UNIT_TEST_SOURCES
+  wrappers_c_test.cc
+  scudo_unit_test_main.cc)
+
+add_scudo_unittest(ScudoCUnitTest
+  SOURCES ${SCUDO_C_UNIT_TEST_SOURCES}
+  ADDITIONAL_RTOBJECTS RTScudoStandaloneCWrappers)
+
+set(SCUDO_CXX_UNIT_TEST_SOURCES
+  wrappers_cpp_test.cc
+  scudo_unit_test_main.cc)
+
+add_scudo_unittest(ScudoCxxUnitTest
+  SOURCES ${SCUDO_CXX_UNIT_TEST_SOURCES}
+  ADDITIONAL_RTOBJECTS RTScudoStandaloneCWrappers RTScudoStandaloneCxxWrappers)
diff --git a/lib/scudo/standalone/tests/wrappers_c_test.cc b/lib/scudo/standalone/tests/wrappers_c_test.cc
new file mode 100644
index 000000000..a5ba80677
--- /dev/null
+++ b/lib/scudo/standalone/tests/wrappers_c_test.cc
@@ -0,0 +1,225 @@
+//===-- wrappers_c_test.cc --------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "platform.h"
+
+#include "gtest/gtest.h"
+
+#include <limits.h>
+#include <malloc.h>
+#include <unistd.h>
+
+// Note that every C allocation function in the test binary will be fulfilled
+// by Scudo (this includes the gtest APIs, etc.), which is a test by itself.
+// But this might also lead to unexpected side-effects, since the allocation and
+// deallocation operations in the TEST functions will coexist with others (see
+// the EXPECT_DEATH comment below).
+
+// We have to use a small quarantine to make sure that our double-free tests
+// trigger. Otherwise EXPECT_DEATH ends up reallocating the chunk that was just
+// freed (this depends on the size obviously) and the following free succeeds.
+extern "C" __attribute__((visibility("default"))) const char *
+__scudo_default_options() {
+  return "quarantine_size_kb=256:thread_local_quarantine_size_kb=128:"
+         "quarantine_max_chunk_size=512";
+}
+
+static const size_t Size = 100U;
+
+TEST(ScudoWrappersCTest, Malloc) {
+  void *P = malloc(Size);
+  EXPECT_NE(P, nullptr);
+  EXPECT_LE(Size, malloc_usable_size(P));
+  EXPECT_EQ(reinterpret_cast<uintptr_t>(P) % FIRST_32_SECOND_64(8U, 16U), 0U);
+  EXPECT_DEATH(
+      free(reinterpret_cast<void *>(reinterpret_cast<uintptr_t>(P) | 1U)), "");
+  free(P);
+  EXPECT_DEATH(free(P), "");
+
+  P = malloc(0U);
+  EXPECT_NE(P, nullptr);
+  free(P);
+
+  errno = 0;
+  EXPECT_EQ(malloc(SIZE_MAX), nullptr);
+  EXPECT_EQ(errno, ENOMEM);
+}
+
+TEST(ScudoWrappersCTest, Calloc) {
+  void *P = calloc(1U, Size);
+  EXPECT_NE(P, nullptr);
+  EXPECT_LE(Size, malloc_usable_size(P));
+  for (size_t I = 0; I < Size; I++)
+    EXPECT_EQ((reinterpret_cast<uint8_t *>(P))[I], 0U);
+  free(P);
+
+  P = calloc(1U, 0U);
+  EXPECT_NE(P, nullptr);
+  free(P);
+  P = calloc(0U, 1U);
+  EXPECT_NE(P, nullptr);
+  free(P);
+
+  errno = 0;
+  EXPECT_EQ(calloc(SIZE_MAX, 1U), nullptr);
+  EXPECT_EQ(errno, ENOMEM);
+  errno = 0;
+  EXPECT_EQ(calloc(static_cast<size_t>(LONG_MAX) + 1U, 2U), nullptr);
+  if (SCUDO_ANDROID)
+    EXPECT_EQ(errno, ENOMEM);
+  errno = 0;
+  EXPECT_EQ(calloc(SIZE_MAX, SIZE_MAX), nullptr);
+  EXPECT_EQ(errno, ENOMEM);
+}
+
+TEST(ScudoWrappersCTest, Memalign) {
+  void *P;
+  for (size_t I = FIRST_32_SECOND_64(2U, 3U); I <= 18U; I++) {
+    const size_t Alignment = 1U << I;
+
+    P = memalign(Alignment, Size);
+    EXPECT_NE(P, nullptr);
+    EXPECT_LE(Size, malloc_usable_size(P));
+    EXPECT_EQ(reinterpret_cast<uintptr_t>(P) % Alignment, 0U);
+    free(P);
+
+    P = nullptr;
+    EXPECT_EQ(posix_memalign(&P, Alignment, Size), 0);
+    EXPECT_NE(P, nullptr);
+    EXPECT_LE(Size, malloc_usable_size(P));
+    EXPECT_EQ(reinterpret_cast<uintptr_t>(P) % Alignment, 0U);
+    free(P);
+  }
+
+  EXPECT_EQ(memalign(4096U, SIZE_MAX), nullptr);
+  EXPECT_EQ(posix_memalign(&P, 15U, Size), EINVAL);
+  EXPECT_EQ(posix_memalign(&P, 4096U, SIZE_MAX), ENOMEM);
+
+  // Android's memalign accepts non power-of-2 alignments, and 0.
+  if (SCUDO_ANDROID) {
+    for (size_t Alignment = 0U; Alignment <= 128U; Alignment++) {
+      P = memalign(Alignment, 1024U);
+      EXPECT_NE(P, nullptr);
+      free(P);
+    }
+  }
+}
+
+TEST(ScudoWrappersCTest, AlignedAlloc) {
+  const size_t Alignment = 4096U;
+  void *P = aligned_alloc(Alignment, Alignment * 4U);
+  EXPECT_NE(P, nullptr);
+  EXPECT_LE(Alignment * 4U, malloc_usable_size(P));
+  EXPECT_EQ(reinterpret_cast<uintptr_t>(P) % Alignment, 0U);
+  free(P);
+
+  errno = 0;
+  P = aligned_alloc(Alignment, Size);
+  EXPECT_EQ(P, nullptr);
+  EXPECT_EQ(errno, EINVAL);
+}
+
+TEST(ScudoWrappersCTest, Realloc) {
+  // realloc(nullptr, N) is malloc(N)
+  void *P = realloc(nullptr, 0U);
+  EXPECT_NE(P, nullptr);
+  free(P);
+
+  P = malloc(Size);
+  EXPECT_NE(P, nullptr);
+  // realloc(P, 0U) is free(P) and returns nullptr
+  EXPECT_EQ(realloc(P, 0U), nullptr);
+
+  P = malloc(Size);
+  EXPECT_NE(P, nullptr);
+  EXPECT_LE(Size, malloc_usable_size(P));
+  memset(P, 0x42, Size);
+
+  P = realloc(P, Size * 2U);
+  EXPECT_NE(P, nullptr);
+  EXPECT_LE(Size * 2U, malloc_usable_size(P));
+  for (size_t I = 0; I < Size; I++)
+    EXPECT_EQ(0x42, (reinterpret_cast<uint8_t *>(P))[I]);
+
+  P = realloc(P, Size / 2U);
+  EXPECT_NE(P, nullptr);
+  EXPECT_LE(Size / 2U, malloc_usable_size(P));
+  for (size_t I = 0; I < Size / 2U; I++)
+    EXPECT_EQ(0x42, (reinterpret_cast<uint8_t *>(P))[I]);
+  free(P);
+
+  EXPECT_DEATH(P = realloc(P, Size), "");
+
+  errno = 0;
+  EXPECT_EQ(realloc(nullptr, SIZE_MAX), nullptr);
+  EXPECT_EQ(errno, ENOMEM);
+  P = malloc(Size);
+  EXPECT_NE(P, nullptr);
+  errno = 0;
+  EXPECT_EQ(realloc(P, SIZE_MAX), nullptr);
+  EXPECT_EQ(errno, ENOMEM);
+  free(P);
+
+  // Android allows realloc of memalign pointers.
+  if (SCUDO_ANDROID) {
+    const size_t Alignment = 1024U;
+    P = memalign(Alignment, Size);
+    EXPECT_NE(P, nullptr);
+    EXPECT_LE(Size, malloc_usable_size(P));
+    EXPECT_EQ(reinterpret_cast<uintptr_t>(P) % Alignment, 0U);
+    memset(P, 0x42, Size);
+
+    P = realloc(P, Size * 2U);
+    EXPECT_NE(P, nullptr);
+    EXPECT_LE(Size * 2U, malloc_usable_size(P));
+    for (size_t I = 0; I < Size; I++)
+      EXPECT_EQ(0x42, (reinterpret_cast<uint8_t *>(P))[I]);
+    free(P);
+  }
+}
+
+#ifndef M_DECAY_TIME
+#define M_DECAY_TIME -100
+#endif
+
+#ifndef M_PURGE
+#define M_PURGE -101
+#endif
+
+TEST(ScudoWrappersCTest, Mallopt) {
+  errno = 0;
+  EXPECT_EQ(mallopt(-1000, 1), 0);
+  // mallopt doesn't set errno.
+  EXPECT_EQ(errno, 0);
+
+  EXPECT_EQ(mallopt(M_PURGE, 0), 1);
+
+  EXPECT_EQ(mallopt(M_DECAY_TIME, 1), 1);
+  EXPECT_EQ(mallopt(M_DECAY_TIME, 0), 1);
+  EXPECT_EQ(mallopt(M_DECAY_TIME, 1), 1);
+  EXPECT_EQ(mallopt(M_DECAY_TIME, 0), 1);
+}
+
+TEST(ScudoWrappersCTest, OtherAlloc) {
+  const size_t PageSize = sysconf(_SC_PAGESIZE);
+
+  void *P = pvalloc(Size);
+  EXPECT_NE(P, nullptr);
+  EXPECT_EQ(reinterpret_cast<uintptr_t>(P) & (PageSize - 1), 0U);
+  EXPECT_LE(PageSize, malloc_usable_size(P));
+  free(P);
+
+  EXPECT_EQ(pvalloc(SIZE_MAX), nullptr);
+
+  P = pvalloc(Size);
+  EXPECT_NE(P, nullptr);
+  EXPECT_EQ(reinterpret_cast<uintptr_t>(P) & (PageSize - 1), 0U);
+  free(P);
+
+  EXPECT_EQ(valloc(SIZE_MAX), nullptr);
+}
diff --git a/lib/scudo/standalone/tests/wrappers_cpp_test.cc b/lib/scudo/standalone/tests/wrappers_cpp_test.cc
new file mode 100644
index 000000000..8d5842a25
--- /dev/null
+++ b/lib/scudo/standalone/tests/wrappers_cpp_test.cc
@@ -0,0 +1,117 @@
+//===-- wrappers_cpp_test.cc ------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "gtest/gtest.h"
+
+#include <condition_variable>
+#include <mutex>
+#include <thread>
+
+// Note that every Cxx allocation function in the test binary will be fulfilled
+// by Scudo. See the comment in the C counterpart of this file.
+
+extern "C" __attribute__((visibility("default"))) const char *
+__scudo_default_options() {
+  return "quarantine_size_kb=256:thread_local_quarantine_size_kb=128:"
+         "quarantine_max_chunk_size=512:dealloc_type_mismatch=true";
+}
+
+template <typename T> static void testCxxNew() {
+  T *P = new T;
+  EXPECT_NE(P, nullptr);
+  memset(P, 0x42, sizeof(T));
+  EXPECT_DEATH(delete[] P, "");
+  delete P;
+  EXPECT_DEATH(delete P, "");
+
+  P = new T;
+  EXPECT_NE(P, nullptr);
+  memset(P, 0x42, sizeof(T));
+  operator delete(P, sizeof(T));
+
+  P = new (std::nothrow) T;
+  EXPECT_NE(P, nullptr);
+  memset(P, 0x42, sizeof(T));
+  delete P;
+
+  const size_t N = 16U;
+  T *A = new T[N];
+  EXPECT_NE(A, nullptr);
+  memset(A, 0x42, sizeof(T) * N);
+  EXPECT_DEATH(delete A, "");
+  delete[] A;
+  EXPECT_DEATH(delete[] A, "");
+
+  A = new T[N];
+  EXPECT_NE(A, nullptr);
+  memset(A, 0x42, sizeof(T) * N);
+  operator delete[](A, sizeof(T) * N);
+
+  A = new (std::nothrow) T[N];
+  EXPECT_NE(A, nullptr);
+  memset(A, 0x42, sizeof(T) * N);
+  delete[] A;
+}
+
+class Pixel {
+public:
+  enum class Color { Red, Green, Blue };
+  int X = 0;
+  int Y = 0;
+  Color C = Color::Red;
+};
+
+TEST(ScudoWrappersCppTest, New) {
+  testCxxNew<bool>();
+  testCxxNew<uint8_t>();
+  testCxxNew<uint16_t>();
+  testCxxNew<uint32_t>();
+  testCxxNew<uint64_t>();
+  testCxxNew<float>();
+  testCxxNew<double>();
+  testCxxNew<long double>();
+  testCxxNew<Pixel>();
+}
+
+static std::mutex Mutex;
+static std::condition_variable Cv;
+static bool Ready = false;
+
+static void stressNew() {
+  std::vector<uintptr_t *> V;
+  {
+    std::unique_lock<std::mutex> Lock(Mutex);
+    while (!Ready)
+      Cv.wait(Lock);
+  }
+  for (size_t I = 0; I < 256U; I++) {
+    const size_t N = std::rand() % 128U;
+    uintptr_t *P = new uintptr_t[N];
+    if (P) {
+      memset(P, 0x42, sizeof(uintptr_t) * N);
+      V.push_back(P);
+    }
+  }
+  while (!V.empty()) {
+    delete[] V.back();
+    V.pop_back();
+  }
+}
+
+TEST(ScudoWrappersCppTest, ThreadedNew) {
+  std::thread Threads[32];
+  for (size_t I = 0U; I < sizeof(Threads) / sizeof(Threads[0]); I++)
+    Threads[I] = std::thread(stressNew);
+  {
+    std::unique_lock<std::mutex> Lock(Mutex);
+    Ready = true;
+    Cv.notify_all();
+  }
+  for (auto &T : Threads)
+    T.join();
+}
diff --git a/lib/scudo/standalone/tsd.h b/lib/scudo/standalone/tsd.h
index d3958ddfc..10cb83f94 100644
--- a/lib/scudo/standalone/tsd.h
+++ b/lib/scudo/standalone/tsd.h
@@ -15,6 +15,11 @@
 
 #include <limits.h> // for PTHREAD_DESTRUCTOR_ITERATIONS
 
+// With some build setups, this might still not be defined.
+#ifndef PTHREAD_DESTRUCTOR_ITERATIONS
+#define PTHREAD_DESTRUCTOR_ITERATIONS 4
+#endif
+
 namespace scudo {
 
 template <class Allocator> struct ALIGNED(SCUDO_CACHE_LINE_SIZE) TSD {
diff --git a/lib/scudo/standalone/wrappers_c.cc b/lib/scudo/standalone/wrappers_c.cc
new file mode 100644
index 000000000..5908c600b
--- /dev/null
+++ b/lib/scudo/standalone/wrappers_c.cc
@@ -0,0 +1,39 @@
+//===-- wrappers_c.cc -------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "platform.h"
+
+// Skip this compilation unit if compiled as part of Bionic.
+#if !SCUDO_ANDROID || !_BIONIC
+
+#include "allocator_config.h"
+#include "wrappers_c.h"
+#include "wrappers_c_checks.h"
+
+#include <stdint.h>
+#include <stdio.h>
+
+static scudo::Allocator<scudo::Config> Allocator;
+// Pointer to the static allocator so that the C++ wrappers can access it.
+// Technically we could have a completely separated heap for C & C++ but in
+// reality the amount of cross pollination between the two is staggering.
+scudo::Allocator<scudo::Config> *AllocatorPtr = &Allocator;
+
+extern "C" {
+
+#define SCUDO_PREFIX(name) name
+#define SCUDO_ALLOCATOR Allocator
+#include "wrappers_c.inc"
+#undef SCUDO_ALLOCATOR
+#undef SCUDO_PREFIX
+
+INTERFACE void __scudo_print_stats(void) { Allocator.printStats(); }
+
+} // extern "C"
+
+#endif // !SCUDO_ANDROID || !_BIONIC
diff --git a/lib/scudo/standalone/wrappers_c.h b/lib/scudo/standalone/wrappers_c.h
new file mode 100644
index 000000000..33a0c53ce
--- /dev/null
+++ b/lib/scudo/standalone/wrappers_c.h
@@ -0,0 +1,52 @@
+//===-- wrappers_c.h --------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SCUDO_WRAPPERS_C_H_
+#define SCUDO_WRAPPERS_C_H_
+
+#include "platform.h"
+#include "stats.h"
+
+// Bionic's struct mallinfo consists of size_t (mallinfo(3) uses int).
+#if SCUDO_ANDROID
+typedef size_t __scudo_mallinfo_data_t;
+#else
+typedef int __scudo_mallinfo_data_t;
+#endif
+
+struct __scudo_mallinfo {
+  __scudo_mallinfo_data_t arena;
+  __scudo_mallinfo_data_t ordblks;
+  __scudo_mallinfo_data_t smblks;
+  __scudo_mallinfo_data_t hblks;
+  __scudo_mallinfo_data_t hblkhd;
+  __scudo_mallinfo_data_t usmblks;
+  __scudo_mallinfo_data_t fsmblks;
+  __scudo_mallinfo_data_t uordblks;
+  __scudo_mallinfo_data_t fordblks;
+  __scudo_mallinfo_data_t keepcost;
+};
+
+// Android sometimes includes malloc.h no matter what, which yields to
+// conflicting return types for mallinfo() if we use our own structure. So if
+// struct mallinfo is declared (#define courtesy of malloc.h), use it directly.
+#if STRUCT_MALLINFO_DECLARED
+#define SCUDO_MALLINFO mallinfo
+#else
+#define SCUDO_MALLINFO __scudo_mallinfo
+#endif
+
+#ifndef M_DECAY_TIME
+#define M_DECAY_TIME -100
+#endif
+
+#ifndef M_PURGE
+#define M_PURGE -101
+#endif
+
+#endif // SCUDO_WRAPPERS_C_H_
diff --git a/lib/scudo/standalone/wrappers_c.inc b/lib/scudo/standalone/wrappers_c.inc
new file mode 100644
index 000000000..2beddc724
--- /dev/null
+++ b/lib/scudo/standalone/wrappers_c.inc
@@ -0,0 +1,176 @@
+//===-- wrappers_c.inc ------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SCUDO_PREFIX
+#error "Define SCUDO_PREFIX prior to including this file!"
+#endif
+
+// malloc-type functions have to be aligned to std::max_align_t. This is
+// distinct from (1U << SCUDO_MIN_ALIGNMENT_LOG), since C++ new-type functions
+// do not have to abide by the same requirement.
+#ifndef SCUDO_MALLOC_ALIGNMENT
+#define SCUDO_MALLOC_ALIGNMENT FIRST_32_SECOND_64(8U, 16U)
+#endif
+
+INTERFACE WEAK void *SCUDO_PREFIX(calloc)(size_t nmemb, size_t size) {
+  scudo::uptr Product;
+  if (UNLIKELY(scudo::checkForCallocOverflow(size, nmemb, &Product))) {
+    if (SCUDO_ALLOCATOR.canReturnNull()) {
+      errno = ENOMEM;
+      return nullptr;
+    }
+    scudo::reportCallocOverflow(nmemb, size);
+  }
+  return scudo::setErrnoOnNull(SCUDO_ALLOCATOR.allocate(
+      Product, scudo::Chunk::Origin::Malloc, SCUDO_MALLOC_ALIGNMENT, true));
+}
+
+INTERFACE WEAK void SCUDO_PREFIX(free)(void *ptr) {
+  SCUDO_ALLOCATOR.deallocate(ptr, scudo::Chunk::Origin::Malloc);
+}
+
+INTERFACE WEAK struct SCUDO_MALLINFO SCUDO_PREFIX(mallinfo)(void) {
+  struct SCUDO_MALLINFO Info = {};
+  scudo::StatCounters Stats;
+  SCUDO_ALLOCATOR.getStats(Stats);
+  Info.uordblks =
+      static_cast<__scudo_mallinfo_data_t>(Stats[scudo::StatAllocated]);
+  return Info;
+}
+
+INTERFACE WEAK void *SCUDO_PREFIX(malloc)(size_t size) {
+  return scudo::setErrnoOnNull(SCUDO_ALLOCATOR.allocate(
+      size, scudo::Chunk::Origin::Malloc, SCUDO_MALLOC_ALIGNMENT));
+}
+
+#if SCUDO_ANDROID
+INTERFACE WEAK size_t SCUDO_PREFIX(malloc_usable_size)(const void *ptr) {
+#else
+INTERFACE WEAK size_t SCUDO_PREFIX(malloc_usable_size)(void *ptr) {
+#endif
+  return SCUDO_ALLOCATOR.getUsableSize(ptr);
+}
+
+INTERFACE WEAK void *SCUDO_PREFIX(memalign)(size_t alignment, size_t size) {
+  // Android rounds up the alignment to a power of two if it isn't one.
+  if (SCUDO_ANDROID) {
+    if (UNLIKELY(!alignment)) {
+      alignment = 1U;
+    } else {
+      if (UNLIKELY(!scudo::isPowerOfTwo(alignment)))
+        alignment = scudo::roundUpToPowerOfTwo(alignment);
+    }
+  } else {
+    if (UNLIKELY(!scudo::isPowerOfTwo(alignment))) {
+      if (SCUDO_ALLOCATOR.canReturnNull()) {
+        errno = EINVAL;
+        return nullptr;
+      }
+      scudo::reportAlignmentNotPowerOfTwo(alignment);
+    }
+  }
+  return SCUDO_ALLOCATOR.allocate(size, scudo::Chunk::Origin::Memalign,
+                                  alignment);
+}
+
+INTERFACE WEAK int SCUDO_PREFIX(posix_memalign)(void **memptr, size_t alignment,
+                                                size_t size) {
+  if (UNLIKELY(scudo::checkPosixMemalignAlignment(alignment))) {
+    if (!SCUDO_ALLOCATOR.canReturnNull())
+      scudo::reportInvalidPosixMemalignAlignment(alignment);
+    return EINVAL;
+  }
+  void *Ptr =
+      SCUDO_ALLOCATOR.allocate(size, scudo::Chunk::Origin::Memalign, alignment);
+  if (UNLIKELY(!Ptr))
+    return ENOMEM;
+  *memptr = Ptr;
+  return 0;
+}
+
+INTERFACE WEAK void *SCUDO_PREFIX(pvalloc)(size_t size) {
+  const scudo::uptr PageSize = scudo::getPageSizeCached();
+  if (UNLIKELY(scudo::checkForPvallocOverflow(size, PageSize))) {
+    if (SCUDO_ALLOCATOR.canReturnNull()) {
+      errno = ENOMEM;
+      return nullptr;
+    }
+    scudo::reportPvallocOverflow(size);
+  }
+  // pvalloc(0) should allocate one page.
+  return scudo::setErrnoOnNull(SCUDO_ALLOCATOR.allocate(
+      size ? scudo::roundUpTo(size, PageSize) : PageSize,
+      scudo::Chunk::Origin::Memalign, PageSize));
+}
+
+INTERFACE WEAK void *SCUDO_PREFIX(realloc)(void *ptr, size_t size) {
+  if (!ptr)
+    return scudo::setErrnoOnNull(SCUDO_ALLOCATOR.allocate(
+        size, scudo::Chunk::Origin::Malloc, SCUDO_MALLOC_ALIGNMENT));
+  if (size == 0) {
+    SCUDO_ALLOCATOR.deallocate(ptr, scudo::Chunk::Origin::Malloc);
+    return nullptr;
+  }
+  return scudo::setErrnoOnNull(
+      SCUDO_ALLOCATOR.reallocate(ptr, size, SCUDO_MALLOC_ALIGNMENT));
+}
+
+INTERFACE WEAK void *SCUDO_PREFIX(valloc)(size_t size) {
+  return scudo::setErrnoOnNull(SCUDO_ALLOCATOR.allocate(
+      size, scudo::Chunk::Origin::Memalign, scudo::getPageSizeCached()));
+}
+
+// Bionic wants a function named PREFIX_iterate and not PREFIX_malloc_iterate
+// which is somewhat inconsistent with the rest, workaround that.
+#if SCUDO_ANDROID && _BIONIC
+#define SCUDO_ITERATE iterate
+#else
+#define SCUDO_ITERATE malloc_iterate
+#endif
+
+INTERFACE WEAK int SCUDO_PREFIX(SCUDO_ITERATE)(
+    uintptr_t base, size_t size,
+    void (*callback)(uintptr_t base, size_t size, void *arg), void *arg) {
+  SCUDO_ALLOCATOR.iterateOverChunks(base, size, callback, arg);
+  return 0;
+}
+
+INTERFACE WEAK void SCUDO_PREFIX(malloc_disable)() {
+  SCUDO_ALLOCATOR.disable();
+}
+
+INTERFACE WEAK void SCUDO_PREFIX(malloc_enable)() { SCUDO_ALLOCATOR.enable(); }
+
+INTERFACE WEAK int SCUDO_PREFIX(mallopt)(int param, UNUSED int value) {
+  if (param == M_DECAY_TIME) {
+    // TODO(kostyak): set release_to_os_interval_ms accordingly.
+    return 1;
+  } else if (param == M_PURGE) {
+    SCUDO_ALLOCATOR.releaseToOS();
+    return 1;
+  }
+  return 0;
+}
+
+INTERFACE WEAK void *SCUDO_PREFIX(aligned_alloc)(size_t alignment,
+                                                 size_t size) {
+  if (UNLIKELY(scudo::checkAlignedAllocAlignmentAndSize(alignment, size))) {
+    if (SCUDO_ALLOCATOR.canReturnNull()) {
+      errno = EINVAL;
+      return nullptr;
+    }
+    scudo::reportInvalidAlignedAllocAlignment(alignment, size);
+  }
+  return scudo::setErrnoOnNull(
+      SCUDO_ALLOCATOR.allocate(size, scudo::Chunk::Origin::Malloc, alignment));
+}
+
+INTERFACE WEAK int SCUDO_PREFIX(malloc_info)(int, FILE *) {
+  errno = ENOTSUP;
+  return -1;
+}
diff --git a/lib/scudo/standalone/wrappers_c_bionic.cc b/lib/scudo/standalone/wrappers_c_bionic.cc
new file mode 100644
index 000000000..f6e863deb
--- /dev/null
+++ b/lib/scudo/standalone/wrappers_c_bionic.cc
@@ -0,0 +1,49 @@
+//===-- wrappers_c_bionic.cc ------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "platform.h"
+
+// This is only used when compiled as part of Bionic.
+#if SCUDO_ANDROID && _BIONIC
+
+#include "allocator_config.h"
+#include "wrappers_c.h"
+#include "wrappers_c_checks.h"
+
+#include <stdint.h>
+#include <stdio.h>
+
+static scudo::Allocator<scudo::AndroidConfig> Allocator;
+static scudo::Allocator<scudo::AndroidSvelteConfig> SvelteAllocator;
+
+extern "C" {
+
+// Regular MallocDispatch definitions.
+#define SCUDO_PREFIX(name) CONCATENATE(scudo_, name)
+#define SCUDO_ALLOCATOR Allocator
+#include "wrappers_c.inc"
+#undef SCUDO_ALLOCATOR
+#undef SCUDO_PREFIX
+
+// Svelte MallocDispatch definitions.
+#define SCUDO_PREFIX(name) CONCATENATE(scudo_svelte_, name)
+#define SCUDO_ALLOCATOR SvelteAllocator
+#include "wrappers_c.inc"
+#undef SCUDO_ALLOCATOR
+#undef SCUDO_PREFIX
+
+// The following is the only function that will end up initializing both
+// allocators, which will result in a slight increase in memory footprint.
+INTERFACE void __scudo_print_stats(void) {
+  Allocator.printStats();
+  SvelteAllocator.printStats();
+}
+
+} // extern "C"
+
+#endif // SCUDO_ANDROID && _BIONIC
diff --git a/lib/scudo/standalone/wrappers_c_checks.h b/lib/scudo/standalone/wrappers_c_checks.h
new file mode 100644
index 000000000..d9de5b261
--- /dev/null
+++ b/lib/scudo/standalone/wrappers_c_checks.h
@@ -0,0 +1,56 @@
+//===-- wrappers_c_checks.h -------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SCUDO_CHECKS_H_
+#define SCUDO_CHECKS_H_
+
+#include "common.h"
+
+#include <errno.h>
+
+namespace scudo {
+
+// A common errno setting logic shared by almost all Scudo C wrappers.
+INLINE void *setErrnoOnNull(void *Ptr) {
+  if (UNLIKELY(!Ptr))
+    errno = ENOMEM;
+  return Ptr;
+}
+
+// Checks return true on failure.
+
+// Checks aligned_alloc() parameters, verifies that the alignment is a power of
+// two and that the size is a multiple of alignment.
+INLINE bool checkAlignedAllocAlignmentAndSize(uptr Alignment, uptr Size) {
+  return Alignment == 0 || !isPowerOfTwo(Alignment) ||
+         !isAligned(Size, Alignment);
+}
+
+// Checks posix_memalign() parameters, verifies that alignment is a power of two
+// and a multiple of sizeof(void *).
+INLINE bool checkPosixMemalignAlignment(uptr Alignment) {
+  return Alignment == 0 || !isPowerOfTwo(Alignment) ||
+         !isAligned(Alignment, sizeof(void *));
+}
+
+// Returns true if calloc(Size, N) overflows on Size*N calculation. The typical
+// way would be to check for (UINTPTR_MAX / Size) < N, but the division ends up
+// being very costly, so use a builtin supported by recent clang & GCC.
+INLINE bool checkForCallocOverflow(uptr Size, uptr N, uptr *Product) {
+  return __builtin_umull_overflow(Size, N, Product);
+}
+
+// Returns true if the size passed to pvalloc overflows when rounded to the next
+// multiple of PageSize.
+INLINE bool checkForPvallocOverflow(uptr Size, uptr PageSize) {
+  return roundUpTo(Size, PageSize) < Size;
+}
+
+} // namespace scudo
+
+#endif // SCUDO_CHECKS_H_
diff --git a/lib/scudo/standalone/wrappers_cpp.cc b/lib/scudo/standalone/wrappers_cpp.cc
new file mode 100644
index 000000000..3ae1cdc05
--- /dev/null
+++ b/lib/scudo/standalone/wrappers_cpp.cc
@@ -0,0 +1,107 @@
+//===-- wrappers_cpp.cc -----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "platform.h"
+
+// Skip this compilation unit if compiled as part of Bionic.
+#if !SCUDO_ANDROID || !_BIONIC
+
+#include "allocator_config.h"
+
+#include <stdint.h>
+
+extern scudo::Allocator<scudo::Config> *AllocatorPtr;
+
+namespace std {
+struct nothrow_t {};
+enum class align_val_t : size_t {};
+} // namespace std
+
+INTERFACE WEAK void *operator new(size_t size) {
+  return AllocatorPtr->allocate(size, scudo::Chunk::Origin::New);
+}
+INTERFACE WEAK void *operator new[](size_t size) {
+  return AllocatorPtr->allocate(size, scudo::Chunk::Origin::NewArray);
+}
+INTERFACE WEAK void *operator new(size_t size,
+                                  std::nothrow_t const &) NOEXCEPT {
+  return AllocatorPtr->allocate(size, scudo::Chunk::Origin::New);
+}
+INTERFACE WEAK void *operator new[](size_t size,
+                                    std::nothrow_t const &) NOEXCEPT {
+  return AllocatorPtr->allocate(size, scudo::Chunk::Origin::NewArray);
+}
+INTERFACE WEAK void *operator new(size_t size, std::align_val_t align) {
+  return AllocatorPtr->allocate(size, scudo::Chunk::Origin::New,
+                                static_cast<scudo::uptr>(align));
+}
+INTERFACE WEAK void *operator new[](size_t size, std::align_val_t align) {
+  return AllocatorPtr->allocate(size, scudo::Chunk::Origin::NewArray,
+                                static_cast<scudo::uptr>(align));
+}
+INTERFACE WEAK void *operator new(size_t size, std::align_val_t align,
+                                  std::nothrow_t const &) NOEXCEPT {
+  return AllocatorPtr->allocate(size, scudo::Chunk::Origin::New,
+                                static_cast<scudo::uptr>(align));
+}
+INTERFACE WEAK void *operator new[](size_t size, std::align_val_t align,
+                                    std::nothrow_t const &) NOEXCEPT {
+  return AllocatorPtr->allocate(size, scudo::Chunk::Origin::NewArray,
+                                static_cast<scudo::uptr>(align));
+}
+
+INTERFACE WEAK void operator delete(void *ptr)NOEXCEPT {
+  AllocatorPtr->deallocate(ptr, scudo::Chunk::Origin::New);
+}
+INTERFACE WEAK void operator delete[](void *ptr) NOEXCEPT {
+  AllocatorPtr->deallocate(ptr, scudo::Chunk::Origin::NewArray);
+}
+INTERFACE WEAK void operator delete(void *ptr, std::nothrow_t const &)NOEXCEPT {
+  AllocatorPtr->deallocate(ptr, scudo::Chunk::Origin::New);
+}
+INTERFACE WEAK void operator delete[](void *ptr,
+                                      std::nothrow_t const &) NOEXCEPT {
+  AllocatorPtr->deallocate(ptr, scudo::Chunk::Origin::NewArray);
+}
+INTERFACE WEAK void operator delete(void *ptr, size_t size)NOEXCEPT {
+  AllocatorPtr->deallocate(ptr, scudo::Chunk::Origin::New, size);
+}
+INTERFACE WEAK void operator delete[](void *ptr, size_t size) NOEXCEPT {
+  AllocatorPtr->deallocate(ptr, scudo::Chunk::Origin::NewArray, size);
+}
+INTERFACE WEAK void operator delete(void *ptr, std::align_val_t align)NOEXCEPT {
+  AllocatorPtr->deallocate(ptr, scudo::Chunk::Origin::New, 0,
+                           static_cast<scudo::uptr>(align));
+}
+INTERFACE WEAK void operator delete[](void *ptr,
+                                      std::align_val_t align) NOEXCEPT {
+  AllocatorPtr->deallocate(ptr, scudo::Chunk::Origin::NewArray, 0,
+                           static_cast<scudo::uptr>(align));
+}
+INTERFACE WEAK void operator delete(void *ptr, std::align_val_t align,
+                                    std::nothrow_t const &)NOEXCEPT {
+  AllocatorPtr->deallocate(ptr, scudo::Chunk::Origin::New, 0,
+                           static_cast<scudo::uptr>(align));
+}
+INTERFACE WEAK void operator delete[](void *ptr, std::align_val_t align,
+                                      std::nothrow_t const &) NOEXCEPT {
+  AllocatorPtr->deallocate(ptr, scudo::Chunk::Origin::NewArray, 0,
+                           static_cast<scudo::uptr>(align));
+}
+INTERFACE WEAK void operator delete(void *ptr, size_t size,
+                                    std::align_val_t align)NOEXCEPT {
+  AllocatorPtr->deallocate(ptr, scudo::Chunk::Origin::New, size,
+                           static_cast<scudo::uptr>(align));
+}
+INTERFACE WEAK void operator delete[](void *ptr, size_t size,
+                                      std::align_val_t align) NOEXCEPT {
+  AllocatorPtr->deallocate(ptr, scudo::Chunk::Origin::NewArray, size,
+                           static_cast<scudo::uptr>(align));
+}
+
+#endif // !SCUDO_ANDROID || !_BIONIC