diff options
Diffstat (limited to 'lib/hwasan')
28 files changed, 648 insertions, 243 deletions
diff --git a/lib/hwasan/CMakeLists.txt b/lib/hwasan/CMakeLists.txt index 20ab94dc0..8fa59199e 100644 --- a/lib/hwasan/CMakeLists.txt +++ b/lib/hwasan/CMakeLists.txt @@ -2,20 +2,23 @@ include_directories(..) # Runtime library sources and build flags. set(HWASAN_RTL_SOURCES - hwasan.cc - hwasan_allocator.cc - hwasan_dynamic_shadow.cc - hwasan_interceptors.cc - hwasan_linux.cc - hwasan_memintrinsics.cc - hwasan_poisoning.cc - hwasan_report.cc - hwasan_thread.cc - hwasan_thread_list.cc + hwasan.cpp + hwasan_allocator.cpp + hwasan_dynamic_shadow.cpp + hwasan_interceptors.cpp + hwasan_interceptors_vfork.S + hwasan_linux.cpp + hwasan_memintrinsics.cpp + hwasan_poisoning.cpp + hwasan_report.cpp + hwasan_tag_mismatch_aarch64.S + hwasan_thread.cpp + hwasan_thread_list.cpp ) set(HWASAN_RTL_CXX_SOURCES - hwasan_new_delete.cc) + hwasan_new_delete.cpp + ) set(HWASAN_RTL_HEADERS hwasan.h @@ -24,6 +27,7 @@ set(HWASAN_RTL_HEADERS hwasan_flags.h hwasan_flags.inc hwasan_interface_internal.h + hwasan_malloc_bisect.h hwasan_mapping.h hwasan_poisoning.h hwasan_report.h @@ -55,7 +59,7 @@ append_list_if(COMPILER_RT_HAS_FTLS_MODEL_INITIAL_EXEC -ftls-model=initial-exec HWASAN_DYNAMIC_CFLAGS) append_list_if(MSVC /DEBUG HWASAN_DYNAMIC_LINK_FLAGS) -set(HWASAN_DYNAMIC_LIBS ${SANITIZER_CXX_ABI_LIBRARY} ${SANITIZER_COMMON_LINK_LIBS}) +set(HWASAN_DYNAMIC_LIBS ${SANITIZER_CXX_ABI_LIBRARIES} ${SANITIZER_COMMON_LINK_LIBS}) append_list_if(COMPILER_RT_HAS_LIBDL dl HWASAN_DYNAMIC_LIBS) append_list_if(COMPILER_RT_HAS_LIBRT rt HWASAN_DYNAMIC_LIBS) diff --git a/lib/hwasan/hwasan.cc b/lib/hwasan/hwasan.cpp index e2bfea5e4..65b755ee2 100644 --- a/lib/hwasan/hwasan.cc +++ b/lib/hwasan/hwasan.cpp @@ -1,9 +1,8 @@ -//===-- hwasan.cc ---------------------------------------------------------===// +//===-- hwasan.cpp --------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -14,6 +13,7 @@ #include "hwasan.h" #include "hwasan_checks.h" +#include "hwasan_dynamic_shadow.h" #include "hwasan_poisoning.h" #include "hwasan_report.h" #include "hwasan_thread.h" @@ -58,7 +58,7 @@ Flags *flags() { } int hwasan_inited = 0; -int hwasan_shadow_inited = 0; +int hwasan_instrumentation_inited = 0; bool hwasan_init_is_running; int hwasan_report_count = 0; @@ -88,6 +88,8 @@ static void InitializeFlags() { cf.check_printf = false; cf.intercept_tls_get_addr = true; cf.exitcode = 99; + // 8 shadow pages ~512kB, small enough to cover common stack sizes. + cf.clear_shadow_mmap_threshold = 4096 * (SANITIZER_ANDROID ? 2 : 8); // Sigtrap is used in error reporting. cf.handle_sigtrap = kHandleSignalExclusive; @@ -142,23 +144,6 @@ static void InitializeFlags() { if (common_flags()->help) parser.PrintFlagDescriptions(); } -void GetStackTrace(BufferedStackTrace *stack, uptr max_s, uptr pc, uptr bp, - void *context, bool request_fast_unwind) { - Thread *t = GetCurrentThread(); - if (!t) { - // the thread is still being created. - stack->size = 0; - return; - } - if (!StackTrace::WillUseFastUnwind(request_fast_unwind)) { - // Block reports from our interceptors during _Unwind_Backtrace. - SymbolizerScope sym_scope; - return stack->Unwind(max_s, pc, bp, context, 0, 0, request_fast_unwind); - } - stack->Unwind(max_s, pc, bp, context, t->stack_top(), t->stack_bottom(), - request_fast_unwind); -} - static void HWAsanCheckFailed(const char *file, int line, const char *cond, u64 v1, u64 v2) { Report("HWAddressSanitizer CHECK failed: %s:%d \"%s\" (0x%zx, 0x%zx)\n", file, @@ -188,17 +173,13 @@ static void HwasanFormatMemoryUsage(InternalScopedString &s) { #if SANITIZER_ANDROID static char *memory_usage_buffer = nullptr; -#define PR_SET_VMA 0x53564d41 -#define PR_SET_VMA_ANON_NAME 0 - static void InitMemoryUsage() { memory_usage_buffer = (char *)MmapOrDie(kMemoryUsageBufferSize, "memory usage string"); CHECK(memory_usage_buffer); memory_usage_buffer[0] = '\0'; - CHECK(internal_prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, - (uptr)memory_usage_buffer, kMemoryUsageBufferSize, - (uptr)memory_usage_buffer) == 0); + DecorateMapping((uptr)memory_usage_buffer, kMemoryUsageBufferSize, + memory_usage_buffer); } void UpdateMemoryUsage() { @@ -245,28 +226,59 @@ const char *GetStackFrameDescr(uptr pc) { return nullptr; } -} // namespace __hwasan - -// Interface. +// Prepare to run instrumented code on the main thread. +void InitInstrumentation() { + if (hwasan_instrumentation_inited) return; -using namespace __hwasan; - -uptr __hwasan_shadow_memory_dynamic_address; // Global interface symbol. - -void __hwasan_shadow_init() { - if (hwasan_shadow_inited) return; if (!InitShadow()) { Printf("FATAL: HWAddressSanitizer cannot mmap the shadow memory.\n"); DumpProcessMap(); Die(); } - hwasan_shadow_inited = 1; + + InitThreads(); + hwasanThreadList().CreateCurrentThread(); + + hwasan_instrumentation_inited = 1; } +} // namespace __hwasan + +void __sanitizer::BufferedStackTrace::UnwindImpl( + uptr pc, uptr bp, void *context, bool request_fast, u32 max_depth) { + using namespace __hwasan; + Thread *t = GetCurrentThread(); + if (!t) { + // the thread is still being created. + size = 0; + return; + } + if (!StackTrace::WillUseFastUnwind(request_fast)) { + // Block reports from our interceptors during _Unwind_Backtrace. + SymbolizerScope sym_scope; + return Unwind(max_depth, pc, bp, context, 0, 0, request_fast); + } + if (StackTrace::WillUseFastUnwind(request_fast)) + Unwind(max_depth, pc, bp, nullptr, t->stack_top(), t->stack_bottom(), true); + else + Unwind(max_depth, pc, 0, context, 0, 0, false); +} + +// Interface. + +using namespace __hwasan; + +uptr __hwasan_shadow_memory_dynamic_address; // Global interface symbol. + void __hwasan_init_frames(uptr beg, uptr end) { InitFrameDescriptors(beg, end); } +void __hwasan_init_static() { + InitShadowGOT(); + InitInstrumentation(); +} + void __hwasan_init() { CHECK(!hwasan_init_is_running); if (hwasan_inited) return; @@ -287,10 +299,11 @@ void __hwasan_init() { DisableCoreDumperIfNecessary(); - __hwasan_shadow_init(); + InitInstrumentation(); - InitThreads(); - hwasanThreadList().CreateCurrentThread(); + // Needs to be called here because flags()->random_tags might not have been + // initialized when InitInstrumentation() was called. + GetCurrentThread()->InitRandomState(); MadviseShadow(); @@ -335,14 +348,14 @@ sptr __hwasan_test_shadow(const void *p, uptr sz) { if (sz == 0) return -1; tag_t ptr_tag = GetTagFromPointer((uptr)p); - if (ptr_tag == 0) - return -1; uptr ptr_raw = UntagAddr(reinterpret_cast<uptr>(p)); uptr shadow_first = MemToShadow(ptr_raw); uptr shadow_last = MemToShadow(ptr_raw + sz - 1); for (uptr s = shadow_first; s <= shadow_last; ++s) - if (*(tag_t*)s != ptr_tag) - return ShadowToMem(s) - ptr_raw; + if (*(tag_t *)s != ptr_tag) { + sptr offset = ShadowToMem(s) - ptr_raw; + return offset < 0 ? 0 : offset; + } return -1; } @@ -467,6 +480,28 @@ void __hwasan_handle_longjmp(const void *sp_dst) { TagMemory(sp, dst - sp, 0); } +void __hwasan_handle_vfork(const void *sp_dst) { + uptr sp = (uptr)sp_dst; + Thread *t = GetCurrentThread(); + CHECK(t); + uptr top = t->stack_top(); + uptr bottom = t->stack_bottom(); + if (top == 0 || bottom == 0 || sp < bottom || sp >= top) { + Report( + "WARNING: HWASan is ignoring requested __hwasan_handle_vfork: " + "stack top: %zx; current %zx; bottom: %zx \n" + "False positive error reports may follow\n", + top, sp, bottom); + return; + } + TagMemory(bottom, sp - bottom, 0); +} + +extern "C" void *__hwasan_extra_spill_area() { + Thread *t = GetCurrentThread(); + return &t->vfork_spill(); +} + void __hwasan_print_memory_usage() { InternalScopedString s(kMemoryUsageBufferSize); HwasanFormatMemoryUsage(s); diff --git a/lib/hwasan/hwasan.h b/lib/hwasan/hwasan.h index ce9e904c5..9cc9490a9 100644 --- a/lib/hwasan/hwasan.h +++ b/lib/hwasan/hwasan.h @@ -1,9 +1,8 @@ //===-- hwasan.h ------------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -71,6 +70,7 @@ extern int hwasan_report_count; bool ProtectRange(uptr beg, uptr end); bool InitShadow(); void InitThreads(); +void InitInstrumentation(); void MadviseShadow(); char *GetProcSelfMaps(); void InitializeInterceptors(); @@ -81,6 +81,7 @@ void HwasanAllocatorThreadFinish(); void *hwasan_malloc(uptr size, StackTrace *stack); void *hwasan_calloc(uptr nmemb, uptr size, StackTrace *stack); void *hwasan_realloc(void *ptr, uptr size, StackTrace *stack); +void *hwasan_reallocarray(void *ptr, uptr nmemb, uptr size, StackTrace *stack); void *hwasan_valloc(uptr size, StackTrace *stack); void *hwasan_pvalloc(uptr size, StackTrace *stack); void *hwasan_aligned_alloc(uptr alignment, uptr size, StackTrace *stack); @@ -104,9 +105,6 @@ struct SymbolizerScope { ~SymbolizerScope() { ExitSymbolizer(); } }; -void GetStackTrace(BufferedStackTrace *stack, uptr max_s, uptr pc, uptr bp, - void *context, bool request_fast_unwind); - // Returns a "chained" origin id, pointing to the given stack trace followed by // the previous origin id. u32 ChainOrigin(u32 id, StackTrace *stack); @@ -115,16 +113,15 @@ const int STACK_TRACE_TAG_POISON = StackTrace::TAG_CUSTOM + 1; #define GET_MALLOC_STACK_TRACE \ BufferedStackTrace stack; \ - if (hwasan_inited) \ - GetStackTrace(&stack, common_flags()->malloc_context_size, \ - StackTrace::GetCurrentPc(), GET_CURRENT_FRAME(), nullptr, \ - common_flags()->fast_unwind_on_malloc) + if (hwasan_inited) \ + stack.Unwind(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME(), \ + nullptr, common_flags()->fast_unwind_on_malloc, \ + common_flags()->malloc_context_size) #define GET_FATAL_STACK_TRACE_PC_BP(pc, bp) \ BufferedStackTrace stack; \ - if (hwasan_inited) \ - GetStackTrace(&stack, kStackTraceMax, pc, bp, nullptr, \ - common_flags()->fast_unwind_on_fatal) + if (hwasan_inited) \ + stack.Unwind(pc, bp, nullptr, common_flags()->fast_unwind_on_fatal) #define GET_FATAL_STACK_TRACE_HERE \ GET_FATAL_STACK_TRACE_PC_BP(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME()) diff --git a/lib/hwasan/hwasan_allocator.cc b/lib/hwasan/hwasan_allocator.cpp index 8487ed7e1..fd5248796 100644 --- a/lib/hwasan/hwasan_allocator.cc +++ b/lib/hwasan/hwasan_allocator.cpp @@ -1,9 +1,8 @@ -//===-- hwasan_allocator.cc ------------------------- ---------------------===// +//===-- hwasan_allocator.cpp ------------------------ ---------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -18,6 +17,7 @@ #include "hwasan.h" #include "hwasan_allocator.h" #include "hwasan_mapping.h" +#include "hwasan_malloc_bisect.h" #include "hwasan_thread.h" #include "hwasan_report.h" @@ -177,10 +177,16 @@ static void *HwasanAllocate(StackTrace *stack, uptr orig_size, uptr alignment, size - orig_size); void *user_ptr = allocated; - if (flags()->tag_in_malloc && - atomic_load_relaxed(&hwasan_allocator_tagging_enabled)) - user_ptr = (void *)TagMemoryAligned( - (uptr)user_ptr, size, t ? t->GenerateRandomTag() : kFallbackAllocTag); + // Tagging can only be skipped when both tag_in_malloc and tag_in_free are + // false. When tag_in_malloc = false and tag_in_free = true malloc needs to + // retag to 0. + if ((flags()->tag_in_malloc || flags()->tag_in_free) && + atomic_load_relaxed(&hwasan_allocator_tagging_enabled)) { + tag_t tag = flags()->tag_in_malloc && malloc_bisect(stack, orig_size) + ? (t ? t->GenerateRandomTag() : kFallbackAllocTag) + : 0; + user_ptr = (void *)TagMemoryAligned((uptr)user_ptr, size, tag); + } if ((orig_size % kShadowAlignment) && (alignment <= kShadowAlignment) && right_align_mode) { @@ -242,7 +248,7 @@ static void HwasanDeallocate(StackTrace *stack, void *tagged_ptr) { Min(TaggedSize(orig_size), (uptr)flags()->max_free_fill_size); internal_memset(aligned_ptr, flags()->free_fill_byte, fill_size); } - if (flags()->tag_in_free && + if (flags()->tag_in_free && malloc_bisect(stack, 0) && atomic_load_relaxed(&hwasan_allocator_tagging_enabled)) TagMemoryAligned(reinterpret_cast<uptr>(aligned_ptr), TaggedSize(orig_size), t ? t->GenerateRandomTag() : kFallbackFreeTag); @@ -335,6 +341,16 @@ void *hwasan_realloc(void *ptr, uptr size, StackTrace *stack) { return SetErrnoOnNull(HwasanReallocate(stack, ptr, size, sizeof(u64))); } +void *hwasan_reallocarray(void *ptr, uptr nmemb, uptr size, StackTrace *stack) { + if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) { + errno = errno_ENOMEM; + if (AllocatorMayReturnNull()) + return nullptr; + ReportReallocArrayOverflow(nmemb, size, stack); + } + return hwasan_realloc(ptr, nmemb * size, stack); +} + void *hwasan_valloc(uptr size, StackTrace *stack) { return SetErrnoOnNull( HwasanAllocate(stack, size, GetPageSizeCached(), false)); diff --git a/lib/hwasan/hwasan_allocator.h b/lib/hwasan/hwasan_allocator.h index 6ab722fa6..3a50a11f3 100644 --- a/lib/hwasan/hwasan_allocator.h +++ b/lib/hwasan/hwasan_allocator.h @@ -1,9 +1,8 @@ //===-- hwasan_allocator.h --------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -62,10 +61,8 @@ struct AP64 { static const uptr kFlags = 0; }; typedef SizeClassAllocator64<AP64> PrimaryAllocator; -typedef SizeClassAllocatorLocalCache<PrimaryAllocator> AllocatorCache; -typedef LargeMmapAllocator<HwasanMapUnmapCallback> SecondaryAllocator; -typedef CombinedAllocator<PrimaryAllocator, AllocatorCache, - SecondaryAllocator> Allocator; +typedef CombinedAllocator<PrimaryAllocator> Allocator; +typedef Allocator::AllocatorCache AllocatorCache; void AllocatorSwallowThreadLocalCache(AllocatorCache *cache); diff --git a/lib/hwasan/hwasan_checks.h b/lib/hwasan/hwasan_checks.h index 688b5e2be..693faa0c2 100644 --- a/lib/hwasan/hwasan_checks.h +++ b/lib/hwasan/hwasan_checks.h @@ -1,9 +1,8 @@ //===-- hwasan_checks.h -----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -15,6 +14,7 @@ #define HWASAN_CHECKS_H #include "hwasan_mapping.h" +#include "sanitizer_common/sanitizer_common.h" namespace __hwasan { template <unsigned X> @@ -23,8 +23,8 @@ __attribute__((always_inline)) static void SigTrap(uptr p) { (void)p; // 0x900 is added to do not interfere with the kernel use of lower values of // brk immediate. - // FIXME: Add a constraint to put the pointer into x0, the same as x86 branch. - asm("brk %0\n\t" ::"n"(0x900 + X)); + register uptr x0 asm("x0") = p; + asm("brk %1\n\t" ::"r"(x0), "n"(0x900 + X)); #elif defined(__x86_64__) // INT3 + NOP DWORD ptr [EAX + X] to pass X to our signal handler, 5 bytes // total. The pointer is passed via rdi. @@ -42,6 +42,25 @@ __attribute__((always_inline)) static void SigTrap(uptr p) { // __builtin_unreachable(); } +// Version with access size which is not power of 2 +template <unsigned X> +__attribute__((always_inline)) static void SigTrap(uptr p, uptr size) { +#if defined(__aarch64__) + register uptr x0 asm("x0") = p; + register uptr x1 asm("x1") = size; + asm("brk %2\n\t" ::"r"(x0), "r"(x1), "n"(0x900 + X)); +#elif defined(__x86_64__) + // Size is stored in rsi. + asm volatile( + "int3\n" + "nopl %c0(%%rax)\n" ::"n"(0x40 + X), + "D"(p), "S"(size)); +#else + __builtin_trap(); +#endif + // __builtin_unreachable(); +} + enum class ErrorAction { Abort, Recover }; enum class AccessType { Load, Store }; @@ -70,7 +89,7 @@ __attribute__((always_inline, nodebug)) static void CheckAddressSized(uptr p, for (tag_t *t = shadow_first; t <= shadow_last; ++t) if (UNLIKELY(ptr_tag != *t)) { SigTrap<0x20 * (EA == ErrorAction::Recover) + - 0x10 * (AT == AccessType::Store) + 0xf>(p); + 0x10 * (AT == AccessType::Store) + 0xf>(p, sz); if (EA == ErrorAction::Abort) __builtin_unreachable(); } diff --git a/lib/hwasan/hwasan_dynamic_shadow.cc b/lib/hwasan/hwasan_dynamic_shadow.cpp index 87670f508..a04751f44 100644 --- a/lib/hwasan/hwasan_dynamic_shadow.cc +++ b/lib/hwasan/hwasan_dynamic_shadow.cpp @@ -1,9 +1,8 @@ -//===-- hwasan_dynamic_shadow.cc --------------------------------*- C++ -*-===// +//===-- hwasan_dynamic_shadow.cpp -------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// @@ -19,6 +18,9 @@ #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_posix.h" +#include <elf.h> +#include <link.h> + // The code in this file needs to run in an unrelocated binary. It should not // access any external symbol, including its own non-hidden globals. @@ -119,10 +121,28 @@ decltype(__hwasan_shadow)* __hwasan_premap_shadow() { INTERFACE_ATTRIBUTE __attribute__((ifunc("__hwasan_premap_shadow"))) void __hwasan_shadow(); +extern __attribute((weak, visibility("hidden"))) ElfW(Rela) __rela_iplt_start[], + __rela_iplt_end[]; + } // extern "C" namespace __hwasan { +void InitShadowGOT() { + // Call the ifunc resolver for __hwasan_shadow and fill in its GOT entry. This + // needs to be done before other ifunc resolvers (which are handled by libc) + // because a resolver might read __hwasan_shadow. + typedef ElfW(Addr) (*ifunc_resolver_t)(void); + for (ElfW(Rela) *r = __rela_iplt_start; r != __rela_iplt_end; ++r) { + ElfW(Addr)* offset = reinterpret_cast<ElfW(Addr)*>(r->r_offset); + ElfW(Addr) resolver = r->r_addend; + if (resolver == reinterpret_cast<ElfW(Addr)>(&__hwasan_premap_shadow)) { + *offset = reinterpret_cast<ifunc_resolver_t>(resolver)(); + break; + } + } +} + uptr FindDynamicShadowStart(uptr shadow_size_bytes) { if (IsPremapShadowAvailable()) return FindPremappedShadowStart(shadow_size_bytes); @@ -133,10 +153,12 @@ uptr FindDynamicShadowStart(uptr shadow_size_bytes) { #else namespace __hwasan { +void InitShadowGOT() {} + uptr FindDynamicShadowStart(uptr shadow_size_bytes) { return MapDynamicShadow(shadow_size_bytes); } } // namespace __hwasan -# + #endif // SANITIZER_ANDROID diff --git a/lib/hwasan/hwasan_dynamic_shadow.h b/lib/hwasan/hwasan_dynamic_shadow.h index b5e9e1dd6..3c2e7c716 100644 --- a/lib/hwasan/hwasan_dynamic_shadow.h +++ b/lib/hwasan/hwasan_dynamic_shadow.h @@ -1,9 +1,8 @@ //===-- hwasan_dynamic_shadow.h ---------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// @@ -21,6 +20,7 @@ namespace __hwasan { uptr FindDynamicShadowStart(uptr shadow_size_bytes); +void InitShadowGOT(); } // namespace __hwasan diff --git a/lib/hwasan/hwasan_flags.h b/lib/hwasan/hwasan_flags.h index 492d5bb98..0a6998f67 100644 --- a/lib/hwasan/hwasan_flags.h +++ b/lib/hwasan/hwasan_flags.h @@ -1,9 +1,8 @@ //===-- hwasan_flags.h ------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/hwasan/hwasan_flags.inc b/lib/hwasan/hwasan_flags.inc index b450ab950..01fdad87a 100644 --- a/lib/hwasan/hwasan_flags.inc +++ b/lib/hwasan/hwasan_flags.inc @@ -1,9 +1,8 @@ //===-- hwasan_flags.inc ------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -86,3 +85,16 @@ HWASAN_FLAG(int, stack_history_size, 1024, "The number of stack frames remembered per thread. " "Affects the quality of stack-related reports, but not the ability " "to find bugs.") + +// Malloc / free bisection. Only tag malloc and free calls when a hash of +// allocation size and stack trace is between malloc_bisect_left and +// malloc_bisect_right (both inclusive). [0, 0] range is special and disables +// bisection (i.e. everything is tagged). Once the range is narrowed down +// enough, use malloc_bisect_dump to see interesting allocations. +HWASAN_FLAG(uptr, malloc_bisect_left, 0, + "Left bound of malloc bisection, inclusive.") +HWASAN_FLAG(uptr, malloc_bisect_right, 0, + "Right bound of malloc bisection, inclusive.") +HWASAN_FLAG(bool, malloc_bisect_dump, false, + "Print all allocations within [malloc_bisect_left, " + "malloc_bisect_right] range ") diff --git a/lib/hwasan/hwasan_interceptors.cc b/lib/hwasan/hwasan_interceptors.cpp index fb0dcb890..e305f6b1b 100644 --- a/lib/hwasan/hwasan_interceptors.cc +++ b/lib/hwasan/hwasan_interceptors.cpp @@ -1,9 +1,8 @@ -//===-- hwasan_interceptors.cc --------------------------------------------===// +//===-- hwasan_interceptors.cpp -------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -179,6 +178,11 @@ void * __sanitizer_realloc(void *ptr, uptr size) { return hwasan_realloc(ptr, size, &stack); } +void * __sanitizer_reallocarray(void *ptr, uptr nmemb, uptr size) { + GET_MALLOC_STACK_TRACE; + return hwasan_reallocarray(ptr, nmemb, size, &stack); +} + void * __sanitizer_malloc(uptr size) { GET_MALLOC_STACK_TRACE; if (UNLIKELY(!hwasan_init_is_running)) @@ -205,6 +209,7 @@ INTERCEPTOR_ALIAS(void, free, void *ptr); INTERCEPTOR_ALIAS(uptr, malloc_usable_size, const void *ptr); INTERCEPTOR_ALIAS(void *, calloc, SIZE_T nmemb, SIZE_T size); INTERCEPTOR_ALIAS(void *, realloc, void *ptr, SIZE_T size); +INTERCEPTOR_ALIAS(void *, reallocarray, void *ptr, SIZE_T nmemb, SIZE_T size); INTERCEPTOR_ALIAS(void *, malloc, SIZE_T size); #if !SANITIZER_FREEBSD && !SANITIZER_NETBSD @@ -228,6 +233,11 @@ INTERCEPTOR(int, pthread_create, void *th, void *attr, } #endif +#if HWASAN_WITH_INTERCEPTORS +DEFINE_REAL(int, vfork); +DECLARE_EXTERN_INTERCEPTOR_AND_WRAPPER(int, vfork); +#endif + static void BeforeFork() { StackDepotLockAll(); } @@ -267,9 +277,12 @@ void InitializeInterceptors() { INTERCEPT_FUNCTION(fork); #if HWASAN_WITH_INTERCEPTORS +#if defined(__linux__) + INTERCEPT_FUNCTION(vfork); +#endif // __linux__ #if !defined(__aarch64__) INTERCEPT_FUNCTION(pthread_create); -#endif +#endif // __aarch64__ INTERCEPT_FUNCTION(realloc); INTERCEPT_FUNCTION(free); #endif diff --git a/lib/hwasan/hwasan_interceptors_vfork.S b/lib/hwasan/hwasan_interceptors_vfork.S new file mode 100644 index 000000000..13d0829c0 --- /dev/null +++ b/lib/hwasan/hwasan_interceptors_vfork.S @@ -0,0 +1,10 @@ +#include "sanitizer_common/sanitizer_asm.h" + +#if defined(__linux__) && HWASAN_WITH_INTERCEPTORS +#define COMMON_INTERCEPTOR_SPILL_AREA __hwasan_extra_spill_area +#define COMMON_INTERCEPTOR_HANDLE_VFORK __hwasan_handle_vfork +#include "sanitizer_common/sanitizer_common_interceptors_vfork_aarch64.inc.S" +#include "sanitizer_common/sanitizer_common_interceptors_vfork_x86_64.inc.S" +#endif + +NO_EXEC_STACK_DIRECTIVE diff --git a/lib/hwasan/hwasan_interface_internal.h b/lib/hwasan/hwasan_interface_internal.h index d3b2087d0..1b10d76c7 100644 --- a/lib/hwasan/hwasan_interface_internal.h +++ b/lib/hwasan/hwasan_interface_internal.h @@ -1,9 +1,8 @@ //===-- hwasan_interface_internal.h -----------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -21,7 +20,7 @@ extern "C" { SANITIZER_INTERFACE_ATTRIBUTE -void __hwasan_shadow_init(); +void __hwasan_init_static(); SANITIZER_INTERFACE_ATTRIBUTE void __hwasan_init(); @@ -101,6 +100,9 @@ SANITIZER_INTERFACE_ATTRIBUTE uptr __hwasan_tag_pointer(uptr p, u8 tag); SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_tag_mismatch(uptr addr, u8 ts); + +SANITIZER_INTERFACE_ATTRIBUTE u8 __hwasan_generate_tag(); // Returns the offset of the first tag mismatch or -1 if the whole range is @@ -118,6 +120,9 @@ SANITIZER_INTERFACE_ATTRIBUTE void __hwasan_handle_longjmp(const void *sp_dst); SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_handle_vfork(const void *sp_dst); + +SANITIZER_INTERFACE_ATTRIBUTE u16 __sanitizer_unaligned_load16(const uu16 *p); SANITIZER_INTERFACE_ATTRIBUTE @@ -193,6 +198,9 @@ SANITIZER_INTERFACE_ATTRIBUTE void * __sanitizer_realloc(void *ptr, uptr size); SANITIZER_INTERFACE_ATTRIBUTE +void * __sanitizer_reallocarray(void *ptr, uptr nmemb, uptr size); + +SANITIZER_INTERFACE_ATTRIBUTE void * __sanitizer_malloc(uptr size); SANITIZER_INTERFACE_ATTRIBUTE diff --git a/lib/hwasan/hwasan_linux.cc b/lib/hwasan/hwasan_linux.cpp index 5b0a8b4ac..d93297648 100644 --- a/lib/hwasan/hwasan_linux.cc +++ b/lib/hwasan/hwasan_linux.cpp @@ -1,9 +1,8 @@ -//===-- hwasan_linux.cc -----------------------------------------*- C++ -*-===// +//===-- hwasan_linux.cpp ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// @@ -39,7 +38,17 @@ #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_procmaps.h" -#if HWASAN_WITH_INTERCEPTORS && !SANITIZER_ANDROID +// Configurations of HWASAN_WITH_INTERCEPTORS and SANITIZER_ANDROID. +// +// HWASAN_WITH_INTERCEPTORS=OFF, SANITIZER_ANDROID=OFF +// Not currently tested. +// HWASAN_WITH_INTERCEPTORS=OFF, SANITIZER_ANDROID=ON +// Integration tests downstream exist. +// HWASAN_WITH_INTERCEPTORS=ON, SANITIZER_ANDROID=OFF +// Tested with check-hwasan on x86_64-linux. +// HWASAN_WITH_INTERCEPTORS=ON, SANITIZER_ANDROID=ON +// Tested with check-hwasan on aarch64-linux-android. +#if !SANITIZER_ANDROID SANITIZER_INTERFACE_ATTRIBUTE THREADLOCAL uptr __hwasan_tls; #endif @@ -219,6 +228,8 @@ bool MemIsApp(uptr p) { } static void HwasanAtExit(void) { + if (common_flags()->print_module_map) + DumpProcessMap(); if (flags()->print_stats && (flags()->atexit || hwasan_report_count > 0)) ReportStats(); if (hwasan_report_count > 0) { @@ -235,7 +246,7 @@ void InstallAtExitHandler() { // ---------------------- TSD ---------------- {{{1 extern "C" void __hwasan_thread_enter() { - hwasanThreadList().CreateCurrentThread(); + hwasanThreadList().CreateCurrentThread()->InitRandomState(); } extern "C" void __hwasan_thread_exit() { @@ -288,7 +299,9 @@ uptr *GetCurrentThreadLongPtr() { #if SANITIZER_ANDROID void AndroidTestTlsSlot() { uptr kMagicValue = 0x010203040A0B0C0D; - *(uptr *)get_android_tls_ptr() = kMagicValue; + uptr *tls_ptr = GetCurrentThreadLongPtr(); + uptr old_value = *tls_ptr; + *tls_ptr = kMagicValue; dlerror(); if (*(uptr *)get_android_tls_ptr() != kMagicValue) { Printf( @@ -296,6 +309,7 @@ void AndroidTestTlsSlot() { "for dlerror().\n"); Die(); } + *tls_ptr = old_value; } #else void AndroidTestTlsSlot() {} @@ -369,22 +383,35 @@ static AccessInfo GetAccessInfo(siginfo_t *info, ucontext_t *uc) { return AccessInfo{addr, size, is_store, !is_store, recover}; } +static void HandleTagMismatch(AccessInfo ai, uptr pc, uptr frame, + ucontext_t *uc, uptr *registers_frame = nullptr) { + InternalMmapVector<BufferedStackTrace> stack_buffer(1); + BufferedStackTrace *stack = stack_buffer.data(); + stack->Reset(); + stack->Unwind(pc, frame, uc, common_flags()->fast_unwind_on_fatal); + + // The second stack frame contains the failure __hwasan_check function, as + // we have a stack frame for the registers saved in __hwasan_tag_mismatch that + // we wish to ignore. This (currently) only occurs on AArch64, as x64 + // implementations use SIGTRAP to implement the failure, and thus do not go + // through the stack saver. + if (registers_frame && stack->trace && stack->size > 0) { + stack->trace++; + stack->size--; + } + + bool fatal = flags()->halt_on_error || !ai.recover; + ReportTagMismatch(stack, ai.addr, ai.size, ai.is_store, fatal, + registers_frame); +} + static bool HwasanOnSIGTRAP(int signo, siginfo_t *info, ucontext_t *uc) { AccessInfo ai = GetAccessInfo(info, uc); if (!ai.is_store && !ai.is_load) return false; - InternalMmapVector<BufferedStackTrace> stack_buffer(1); - BufferedStackTrace *stack = stack_buffer.data(); - stack->Reset(); SignalContext sig{info, uc}; - GetStackTrace(stack, kStackTraceMax, StackTrace::GetNextInstructionPc(sig.pc), - sig.bp, uc, common_flags()->fast_unwind_on_fatal); - - ++hwasan_report_count; - - bool fatal = flags()->halt_on_error || !ai.recover; - ReportTagMismatch(stack, ai.addr, ai.size, ai.is_store, fatal); + HandleTagMismatch(ai, StackTrace::GetNextInstructionPc(sig.pc), sig.bp, uc); #if defined(__aarch64__) uc->uc_mcontext.pc += 4; @@ -395,10 +422,25 @@ static bool HwasanOnSIGTRAP(int signo, siginfo_t *info, ucontext_t *uc) { return true; } +// Entry point stub for interoperability between __hwasan_tag_mismatch (ASM) and +// the rest of the mismatch handling code (C++). +extern "C" void __hwasan_tag_mismatch_stub(uptr addr, uptr access_info, + uptr *registers_frame) { + AccessInfo ai; + ai.is_store = access_info & 0x10; + ai.recover = false; + ai.addr = addr; + ai.size = 1 << (access_info & 0xf); + + HandleTagMismatch(ai, (uptr)__builtin_return_address(0), + (uptr)__builtin_frame_address(0), nullptr, registers_frame); + __builtin_unreachable(); +} + static void OnStackUnwind(const SignalContext &sig, const void *, BufferedStackTrace *stack) { - GetStackTrace(stack, kStackTraceMax, StackTrace::GetNextInstructionPc(sig.pc), - sig.bp, sig.context, common_flags()->fast_unwind_on_fatal); + stack->Unwind(StackTrace::GetNextInstructionPc(sig.pc), sig.bp, sig.context, + common_flags()->fast_unwind_on_fatal); } void HwasanOnDeadlySignal(int signo, void *info, void *context) { diff --git a/lib/hwasan/hwasan_malloc_bisect.h b/lib/hwasan/hwasan_malloc_bisect.h new file mode 100644 index 000000000..eaf124aab --- /dev/null +++ b/lib/hwasan/hwasan_malloc_bisect.h @@ -0,0 +1,50 @@ +//===-- hwasan_malloc_bisect.h ----------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HWAddressSanitizer. +// +//===----------------------------------------------------------------------===// + +#include "sanitizer_common/sanitizer_hash.h" +#include "hwasan.h" + +namespace __hwasan { + +static u32 malloc_hash(StackTrace *stack, uptr orig_size) { + uptr len = Min(stack->size, (unsigned)7); + MurMur2HashBuilder H(len); + H.add(orig_size); + // Start with frame #1 to skip __sanitizer_malloc frame, which is + // (a) almost always the same (well, could be operator new or new[]) + // (b) can change hashes when compiler-rt is rebuilt, invalidating previous + // bisection results. + // Because of ASLR, use only offset inside the page. + for (uptr i = 1; i < len; ++i) H.add(((u32)stack->trace[i]) & 0xFFF); + return H.get(); +} + +static INLINE bool malloc_bisect(StackTrace *stack, uptr orig_size) { + uptr left = flags()->malloc_bisect_left; + uptr right = flags()->malloc_bisect_right; + if (LIKELY(left == 0 && right == 0)) + return true; + if (!stack) + return true; + // Allow malloc_bisect_right > (u32)(-1) to avoid spelling the latter in + // decimal. + uptr h = (uptr)malloc_hash(stack, orig_size); + if (h < left || h > right) + return false; + if (flags()->malloc_bisect_dump) { + Printf("[alloc] %u %zu\n", h, orig_size); + stack->Print(); + } + return true; +} + +} // namespace __hwasan diff --git a/lib/hwasan/hwasan_mapping.h b/lib/hwasan/hwasan_mapping.h index e5e23dc60..a86ad7ca8 100644 --- a/lib/hwasan/hwasan_mapping.h +++ b/lib/hwasan/hwasan_mapping.h @@ -1,9 +1,8 @@ //===-- hwasan_mapping.h ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/lib/hwasan/hwasan_memintrinsics.cc b/lib/hwasan/hwasan_memintrinsics.cpp index 9cb844e45..e82d77a1b 100644 --- a/lib/hwasan/hwasan_memintrinsics.cc +++ b/lib/hwasan/hwasan_memintrinsics.cpp @@ -1,9 +1,8 @@ -//===-- hwasan_memintrinsics.cc ---------------------------------*- C++ -*-===// +//===-- hwasan_memintrinsics.cpp --------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/lib/hwasan/hwasan_new_delete.cc b/lib/hwasan/hwasan_new_delete.cpp index f2e8faf5d..438a3699a 100644 --- a/lib/hwasan/hwasan_new_delete.cc +++ b/lib/hwasan/hwasan_new_delete.cpp @@ -1,9 +1,8 @@ -//===-- hwasan_new_delete.cc ----------------------------------------------===// +//===-- hwasan_new_delete.cpp ---------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/hwasan/hwasan_poisoning.cc b/lib/hwasan/hwasan_poisoning.cc deleted file mode 100644 index 9c8e16b12..000000000 --- a/lib/hwasan/hwasan_poisoning.cc +++ /dev/null @@ -1,37 +0,0 @@ -//===-- hwasan_poisoning.cc -------------------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file is a part of HWAddressSanitizer. -// -//===----------------------------------------------------------------------===// - -#include "hwasan_poisoning.h" - -#include "hwasan_mapping.h" -#include "interception/interception.h" -#include "sanitizer_common/sanitizer_common.h" - -namespace __hwasan { - -uptr TagMemoryAligned(uptr p, uptr size, tag_t tag) { - CHECK(IsAligned(p, kShadowAlignment)); - CHECK(IsAligned(size, kShadowAlignment)); - uptr shadow_start = MemToShadow(p); - uptr shadow_size = MemToShadowSize(size); - internal_memset((void *)shadow_start, tag, shadow_size); - return AddTagToPointer(p, tag); -} - -uptr TagMemory(uptr p, uptr size, tag_t tag) { - uptr start = RoundDownTo(p, kShadowAlignment); - uptr end = RoundUpTo(p + size, kShadowAlignment); - return TagMemoryAligned(start, end - start, tag); -} - -} // namespace __hwasan diff --git a/lib/hwasan/hwasan_poisoning.cpp b/lib/hwasan/hwasan_poisoning.cpp new file mode 100644 index 000000000..2a0816428 --- /dev/null +++ b/lib/hwasan/hwasan_poisoning.cpp @@ -0,0 +1,52 @@ +//===-- hwasan_poisoning.cpp ------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HWAddressSanitizer. +// +//===----------------------------------------------------------------------===// + +#include "hwasan_poisoning.h" + +#include "hwasan_mapping.h" +#include "interception/interception.h" +#include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_linux.h" + +namespace __hwasan { + +uptr TagMemoryAligned(uptr p, uptr size, tag_t tag) { + CHECK(IsAligned(p, kShadowAlignment)); + CHECK(IsAligned(size, kShadowAlignment)); + uptr shadow_start = MemToShadow(p); + uptr shadow_size = MemToShadowSize(size); + + uptr page_size = GetPageSizeCached(); + uptr page_start = RoundUpTo(shadow_start, page_size); + uptr page_end = RoundDownTo(shadow_start + shadow_size, page_size); + uptr threshold = common_flags()->clear_shadow_mmap_threshold; + if (SANITIZER_LINUX && + UNLIKELY(page_end >= page_start + threshold && tag == 0)) { + internal_memset((void *)shadow_start, tag, page_start - shadow_start); + internal_memset((void *)page_end, tag, + shadow_start + shadow_size - page_end); + // For an anonymous private mapping MADV_DONTNEED will return a zero page on + // Linux. + ReleaseMemoryPagesToOSAndZeroFill(page_start, page_end); + } else { + internal_memset((void *)shadow_start, tag, shadow_size); + } + return AddTagToPointer(p, tag); +} + +uptr TagMemory(uptr p, uptr size, tag_t tag) { + uptr start = RoundDownTo(p, kShadowAlignment); + uptr end = RoundUpTo(p + size, kShadowAlignment); + return TagMemoryAligned(start, end - start, tag); +} + +} // namespace __hwasan diff --git a/lib/hwasan/hwasan_poisoning.h b/lib/hwasan/hwasan_poisoning.h index 0dbf9d8ed..61751f7d2 100644 --- a/lib/hwasan/hwasan_poisoning.h +++ b/lib/hwasan/hwasan_poisoning.h @@ -1,9 +1,8 @@ //===-- hwasan_poisoning.h --------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/hwasan/hwasan_report.cc b/lib/hwasan/hwasan_report.cpp index ea3e4096d..fa2fff742 100644 --- a/lib/hwasan/hwasan_report.cc +++ b/lib/hwasan/hwasan_report.cpp @@ -1,9 +1,8 @@ -//===-- hwasan_report.cc --------------------------------------------------===// +//===-- hwasan_report.cpp -------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -15,6 +14,7 @@ #include "hwasan.h" #include "hwasan_allocator.h" #include "hwasan_mapping.h" +#include "hwasan_report.h" #include "hwasan_thread.h" #include "hwasan_thread_list.h" #include "sanitizer_common/sanitizer_allocator_internal.h" @@ -35,15 +35,21 @@ class ScopedReport { ScopedReport(bool fatal = false) : error_message_(1), fatal(fatal) { BlockingMutexLock lock(&error_message_lock_); error_message_ptr_ = fatal ? &error_message_ : nullptr; + ++hwasan_report_count; } ~ScopedReport() { - BlockingMutexLock lock(&error_message_lock_); - if (fatal) { - SetAbortMessage(error_message_.data()); - Die(); + { + BlockingMutexLock lock(&error_message_lock_); + if (fatal) + SetAbortMessage(error_message_.data()); + error_message_ptr_ = nullptr; } - error_message_ptr_ = nullptr; + if (common_flags()->print_module_map >= 2 || + (fatal && common_flags()->print_module_map)) + DumpProcessMap(); + if (fatal) + Die(); } static void MaybeAppendToErrorMessage(const char *msg) { @@ -247,8 +253,8 @@ void PrintAddressDescription( uptr pc_mask = (1ULL << 48) - 1; uptr pc = record & pc_mask; if (SymbolizedStack *frame = Symbolizer::GetOrInit()->SymbolizePC(pc)) { - frame_desc.append(" sp: 0x%zx pc: %p ", sp, pc); - RenderFrame(&frame_desc, "in %f %s:%l\n", 0, frame->info, + frame_desc.append(" sp: 0x%zx ", sp); + RenderFrame(&frame_desc, "#%n %p %F %L\n", 0, frame->info, common_flags()->symbolize_vs_style, common_flags()->strip_path_prefix); frame->ClearAll(); @@ -384,7 +390,7 @@ void ReportTailOverwritten(StackTrace *stack, uptr tagged_addr, uptr orig_size, } void ReportTagMismatch(StackTrace *stack, uptr tagged_addr, uptr access_size, - bool is_store, bool fatal) { + bool is_store, bool fatal, uptr *registers_frame) { ScopedReport R(fatal); SavedStackAllocations current_stack_allocations( GetCurrentThread()->stack_allocations()); @@ -400,13 +406,21 @@ void ReportTagMismatch(StackTrace *stack, uptr tagged_addr, uptr access_size, Thread *t = GetCurrentThread(); + sptr offset = + __hwasan_test_shadow(reinterpret_cast<void *>(tagged_addr), access_size); + CHECK(offset >= 0 && offset < static_cast<sptr>(access_size)); tag_t ptr_tag = GetTagFromPointer(tagged_addr); - tag_t *tag_ptr = reinterpret_cast<tag_t*>(MemToShadow(untagged_addr)); + tag_t *tag_ptr = + reinterpret_cast<tag_t *>(MemToShadow(untagged_addr + offset)); tag_t mem_tag = *tag_ptr; + Printf("%s", d.Access()); Printf("%s of size %zu at %p tags: %02x/%02x (ptr/mem) in thread T%zd\n", is_store ? "WRITE" : "READ", access_size, untagged_addr, ptr_tag, mem_tag, t->unique_id()); + if (offset != 0) + Printf("Invalid access starting at offset [%zu, %zu)\n", offset, + Min(access_size, static_cast<uptr>(offset) + (1 << kShadowScale))); Printf("%s", d.Default()); stack->Print(); @@ -417,7 +431,37 @@ void ReportTagMismatch(StackTrace *stack, uptr tagged_addr, uptr access_size, PrintTagsAroundAddr(tag_ptr); + if (registers_frame) + ReportRegisters(registers_frame, pc); + ReportErrorSummary(bug_type, stack); } +// See the frame breakdown defined in __hwasan_tag_mismatch (from +// hwasan_tag_mismatch_aarch64.S). +void ReportRegisters(uptr *frame, uptr pc) { + Printf("Registers where the failure occurred (pc %p):\n", pc); + + // We explicitly print a single line (4 registers/line) each iteration to + // reduce the amount of logcat error messages printed. Each Printf() will + // result in a new logcat line, irrespective of whether a newline is present, + // and so we wish to reduce the number of Printf() calls we have to make. + Printf(" x0 %016llx x1 %016llx x2 %016llx x3 %016llx\n", + frame[0], frame[1], frame[2], frame[3]); + Printf(" x4 %016llx x5 %016llx x6 %016llx x7 %016llx\n", + frame[4], frame[5], frame[6], frame[7]); + Printf(" x8 %016llx x9 %016llx x10 %016llx x11 %016llx\n", + frame[8], frame[9], frame[10], frame[11]); + Printf(" x12 %016llx x13 %016llx x14 %016llx x15 %016llx\n", + frame[12], frame[13], frame[14], frame[15]); + Printf(" x16 %016llx x17 %016llx x18 %016llx x19 %016llx\n", + frame[16], frame[17], frame[18], frame[19]); + Printf(" x20 %016llx x21 %016llx x22 %016llx x23 %016llx\n", + frame[20], frame[21], frame[22], frame[23]); + Printf(" x24 %016llx x25 %016llx x26 %016llx x27 %016llx\n", + frame[24], frame[25], frame[26], frame[27]); + Printf(" x28 %016llx x29 %016llx x30 %016llx\n", + frame[28], frame[29], frame[30]); +} + } // namespace __hwasan diff --git a/lib/hwasan/hwasan_report.h b/lib/hwasan/hwasan_report.h index 10fb20cc5..f03eb7a69 100644 --- a/lib/hwasan/hwasan_report.h +++ b/lib/hwasan/hwasan_report.h @@ -1,9 +1,8 @@ //===-- hwasan_report.h -----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// @@ -23,11 +22,11 @@ namespace __hwasan { void ReportStats(); void ReportTagMismatch(StackTrace *stack, uptr addr, uptr access_size, - bool is_store, bool fatal); + bool is_store, bool fatal, uptr *registers_frame); void ReportInvalidFree(StackTrace *stack, uptr addr); void ReportTailOverwritten(StackTrace *stack, uptr addr, uptr orig_size, uptr tail_size, const u8 *expected); - +void ReportRegisters(uptr *registers_frame, uptr pc); void ReportAtExitStatistics(); diff --git a/lib/hwasan/hwasan_tag_mismatch_aarch64.S b/lib/hwasan/hwasan_tag_mismatch_aarch64.S new file mode 100644 index 000000000..92f627480 --- /dev/null +++ b/lib/hwasan/hwasan_tag_mismatch_aarch64.S @@ -0,0 +1,106 @@ +#include "sanitizer_common/sanitizer_asm.h" + +// The content of this file is AArch64-only: +#if defined(__aarch64__) + +// The responsibility of the HWASan entry point in compiler-rt is to primarily +// readjust the stack from the callee and save the current register values to +// the stack. +// This entry point function should be called from a __hwasan_check_* symbol. +// These are generated during a lowering pass in the backend, and are found in +// AArch64AsmPrinter::EmitHwasanMemaccessSymbols(). Please look there for +// further information. +// The __hwasan_check_* caller of this function should have expanded the stack +// and saved the previous values of x0, x1, x29, and x30. This function will +// "consume" these saved values and treats it as part of its own stack frame. +// In this sense, the __hwasan_check_* callee and this function "share" a stack +// frame. This allows us to omit having unwinding information (.cfi_*) present +// in every __hwasan_check_* function, therefore reducing binary size. This is +// particularly important as hwasan_check_* instances are duplicated in every +// translation unit where HWASan is enabled. +// This function calls HwasanTagMismatch to step back into the C++ code that +// completes the stack unwinding and error printing. This function is is not +// permitted to return. + + +// Frame from __hwasan_check_: +// | ... | +// | ... | +// | Previous stack frames... | +// +=================================+ +// | Unused 8-bytes for maintaining | +// | 16-byte SP alignment. | +// +---------------------------------+ +// | Return address (x30) for caller | +// | of __hwasan_check_*. | +// +---------------------------------+ +// | Frame address (x29) for caller | +// | of __hwasan_check_* | +// +---------------------------------+ <-- [SP + 232] +// | ... | +// | | +// | Stack frame space for x2 - x28. | +// | | +// | ... | +// +---------------------------------+ <-- [SP + 16] +// | | +// | Saved x1, as __hwasan_check_* | +// | clobbers it. | +// +---------------------------------+ +// | Saved x0, likewise above. | +// +---------------------------------+ <-- [x30 / SP] + +// This function takes two arguments: +// * x0: The address of read/write instruction that caused HWASan check fail. +// * x1: The tag size. + +.section .text +.file "hwasan_tag_mismatch_aarch64.S" +.global __hwasan_tag_mismatch +.type __hwasan_tag_mismatch, %function +__hwasan_tag_mismatch: + CFI_STARTPROC + + // Set the CFA to be the return address for caller of __hwasan_check_*. Note + // that we do not emit CFI predicates to describe the contents of this stack + // frame, as this proxy entry point should never be debugged. The contents + // are static and are handled by the unwinder after calling + // __hwasan_tag_mismatch. The frame pointer is already correctly setup + // by __hwasan_check_*. + add x29, sp, #232 + CFI_DEF_CFA(w29, 24) + CFI_OFFSET(w30, -16) + CFI_OFFSET(w29, -24) + + // Save the rest of the registers into the preallocated space left by + // __hwasan_check. + str x28, [sp, #224] + stp x26, x27, [sp, #208] + stp x24, x25, [sp, #192] + stp x22, x23, [sp, #176] + stp x20, x21, [sp, #160] + stp x18, x19, [sp, #144] + stp x16, x17, [sp, #128] + stp x14, x15, [sp, #112] + stp x12, x13, [sp, #96] + stp x10, x11, [sp, #80] + stp x8, x9, [sp, #64] + stp x6, x7, [sp, #48] + stp x4, x5, [sp, #32] + stp x2, x3, [sp, #16] + + // Pass the address of the frame to __hwasan_tag_mismatch_stub, so that it can + // extract the saved registers from this frame without having to worry about + // finding this frame. + mov x2, sp + + bl __hwasan_tag_mismatch_stub + CFI_ENDPROC + +.Lfunc_end0: + .size __hwasan_tag_mismatch, .Lfunc_end0-__hwasan_tag_mismatch + +#endif // defined(__aarch64__) + +// We do not need executable stack. +NO_EXEC_STACK_DIRECTIVE diff --git a/lib/hwasan/hwasan_thread.cc b/lib/hwasan/hwasan_thread.cpp index 631c2813e..46dcddd42 100644 --- a/lib/hwasan/hwasan_thread.cc +++ b/lib/hwasan/hwasan_thread.cpp @@ -25,10 +25,13 @@ static u32 RandomSeed() { return seed; } +void Thread::InitRandomState() { + random_state_ = flags()->random_tags ? RandomSeed() : unique_id_; +} + void Thread::Init(uptr stack_buffer_start, uptr stack_buffer_size) { static u64 unique_id; unique_id_ = unique_id++; - random_state_ = flags()->random_tags ? RandomSeed() : unique_id_; if (auto sz = flags()->heap_history_size) heap_allocations_ = HeapAllocationsRingBuffer::New(sz); diff --git a/lib/hwasan/hwasan_thread.h b/lib/hwasan/hwasan_thread.h index 4830473f4..6fa592bfa 100644 --- a/lib/hwasan/hwasan_thread.h +++ b/lib/hwasan/hwasan_thread.h @@ -1,9 +1,8 @@ //===-- hwasan_thread.h -----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -25,6 +24,7 @@ typedef __sanitizer::CompactRingBuffer<uptr> StackAllocationsRingBuffer; class Thread { public: void Init(uptr stack_buffer_start, uptr stack_buffer_size); // Must be called from the thread itself. + void InitRandomState(); void Destroy(); uptr stack_top() { return stack_top_; } @@ -67,11 +67,14 @@ class Thread { Print("Thread: "); } + uptr &vfork_spill() { return vfork_spill_; } + private: // NOTE: There is no Thread constructor. It is allocated // via mmap() and *must* be valid in zero-initialized state. void ClearShadowForThreadStackAndTLS(); void Print(const char *prefix); + uptr vfork_spill_; uptr stack_top_; uptr stack_bottom_; uptr tls_begin_; diff --git a/lib/hwasan/hwasan_thread_list.cc b/lib/hwasan/hwasan_thread_list.cpp index a31eee84e..a31eee84e 100644 --- a/lib/hwasan/hwasan_thread_list.cc +++ b/lib/hwasan/hwasan_thread_list.cpp diff --git a/lib/hwasan/hwasan_thread_list.h b/lib/hwasan/hwasan_thread_list.h index 53747b51f..914b632d9 100644 --- a/lib/hwasan/hwasan_thread_list.h +++ b/lib/hwasan/hwasan_thread_list.h @@ -1,9 +1,8 @@ //===-- hwasan_thread_list.h ------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -109,38 +108,52 @@ struct ThreadStats { class HwasanThreadList { public: HwasanThreadList(uptr storage, uptr size) - : free_space_(storage), - free_space_end_(storage + size), - ring_buffer_size_(RingBufferSize()) {} + : free_space_(storage), free_space_end_(storage + size) { + // [storage, storage + size) is used as a vector of + // thread_alloc_size_-sized, ring_buffer_size_*2-aligned elements. + // Each element contains + // * a ring buffer at offset 0, + // * a Thread object at offset ring_buffer_size_. + ring_buffer_size_ = RingBufferSize(); + thread_alloc_size_ = + RoundUpTo(ring_buffer_size_ + sizeof(Thread), ring_buffer_size_ * 2); + } Thread *CreateCurrentThread() { Thread *t; { SpinMutexLock l(&list_mutex_); t = free_list_.Pop(); - if (t) - internal_memset((void *)t, 0, sizeof(Thread) + ring_buffer_size_); - else + if (t) { + uptr start = (uptr)t - ring_buffer_size_; + internal_memset((void *)start, 0, ring_buffer_size_ + sizeof(Thread)); + } else { t = AllocThread(); + } live_list_.Push(t); } - t->Init((uptr)(t + 1), ring_buffer_size_); + t->Init((uptr)t - ring_buffer_size_, ring_buffer_size_); AddThreadStats(t); return t; } + void DontNeedThread(Thread *t) { + uptr start = (uptr)t - ring_buffer_size_; + ReleaseMemoryPagesToOS(start, start + thread_alloc_size_); + } + void ReleaseThread(Thread *t) { - // FIXME: madvise away the ring buffer? RemoveThreadStats(t); t->Destroy(); SpinMutexLock l(&list_mutex_); live_list_.Remove(t); free_list_.Push(t); + DontNeedThread(t); } Thread *GetThreadByBufferAddress(uptr p) { - uptr align = ring_buffer_size_ * 2; - return (Thread *)(RoundDownTo(p, align) - sizeof(Thread)); + return (Thread *)(RoundDownTo(p, ring_buffer_size_ * 2) + + ring_buffer_size_); } uptr MemoryUsedPerThread() { @@ -176,15 +189,17 @@ class HwasanThreadList { private: Thread *AllocThread() { uptr align = ring_buffer_size_ * 2; - uptr ring_buffer_start = RoundUpTo(free_space_ + sizeof(Thread), align); - free_space_ = ring_buffer_start + ring_buffer_size_; + CHECK(IsAligned(free_space_, align)); + Thread *t = (Thread *)(free_space_ + ring_buffer_size_); + free_space_ += thread_alloc_size_; CHECK(free_space_ <= free_space_end_ && "out of thread memory"); - return (Thread *)(ring_buffer_start - sizeof(Thread)); + return t; } uptr free_space_; uptr free_space_end_; uptr ring_buffer_size_; + uptr thread_alloc_size_; ThreadListHead free_list_; ThreadListHead live_list_; |